From 3f3e5b9ed0d9e5d91f337f59b6f78147f38fd6d4 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 28 Nov 2023 10:17:05 +0100 Subject: [PATCH 1/8] add key_post_processor, need to accept optional additional args after first argument being bytes --- boaconstructor/__init__.py | 12 +++++--- boaconstructor/storage.py | 58 ++++++++++++++++++++++++++++++++++++ examples/nep17/test_nep17.py | 5 ++-- 3 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 boaconstructor/storage.py diff --git a/boaconstructor/__init__.py b/boaconstructor/__init__.py index 3eda817..bb02e41 100644 --- a/boaconstructor/__init__.py +++ b/boaconstructor/__init__.py @@ -4,7 +4,7 @@ import signal import re import inspect -from typing import Optional, TypeVar, Type, cast, Sequence +from typing import Optional, TypeVar, Type, cast, Sequence, Callable, ParamSpec, Concatenate from neo3.core import types, cryptography from neo3.wallet import account from neo3.api.wrappers import GenericContract, NEP17Contract, ChainFacade @@ -23,6 +23,7 @@ ASSERT_REASON = re.compile(r".*Reason: (.*)") +P = ParamSpec('P') class AssertException(Exception): pass @@ -245,6 +246,7 @@ async def get_storage( *, target_contract: Optional[types.UInt160] = None, remove_prefix: bool = False, + key_post_processor: Optional[Callable[[bytes, Optional[P]], T]] = None ) -> dict[bytes, bytes]: """ Gets the entries in the storage of the contract specified by `contract_hash` @@ -253,6 +255,7 @@ async def get_storage( prefix: prefix to filter the entries in the storage. Return the entire storage if not set. target_contract: gets the storage of a different contract than the one under test. e.g. NeoToken remove_prefix: whether the prefix should be removed from the output keys. False by default. + key_post_processor: a function to post process the storage key before placing it in the dictionary """ if target_contract is None: contract = GenericContract(cls.contract_hash) @@ -266,9 +269,10 @@ async def get_storage( async with noderpc.NeoRpcClient(cls.node.facade.rpc_host) as rpc_client: async for k, v in rpc_client.find_states(contract.hash, prefix): if remove_prefix: - results[k.removeprefix(prefix)] = v - else: - results[k] = v + k = k.removeprefix(prefix) + if key_post_processor is not None: + k = key_post_processor(k) + results[k] = v return results diff --git a/boaconstructor/storage.py b/boaconstructor/storage.py new file mode 100644 index 0000000..6a5d727 --- /dev/null +++ b/boaconstructor/storage.py @@ -0,0 +1,58 @@ +from neo3.core import types, cryptography +from neo3.wallet import utils as walletutils +from neo3.wallet.types import NeoAddress + + +def as_uint160(data: bytes, *args) -> types.UInt160: + """ + Convert the data to a UInt160 + + Args: + data: a serialized UInt160 + """ + return types.UInt160(data) + + +def as_uint256(data: bytes) -> types.UInt256: + """ + Convert the data to a UInt256 + + Args: + data: a serialized UInt256 + """ + return types.UInt256(data) + + +def as_int(data: bytes) -> int: + """ + Convert the data to an integer + """ + return int(types.BigInteger(data)) + + +def as_str(data: bytes) -> str: + """ + Convert the data to a UTF-8 encoded string + """ + return data.decode() + + +def as_address(data: bytes) -> NeoAddress: + """ + Convert the data to a Neo address string + + Args: + data: a serialized UInt160 + """ + return walletutils.script_hash_to_address(types.UInt160(data)) + + +def as_public_key(data: bytes) -> cryptography.ECPoint: + """ + Convert the data to a public key + + Args: + data: a serialized compressed public key + """ + return cryptography.ECPoint.deserialize_from_bytes(data) + diff --git a/examples/nep17/test_nep17.py b/examples/nep17/test_nep17.py index 8b898e5..6b9cd99 100644 --- a/examples/nep17/test_nep17.py +++ b/examples/nep17/test_nep17.py @@ -70,9 +70,8 @@ async def test_02_balance_of(self): signing_account=self.user1, ) self.assertTrue(success) - - storage = await self.get_storage(prefix=self.balance_prefix, remove_prefix=True) - + from boaconstructor import storage as stor + storage = await self.get_storage(prefix=self.balance_prefix, key_post_processor=stor.as_uint160) result, _ = await self.call( "balanceOf", [self.user1.script_hash], return_type=int ) From b44b395c9ad647be59123eed802d9fad7b3bb1f9 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 28 Nov 2023 10:46:57 +0100 Subject: [PATCH 2/8] use callback protocol for PostProcessor --- boaconstructor/__init__.py | 6 +++--- boaconstructor/storage.py | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/boaconstructor/__init__.py b/boaconstructor/__init__.py index bb02e41..c2730cf 100644 --- a/boaconstructor/__init__.py +++ b/boaconstructor/__init__.py @@ -4,7 +4,7 @@ import signal import re import inspect -from typing import Optional, TypeVar, Type, cast, Sequence, Callable, ParamSpec, Concatenate +from typing import Optional, TypeVar, Type, cast, Sequence from neo3.core import types, cryptography from neo3.wallet import account from neo3.api.wrappers import GenericContract, NEP17Contract, ChainFacade @@ -18,12 +18,12 @@ from neo3.contracts import nef, manifest from dataclasses import dataclass from boaconstructor.node import NeoGoNode, Node +from boaconstructor.storage import PostProcessor __version__ = "0.1.3" ASSERT_REASON = re.compile(r".*Reason: (.*)") -P = ParamSpec('P') class AssertException(Exception): pass @@ -246,7 +246,7 @@ async def get_storage( *, target_contract: Optional[types.UInt160] = None, remove_prefix: bool = False, - key_post_processor: Optional[Callable[[bytes, Optional[P]], T]] = None + key_post_processor: Optional[PostProcessor] = None ) -> dict[bytes, bytes]: """ Gets the entries in the storage of the contract specified by `contract_hash` diff --git a/boaconstructor/storage.py b/boaconstructor/storage.py index 6a5d727..72de6a5 100644 --- a/boaconstructor/storage.py +++ b/boaconstructor/storage.py @@ -1,9 +1,18 @@ from neo3.core import types, cryptography from neo3.wallet import utils as walletutils from neo3.wallet.types import NeoAddress +from typing_extensions import Protocol +from typing import Any, TypeVar +T = TypeVar('T') -def as_uint160(data: bytes, *args) -> types.UInt160: + +class PostProcessor(Protocol): + def __call__(self, data: bytes, *args: Any) -> Any: + ... + + +def as_uint160(data: bytes, *_: Any) -> types.UInt160: """ Convert the data to a UInt160 @@ -13,7 +22,7 @@ def as_uint160(data: bytes, *args) -> types.UInt160: return types.UInt160(data) -def as_uint256(data: bytes) -> types.UInt256: +def as_uint256(data: bytes, *_: Any) -> types.UInt256: """ Convert the data to a UInt256 @@ -23,21 +32,21 @@ def as_uint256(data: bytes) -> types.UInt256: return types.UInt256(data) -def as_int(data: bytes) -> int: +def as_int(data: bytes, *_: Any) -> int: """ Convert the data to an integer """ return int(types.BigInteger(data)) -def as_str(data: bytes) -> str: +def as_str(data: bytes, *_: Any) -> str: """ Convert the data to a UTF-8 encoded string """ return data.decode() -def as_address(data: bytes) -> NeoAddress: +def as_address(data: bytes, *_: Any) -> NeoAddress: """ Convert the data to a Neo address string @@ -47,7 +56,7 @@ def as_address(data: bytes) -> NeoAddress: return walletutils.script_hash_to_address(types.UInt160(data)) -def as_public_key(data: bytes) -> cryptography.ECPoint: +def as_public_key(data: bytes, *_: Any) -> cryptography.ECPoint: """ Convert the data to a public key From 9187ca390e8b6cdc5926d04030a5fc2bc14598c2 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 28 Nov 2023 11:00:55 +0100 Subject: [PATCH 3/8] use key_post_processor in NEP17 example and fix all type errors --- boaconstructor/__init__.py | 4 ++-- examples/nep17/test_nep17.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/boaconstructor/__init__.py b/boaconstructor/__init__.py index c2730cf..df301a0 100644 --- a/boaconstructor/__init__.py +++ b/boaconstructor/__init__.py @@ -4,7 +4,7 @@ import signal import re import inspect -from typing import Optional, TypeVar, Type, cast, Sequence +from typing import Optional, TypeVar, Type, Sequence from neo3.core import types, cryptography from neo3.wallet import account from neo3.api.wrappers import GenericContract, NEP17Contract, ChainFacade @@ -246,7 +246,7 @@ async def get_storage( *, target_contract: Optional[types.UInt160] = None, remove_prefix: bool = False, - key_post_processor: Optional[PostProcessor] = None + key_post_processor: Optional[PostProcessor] = None, ) -> dict[bytes, bytes]: """ Gets the entries in the storage of the contract specified by `contract_hash` diff --git a/examples/nep17/test_nep17.py b/examples/nep17/test_nep17.py index 6b9cd99..d4325e3 100644 --- a/examples/nep17/test_nep17.py +++ b/examples/nep17/test_nep17.py @@ -4,11 +4,13 @@ AssertException, SmartContractTestCase, Nep17TransferEvent, + storage as _storage, ) from neo3.api.wrappers import NEP17Contract from neo3.wallet import account from neo3.core import types from neo3.contracts.contract import CONTRACT_HASHES +from typing import cast NEO = CONTRACT_HASHES.NEO_TOKEN GAS = CONTRACT_HASHES.GAS_TOKEN @@ -19,6 +21,7 @@ class Nep17ContractTest(SmartContractTestCase): user1: account.Account user2: account.Account balance_prefix: bytes = b"b" + contract: NEP17Contract @classmethod def setUpClass(cls) -> None: @@ -34,7 +37,7 @@ def setUpClass(cls) -> None: @classmethod async def asyncSetupClass(cls) -> None: - cls.genesis = cls.node.wallet.account_get_by_label("committee") + cls.genesis = cls.node.wallet.account_get_by_label("committee") # type: ignore cls.contract_hash = await cls.deploy("./resources/nep17.nef", cls.genesis) cls.contract = NEP17Contract(cls.contract_hash) await cls.transfer(GAS, cls.genesis.script_hash, cls.user1.script_hash, 100) @@ -70,13 +73,19 @@ async def test_02_balance_of(self): signing_account=self.user1, ) self.assertTrue(success) - from boaconstructor import storage as stor - storage = await self.get_storage(prefix=self.balance_prefix, key_post_processor=stor.as_uint160) + storage = cast( + dict[types.UInt160, bytes], + await self.get_storage( + prefix=self.balance_prefix, + remove_prefix=True, + key_post_processor=_storage.as_uint160, + ), + ) result, _ = await self.call( "balanceOf", [self.user1.script_hash], return_type=int ) self.assertEqual(expected, result) - balance_key = self.user1.script_hash.to_array() + balance_key = self.user1.script_hash self.assertIn(balance_key, storage) self.assertEqual(types.BigInteger(expected).to_array(), storage[balance_key]) @@ -85,7 +94,7 @@ async def test_02_balance_of(self): unknown_account = types.UInt160(b"\x01" * 20) result, _ = await self.call("balanceOf", [unknown_account], return_type=int) self.assertEqual(expected, result) - balance_key = unknown_account.to_array() + balance_key = unknown_account self.assertNotIn(balance_key, storage) # now test invalid account @@ -115,7 +124,14 @@ async def test_03_transfer_success(self): self.assertEqual(1, len(notifications)) self.assertEqual("Transfer", notifications[0].event_name) - storage = await self.get_storage(prefix=self.balance_prefix, remove_prefix=True) + storage = cast( + dict[types.UInt160, bytes], + await self.get_storage( + prefix=self.balance_prefix, + remove_prefix=True, + key_post_processor=_storage.as_uint160, + ), + ) # test we emitted the correct transfer event event = Nep17TransferEvent.from_notification(notifications[0]) @@ -130,12 +146,10 @@ async def test_03_transfer_success(self): self.assertEqual(user1_balance, user2_balance) # test storage updates - user1_balance_key = self.user1.script_hash.to_array() - user2_balance_key = self.user2.script_hash.to_array() - self.assertNotIn(user1_balance_key, storage) - self.assertIn(user2_balance_key, storage) + self.assertNotIn(self.user1.script_hash, storage) + self.assertIn(self.user2.script_hash, storage) self.assertEqual( - types.BigInteger(user1_balance).to_array(), storage[user2_balance_key] + types.BigInteger(user1_balance).to_array(), storage[self.user2.script_hash] ) async def test_onnep17(self): From 58713aaeba555322c750dcc5917e01592430c618 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 28 Nov 2023 14:05:20 +0100 Subject: [PATCH 4/8] delete unused --- boaconstructor/storage.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/boaconstructor/storage.py b/boaconstructor/storage.py index 72de6a5..023456c 100644 --- a/boaconstructor/storage.py +++ b/boaconstructor/storage.py @@ -2,9 +2,7 @@ from neo3.wallet import utils as walletutils from neo3.wallet.types import NeoAddress from typing_extensions import Protocol -from typing import Any, TypeVar - -T = TypeVar('T') +from typing import Any class PostProcessor(Protocol): @@ -64,4 +62,3 @@ def as_public_key(data: bytes, *_: Any) -> cryptography.ECPoint: data: a serialized compressed public key """ return cryptography.ECPoint.deserialize_from_bytes(data) - From 680aa0348baeffcff0a5368631caeec00b4779cd Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Wed, 29 Nov 2023 11:56:01 +0100 Subject: [PATCH 5/8] fix type checking of amm example --- examples/amm/test_amm.py | 9 +++++---- pyproject.toml | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/amm/test_amm.py b/examples/amm/test_amm.py index a060dde..70b3e3d 100644 --- a/examples/amm/test_amm.py +++ b/examples/amm/test_amm.py @@ -20,6 +20,7 @@ class AmmContractTest(SmartContractTestCase): zgas_contract: GenericContract zneo_contract_hash: types.UInt160 zneo_contract: GenericContract + contract: GenericContract @classmethod def setUpClass(cls) -> None: @@ -35,7 +36,7 @@ def setUpClass(cls) -> None: @classmethod async def asyncSetupClass(cls) -> None: - cls.genesis = cls.node.wallet.account_get_by_label("committee") + cls.genesis = cls.node.wallet.account_get_by_label("committee") # type: ignore await cls.transfer(GAS, cls.genesis.script_hash, cls.owner.script_hash, 100) await cls.transfer(GAS, cls.genesis.script_hash, cls.user.script_hash, 100) @@ -338,7 +339,7 @@ async def test_07_add_liquidity(self): self.assertEqual(1, len(transfer_events)) self.assertEqual(3, len(transfer_events[0].state.as_list())) sender, receiver, amount = transfer_events[0].state.as_list() - self.assertEqual(None, sender.as_none()) + self.assertIsNone(sender.as_none()) self.assertEqual(self.user.script_hash, receiver.as_uint160()) self.assertEqual(liquidity, amount.as_int()) @@ -434,7 +435,7 @@ async def test_07_add_liquidity(self): self.assertEqual(1, len(transfer_events)) self.assertEqual(3, len(transfer_events[0].state.as_list())) sender, receiver, amount = transfer_events[0].state.as_list() - self.assertEqual(None, sender.as_none()) + self.assertIsNone(sender.as_none()) self.assertEqual(self.user.script_hash, receiver.as_uint160()) self.assertEqual(liquidity, amount.as_int()) @@ -638,7 +639,7 @@ async def test_08_remove_liquidity(self): self.assertEqual(3, len(transfer_events[0].state.as_list())) sender, receiver, amount = transfer_events[0].state.as_list() self.assertEqual(self.user.script_hash, sender.as_uint160()) - self.assertEqual(None, receiver.as_none()) + self.assertIsNone(receiver.as_none()) self.assertEqual(liquidity, amount.as_int()) self.assertEqual(1, len(sync_events)) diff --git a/pyproject.toml b/pyproject.toml index f4ca361..d575e02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ tag = "v0.102.0" [tool.mypy] check_untyped_defs = true +disable_error_code = "func-returns-value" [tool.bumpversion] current_version = "0.1.3" From 1f865783c1b441057a33aeb7f8a72cf40b86f02e Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Wed, 29 Nov 2023 11:56:16 +0100 Subject: [PATCH 6/8] add module description --- boaconstructor/storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/boaconstructor/storage.py b/boaconstructor/storage.py index 023456c..cf05232 100644 --- a/boaconstructor/storage.py +++ b/boaconstructor/storage.py @@ -1,3 +1,7 @@ +""" +Common post processor functions for the get_storage() method. +""" + from neo3.core import types, cryptography from neo3.wallet import utils as walletutils from neo3.wallet.types import NeoAddress From 4e35b6d5cd7efed079eb46c75934da2f563faba7 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Wed, 29 Nov 2023 12:05:10 +0100 Subject: [PATCH 7/8] add values_post_processor --- boaconstructor/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/boaconstructor/__init__.py b/boaconstructor/__init__.py index df301a0..d262935 100644 --- a/boaconstructor/__init__.py +++ b/boaconstructor/__init__.py @@ -247,6 +247,7 @@ async def get_storage( target_contract: Optional[types.UInt160] = None, remove_prefix: bool = False, key_post_processor: Optional[PostProcessor] = None, + values_post_processor: Optional[PostProcessor] = None, ) -> dict[bytes, bytes]: """ Gets the entries in the storage of the contract specified by `contract_hash` @@ -255,7 +256,8 @@ async def get_storage( prefix: prefix to filter the entries in the storage. Return the entire storage if not set. target_contract: gets the storage of a different contract than the one under test. e.g. NeoToken remove_prefix: whether the prefix should be removed from the output keys. False by default. - key_post_processor: a function to post process the storage key before placing it in the dictionary + key_post_processor: a function to post process the storage key before placing it in the dictionary. + values_post_processor: a function to post process the storage value before placing it in the dictionary. """ if target_contract is None: contract = GenericContract(cls.contract_hash) @@ -272,6 +274,8 @@ async def get_storage( k = k.removeprefix(prefix) if key_post_processor is not None: k = key_post_processor(k) + if values_post_processor is not None: + v = values_post_processor(v) results[k] = v return results From 08e97e4f38663e1b3fffc83f171377410567f933 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Fri, 1 Dec 2023 17:20:27 +0100 Subject: [PATCH 8/8] add stdlib deserialize function --- boaconstructor/storage.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/boaconstructor/storage.py b/boaconstructor/storage.py index cf05232..a016a96 100644 --- a/boaconstructor/storage.py +++ b/boaconstructor/storage.py @@ -5,6 +5,7 @@ from neo3.core import types, cryptography from neo3.wallet import utils as walletutils from neo3.wallet.types import NeoAddress +from neo3.api.helpers import stdlib from typing_extensions import Protocol from typing import Any @@ -66,3 +67,13 @@ def as_public_key(data: bytes, *_: Any) -> cryptography.ECPoint: data: a serialized compressed public key """ return cryptography.ECPoint.deserialize_from_bytes(data) + + +def stdlib_deserialize(data: bytes, *_: Any) -> Any: + """ + Deserialize the data using the Binary Deserialize logic of the StdLib native contract + + Args: + data: data that has been serialized using the StdLib native contract binary serialize method + """ + return stdlib.binary_deserialize(data)