From 3f3431386e6ae96ede6a3dd9407a59e4ade701f7 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Tue, 31 Oct 2023 10:58:13 +0100 Subject: [PATCH] api: add helper to deserialize data that is serialized using the StdLib native contract --- neo3/api/helpers/stdlib.py | 78 +++++++++++++++++++++++++++++++ neo3/vm.py | 16 +++++++ tests/api/helpers/__init__.py | 0 tests/api/helpers/test_stdlib.py | 79 ++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+) create mode 100644 neo3/api/helpers/stdlib.py create mode 100644 tests/api/helpers/__init__.py create mode 100644 tests/api/helpers/test_stdlib.py diff --git a/neo3/api/helpers/stdlib.py b/neo3/api/helpers/stdlib.py new file mode 100644 index 00000000..d5ca2287 --- /dev/null +++ b/neo3/api/helpers/stdlib.py @@ -0,0 +1,78 @@ +""" +This module holds helper functions for data that has been serialized using the StdLib native contract +""" + +from neo3 import vm +from typing import NamedTuple, cast, Any +from neo3.core import serialization, types + + +class PlaceHolder(NamedTuple): + type: vm.StackItemType + count: int # type: ignore + + +def binary_deserialize(data: bytes): + """ + Deserialize data that has been serialized using StdLib.serialize() + + This is the equivalent of the StdLib.deserialize() + https://github.com/neo-project/neo/blob/29fab8d3f8f21046a95232b29053c08f9d81f0e3/src/Neo/SmartContract/Native/StdLib.cs#L39 + and can be used to deserialize data from smart contract storage that was serialized when stored. + """ + # https://github.com/neo-project/neo-vm/blob/859417ad8ff25c2e4a432b6b5b628149875b3eb9/src/Neo.VM/ExecutionEngineLimits.cs#L39 + max_size = 0xFFFF * 2 + if len(data) == 0: + raise ValueError("Nothing to deserialize") + + deserialized: list[Any | PlaceHolder] = [] + to_deserialize = 1 + with serialization.BinaryReader(data) as reader: + while not to_deserialize == 0: + to_deserialize -= 1 + item_type = vm.StackItemType(reader.read_byte()[0]) + if item_type == vm.StackItemType.ANY: + deserialized.append(None) + elif item_type == vm.StackItemType.BOOLEAN: + deserialized.append(reader.read_bool()) + elif item_type == vm.StackItemType.INTEGER: + # https://github.com/neo-project/neo-vm/blob/859417ad8ff25c2e4a432b6b5b628149875b3eb9/src/Neo.VM/Types/Integer.cs#L27 + deserialized.append(int(types.BigInteger(reader.read_var_bytes(32)))) + elif item_type in [vm.StackItemType.BYTESTRING, vm.StackItemType.BUFFER]: + deserialized.append(reader.read_var_bytes(len(data))) + elif item_type in [vm.StackItemType.ARRAY, vm.StackItemType.STRUCT]: + count = reader.read_var_int(max_size) + deserialized.append(PlaceHolder(item_type, count)) + to_deserialize += count + elif item_type == vm.StackItemType.MAP: + count = reader.read_var_int(max_size) + deserialized.append(PlaceHolder(item_type, count)) + to_deserialize += count * 2 + else: + raise ValueError("unreachable") + + temp: list = [] + while len(deserialized) > 0: + item = deserialized.pop() + if type(item) == PlaceHolder: + item = cast(PlaceHolder, item) + if item.type == vm.StackItemType.ARRAY: + array = [] + for _ in range(0, item.count): + array.append(temp.pop()) + temp.append(array) + elif item.type == vm.StackItemType.STRUCT: + struct = [] + for _ in range(0, item.count): + struct.append(temp.pop()) + temp.append(struct) + elif item.type == vm.StackItemType.MAP: + m = dict() + for _ in range(0, item.count): + k = temp.pop() + v = temp.pop() + m[k] = v + temp.append(m) + else: + temp.append(item) + return temp.pop() diff --git a/neo3/vm.py b/neo3/vm.py index 450b5e46..25c0003c 100644 --- a/neo3/vm.py +++ b/neo3/vm.py @@ -10,6 +10,22 @@ from collections.abc import Sequence +class StackItemType(IntEnum): + """ + StackItemType as defined inside the virtual machine + """ + + ANY = 0x0 + POINTER = 0x10 + BOOLEAN = 0x20 + INTEGER = 0x21 + BYTESTRING = 0x28 + BUFFER = 0x30 + ARRAY = 0x40 + STRUCT = 0x41 + MAP = 0x48 + + def _syscall_name_to_int(name: str) -> int: return int.from_bytes( hashlib.sha256(name.encode()).digest()[:4], "little", signed=False diff --git a/tests/api/helpers/__init__.py b/tests/api/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/api/helpers/test_stdlib.py b/tests/api/helpers/test_stdlib.py new file mode 100644 index 00000000..bcb28081 --- /dev/null +++ b/tests/api/helpers/test_stdlib.py @@ -0,0 +1,79 @@ +import unittest +from neo3.api.helpers import stdlib +from neo3.core import types + + +class TestStdLibHelpers(unittest.TestCase): + def test_with_map(self): + """ + C# reference code + var m = new Map(new ReferenceCounter()); + var i = new Integer(new BigInteger(1)); + var i2 = new Integer(new BigInteger(2)); + var b = new Neo.VM.Types.Boolean(true); + m[i] = b; + m[i2] = b; + """ + # moved outside of multiline comment because pycharm is broken: https://youtrack.jetbrains.com/issue/PY-43117 + # Console.WriteLine($"b'\\x{BitConverter.ToString(BinarySerializer.Serialize(m, 999)).Replace("-", @"\x")}'"); + + data = b"\x48\x02\x21\x01\x01\x20\x01\x21\x01\x02\x20\x01" + + expected = {1: True, 2: True} + results: dict = stdlib.binary_deserialize(data) + + self.assertEqual(expected, results) + + def test_with_array(self): + """ + var a = new Neo.VM.Types.Array(); + var i = new Integer(new BigInteger(1)); + var i2 = new Integer(new BigInteger(2)); + a.Add(i); + a.Add(i2); + """ + # Console.WriteLine($"b'\\x{BitConverter.ToString(BinarySerializer.Serialize(a, 999)).Replace("-", @"\x")}'"); + # moved outside of multiline comment because pycharm is broken: https://youtrack.jetbrains.com/issue/PY-43117 + data = b"\x40\x02\x21\x01\x01\x21\x01\x02" + expected = [1, 2] + results: dict = stdlib.binary_deserialize(data) + self.assertEqual(expected, results) + + def test_with_null(self): + data = b"\x00" + expected = None + results: dict = stdlib.binary_deserialize(data) + self.assertEqual(expected, results) + + def test_deserialize_bytestring(self): + data = b"\x28\x02\x01\x02" + expected = b"\x01\x02" + results: dict = stdlib.binary_deserialize(data) + self.assertEqual(expected, results) + + def test_deserialize_buffer(self): + data = b"\x30\x02\x01\x02" + expected = b"\x01\x02" + results: dict = stdlib.binary_deserialize(data) + self.assertEqual(expected, results) + + def test_deserialize_struct(self): + # struct with 2 integers (1,2) + data = b"\x41\x02\x21\x01\x01\x21\x01\x02" + expected = [1, 2] + results: dict = stdlib.binary_deserialize(data) + self.assertEqual(expected, results) + + def test_invalid(self): + data = b"\xFF" # invalid stack item type + with self.assertRaises(ValueError) as context: + stdlib.binary_deserialize(data) + self.assertIn("not a valid StackItemType", str(context.exception)) + + with self.assertRaises(ValueError) as context: + stdlib.binary_deserialize(b"") + self.assertEqual("Nothing to deserialize", str(context.exception)) + + +if __name__ == "__main__": + unittest.main()