api: add helper to deserialize data that is serialized using the StdL…

…ib native contract

neo3/api/helpers/stdlib.py

@@ -0,0 +1,78 @@
+"""
+This module holds helper functions for data that has been serialized using the StdLib native contract
+"""
+
+from neo3 import vm
+from typing import NamedTuple, cast, Any
+from neo3.core import serialization, types
+
+
+class PlaceHolder(NamedTuple):
+    type: vm.StackItemType
+    count: int  # type: ignore
+
+
+def binary_deserialize(data: bytes):
+    """
+    Deserialize data that has been serialized using StdLib.serialize()
+
+    This is the equivalent of the StdLib.deserialize()
+    https://github.com/neo-project/neo/blob/29fab8d3f8f21046a95232b29053c08f9d81f0e3/src/Neo/SmartContract/Native/StdLib.cs#L39
+    and can be used to deserialize data from smart contract storage that was serialized when stored.
+    """
+    # https://github.com/neo-project/neo-vm/blob/859417ad8ff25c2e4a432b6b5b628149875b3eb9/src/Neo.VM/ExecutionEngineLimits.cs#L39
+    max_size = 0xFFFF * 2
+    if len(data) == 0:
+        raise ValueError("Nothing to deserialize")
+
+    deserialized: list[Any | PlaceHolder] = []
+    to_deserialize = 1
+    with serialization.BinaryReader(data) as reader:
+        while not to_deserialize == 0:
+            to_deserialize -= 1
+            item_type = vm.StackItemType(reader.read_byte()[0])
+            if item_type == vm.StackItemType.ANY:
+                deserialized.append(None)
+            elif item_type == vm.StackItemType.BOOLEAN:
+                deserialized.append(reader.read_bool())
+            elif item_type == vm.StackItemType.INTEGER:
+                # https://github.com/neo-project/neo-vm/blob/859417ad8ff25c2e4a432b6b5b628149875b3eb9/src/Neo.VM/Types/Integer.cs#L27
+                deserialized.append(int(types.BigInteger(reader.read_var_bytes(32))))
+            elif item_type in [vm.StackItemType.BYTESTRING, vm.StackItemType.BUFFER]:
+                deserialized.append(reader.read_var_bytes(len(data)))
+            elif item_type in [vm.StackItemType.ARRAY, vm.StackItemType.STRUCT]:
+                count = reader.read_var_int(max_size)
+                deserialized.append(PlaceHolder(item_type, count))
+                to_deserialize += count
+            elif item_type == vm.StackItemType.MAP:
+                count = reader.read_var_int(max_size)
+                deserialized.append(PlaceHolder(item_type, count))
+                to_deserialize += count * 2
+            else:
+                raise ValueError("unreachable")
+
+    temp: list = []
+    while len(deserialized) > 0:
+        item = deserialized.pop()
+        if type(item) == PlaceHolder:
+            item = cast(PlaceHolder, item)
+            if item.type == vm.StackItemType.ARRAY:
+                array = []
+                for _ in range(0, item.count):
+                    array.append(temp.pop())
+                temp.append(array)
+            elif item.type == vm.StackItemType.STRUCT:
+                struct = []
+                for _ in range(0, item.count):
+                    struct.append(temp.pop())
+                temp.append(struct)
+            elif item.type == vm.StackItemType.MAP:
+                m = dict()
+                for _ in range(0, item.count):
+                    k = temp.pop()
+                    v = temp.pop()
+                    m[k] = v
+                temp.append(m)
+        else:
+            temp.append(item)
+    return temp.pop()

neo3/vm.py

@@ -10,6 +10,22 @@
 from collections.abc import Sequence
 
 
+class StackItemType(IntEnum):
+    """
+    StackItemType as defined inside the virtual machine
+    """
+
+    ANY = 0x0
+    POINTER = 0x10
+    BOOLEAN = 0x20
+    INTEGER = 0x21
+    BYTESTRING = 0x28
+    BUFFER = 0x30
+    ARRAY = 0x40
+    STRUCT = 0x41
+    MAP = 0x48
+
+
 def _syscall_name_to_int(name: str) -> int:
     return int.from_bytes(
         hashlib.sha256(name.encode()).digest()[:4], "little", signed=False

tests/api/helpers/test_stdlib.py

@@ -0,0 +1,79 @@
+import unittest
+from neo3.api.helpers import stdlib
+from neo3.core import types
+
+
+class TestStdLibHelpers(unittest.TestCase):
+    def test_with_map(self):
+        """
+        C# reference code
+            var m = new Map(new ReferenceCounter());
+            var i = new Integer(new BigInteger(1));
+            var i2 = new Integer(new BigInteger(2));
+            var b = new Neo.VM.Types.Boolean(true);
+            m[i] = b;
+            m[i2] = b;
+        """
+        # moved outside of multiline comment because pycharm is broken: https://youtrack.jetbrains.com/issue/PY-43117
+        # Console.WriteLine($"b'\\x{BitConverter.ToString(BinarySerializer.Serialize(m, 999)).Replace("-", @"\x")}'");
+
+        data = b"\x48\x02\x21\x01\x01\x20\x01\x21\x01\x02\x20\x01"
+
+        expected = {1: True, 2: True}
+        results: dict = stdlib.binary_deserialize(data)
+
+        self.assertEqual(expected, results)
+
+    def test_with_array(self):
+        """
+            var a = new Neo.VM.Types.Array();
+        var i = new Integer(new BigInteger(1));
+        var i2 = new Integer(new BigInteger(2));
+        a.Add(i);
+        a.Add(i2);
+        """
+        # Console.WriteLine($"b'\\x{BitConverter.ToString(BinarySerializer.Serialize(a, 999)).Replace("-", @"\x")}'");
+        # moved outside of multiline comment because pycharm is broken: https://youtrack.jetbrains.com/issue/PY-43117
+        data = b"\x40\x02\x21\x01\x01\x21\x01\x02"
+        expected = [1, 2]
+        results: dict = stdlib.binary_deserialize(data)
+        self.assertEqual(expected, results)
+
+    def test_with_null(self):
+        data = b"\x00"
+        expected = None
+        results: dict = stdlib.binary_deserialize(data)
+        self.assertEqual(expected, results)
+
+    def test_deserialize_bytestring(self):
+        data = b"\x28\x02\x01\x02"
+        expected = b"\x01\x02"
+        results: dict = stdlib.binary_deserialize(data)
+        self.assertEqual(expected, results)
+
+    def test_deserialize_buffer(self):
+        data = b"\x30\x02\x01\x02"
+        expected = b"\x01\x02"
+        results: dict = stdlib.binary_deserialize(data)
+        self.assertEqual(expected, results)
+
+    def test_deserialize_struct(self):
+        # struct with 2 integers (1,2)
+        data = b"\x41\x02\x21\x01\x01\x21\x01\x02"
+        expected = [1, 2]
+        results: dict = stdlib.binary_deserialize(data)
+        self.assertEqual(expected, results)
+
+    def test_invalid(self):
+        data = b"\xFF"  # invalid stack item type
+        with self.assertRaises(ValueError) as context:
+            stdlib.binary_deserialize(data)
+        self.assertIn("not a valid StackItemType", str(context.exception))
+
+        with self.assertRaises(ValueError) as context:
+            stdlib.binary_deserialize(b"")
+        self.assertEqual("Nothing to deserialize", str(context.exception))
+
+
+if __name__ == "__main__":
+    unittest.main()