From d98384170e6a229ac822311b32a06b56e712e5d8 Mon Sep 17 00:00:00 2001
From: Matthew Larson <matthewjlar@gmail.com>
Date: Wed, 1 Nov 2023 11:35:24 -0500
Subject: [PATCH] Fix jsonToArray on None array

---
 hsds/attr_sn.py               | 12 ++++-----
 hsds/util/arrayUtil.py        | 50 ++++++++++++++++++++---------------
 tests/unit/array_util_test.py | 11 ++++++++
 3 files changed, 46 insertions(+), 27 deletions(-)

diff --git a/hsds/attr_sn.py b/hsds/attr_sn.py
index da78f4cb..f5134705 100755
--- a/hsds/attr_sn.py
+++ b/hsds/attr_sn.py
@@ -374,8 +374,8 @@ async def PUT_Attribute(request):
         log.debug(f"attribute value: {value}")
         try:
             arr = jsonToArray(np_dims, arr_dtype, value)
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
+        except ValueError as e:
+            msg = f"Bad Request: input data doesn't match selection: {e}"
             log.warn(msg)
             raise HTTPBadRequest(reason=msg)
         log.debug(f"Got: {arr.size} array elements")
@@ -540,8 +540,8 @@ async def GET_AttributeValue(request):
         np_shape = getShapeDims(shape_json)
         try:
             arr = jsonToArray(np_shape, arr_dtype, dn_json["value"])
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
+        except ValueError as e:
+            msg = f"Bad Request: input data doesn't match selection: {e}"
             log.warn(msg)
             raise HTTPBadRequest(reason=msg)
         output_data = arr.tobytes()
@@ -714,8 +714,8 @@ async def PUT_AttributeValue(request):
         # validate that the value agrees with type/shape
         try:
             arr = jsonToArray(np_shape, np_dtype, value)
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
+        except ValueError as e:
+            msg = f"Bad Request: input data doesn't match selection: {e}"
             log.warn(msg)
             raise HTTPBadRequest(reason=msg)
     log.debug(f"Got: {arr.size} array elements")
diff --git a/hsds/util/arrayUtil.py b/hsds/util/arrayUtil.py
index 5bd7e0ab..2c5e66ab 100644
--- a/hsds/util/arrayUtil.py
+++ b/hsds/util/arrayUtil.py
@@ -14,6 +14,7 @@
 import base64
 import binascii
 import numpy as np
+from aiohttp.web_exceptions import HTTPBadRequest
 
 MAX_VLEN_ELEMENT = 1_000_000  # restrict largest vlen element to one million
 
@@ -46,7 +47,10 @@ def bytesArrayToList(data):
         for item in data:
             out.append(bytesArrayToList(item))  # recursive call
     elif type(data) is bytes:
-        out = data.decode("utf-8")
+        try:
+            out = data.decode("utf-8")
+        except UnicodeDecodeError as e:
+            raise HTTPBadRequest(msg=f"Can't decode bytes data: {e}")
     else:
         out = data
 
@@ -106,6 +110,13 @@ def fillVlenArray(rank, data, arr, index):
                 index += 1
         return index
 
+    if (data_json is None):
+        return np.array([]).astype(data_dtype)
+
+    if (isinstance(data_json, (list, tuple))):
+        if None in data_json:
+            return np.array([]).astype(data_dtype)
+
     # need some special conversion for compound types --
     # each element must be a tuple, but the JSON decoder
     # gives us a list instead.
@@ -124,27 +135,24 @@ def fillVlenArray(rank, data, arr, index):
     else:
         data_json = [data_json,]  # listify
 
-    if not (None in data_json):
-        if isVlen(data_dtype):
-            arr = np.zeros((npoints,), dtype=data_dtype)
-            fillVlenArray(np_shape_rank, data_json, arr, 0)
-        else:
-            try:
-                arr = np.array(data_json, dtype=data_dtype)
-            except UnicodeEncodeError as ude:
-                msg = "Unable to encode data"
-                raise ValueError(msg) from ude
-        # raise an exception of the array shape doesn't match the selection shape
-        # allow if the array is a scalar and the selection shape is one element,
-        # numpy is ok with this
-        if arr.size != npoints:
-            msg = "Input data doesn't match selection number of elements"
-            msg += f" Expected {npoints}, but received: {arr.size}"
-            raise ValueError(msg)
-        if arr.shape != data_shape:
-            arr = arr.reshape(data_shape)  # reshape to match selection
+    if isVlen(data_dtype):
+        arr = np.zeros((npoints,), dtype=data_dtype)
+        fillVlenArray(np_shape_rank, data_json, arr, 0)
     else:
-        arr = np.array([]).astype(data_dtype)
+        try:
+            arr = np.array(data_json, dtype=data_dtype)
+        except UnicodeEncodeError as ude:
+            msg = "Unable to encode data"
+            raise ValueError(msg) from ude
+    # raise an exception of the array shape doesn't match the selection shape
+    # allow if the array is a scalar and the selection shape is one element,
+    # numpy is ok with this
+    if arr.size != npoints:
+        msg = "Input data doesn't match selection number of elements"
+        msg += f" Expected {npoints}, but received: {arr.size}"
+        raise ValueError(msg)
+    if arr.shape != data_shape:
+        arr = arr.reshape(data_shape)  # reshape to match selection
 
     return arr
 
diff --git a/tests/unit/array_util_test.py b/tests/unit/array_util_test.py
index c734e045..19be071e 100644
--- a/tests/unit/array_util_test.py
+++ b/tests/unit/array_util_test.py
@@ -816,6 +816,17 @@ def testGetBroadcastShape(self):
         bcshape = getBroadcastShape([2, 3, 5], 15)
         self.assertEqual(bcshape, [3, 5])
 
+    def testJsonToArrayOnNoneCompoundArray(self):
+        # compound type
+        dt = np.dtype([("a", "i4"), ("b", "S5")])
+        shape = [1,]
+        data = None
+
+        arr = jsonToArray(shape, dt, data)
+
+        self.assertEqual(len(arr), 0)
+        self.assertEqual(arr.dtype, dt)
+
 
 if __name__ == "__main__":
     # setup test files