Fix jsonToArray on None array

hsds/attr_sn.py

@@ -374,8 +374,8 @@ async def PUT_Attribute(request):
         log.debug(f"attribute value: {value}")
         try:
             arr = jsonToArray(np_dims, arr_dtype, value)
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
+        except ValueError as e:
+            msg = f"Bad Request: input data doesn't match selection: {e}"
             log.warn(msg)
             raise HTTPBadRequest(reason=msg)
         log.debug(f"Got: {arr.size} array elements")
@@ -540,8 +540,8 @@ async def GET_AttributeValue(request):
         np_shape = getShapeDims(shape_json)
         try:
             arr = jsonToArray(np_shape, arr_dtype, dn_json["value"])
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
+        except ValueError as e:
+            msg = f"Bad Request: input data doesn't match selection: {e}"
             log.warn(msg)
             raise HTTPBadRequest(reason=msg)
         output_data = arr.tobytes()
@@ -719,8 +719,8 @@ async def PUT_AttributeValue(request):
         # validate that the value agrees with type/shape
         try:
             arr = jsonToArray(np_shape, np_dtype, value)
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
+        except ValueError as e:
+            msg = f"Bad Request: input data doesn't match selection: {e}"
             log.warn(msg)
             raise HTTPBadRequest(reason=msg)
     log.debug(f"Got: {arr.size} array elements")

hsds/util/arrayUtil.py

@@ -46,7 +46,10 @@ def bytesArrayToList(data):
         for item in data:
             out.append(bytesArrayToList(item))  # recursive call
     elif type(data) is bytes:
-        out = data.decode("utf-8")
+        try:
+            out = data.decode("utf-8")
+        except UnicodeDecodeError as err:
+            raise err
     else:
         out = data
 
@@ -125,6 +128,13 @@ def fillVlenArray(rank, data, arr, index):
                 index += 1
         return index
 
+    if (data_json is None):
+        return np.array([]).astype(data_dtype)
+
+    if (isinstance(data_json, (list, tuple))):
+        if None in data_json:
+            return np.array([]).astype(data_dtype)
+
     # need some special conversion for compound types --
     # each element must be a tuple, but the JSON decoder
     # gives us a list instead.
@@ -145,27 +155,24 @@ def fillVlenArray(rank, data, arr, index):
             data_json = data_json.encode("utf8")
         data_json = [data_json,]  # listify
 
-    if not (None in data_json):
-        if isVlen(data_dtype):
-            arr = np.zeros((npoints,), dtype=data_dtype)
-            fillVlenArray(np_shape_rank, data_json, arr, 0)
-        else:
-            try:
-                arr = np.array(data_json, dtype=data_dtype)
-            except UnicodeEncodeError as ude:
-                msg = "Unable to encode data"
-                raise ValueError(msg) from ude
-        # raise an exception of the array shape doesn't match the selection shape
-        # allow if the array is a scalar and the selection shape is one element,
-        # numpy is ok with this
-        if arr.size != npoints:
-            msg = "Input data doesn't match selection number of elements"
-            msg += f" Expected {npoints}, but received: {arr.size}"
-            raise ValueError(msg)
-        if arr.shape != data_shape:
-            arr = arr.reshape(data_shape)  # reshape to match selection
+    if isVlen(data_dtype):
+        arr = np.zeros((npoints,), dtype=data_dtype)
+        fillVlenArray(np_shape_rank, data_json, arr, 0)
     else:
-        arr = np.array([]).astype(data_dtype)
+        try:
+            arr = np.array(data_json, dtype=data_dtype)
+        except UnicodeEncodeError as ude:
+            msg = "Unable to encode data"
+            raise ValueError(msg) from ude
+    # raise an exception of the array shape doesn't match the selection shape
+    # allow if the array is a scalar and the selection shape is one element,
+    # numpy is ok with this
+    if arr.size != npoints:
+        msg = "Input data doesn't match selection number of elements"
+        msg += f" Expected {npoints}, but received: {arr.size}"
+        raise ValueError(msg)
+    if arr.shape != data_shape:
+        arr = arr.reshape(data_shape)  # reshape to match selection
 
     return arr
 

tests/unit/array_util_test.py

@@ -806,6 +806,17 @@ def testGetBroadcastShape(self):
         bcshape = getBroadcastShape([2, 3, 5], 15)
         self.assertEqual(bcshape, [3, 5])
 
+    def testJsonToArrayOnNoneCompoundArray(self):
+        # compound type
+        dt = np.dtype([("a", "i4"), ("b", "S5")])
+        shape = [1,]
+        data = None
+
+        arr = jsonToArray(shape, dt, data)
+
+        self.assertEqual(len(arr), 0)
+        self.assertEqual(arr.dtype, dt)
+
 
 if __name__ == "__main__":
     # setup test files