Fix jsonToArray on None array

hsds/attr_sn.py

@@ -374,10 +374,13 @@ async def PUT_Attribute(request):
         log.debug(f"attribute value: {value}")
         try:
             arr = jsonToArray(np_dims, arr_dtype, value)
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
-            log.warn(msg)
-            raise HTTPBadRequest(reason=msg)
+        except ValueError as e:
+            if value is None:
+                arr = np.array([]).astype(arr_dtype)
+            else:
+                msg = f"Bad Request: input data doesn't match selection: {e}"
+                log.warn(msg)
+                raise HTTPBadRequest(reason=msg)
         log.debug(f"Got: {arr.size} array elements")
     else:
         value = None
@@ -540,10 +543,13 @@ async def GET_AttributeValue(request):
         np_shape = getShapeDims(shape_json)
         try:
             arr = jsonToArray(np_shape, arr_dtype, dn_json["value"])
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
-            log.warn(msg)
-            raise HTTPBadRequest(reason=msg)
+        except ValueError as e:
+            if dn_json["value"] is None:
+                arr = np.array([]).astype(arr_dtype)
+            else:
+                msg = f"Bad Request: input data doesn't match selection: {e}"
+                log.warn(msg)
+                raise HTTPBadRequest(reason=msg)
         output_data = arr.tobytes()
         msg = f"GET AttributeValue - returning {len(output_data)} "
         msg += "bytes binary data"
@@ -697,7 +703,13 @@ async def PUT_AttributeValue(request):
         arr = arr.reshape(np_shape)  # conform to selection shape
         # convert to JSON for transmission to DN
         data = arr.tolist()
-        value = bytesArrayToList(data)
+
+        try:
+            value = bytesArrayToList(data)
+        except ValueError as err:
+            msg = f"Cannot decode bytes to list: {err}"
+            raise HTTPBadRequest(reason=msg)
+
         if attr_shape["class"] == "H5S_SCALAR":
             # just send the value, not a list
             value = value[0]
@@ -719,10 +731,13 @@ async def PUT_AttributeValue(request):
         # validate that the value agrees with type/shape
         try:
             arr = jsonToArray(np_shape, np_dtype, value)
-        except ValueError:
-            msg = "Bad Request: input data doesn't match selection"
-            log.warn(msg)
-            raise HTTPBadRequest(reason=msg)
+        except ValueError as e:
+            if value is None:
+                arr = np.array([]).astype(np_dtype)
+            else:
+                msg = f"Bad Request: input data doesn't match selection: {e}"
+                log.warn(msg)
+                raise HTTPBadRequest(reason=msg)
     log.debug(f"Got: {arr.size} array elements")
 
     # ready to add attribute now

hsds/chunk_sn.py

@@ -288,7 +288,11 @@ async def PUT_Value(request):
             rsp_json = {}
             data = arr_rsp.tolist()
             log.debug(f"got rsp data {len(data)} points")
-            json_query_data = bytesArrayToList(data)
+            try:
+                json_query_data = bytesArrayToList(data)
+            except ValueError as err:
+                msg = f"Cannot decode provided bytes to list: {err}"
+                raise HTTPBadRequest(reason=msg)
             rsp_json["value"] = json_query_data
             rsp_json["hrefs"] = get_hrefs(request, dset_json)
             resp = await jsonResponse(request, rsp_json)
@@ -1020,8 +1024,11 @@ async def GET_Value(request):
                 arr = squeezeArray(arr)
 
             data = arr.tolist()
-            json_data = bytesArrayToList(data)
-
+            try:
+                json_data = bytesArrayToList(data)
+            except ValueError as err:
+                msg = f"Cannot decode bytes to list: {err}"
+                raise HTTPBadRequest(reason=msg)
             datashape = dset_json["shape"]
 
             if datashape["class"] == "H5S_SCALAR":
@@ -1279,7 +1286,11 @@ async def POST_Value(request):
             resp_json = {}
             data = arr_rsp.tolist()
             log.debug(f"got rsp data {len(data)} points")
-            json_data = bytesArrayToList(data)
+            try:
+                json_data = bytesArrayToList(data)
+            except ValueError as err:
+                msg = f"Cannot decode bytes to list: {err}"
+                raise HTTPBadRequest(reason=msg)
             resp_json["value"] = json_data
             resp_json["hrefs"] = get_hrefs(request, dset_json)
             resp_body = await jsonResponse(

hsds/chunklocator.py

@@ -215,7 +215,10 @@ def main():
                 log.warn(msg)
                 sys.exit(-1)
             log.info(f"got chunk array shape: {arr.shape}")
-            json_data = bytesArrayToList(arr)
+            try:
+                json_data = bytesArrayToList(arr)
+            except ValueError as err:
+                raise err
             # print list data to stdout
             print(json_data)
             log.info(f"got {len(json_data)} json elements")

hsds/util/arrayUtil.py

@@ -44,9 +44,16 @@ def bytesArrayToList(data):
     if is_list:
         out = []
         for item in data:
-            out.append(bytesArrayToList(item))  # recursive call
+            try:
+                rec_item = bytesArrayToList(item)  # recursive call
+                out.append(rec_item)
+            except ValueError as err:
+                raise err
     elif type(data) is bytes:
-        out = data.decode("utf-8")
+        try:
+            out = data.decode("utf-8")
+        except UnicodeDecodeError as err:
+            raise ValueError(err)
     else:
         out = data
 
@@ -125,6 +132,13 @@ def fillVlenArray(rank, data, arr, index):
                 index += 1
         return index
 
+    if data_json is None:
+        return np.array([]).astype(data_dtype)
+
+    if isinstance(data_json, (list, tuple)):
+        if None in data_json:
+            return np.array([]).astype(data_dtype)
+
     # need some special conversion for compound types --
     # each element must be a tuple, but the JSON decoder
     # gives us a list instead.
@@ -145,27 +159,24 @@ def fillVlenArray(rank, data, arr, index):
             data_json = data_json.encode("utf8")
         data_json = [data_json,]  # listify
 
-    if not (None in data_json):
-        if isVlen(data_dtype):
-            arr = np.zeros((npoints,), dtype=data_dtype)
-            fillVlenArray(np_shape_rank, data_json, arr, 0)
-        else:
-            try:
-                arr = np.array(data_json, dtype=data_dtype)
-            except UnicodeEncodeError as ude:
-                msg = "Unable to encode data"
-                raise ValueError(msg) from ude
-        # raise an exception of the array shape doesn't match the selection shape
-        # allow if the array is a scalar and the selection shape is one element,
-        # numpy is ok with this
-        if arr.size != npoints:
-            msg = "Input data doesn't match selection number of elements"
-            msg += f" Expected {npoints}, but received: {arr.size}"
-            raise ValueError(msg)
-        if arr.shape != data_shape:
-            arr = arr.reshape(data_shape)  # reshape to match selection
+    if isVlen(data_dtype):
+        arr = np.zeros((npoints,), dtype=data_dtype)
+        fillVlenArray(np_shape_rank, data_json, arr, 0)
     else:
-        arr = np.array([]).astype(data_dtype)
+        try:
+            arr = np.array(data_json, dtype=data_dtype)
+        except UnicodeEncodeError as ude:
+            msg = "Unable to encode data"
+            raise ValueError(msg) from ude
+    # raise an exception of the array shape doesn't match the selection shape
+    # allow if the array is a scalar and the selection shape is one element,
+    # numpy is ok with this
+    if arr.size != npoints:
+        msg = "Input data doesn't match selection number of elements"
+        msg += f" Expected {npoints}, but received: {arr.size}"
+        raise ValueError(msg)
+    if arr.shape != data_shape:
+        arr = arr.reshape(data_shape)  # reshape to match selection
 
     return arr
 

hsds/util/chunkUtil.py

@@ -9,43 +9,6 @@
 PRIMES = [29, 31, 37, 41, 43, 47, 53, 59, 61, 67]  # for chunk partitioning
 
 
-"""
-Convert list that may contain bytes type elements to list of string elements
-
-TBD: code copy from arrayUtil.py
-"""
-
-
-def _bytesArrayToList(data):
-    if type(data) in (bytes, str):
-        is_list = False
-    elif isinstance(data, (np.ndarray, np.generic)):
-        if len(data.shape) == 0:
-            is_list = False
-            data = data.tolist()  # tolist will return a scalar in this case
-            if type(data) in (list, tuple):
-                is_list = True
-            else:
-                is_list = False
-        else:
-            is_list = True
-    elif type(data) in (list, tuple):
-        is_list = True
-    else:
-        is_list = False
-
-    if is_list:
-        out = []
-        for item in data:
-            out.append(_bytesArrayToList(item))  # recursive call
-    elif type(data) is bytes:
-        out = data.decode("utf-8")
-    else:
-        out = data
-
-    return out
-
-
 def getChunkSize(layout, type_size):
     """Return chunk size given layout.
     i.e. just the product of the values in the list.

tests/unit/array_util_test.py

@@ -806,6 +806,17 @@ def testGetBroadcastShape(self):
         bcshape = getBroadcastShape([2, 3, 5], 15)
         self.assertEqual(bcshape, [3, 5])
 
+    def testJsonToArrayOnNoneCompoundArray(self):
+        # compound type
+        dt = np.dtype([("a", "i4"), ("b", "S5")])
+        shape = [1,]
+        data = None
+
+        arr = jsonToArray(shape, dt, data)
+
+        self.assertEqual(len(arr), 0)
+        self.assertEqual(arr.dtype, dt)
+
 
 if __name__ == "__main__":
     # setup test files