From 776171a852e5ed6856d2f063019c3905464fa921 Mon Sep 17 00:00:00 2001 From: jreadey Date: Fri, 26 Jan 2024 16:09:43 +0000 Subject: [PATCH] fix for point selection of vlen type --- hsds/chunk_sn.py | 2 +- tests/integ/pointsel_test.py | 1 - tests/integ/vlen_test.py | 199 +++++++++++++++++++++++++++++------ 3 files changed, 170 insertions(+), 32 deletions(-) diff --git a/hsds/chunk_sn.py b/hsds/chunk_sn.py index 3e652c45..d01359ff 100755 --- a/hsds/chunk_sn.py +++ b/hsds/chunk_sn.py @@ -1438,7 +1438,7 @@ async def POST_Value(request): log.debug(f"arr shape: {arr_rsp.shape}") if response_type == "binary": log.debug("preparing binary response") - output_data = arr_rsp.tobytes() + output_data = arrayToBytes(arr_rsp) msg = f"POST Value - returning {len(output_data)} bytes binary data" log.debug(msg) await resp.write(output_data) diff --git a/tests/integ/pointsel_test.py b/tests/integ/pointsel_test.py index 1a75423c..a074bb30 100755 --- a/tests/integ/pointsel_test.py +++ b/tests/integ/pointsel_test.py @@ -1743,7 +1743,6 @@ def testPostCompoundDataset(self): rspJson = json.loads(rsp.text) self.assertTrue("value" in rspJson) ret_value = np.array(rspJson["value"], dtype=int) - print(f"ret value = {ret_value}") self.assertTrue(np.array_equal(ret_value[:, 0], np.full(shape=num_elements, fill_value=10000, dtype=int))) for i in range(2, 5): diff --git a/tests/integ/vlen_test.py b/tests/integ/vlen_test.py index 074fa26f..fd578b11 100755 --- a/tests/integ/vlen_test.py +++ b/tests/integ/vlen_test.py @@ -58,9 +58,7 @@ def testPutVLenInt(self): } payload = { "type": vlen_type, - "shape": [ - 4, - ], + "shape": [4, ], } req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) @@ -116,7 +114,6 @@ def testPutVLenInt(self): body = {"points": points} # read selected points rsp = self.session.post(req, data=json.dumps(body), headers=headers) - # point select not supported on zero-dimensional datasets self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) self.assertTrue("value" in rspJson) @@ -132,9 +129,7 @@ def testPutVLenIntBinary(self): count = 4 test_values = [] for i in range(count): - e = [ - 1, - ] + e = [1,] for j in range(0, i): e.append(j + 2) test_values.append(e) @@ -162,9 +157,7 @@ def testPutVLenIntBinary(self): } payload = { "type": vlen_type, - "shape": [ - count, - ], + "shape": [count, ], } req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) @@ -211,13 +204,7 @@ def testPutVLenIntBinary(self): self.assertEqual(rsp.headers["Content-Type"], "application/octet-stream") data = rsp.content self.assertEqual(len(data), 56) - arr = bytesToArray( - data, - dt, - [ - count, - ], - ) + arr = bytesToArray(data, dt, [count, ]) for i in range(count): self.assertEqual(value[i], test_values[i]) @@ -332,9 +319,7 @@ def testPutVLenString(self): } payload = { "type": vlen_type, - "shape": [ - 4, - ], + "shape": [4, ], } req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) @@ -368,6 +353,109 @@ def testPutVLenString(self): for i in range(4): self.assertEqual(value[i], data[i]) + # read a point selection + points = [1, 3] + body = {"points": points} + # read selected points + rsp = self.session.post(req, data=json.dumps(body), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("value" in rspJson) + value = rspJson["value"] + self.assertEqual(len(value), 2) + self.assertEqual(value[0], data[1]) + self.assertEqual(value[1], data[3]) + + def testPutVLenStringBinary(self): + # Test PUT value for 1d attribute with variable length string types + print("testPutVLenStringBinary", self.base_domain) + + headers = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_req = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_req["Content-Type"] = "application/octet-stream" + headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_rsp["accept"] = "application/octet-stream" + headers_bin_reqrsp = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_reqrsp["accept"] = "application/octet-stream" + headers_bin_reqrsp["Content-Type"] = "application/octet-stream" + + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create dataset + vlen_type = { + "class": "H5T_STRING", + "charSet": "H5T_CSET_ASCII", + "strPad": "H5T_STR_NULLTERM", + "length": "H5T_VARIABLE", + } + payload = { + "type": vlen_type, + "shape": [4, ], + } + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # create dataset + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + self.assertTrue(helper.validateId(dset_uuid)) + + # link new dataset as 'dset' + name = "dset" + req = self.endpoint + "/groups/" + root_uuid + "/links/" + name + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # write values to dataset + data = ["This is", "a variable length", "string", "array"] + + dt_str = createDataType(vlen_type) + + # create numpy vlen array + + arr = np.zeros((4,), dtype=dt_str) + for i in range(4): + arr[i] = data[i] + + # write as binary data + bin_data = arrayToBytes(arr) + + req = self.endpoint + "/datasets/" + dset_uuid + "/value" + rsp = self.session.put(req, data=bin_data, headers=headers_bin_req) + self.assertEqual(rsp.status_code, 200) + + # read as binary + rsp = self.session.get(req, headers=headers_bin_rsp) + self.assertEqual(rsp.status_code, 200) + self.assertEqual(rsp.headers["Content-Type"], "application/octet-stream") + bin_data = rsp.content + arr = bytesToArray(bin_data, dt_str, [4,]) + for i in range(4): + self.assertEqual(arr[i].decode(), data[i]) + + # prepare a binary list of points to send + points = [1, 3] + arr_points = np.asarray(points, dtype="u8") # must use unsigned 64-bit int + req_data = arr_points.tobytes() + + # read selected points with binary request, binary response + rsp = self.session.post(req, data=req_data, headers=headers_bin_reqrsp) + self.assertEqual(rsp.status_code, 200) + self.assertEqual(rsp.headers["Content-Type"], "application/octet-stream") + bin_data = rsp.content + arr = bytesToArray(bin_data, dt_str, [2,]) + + self.assertEqual(arr.shape[0], 2) + self.assertEqual(arr[0].decode(), data[1]) + self.assertEqual(arr[1].decode(), data[3]) + def testPutVLenCompound(self): # Test PUT value for 1d attribute with variable length int types print("testPutVLenCompound", self.base_domain) @@ -457,6 +545,19 @@ def testPutVLenCompound(self): value = rspJson["value"] self.assertEqual(len(value), count) + # read a point selection + points = [1, 3] + body = {"points": points} + # read selected points + rsp = self.session.post(req, data=json.dumps(body), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("value" in rspJson) + value = rspJson["value"] + self.assertEqual(len(value), 2) + self.assertEqual(value[0], data[1]) + self.assertEqual(value[1], data[3]) + def testPutVLenCompoundBinary(self): # Test PUT value for 1d attribute with variable length int types print("testPutVLenCompoundBinary", self.base_domain) @@ -466,6 +567,9 @@ def testPutVLenCompoundBinary(self): headers_bin_req["Content-Type"] = "application/octet-stream" headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain) headers_bin_rsp["accept"] = "application/octet-stream" + headers_bin_reqrsp = helper.getRequestHeaders(domain=self.base_domain) + headers_bin_reqrsp["accept"] = "application/octet-stream" + headers_bin_reqrsp["Content-Type"] = "application/octet-stream" req = self.endpoint + "/" @@ -509,9 +613,7 @@ def testPutVLenCompoundBinary(self): datatype = {"class": "H5T_COMPOUND", "fields": fields} payload = { "type": datatype, - "shape": [ - count, - ], + "shape": [count, ], } req = self.endpoint + "/datasets" rsp = self.session.post(req, data=json.dumps(payload), headers=headers) @@ -564,13 +666,50 @@ def testPutVLenCompoundBinary(self): self.assertEqual(rsp.headers["Content-Type"], "application/octet-stream") data = rsp.content self.assertEqual(len(data), 192) - arr = bytesToArray( - data, - dt_compound, - [ - count, - ], - ) + arr_rsp = bytesToArray(data, dt_compound, [count,]) + for i in range(count): + req_row = arr[i] + rsp_row = arr_rsp[i] + self.assertEqual(rsp_row["VALUE1"], req_row["VALUE1"]) + self.assertEqual(rsp_row["VALUE2"], req_row["VALUE2"]) + req_value3 = req_row["VALUE3"] + rsp_value3 = rsp_row["VALUE3"] + self.assertEqual(len(req_value3), len(rsp_value3)) + for j in range(len(req_value3)): + req_item = req_value3[j] + rsp_item = rsp_value3[j] + # strings are showing up as bytes in the response + self.assertEqual(req_item, rsp_item.decode()) + + # prepare a binary list of points to send + points = [1, 3] + arr_points = np.asarray(points, dtype="u8") # must use unsigned 64-bit int + req_data = arr_points.tobytes() + + # read selected points with binary request, binary response + rsp = self.session.post(req, data=req_data, headers=headers_bin_reqrsp) + self.assertEqual(rsp.status_code, 200) + self.assertEqual(rsp.headers["Content-Type"], "application/octet-stream") + bin_data = rsp.content + arr_rsp = bytesToArray(bin_data, dt_compound, [2,]) + self.assertEqual(arr_rsp.shape[0], 2) + + for i in range(2): + if i == 0: + req_row = arr[1] + else: + req_row = arr[3] + rsp_row = arr_rsp[i] + self.assertEqual(rsp_row["VALUE1"], req_row["VALUE1"]) + self.assertEqual(rsp_row["VALUE2"], req_row["VALUE2"]) + req_value3 = req_row["VALUE3"] + rsp_value3 = rsp_row["VALUE3"] + self.assertEqual(len(req_value3), len(rsp_value3)) + for j in range(len(req_value3)): + req_item = req_value3[j] + rsp_item = rsp_value3[j] + # strings are showing up as bytes in the response + self.assertEqual(req_item, rsp_item.decode()) if __name__ == "__main__":