diff --git a/hsds/attr_dn.py b/hsds/attr_dn.py index d80ca322..8dd44da3 100755 --- a/hsds/attr_dn.py +++ b/hsds/attr_dn.py @@ -15,11 +15,11 @@ import time from bisect import bisect_left -from aiohttp.web_exceptions import HTTPBadRequest, HTTPConflict, HTTPNotFound +from aiohttp.web_exceptions import HTTPBadRequest, HTTPConflict, HTTPNotFound, HTTPGone from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.web import json_response -from .util.attrUtil import validateAttributeName +from .util.attrUtil import validateAttributeName, isEqualAttr from .util.hdf5dtype import getItemSize, createDataType from .util.dsetUtil import getShapeDims from .util.arrayUtil import arrayToBytes, jsonToArray, decodeData @@ -270,21 +270,31 @@ async def POST_Attributes(request): if encoding: kwargs["encoding"] = encoding + missing_names = set() + for attr_name in titles: if attr_name not in attr_dict: + missing_names.add(attr_name) continue des_attr = _getAttribute(attr_name, obj_json, **kwargs) attr_list.append(des_attr) resp_json = {"attributes": attr_list} - if not attr_list: - msg = f"POST attributes - requested {len(titles)} but none were found" - log.warn(msg) - raise HTTPNotFound() - if len(attr_list) != len(titles): + + if missing_names: msg = f"POST attributes - requested {len(titles)} attributes but only " msg += f"{len(attr_list)} were found" log.warn(msg) + # one or more attributes not found, check to see if any + # had been previously deleted + deleted_attrs = app["deleted_attrs"] + if obj_id in deleted_attrs: + attr_delete_set = deleted_attrs[obj_id] + for attr_name in missing_names: + if attr_name in attr_delete_set: + log.info(f"attribute: {attr_name} was previously deleted, returning 410") + raise HTTPGone() + log.info("one or mores attributes not found, returning 404") raise HTTPNotFound() log.debug(f"POST attributes returning: {resp_json}") resp = json_response(resp_json) @@ -392,18 +402,28 @@ async def PUT_Attributes(request): attributes = obj_json["attributes"] - # check for conflicts, also set timestamp create_time = time.time() - new_attribute = False # set this if we have any new attributes + # check for conflicts + new_attributes = set() # attribute names that are new or replacements for attr_name in items: attribute = items[attr_name] if attr_name in attributes: log.debug(f"attribute {attr_name} exists") - if replace: + old_item = attributes[attr_name] + try: + is_dup = isEqualAttr(attribute, old_item) + except TypeError: + log.error(f"isEqualAttr TypeError - new: {attribute} old: {old_item}") + raise HTTPInternalServerError() + if is_dup: + log.debug(f"duplicate attribute: {attr_name}") + continue + elif replace: # don't change the create timestamp log.debug(f"attribute {attr_name} exists, but will be updated") old_item = attributes[attr_name] attribute["created"] = old_item["created"] + new_attributes.add(attr_name) else: # Attribute already exists, return a 409 msg = f"Attempt to overwrite attribute: {attr_name} " @@ -414,18 +434,30 @@ async def PUT_Attributes(request): # set the timestamp log.debug(f"new attribute {attr_name}") attribute["created"] = create_time - new_attribute = True + new_attributes.add(attr_name) - # ok - all set, create the attributes - for attr_name in items: + # if any of the attribute names was previously deleted, + # remove from the deleted set + deleted_attrs = app["deleted_attrs"] + if obj_id in deleted_attrs: + attr_delete_set = deleted_attrs[obj_id] + else: + attr_delete_set = set() + + # ok - all set, add the attributes + for attr_name in new_attributes: log.debug(f"adding attribute {attr_name}") attr_json = items[attr_name] attributes[attr_name] = attr_json - - # write back to S3, save to metadata cache - await save_metadata_obj(app, obj_id, obj_json, bucket=bucket) - - if new_attribute: + if attr_name in attr_delete_set: + attr_delete_set.remove(attr_name) + + if new_attributes: + # update the obj lastModified + now = time.time() + obj_json["lastModified"] = now + # write back to S3, save to metadata cache + await save_metadata_obj(app, obj_id, obj_json, bucket=bucket) status = 201 else: status = 200 @@ -490,15 +522,35 @@ async def DELETE_Attributes(request): # return a list of attributes based on sorted dictionary keys attributes = obj_json["attributes"] + # add attribute names to deleted set, so we can return a 410 if they + # are requested in the future + deleted_attrs = app["deleted_attrs"] + if obj_id in deleted_attrs: + attr_delete_set = deleted_attrs[obj_id] + else: + attr_delete_set = set() + deleted_attrs[obj_id] = attr_delete_set + + save_obj = False # set to True if anything is actually modified for attr_name in attr_names: + if attr_name in attr_delete_set: + log.warn(f"attribute {attr_name} already deleted") + continue + if attr_name not in attributes: - msg = f"Attribute {attr_name} not found in objid: {obj_id}" + msg = f"Attribute {attr_name} not found in obj id: {obj_id}" log.warn(msg) raise HTTPNotFound() del attributes[attr_name] - - await save_metadata_obj(app, obj_id, obj_json, bucket=bucket) + attr_delete_set.add(attr_name) + save_obj = True + + if save_obj: + # update the object lastModified + now = time.time() + obj_json["lastModified"] = now + await save_metadata_obj(app, obj_id, obj_json, bucket=bucket) resp_json = {} resp = json_response(resp_json) diff --git a/hsds/datanode.py b/hsds/datanode.py index 7e8c9a9d..50bb0307 100644 --- a/hsds/datanode.py +++ b/hsds/datanode.py @@ -299,6 +299,8 @@ def create_app(): } app["chunk_cache"] = LruCache(**kwargs) app["deleted_ids"] = set() + app["deleted_attrs"] = {} # map of objectid to set of deleted attribute names + app["deleted_links"] = {} # map of objecctid to set of deleted link names # map of objids to timestamp and bucket of which they were last updated app["dirty_ids"] = {} # map of dataset ids to deflate levels (if compressed) diff --git a/hsds/domain_crawl.py b/hsds/domain_crawl.py index 8697e267..f2d672b5 100644 --- a/hsds/domain_crawl.py +++ b/hsds/domain_crawl.py @@ -19,7 +19,7 @@ from aiohttp.web_exceptions import HTTPInternalServerError, HTTPNotFound, HTTPGone from .util.idUtil import getCollectionForId, getDataNodeUrl -from .servicenode_lib import getObjectJson, getAttributes, putAttributes, getLinks +from .servicenode_lib import getObjectJson, getAttributes, putAttributes, getLinks, putLinks from . import hsds_logger as log @@ -295,6 +295,30 @@ async def get_links(self, grp_id, titles=None): else: log.debug(f"link: {link_id} already in object dict") + async def put_links(self, grp_id, link_items): + # write the given links for the obj_id + log.debug(f"put_links for {grp_id}, {len(link_items)} links") + req = getDataNodeUrl(self._app, grp_id) + req += f"/groups/{grp_id}/links" + kwargs = {} + if "bucket" in self._params: + kwargs["bucket"] = self._params["bucket"] + status = None + try: + status = await putLinks(self._app, grp_id, link_items, **kwargs) + except HTTPConflict: + log.warn("DomainCrawler - got HTTPConflict from http_put") + status = 409 + except HTTPServiceUnavailable: + status = 503 + except HTTPInternalServerError: + status = 500 + except Exception as e: + log.error(f"unexpected exception {e}") + + log.debug(f"DomainCrawler fetch for {grp_id} - returning status: {status}") + self._obj_dict[grp_id] = {"status": status} + def get_status(self): """ return the highest status of any of the returned objects """ status = None @@ -419,7 +443,16 @@ async def fetch(self, obj_id): log.debug(f"DomainCrawler - get link titles: {link_titles}") await self.get_links(obj_id, link_titles) + elif self._action == "put_link": + log.debug("DomainCrawlwer - put links") + # write links + if self._objs and obj_id not in self._objs: + log.error(f"couldn't find {obj_id} in self._objs") + return + link_items = self._objs[obj_id] + log.debug(f"got {len(link_items)} link items for {obj_id}") + await self.put_links(obj_id, link_items) else: msg = f"DomainCrawler: unexpected action: {self._action}" log.error(msg) diff --git a/hsds/domain_sn.py b/hsds/domain_sn.py index d7778ecd..785a9528 100755 --- a/hsds/domain_sn.py +++ b/hsds/domain_sn.py @@ -624,6 +624,131 @@ async def getScanTime(app, root_id, bucket=None): return root_scan +async def POST_Domain(request): + """ return object defined by h5path list """ + + log.request(request) + app = request.app + params = request.rel_url.query + log.debug(f"POST_Domain query params: {params}") + + include_links = False + include_attrs = False + follow_soft_links = False + follow_external_links = False + + if "include_links" in params and params["include_links"]: + include_links = True + if "include_attrs" in params and params["include_attrs"]: + include_attrs = True + if "follow_soft_links" in params and params["follow_soft_links"]: + follow_soft_links = True + if "follow_external_links" in params and params["follow_external_links"]: + follow_external_links = True + + if not request.has_body: + msg = "POST Domain with no body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + try: + body = await request.json() + except json.JSONDecodeError: + msg = "Unable to load JSON body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + if "h5paths" in body: + h5paths = body["h5paths"] + if not isinstance(h5paths, list): + msg = f"expected list for h5paths but got: {type(h5paths)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + msg = "expected h5paths key in body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + (username, pswd) = getUserPasswordFromRequest(request) + if username is None and app["allow_noauth"]: + username = "default" + else: + await validateUserPassword(app, username, pswd) + + domain = None + try: + domain = getDomainFromRequest(request) + except ValueError: + log.warn(f"Invalid domain: {domain}") + raise HTTPBadRequest(reason="Invalid domain name") + + bucket = getBucketForDomain(domain) + log.debug(f"GET_Domain domain: {domain} bucket: {bucket}") + + if not bucket: + # no bucket defined, raise 400 + msg = "Bucket not provided" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + if bucket: + checkBucketAccess(app, bucket) + + if not domain: + msg = "no domain given" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + log.info(f"got domain: {domain}") + + domain_json = await getDomainJson(app, domain, reload=True) + + if domain_json is None: + log.warn(f"domain: {domain} not found") + raise HTTPNotFound() + + if "acls" not in domain_json: + log.error("No acls key found in domain") + raise HTTPInternalServerError() + + log.debug(f"got domain_json: {domain_json}") + # validate that the requesting user has permission to read this domain + # aclCheck throws exception if not authorized + aclCheck(app, domain_json, "read", username) + + json_objs = {} + + for h5path in h5paths: + root_id = domain_json["root"] + + # getObjectIdByPath throws 404 if not found + obj_id, domain, _ = await getObjectIdByPath( + app, root_id, h5path, bucket=bucket, domain=domain, + follow_soft_links=follow_soft_links, + follow_external_links=follow_external_links) + log.info(f"get obj_id: {obj_id} from h5path: {h5path}") + # get authoritative state for object from DN (even if + # it's in the meta_cache). + kwargs = {"refresh": True, "bucket": bucket, + "include_attrs": include_attrs, "include_links": include_links} + log.debug(f"kwargs for getObjectJson: {kwargs}") + + obj_json = await getObjectJson(app, obj_id, **kwargs) + + obj_json = respJsonAssemble(obj_json, params, obj_id) + + obj_json["domain"] = getPathForDomain(domain) + + # client may not know class of object retrieved via path + obj_json["class"] = getObjectClass(obj_id) + + json_objs[h5path] = obj_json + + jsonRsp = {"h5paths": json_objs} + resp = await jsonResponse(request, jsonRsp) + log.response(request, resp=resp) + return resp + + async def PUT_Domain(request): """HTTP method to create a new domain""" log.request(request) diff --git a/hsds/link_dn.py b/hsds/link_dn.py index 5a8651c8..d61d33d0 100755 --- a/hsds/link_dn.py +++ b/hsds/link_dn.py @@ -17,12 +17,12 @@ from copy import copy from bisect import bisect_left -from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound, HTTPConflict +from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound, HTTPGone, HTTPConflict from aiohttp.web_exceptions import HTTPInternalServerError from aiohttp.web import json_response from .util.idUtil import isValidUuid -from .util.linkUtil import validateLinkName +from .util.linkUtil import validateLinkName, getLinkClass, isEqualLink from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj from . import hsds_logger as log @@ -179,10 +179,12 @@ async def POST_Links(request): link_list = [] # links to be returned + missing_names = set() for title in titles: if title not in links: + missing_names.add(title) log.info(f"Link name {title} not found in group: {group_id}") - raise HTTPNotFound() + continue link_json = links[title] item = {} if "class" not in link_json: @@ -223,15 +225,20 @@ async def POST_Links(request): link_list.append(item) - if not link_list: - msg = f"POST_links - requested {len(titles)} but none were found" - log.warn(msg) - raise HTTPNotFound() - - if len(link_list) != len(titles): + if missing_names: msg = f"POST_links - requested {len(titles)} links but only " msg += f"{len(link_list)} were found" log.warn(msg) + # one or more links not found, check to see if any + # had been previously deleted + deleted_links = app["deleted_links"] + if group_id in deleted_links: + link_delete_set = deleted_links[group_id] + for link_name in missing_names: + if link_name in link_delete_set: + log.info(f"link: {link_name} was previously deleted, returning 410") + raise HTTPGone() + log.info("one or more links not found, returning 404") raise HTTPNotFound() rspJson = {"links": link_list} @@ -270,12 +277,12 @@ async def PUT_Links(request): for title in items: validateLinkName(title) item = items[title] - - if "id" in item: - if not isValidUuid(item["id"]): - msg = f"invalid uuid for {title}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + try: + link_class = getLinkClass(item) + except ValueError: + raise HTTPBadRequest(reason="invalid link") + if "class" not in item: + item["class"] = link_class if "bucket" in params: bucket = params["bucket"] @@ -295,44 +302,48 @@ async def PUT_Links(request): raise HTTPInternalServerError() links = group_json["links"] - dup_titles = [] + new_links = set() for title in items: if title in links: link_json = items[title] existing_link = links[title] - for prop in ("class", "id", "h5path", "h5domain"): - if prop in link_json: - if prop not in existing_link: - msg = f"PUT Link - prop {prop} not found in existing " - msg += "link, returning 409" - log.warn(msg) - raise HTTPConflict() - - if link_json[prop] != existing_link[prop]: - msg = f"PUT Links - prop {prop} value is different, old: " - msg += f"{existing_link[prop]}, new: {link_json[prop]}, " - msg += "returning 409" - log.warn(msg) - raise HTTPConflict() - msg = f"Link name {title} already found in group: {group_id}" - log.warn(msg) - dup_titles.append(title) - - for title in dup_titles: - del items[title] - - if items: + try: + is_dup = isEqualLink(link_json, existing_link) + except TypeError: + log.error(f"isEqualLink TypeError - new: {link_json}, old: {existing_link}") + raise HTTPInternalServerError() + + if is_dup: + # TBD: replace param for links? + continue # dup + else: + msg = f"link {title} already exists, returning 409" + log.warn(msg) + raise HTTPConflict() + else: + new_links.add(title) - now = time.time() + # if any of the attribute names was previously deleted, + # remove from the deleted set + deleted_links = app["deleted_links"] + if group_id in deleted_links: + link_delete_set = deleted_links[group_id] + else: + link_delete_set = set() - # add the links - for title in items: - item = items[title] - item["created"] = now - links[title] = item + create_time = time.time() + for title in new_links: + item = items[title] + item["created"] = create_time + links[title] = item + log.debug(f"added link {title}: {item}") + if title in link_delete_set: + link_delete_set.remove(title) + if new_links: # update the group lastModified - group_json["lastModified"] = now + group_json["lastModified"] = create_time + log.debug(f"tbd: group_json: {group_json}") # write back to S3, save to metadata cache await save_metadata_obj(app, group_id, group_json, bucket=bucket) @@ -343,7 +354,7 @@ async def PUT_Links(request): status = 200 # put the status in the JSON response since the http_put function - # used the the SN won't return it + # used by the the SN won't return it resp_json = {"status": status} resp = json_response(resp_json, status=status) @@ -398,22 +409,36 @@ async def DELETE_Links(request): links = group_json["links"] + # add link titles to deleted set, so we can return a 410 if they + # are requested in the future + deleted_links = app["deleted_links"] + if group_id in deleted_links: + link_delete_set = deleted_links[group_id] + else: + link_delete_set = set() + deleted_links[group_id] = link_delete_set + + save_obj = False # set to True if anything actually updated for title in titles: if title not in links: + if title in link_delete_set: + log.warn(f"Link name {title} has already been deleted") + continue msg = f"Link name {title} not found in group: {group_id}" log.warn(msg) raise HTTPNotFound() - # now delete the links - for title in titles: del links[title] # remove the link from dictionary + link_delete_set.add(title) + save_obj = True - # update the group lastModified - now = time.time() - group_json["lastModified"] = now + if save_obj: + # update the group lastModified + now = time.time() + group_json["lastModified"] = now - # write back to S3 - await save_metadata_obj(app, group_id, group_json, bucket=bucket) + # write back to S3 + await save_metadata_obj(app, group_id, group_json, bucket=bucket) resp_json = {} diff --git a/hsds/link_sn.py b/hsds/link_sn.py index 3ccad7fa..7d29acb0 100755 --- a/hsds/link_sn.py +++ b/hsds/link_sn.py @@ -22,9 +22,9 @@ from .util.authUtil import getUserPasswordFromRequest, validateUserPassword from .util.domainUtil import getDomainFromRequest, isValidDomain, verifyRoot from .util.domainUtil import getBucketForDomain -from .util.linkUtil import validateLinkName +from .util.linkUtil import validateLinkName, getLinkClass from .servicenode_lib import getDomainJson, validateAction -from .servicenode_lib import getLink, putLink, getLinks, deleteLinks +from .servicenode_lib import getLink, putLink, putLinks, getLinks, deleteLinks from .domain_crawl import DomainCrawler from . import hsds_logger as log @@ -134,7 +134,10 @@ async def GET_Link(request): log.warn(msg) raise HTTPBadRequest(reason=msg) link_title = request.match_info.get("title") - validateLinkName(link_title) + try: + validateLinkName(link_title) + except ValueError: + raise HTTPBadRequest(reason="invalid link name") username, pswd = getUserPasswordFromRequest(request) if username is None and app["allow_noauth"]: @@ -256,6 +259,191 @@ async def PUT_Link(request): return resp +async def PUT_Links(request): + """HTTP method to create a new links """ + log.request(request) + params = request.rel_url.query + app = request.app + status = None + + log.debug("PUT_Links") + + username, pswd = getUserPasswordFromRequest(request) + # write actions need auth + await validateUserPassword(app, username, pswd) + + if not request.has_body: + msg = "PUT_Links with no body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + try: + body = await request.json() + except JSONDecodeError: + msg = "Unable to load JSON body" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + domain = getDomainFromRequest(request) + if not isValidDomain(domain): + msg = f"Invalid domain: {domain}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + bucket = getBucketForDomain(domain) + log.debug(f"got bucket: {bucket}") + + # get domain JSON + domain_json = await getDomainJson(app, domain) + verifyRoot(domain_json) + + req_grp_id = request.match_info.get("id") + if not req_grp_id: + req_grp_id = domain_json["root"] + + if "links" in body: + link_items = body["links"] + if not isinstance(link_items, dict): + msg = f"PUT_Links expected dict for for links body, but got: {type(link_items)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + # validate the links + for title in link_items: + try: + validateLinkName(title) + link_item = link_items[title] + getLinkClass(link_item) + except ValueError: + raise HTTPBadRequest(reason="invalid link item") + else: + link_items = None + + if link_items: + log.debug(f"PUT Links {len(link_items)} links to add") + else: + log.debug("no links defined yet") + + # next, sort out where these attributes are going to + + grp_ids = {} + if "grp_ids" in body: + body_ids = body["grp_ids"] + if isinstance(body_ids, list): + # multi cast the links - each link in link_items + # will be written to each of the objects identified by obj_id + if not link_items: + msg = "no links provided" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + for grp_id in body_ids: + if not isValidUuid(grp_id): + msg = f"Invalid object id: {grp_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + grp_ids[grp_id] = link_items + + msg = f"{len(link_items)} links will be multicast to " + msg += f"{len(grp_ids)} objects" + log.info(msg) + elif isinstance(body_ids, dict): + # each value is body_ids is a set of links to write to the object + # unlike the above case, different attributes can be written to + # different objects + if link_items: + msg = "links defined outside the obj_ids dict" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + for grp_id in body_ids: + if not isValidUuid(grp_id): + msg = f"Invalid object id: {grp_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + id_json = body_ids[grp_id] + + if "links" not in id_json: + msg = f"PUT_links with no links for grp_id: {grp_id}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + link_items = id_json["links"] + if not isinstance(link_items, dict): + msg = f"PUT_Links expected dict for grp_id {grp_id}, " + msg += f"but got: {type(link_items)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + # validate link items + for title in link_items: + try: + validateLinkName(title) + link_item = link_items[title] + getLinkClass(link_item) + except ValueError: + raise HTTPBadRequest(reason="invalid link item") + grp_ids[grp_id] = link_items + + # write different attributes to different objects + msg = f"PUT_Links over {len(grp_ids)} objects" + else: + msg = f"unexpected type for grp_ids: {type(grp_ids)}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + # use the object id from the request + grp_id = request.match_info.get("id") + if not grp_id: + msg = "Missing object id" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + grp_ids[grp_id] = link_items # make it look like a list for consistency + + log.debug(f"got {len(grp_ids)} grp_ids") + + # TBD - verify that the grp_id belongs to the given domain + await validateAction(app, domain, req_grp_id, username, "create") + + kwargs = {"bucket": bucket} + if params.get("replace"): + kwargs["replace"] = True + + count = len(grp_ids) + if count == 0: + msg = "no grp_ids defined" + log.warn(f"PUT_Attributes: {msg}") + raise HTTPBadRequest(reason=msg) + elif count == 1: + # just send one PUT Attributes request to the dn + grp_id = list(grp_ids.keys())[0] + link_json = grp_ids[grp_id] + log.debug(f"got link_json: {link_json}") + + status = await putLinks(app, grp_id, link_json, **kwargs) + + else: + # put multi obj + + # mixin some additonal kwargs + crawler_params = {"follow_links": False} + if bucket: + crawler_params["bucket"] = bucket + + kwargs = {"action": "put_link", "raise_error": True, "params": crawler_params} + crawler = DomainCrawler(app, grp_ids, **kwargs) + + # will raise exception on not found, server busy, etc. + await crawler.crawl() + + status = crawler.get_status() + + log.info("DomainCrawler done for put_links action") + + # link creation successful + log.debug(f"PUT_Links returning status: {status}") + req_rsp = {} + resp = await jsonResponse(request, req_rsp, status=status) + log.response(request, resp=resp) + return resp + + async def DELETE_Links(request): """HTTP method to delete multiple links """ log.request(request) @@ -284,7 +472,10 @@ async def DELETE_Links(request): titles = titles_param.split(separator) for title in titles: - validateLinkName(title) + try: + validateLinkName(title) + except ValueError: + raise HTTPBadRequest(reason="invalid link name") username, pswd = getUserPasswordFromRequest(request) await validateUserPassword(app, username, pswd) @@ -392,7 +583,10 @@ async def POST_Links(request): log.debug(f"getting all links for {group_id}") elif isinstance(titles, list): for title in titles: - validateLinkName(title) # raises HTTPBadRequest if invalid + try: + validateLinkName(title) + except ValueError: + raise HTTPBadRequest(reason="invalid link name") else: msg = f"expected list for titles but got: {type(titles)}" log.warn(msg) diff --git a/hsds/servicenode.py b/hsds/servicenode.py index 0f99f1b0..2ea85319 100755 --- a/hsds/servicenode.py +++ b/hsds/servicenode.py @@ -25,13 +25,14 @@ from .basenode import healthCheck, baseInit from . import hsds_logger as log from .util.authUtil import initUserDB, initGroupDB, setPassword -from .domain_sn import GET_Domain, PUT_Domain, DELETE_Domain, GET_Domains +from .domain_sn import GET_Domain, PUT_Domain, DELETE_Domain, GET_Domains, POST_Domain from .domain_sn import GET_Datasets, GET_Groups, GET_Datatypes from .domain_sn import GET_ACL, GET_ACLs, PUT_ACL from .group_sn import GET_Group, POST_Group, DELETE_Group -from .link_sn import GET_Links, POST_Links, GET_Link, PUT_Link, DELETE_Link, DELETE_Links -from .attr_sn import GET_Attributes, GET_Attribute, PUT_Attribute, PUT_Attributes, DELETE_Attribute -from .attr_sn import DELETE_Attributes, GET_AttributeValue, PUT_AttributeValue, POST_Attributes +from .link_sn import GET_Links, POST_Links, GET_Link, PUT_Link, PUT_Links +from .link_sn import DELETE_Link, DELETE_Links +from .attr_sn import GET_Attributes, GET_Attribute, PUT_Attribute, PUT_Attributes, POST_Attributes +from .attr_sn import DELETE_Attributes, DELETE_Attribute, GET_AttributeValue, PUT_AttributeValue from .ctype_sn import GET_Datatype, POST_Datatype, DELETE_Datatype from .dset_sn import GET_Dataset, POST_Dataset, DELETE_Dataset from .dset_sn import GET_DatasetShape, PUT_DatasetShape, GET_DatasetType @@ -52,6 +53,7 @@ async def init(): app.router.add_route("GET", path, GET_Domain) app.router.add_route("DELETE", path, DELETE_Domain) app.router.add_route("PUT", path, PUT_Domain) + app.router.add_route("POST", path, POST_Domain) path = "/domains" app.router.add_route("GET", path, GET_Domains) @@ -83,6 +85,7 @@ async def init(): path = "/groups/{id}/links" app.router.add_route("GET", path, GET_Links) app.router.add_route("POST", path, POST_Links) + app.router.add_route("PUT", path, PUT_Links) app.router.add_route("DELETE", path, DELETE_Links) path = "/groups/{id}/links/{title}" diff --git a/hsds/servicenode_lib.py b/hsds/servicenode_lib.py index be009a90..fad9d1ea 100644 --- a/hsds/servicenode_lib.py +++ b/hsds/servicenode_lib.py @@ -24,7 +24,7 @@ from .util.arrayUtil import encodeData from .util.idUtil import getDataNodeUrl, getCollectionForId from .util.idUtil import isSchema2Id, getS3Key, isValidUuid -from .util.linkUtil import h5Join, validateLinkName +from .util.linkUtil import h5Join, validateLinkName, getLinkClass from .util.storUtil import getStorJSONObj, isStorObj from .util.authUtil import aclCheck from .util.httpUtil import http_get, http_put, http_post, http_delete @@ -390,7 +390,10 @@ async def putLink(app, group_id, title, tgt_id=None, h5path=None, h5domain=None, """ create a new link. Return 201 if this is a new link, or 200 if it's a duplicate of an existing link. """ - validateLinkName(title) + try: + validateLinkName(title) + except ValueError: + raise HTTPBadRequest(reason="invalid link name") if h5path and tgt_id: msg = "putLink - provide tgt_id or h5path, but not both" @@ -398,27 +401,17 @@ async def putLink(app, group_id, title, tgt_id=None, h5path=None, h5domain=None, raise HTTPBadRequest(reason=msg) link_json = {} - if tgt_id: - if not isValidUuid(tgt_id): - msg = f"putLink with invalid id: {tgt_id}" - log.warn(msg) - raise HTTPBadRequest(reason=msg) link_json["id"] = tgt_id - link_class = "H5L_TYPE_HARD" - elif h5path: + if h5path: link_json["h5path"] = h5path - # could be hard or soft link - if h5domain: - link_json["h5domain"] = h5domain - link_class = "H5L_TYPE_EXTERNAL" - else: - # soft link - link_class = "H5L_TYPE_SOFT" - else: - msg = "PUT Link with no id or h5path keys" - log.warn(msg) - raise HTTPBadRequest(reason=msg) + if h5domain: + link_json["h5domain"] = h5domain + + try: + link_class = getLinkClass(link_json) + except ValueError: + raise HTTPBadRequest(reason="invalid link") link_json["class"] = link_class @@ -474,6 +467,54 @@ async def putExternalLink(app, group_id, title, h5path=None, h5domain=None, buck return status +async def putLinks(app, group_id, items, bucket=None): + """ create a new links. Return 201 if any item is a new link, + or 200 if it's a duplicate of an existing link. """ + + isValidUuid(group_id, obj_class="group") + group_json = None + + # validate input + for title in items: + try: + validateLinkName(title) + item = items[title] + link_class = getLinkClass(item) + except ValueError: + # invalid link + raise HTTPBadRequest(reason="invalid link") + + if link_class == "H5L_TYPE_HARD": + tgt_id = item["id"] + isValidUuid(tgt_id) + # for hard links, verify that the referenced id exists and is in + # this domain + ref_json = await getObjectJson(app, tgt_id, bucket=bucket) + if not group_json: + # just need to fetch this once + group_json = await getObjectJson(app, group_id, bucket=bucket) + if ref_json["root"] != group_json["root"]: + msg = "Hard link must reference an object in the same domain" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + + # ready to add links now + req = getDataNodeUrl(app, group_id) + req += "/groups/" + group_id + "/links" + log.debug(f"PUT links - PUT request: {req}") + params = {"bucket": bucket} + + data = {"links": items} + + put_rsp = await http_put(app, req, data=data, params=params) + log.debug(f"PUT Link resp: {put_rsp}") + if "status" in put_rsp: + status = put_rsp["status"] + else: + status = 201 + return status + + async def deleteLinks(app, group_id, titles=None, separator="/", bucket=None): """ delete the requested set of links from the given object """ diff --git a/hsds/util/attrUtil.py b/hsds/util/attrUtil.py index 68ef2cdc..e6afd26e 100755 --- a/hsds/util/attrUtil.py +++ b/hsds/util/attrUtil.py @@ -51,3 +51,32 @@ def validateAttributeName(name): msg = f"attribute name must be a string, but got: {type(name)}" log.warn(msg) raise HTTPBadRequest(reason=msg) + + +def isEqualAttr(attr1, attr2): + """ compare to attributes, return True if the same, False if differnt """ + for obj in (attr1, attr2): + if not isinstance(obj, dict): + raise TypeError(f"unexpected type: {type(obj)}") + if "type" not in obj: + raise TypeError("expected type key for attribute") + if "shape" not in obj: + raise TypeError("expected shape key for attribute") + # value is optional (not set for null space attributes) + if attr1["type"] != attr2["type"]: + return False + if attr1["shape"] != attr2["shape"]: + return False + shape_class = attr1["shape"].get("class") + if shape_class == "H5S_NULL": + return True # nothing else to compare + for obj in (attr1, attr2): + if "value" not in obj: + raise TypeError("expected value key for attribute") + return attr1["value"] == attr2["value"] + + if not isinstance(attr1, dict): + raise TypeError(f"unexpected type: {type(attr1)}") + return True + if not attr1 and not attr2: + return True diff --git a/hsds/util/linkUtil.py b/hsds/util/linkUtil.py index b16133d1..3469a8a1 100644 --- a/hsds/util/linkUtil.py +++ b/hsds/util/linkUtil.py @@ -13,20 +13,112 @@ # linkdUtil: # link related functions # -from aiohttp.web_exceptions import HTTPBadRequest from .. import hsds_logger as log def validateLinkName(name): + """ verify the link name is valid """ if not isinstance(name, str): msg = "Unexpected type for link name" - log.error(msg) - raise HTTPBadRequest(reason=msg) + log.warn(msg) + raise ValueError(msg) if name.find("/") >= 0: msg = "link name contains slash" - log.error(msg) - raise HTTPBadRequest(reason=msg) + log.warn(msg) + raise ValueError(msg) + + +def getLinkClass(link_json): + """ verify this is a valid link + returns the link class """ + if "class" in link_json: + link_class = link_json["class"] + else: + link_class = None + if "h5path" in link_json and "id" in link_json: + msg = "link tgt_id and h5path both set" + log.warn(msg) + raise ValueError(msg) + if "id" in link_json: + tgt_id = link_json["id"] + if not isinstance(tgt_id, str) or len(tgt_id) < 38: + msg = f"link with invalid id: {tgt_id}" + log.warn(msg) + raise ValueError(msg) + if tgt_id[:2] not in ("g-", "t-", "d-"): + msg = "link tgt must be group, datatype or dataset uuid" + log.warn(msg) + raise ValueError(msg) + if link_class: + if link_class != "H5L_TYPE_HARD": + msg = f"expected link class to be H5L_TYPE_HARD but got: {link_class}" + log.warn(msg) + raise ValueError(msg) + else: + link_class = "H5L_TYPE_HARD" + elif "h5path" in link_json: + h5path = link_json["h5path"] + log.debug(f"link path: {h5path}") + if "h5domain" in link_json: + if link_class: + if link_class != "H5L_TYPE_EXTERNAL": + msg = f"expected link class to be H5L_TYPE_EXTERNAL but got: {link_class}" + log.warn(msg) + raise ValueError(msg) + else: + link_class = "H5L_TYPE_EXTERNAL" + else: + if link_class: + if link_class != "H5L_TYPE_SOFT": + msg = f"expected link class to be H5L_TYPE_SOFT but got: {link_class}" + log.warn(msg) + raise ValueError(msg) + else: + link_class = "H5L_TYPE_SOFT" + else: + msg = "link with no id or h5path" + log.warn(msg) + raise ValueError(msg) + + return link_class + + +def isEqualLink(link1, link2): + """ Return True if the two links are the same """ + + for obj in (link1, link2): + if not isinstance(obj, dict): + raise TypeError(f"unexpected type: {type(obj)}") + if "class" not in obj: + raise TypeError("expected class key for link") + if link1["class"] != link2["class"]: + return False # different link types + link_class = link1["class"] + if link_class == "H5L_TYPE_HARD": + for obj in (link1, link2): + if "id" not in obj: + raise TypeError(f"expected id key for link: {obj}") + if link1["id"] != link2["id"]: + return False + elif link_class == "H5L_TYPE_SOFT": + for obj in (link1, link2): + if "h5path" not in obj: + raise TypeError(f"expected h5path key for link: {obj}") + if link1["h5path"] != link2["h5path"]: + return False + elif link_class == "H5L_TYPE_EXTERNAL": + for obj in (link1, link2): + for k in ("h5path", "h5domain"): + if k not in obj: + raise TypeError(f"expected {k} key for link: {obj}") + if link1["h5path"] != link2["h5path"]: + return False + if link1["h5domain"] != link2["h5domain"]: + return False + else: + raise TypeError(f"unexpected link class: {link_class}") + return True def h5Join(path, paths): diff --git a/tests/integ/attr_test.py b/tests/integ/attr_test.py index 78c7a099..7376f9a8 100644 --- a/tests/integ/attr_test.py +++ b/tests/integ/attr_test.py @@ -245,6 +245,7 @@ def testObjAttr(self): rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 404) # not found attr_payload = {"type": "H5T_STD_I32LE", "value": 42} + attr_payload2 = {"type": "H5T_STD_I32LE", "value": 84} # try adding the attribute as a different user user2_name = config.get("user2_name") @@ -266,6 +267,14 @@ def testObjAttr(self): rsp = self.session.put(req, data=json.dumps(attr_payload), headers=headers) self.assertEqual(rsp.status_code, 201) # created + # try resending + rsp = self.session.put(req, data=json.dumps(attr_payload), headers=headers) + self.assertEqual(rsp.status_code, 200) # ok + + # try with a different value + rsp = self.session.put(req, data=json.dumps(attr_payload2), headers=headers) + self.assertEqual(rsp.status_code, 409) # conflict + # read the attribute we just created rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) # create attribute @@ -287,11 +296,11 @@ def testObjAttr(self): rspJson = json.loads(rsp.text) self.assertEqual(rspJson["attributeCount"], 1) # one attribute - # try creating the attribute again - should return 409 + # try creating the attribute again - should return 200 req = f"{self.endpoint}/{col_name}/{obj1_id}/attributes/{attr_name}" rsp = self.session.put(req, data=json.dumps(attr_payload), headers=headers) - self.assertEqual(rsp.status_code, 409) # conflict + self.assertEqual(rsp.status_code, 200) # OK # set the replace param and we should get a 200 params = {"replace": 1} @@ -327,6 +336,10 @@ def testEmptyShapeAttr(self): rsp = self.session.put(req, headers=headers, data=json.dumps(attr_payload)) self.assertEqual(rsp.status_code, 201) # created + # retry + rsp = self.session.put(req, headers=headers, data=json.dumps(attr_payload)) + self.assertEqual(rsp.status_code, 200) # OK + # read back the attribute rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) # OK @@ -411,6 +424,10 @@ def testNoShapeAttr(self): rsp = self.session.put(req, headers=headers, data=json.dumps(attr_payload)) self.assertEqual(rsp.status_code, 201) # created + # try re-sending the put. Should return 200 + rsp = self.session.put(req, headers=headers, data=json.dumps(attr_payload)) + self.assertEqual(rsp.status_code, 200) + # read back the attribute rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) # OK @@ -465,6 +482,10 @@ def testPutFixedString(self): rsp = self.session.put(req, data=json.dumps(data), headers=headers) self.assertEqual(rsp.status_code, 201) + # try re-sending the put. Should return 200 + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 200) + # read attr rsp = self.session.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) @@ -2027,9 +2048,9 @@ def testPutAttributeMultiple(self): self.assertEqual(len(attr_value), extent) self.assertEqual(attr_value, [i * 10 + j for j in range(extent)]) - # try writing again, should get 409 + # try writing again, should get 200 rsp = self.session.put(req, data=json.dumps(data), headers=headers) - self.assertEqual(rsp.status_code, 409) + self.assertEqual(rsp.status_code, 200) # write attributes to the three group objects data = {"obj_ids": grp_ids, "attributes": attributes} @@ -2090,10 +2111,10 @@ def testPutAttributeMultiple(self): self.assertEqual(len(attr_value), extent) self.assertEqual(attr_value, expected_value) - # try writing again, should get 409 + # try writing again, should get 200 req = self.endpoint + "/groups/" + root_id + "/attributes" rsp = self.session.put(req, data=json.dumps(data), headers=headers) - self.assertEqual(rsp.status_code, 409) + self.assertEqual(rsp.status_code, 200) def testDeleteAttributesMultiple(self): print("testDeleteAttributesMultiple", self.base_domain) @@ -2146,7 +2167,7 @@ def testDeleteAttributesMultiple(self): for i in range(attr_count): req = self.endpoint + "/groups/" + grp_id + "/attributes/" + attr_names[i] rsp = self.session.get(req, headers=headers) - self.assertEqual(rsp.status_code, 404) + self.assertEqual(rsp.status_code, 410) # Create another batch of attributes for i in range(attr_count): @@ -2168,7 +2189,7 @@ def testDeleteAttributesMultiple(self): for i in range(attr_count): req = self.endpoint + "/groups/" + grp_id + "/attributes/" + attr_names[i] rsp = self.session.get(req, headers=headers) - self.assertEqual(rsp.status_code, 404) + self.assertEqual(rsp.status_code, 410) if __name__ == "__main__": diff --git a/tests/integ/domain_test.py b/tests/integ/domain_test.py index e339aa6e..1f69749a 100755 --- a/tests/integ/domain_test.py +++ b/tests/integ/domain_test.py @@ -189,6 +189,35 @@ def testGetDomain(self): rsp = self.session.get(req, params=params, headers=headers) self.assertEqual(rsp.status_code, 400) + def testPostDomain(self): + domain = helper.getTestDomain("tall.h5") + print("testPostDomain", domain) + headers = helper.getRequestHeaders(domain=domain) + + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + if rsp.status_code != 200: + msg = f"WARNING: Failed to get domain: {domain}. Is test data setup?" + print(msg) + return # abort rest of test + domainJson = json.loads(rsp.text) + self.assertTrue("root" in domainJson) + root_id = domainJson["root"] + + # Get group at /g1/g1.1 by using h5path + data = {"h5paths": ["/g1/g1.1", ]} + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("h5paths" in rspJson) + rsp_paths = rspJson["h5paths"] + self.assertTrue("/g1/g1.1" in rsp_paths) + obj_json = rsp_paths["/g1/g1.1"] + g11id = helper.getUUIDByPath(domain, "/g1/g1.1", session=self.session) + self.assertEqual(g11id, obj_json["id"]) + self.assertTrue("root" in obj_json) + self.assertEqual(root_id, obj_json["root"]) + def testGetByPath(self): domain = helper.getTestDomain("tall.h5") print("testGetByPath", domain) diff --git a/tests/integ/link_test.py b/tests/integ/link_test.py index 1633b95c..faad65a7 100755 --- a/tests/integ/link_test.py +++ b/tests/integ/link_test.py @@ -23,6 +23,7 @@ def __init__(self, *args, **kwargs): super(LinkTest, self).__init__(*args, **kwargs) self.base_domain = helper.getTestDomainName(self.__class__.__name__) helper.setupDomain(self.base_domain, folder=True) + self.endpoint = helper.getEndpoint() def setUp(self): self.session = helper.getSession() @@ -861,6 +862,97 @@ def testRootH5Path(self): self.assertTrue(k in cprops) self.assertEqual(cprops[k], creation_props[k]) + def testNonURLEncodableLinkName(self): + domain = self.base_domain + "/testNonURLEncodableLinkName.h5" + print("testNonURLEncodableLinkName", domain) + helper.setupDomain(domain) + + headers = helper.getRequestHeaders(domain=domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create a subgroup + req = self.endpoint + "/groups" + rsp = self.session.post(req, headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + grp_id = rspJson["id"] + self.assertTrue(helper.validateId(grp_id)) + + # link as "grp1" + grp_name = "grp1" + req = self.endpoint + "/groups/" + root_uuid + "/links/" + grp_name + payload = {"id": grp_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # created + + link_name = "#link1#" + data = {"h5path": "somewhere"} + req = self.endpoint + "/groups/" + grp_id + "/links" # request without name + bad_req = f"{req}/{link_name}" # this request will fail because of the hash char + + # create link + rsp = self.session.put(bad_req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 404) # regular put doesn't work + + links = {link_name: data} + body = {"links": links} + + rsp = self.session.put(req, data=json.dumps(body), headers=headers) + self.assertEqual(rsp.status_code, 201) # this is ok + + # get all links and verify the one we created is there + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + rsp_links = rspJson["links"] + self.assertEqual(len(rsp_links), 1) + rsp_link = rsp_links[0] + self.assertTrue("title" in rsp_link) + self.assertEqual(rsp_link["title"], link_name) + + # try doing a get on this specific link + rsp = self.session.get(bad_req, headers=headers) + self.assertEqual(rsp.status_code, 404) # can't do a get with the link name + + # do a post request with the link name + link_names = [link_name, ] + data = {"titles": link_names} + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + rsp_links = rspJson["links"] + self.assertEqual(len(rsp_links), 1) + rsp_links = rsp_links[0] + + self.assertTrue("title" in rsp_link) + self.assertEqual(rsp_link["title"], link_name) + + # try deleting the link by name + rsp = self.session.delete(bad_req, headers=headers) + self.assertEqual(rsp.status_code, 404) # not found + + # send link name as a query param + params = {"titles": link_names} + rsp = self.session.delete(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + + # verify the link is gone + rsp = self.session.get(req, headers=headers, params=params) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + rsp_links = rspJson["links"] + self.assertEqual(len(rsp_links), 0) + def testPostLinkSingle(self): domain = helper.getTestDomain("tall.h5") print("testPostLinkSingle", domain) @@ -1052,6 +1144,273 @@ def testPostLinkMultiple(self): else: self.assertTrue(False) # unexpected + def testPutLinkMultiple(self): + domain = self.base_domain + "/testPutLinkMultiple.h5" + helper.setupDomain(domain) + print("testPutLinkMultiple", domain) + headers = helper.getRequestHeaders(domain=domain) + req = self.endpoint + "/" + + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_id = rspJson["root"] + + # create a group + req = self.endpoint + "/groups" + rsp = self.session.post(req, headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + grpA_id = rspJson["id"] + self.assertTrue(helper.validateId(grpA_id)) + + # link new obj as '/grpA' + req = self.endpoint + "/groups/" + root_id + "/links/grpA" + payload = {"id": grpA_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # created + + # create some groups under grp1 + grp_count = 3 + + grp_names = [f"grp{(i+1):04d}" for i in range(grp_count)] + grp_ids = [] + + for grp_name in grp_names: + # create sub_groups + req = self.endpoint + "/groups" + rsp = self.session.post(req, headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + grp_id = rspJson["id"] + self.assertTrue(helper.validateId(grp_id)) + grp_ids.append(grp_id) + + # create some links + links = {} + for i in range(grp_count): + title = grp_names[i] + links[title] = {"id": grp_ids[i]} + + # add a soft and external link as well + links["softlink"] = {"h5path": "a_path"} + links["extlink"] = {"h5path": "another_path", "h5domain": "/a_domain"} + link_count = len(links) + + # write links to the grpA + data = {"links": links} + req = self.endpoint + "/groups/" + grpA_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # do a get on the links + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + ret_links = rspJson["links"] + self.assertEqual(len(ret_links), link_count) + for link in ret_links: + self.assertTrue("title" in link) + title = link["title"] + self.assertTrue("class" in link) + link_class = link["class"] + if link_class == "H5L_TYPE_HARD": + self.assertTrue("id" in link) + self.assertTrue(link["id"] in grp_ids) + self.assertTrue(title in grp_names) + elif link_class == "H5L_TYPE_SOFT": + self.assertTrue("h5path" in link) + h5path = link["h5path"] + self.assertEqual(h5path, "a_path") + elif link_class == "H5L_TYPE_EXTERNAL": + self.assertTrue("h5path" in link) + h5path = link["h5path"] + self.assertEqual(h5path, "another_path") + self.assertTrue("h5domain" in link) + h5domain = link["h5domain"] + self.assertEqual(h5domain, "/a_domain") + else: + self.assertTrue(False) # unexpected + + # try writing again, should get 200 (no new links) + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 200) + + # write some links to three group objects + links = {} + links["hardlink_multicast"] = {"id": root_id} + links["softlink_multicast"] = {"h5path": "multi_path"} + links["extlink_multicast"] = {"h5path": "multi_path", "h5domain": "/another_domain"} + link_count = len(links) + data = {"links": links, "grp_ids": grp_ids} + req = self.endpoint + "/groups/" + root_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # check that the links got created + for grp_id in grp_ids: + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + ret_links = rspJson["links"] + self.assertEqual(len(ret_links), 3) + for ret_link in ret_links: + self.assertTrue("class" in ret_link) + link_class = ret_link["class"] + if link_class == "H5L_TYPE_HARD": + self.assertTrue("id" in ret_link) + self.assertEqual(ret_link["id"], root_id) + elif link_class == "H5L_TYPE_SOFT": + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], "multi_path") + elif link_class == "H5L_TYPE_EXTERNAL": + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], "multi_path") + self.assertTrue("h5domain" in ret_link) + self.assertEqual(ret_link["h5domain"], "/another_domain") + else: + self.assertTrue(False) # unexpected + + # write different links to three group objects + link_data = {} + for i in range(grp_count): + grp_id = grp_ids[i] + links = {} + links[f"hardlink_{i}"] = {"id": root_id} + links[f"softlink_{i}"] = {"h5path": f"multi_path_{i}"} + ext_link = {"h5path": f"multi_path_{i}", "h5domain": f"/another_domain/{i}"} + links[f"extlink_{i}"] = ext_link + link_data[grp_id] = {"links": links} + + data = {"grp_ids": link_data} + req = self.endpoint + "/groups/" + root_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # check that the new links got created + for i in range(grp_count): + grp_id = grp_ids[i] + titles = [f"hardlink_{i}", f"softlink_{i}", f"extlink_{i}", ] + data = {"titles": titles} + # do a post to just return the links we are interested in + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.post(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("links" in rspJson) + ret_links = rspJson["links"] + self.assertEqual(len(ret_links), len(titles)) + for j in range(len(titles)): + ret_link = ret_links[j] + self.assertTrue("class" in ret_link) + link_class = ret_link["class"] + self.assertTrue("title" in ret_link) + link_title = ret_link["title"] + if link_class == "H5L_TYPE_HARD": + self.assertEqual(link_title, f"hardlink_{i}") + self.assertTrue("id" in ret_link) + self.assertEqual(ret_link["id"], root_id) + elif link_class == "H5L_TYPE_SOFT": + self.assertEqual(link_title, f"softlink_{i}") + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], f"multi_path_{i}") + elif link_class == "H5L_TYPE_EXTERNAL": + self.assertEqual(link_title, f"extlink_{i}") + self.assertTrue("h5path" in ret_link) + self.assertEqual(ret_link["h5path"], f"multi_path_{i}") + self.assertTrue("h5domain" in ret_link) + self.assertEqual(ret_link["h5domain"], f"/another_domain/{i}") + else: + self.assertTrue(False) # unexpected + + def testDeleteLinkMultiple(self): + domain = self.base_domain + "/testDeleteLinkMultiple.h5" + helper.setupDomain(domain) + + print("testDeleteLinkMultiple", self.base_domain) + + headers = helper.getRequestHeaders(domain=domain) + req = self.endpoint + "/" + + # Get root uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + root_uuid = rspJson["root"] + helper.validateId(root_uuid) + + # create a subgroup + req = self.endpoint + "/groups" + rsp = self.session.post(req, headers=headers) + self.assertEqual(rsp.status_code, 201) + rspJson = json.loads(rsp.text) + grp_id = rspJson["id"] + self.assertTrue(helper.validateId(grp_id)) + + # link as "grp1" + grp_name = "grp1" + req = self.endpoint + "/groups/" + root_uuid + "/links/" + grp_name + payload = {"id": grp_id} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # created + + # create some links + titles = [] + links = {} + title = "root" + links[title] = {"id": root_uuid} + titles.append(title) + + # add a soft and external link as well + title = "softlink" + links[title] = {"h5path": "a_path"} + titles.append(title) + title = "extlink" + links[title] = {"h5path": "another_path", "h5domain": "/a_domain"} + titles.append(title) + link_count = len(links) + + # write links to the grp1 + data = {"links": links} + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # Delete all by parameter + separator = '/' + params = {"titles": separator.join(titles)} + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.delete(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + + # Attempt to read deleted links + for i in range(link_count): + req = self.endpoint + "/groups/" + grp_id + "/links/" + titles[i] + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 410) + + # re-create links + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.put(req, data=json.dumps(data), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # Delete with custom separator + separator = ':' + params = {"titles": separator.join(titles)} + params["separator"] = ":" + req = self.endpoint + "/groups/" + grp_id + "/links" + rsp = self.session.delete(req, params=params, headers=headers) + self.assertEqual(rsp.status_code, 200) + + # Attempt to read + for i in range(link_count): + req = self.endpoint + "/groups/" + grp_id + "/links/" + titles[i] + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 410) + if __name__ == "__main__": # setup test files