diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..42254de --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +ras_stac_venv/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8c45fb3..4b384a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.0 +FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.5 RUN apt-get update && \ apt-get install jq -y && \ @@ -11,12 +11,11 @@ RUN pip3 install -r requirements.txt WORKDIR /plugins # Copy plugin utils -COPY ras_stac/utils utils +COPY ras_stac/utils ras_stac/utils # Copy plugin functions -COPY ras_stac/ras_geom_hdf.py . -COPY ras_stac/ras_plan_hdf.py . -COPY ras_stac/ras_plan_dg.py . +COPY ras_stac/ras_geom_hdf.py ras_stac/ +COPY ras_stac/ras_plan_hdf.py ras_stac/ +COPY ras_stac/ras_plan_dg.py ras_stac/ -# Copy plugin main functions -COPY ras_stac/plugins . +COPY tests tests diff --git a/README.md b/README.md index 2fcfb4a..465a492 100644 --- a/README.md +++ b/README.md @@ -19,4 +19,9 @@ This repository contains code for developing STAC items from HEC-RAS models. Cur 3. Run the `populate-sample-data.sh` script to test set-up, connetivity, and view a sample stac catalog created using this library. -**NOTE** It is recommended that ras-stac not be run in a container for testing and development due to networking issues that complicate use of these tools, when using minio. +**NOTE** It is recommended that ras-stac not be run in a container for testing and development of its full codebase due to networking issues that complicate use of these tools, when using minio. + + +## Core tests (can run locally in Docker) + +When `docker-test.sh` is executed, the Docker image is built and `pytest` is invoked to run the Python test scripts. This leverages test data that is included in this repository. This does not use cloud storage, s3, minio, nor other forms of network data. diff --git a/docker-test.sh b/docker-test.sh new file mode 100755 index 0000000..5f5e509 --- /dev/null +++ b/docker-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail +set -x + +docker build -t ras-stac . +docker run --rm -it -w /plugins/tests ras-stac pytest diff --git a/ras_stac/ras_geom_hdf.py b/ras_stac/ras_geom_hdf.py index 1f91c4d..b4882cd 100644 --- a/ras_stac/ras_geom_hdf.py +++ b/ras_stac/ras_geom_hdf.py @@ -8,7 +8,7 @@ from typing import List from .utils.common import check_params, GEOM_HDF_IGNORE_PROPERTIES -from .utils.ras_stac import create_model_item, new_geom_assets, ras_geom_asset_info +from .utils.ras_utils import RasStacGeom, new_geom_assets, ras_geom_asset_info from .utils.s3_utils import ( verify_safe_prefix, s3_key_public_url_converter, @@ -16,6 +16,7 @@ init_s3_resources, get_basic_object_metadata, copy_item_to_s3, + read_ras_geom_from_s3, ) logging.getLogger("boto3").setLevel(logging.WARNING) @@ -56,12 +57,12 @@ def new_geom_item( _, s3_client, s3_resource = init_s3_resources(minio_mode=minio_mode) bucket = s3_resource.Bucket(bucket_name) # Create geometry item + geom_hdf_obj, ras_model_name = read_ras_geom_from_s3(geom_hdf, minio_mode) + ras_stac_geom = RasStacGeom(geom_hdf_obj) if item_props_to_remove: - item = create_model_item(geom_hdf, item_props_to_remove, minio_mode=minio_mode) + item = ras_stac_geom.to_item(item_props_to_remove, ras_model_name) else: - item = create_model_item( - geom_hdf, GEOM_HDF_IGNORE_PROPERTIES, minio_mode=minio_mode - ) + item = ras_stac_geom.to_item(GEOM_HDF_IGNORE_PROPERTIES, ras_model_name) if item_props_to_add: item.properties.update(item_props_to_add) diff --git a/ras_stac/ras_plan_dg.py b/ras_stac/ras_plan_dg.py index 03c54ab..517a4eb 100644 --- a/ras_stac/ras_plan_dg.py +++ b/ras_stac/ras_plan_dg.py @@ -8,7 +8,7 @@ from .utils.common import check_params from .utils.dg_utils import create_depth_grid_item -from utils.ras_stac import ras_plan_asset_info +from .utils.ras_utils import ras_plan_asset_info from .utils.s3_utils import ( verify_safe_prefix, s3_key_public_url_converter, diff --git a/ras_stac/ras_plan_hdf.py b/ras_stac/ras_plan_hdf.py index c6a1cf4..4a3a192 100644 --- a/ras_stac/ras_plan_hdf.py +++ b/ras_stac/ras_plan_hdf.py @@ -7,11 +7,7 @@ from typing import List from .utils.common import check_params, PLAN_HDF_IGNORE_PROPERTIES -from .utils.ras_stac import ( - get_simulation_metadata, - create_model_simulation_item, - ras_plan_asset_info, -) +from .utils.ras_utils import ras_plan_asset_info, RasStacPlan from .utils.s3_utils import ( verify_safe_prefix, s3_key_public_url_converter, @@ -19,6 +15,7 @@ init_s3_resources, get_basic_object_metadata, copy_item_to_s3, + read_ras_plan_from_s3, ) @@ -66,16 +63,18 @@ def new_plan_item( geom_item = pystac.Item.from_file(geom_item_public_url) logging.info("fetching plan metadata") - plan_meta = get_simulation_metadata(plan_hdf, sim_id, minio_mode=minio_mode) + plan_hdf_obj = read_ras_plan_from_s3(plan_hdf, minio_mode) + ras_stac_plan = RasStacPlan(plan_hdf_obj) + plan_meta = ras_stac_plan.get_simulation_metadata(sim_id) if plan_meta: try: logging.info("creating plan item") if item_props_to_remove: - plan_item = create_model_simulation_item( + plan_item = ras_stac_plan.to_item( geom_item, plan_meta, sim_id, item_props_to_remove ) else: - plan_item = create_model_simulation_item( + plan_item = ras_stac_plan.to_item( geom_item, plan_meta, sim_id, PLAN_HDF_IGNORE_PROPERTIES ) except TypeError: diff --git a/ras_stac/utils/dg_utils.py b/ras_stac/utils/dg_utils.py index 1aa2317..6943fa1 100644 --- a/ras_stac/utils/dg_utils.py +++ b/ras_stac/utils/dg_utils.py @@ -3,8 +3,9 @@ import os import pystac import rasterio +import rasterio.warp -from datetime import datetime +from datetime import datetime, timezone from mypy_boto3_s3.service_resource import Object from pathlib import Path from rasterio.session import AWSSession @@ -91,9 +92,9 @@ def bbox_to_polygon(bbox) -> Polygon: return Polygon( [ [min_x, min_y], - [min_x, max_y], - [max_x, max_y], [max_x, min_y], + [max_x, max_y], + [min_x, max_y], ] ) @@ -125,7 +126,7 @@ def create_depth_grid_item( id=item_id, properties={}, bbox=bbox, - datetime=datetime.now(), + datetime=datetime.now(timezone.utc), geometry=json.loads(to_geojson(geometry)), ) # non_null = not raster_is_all_null(depth_grid.key) diff --git a/ras_stac/utils/ras_hdf.py b/ras_stac/utils/ras_hdf.py deleted file mode 100644 index 2011fed..0000000 --- a/ras_stac/utils/ras_hdf.py +++ /dev/null @@ -1,218 +0,0 @@ -import logging -from dotenv import load_dotenv, find_dotenv - -from rashdf import RasPlanHdf, RasGeomHdf -from rashdf.utils import parse_duration - -logging.getLogger("boto3").setLevel(logging.WARNING) -logging.getLogger("botocore").setLevel(logging.WARNING) - -load_dotenv(find_dotenv()) - - -def to_snake_case(text): - """ - Convert a string to snake case, removing punctuation and other symbols. - - Parameters: - text (str): The string to be converted. - - Returns: - str: The snake case version of the string. - """ - import re - - # Remove all non-word characters (everything except numbers and letters) - text = re.sub(r"[^\w\s]", "", text) - - # Replace all runs of whitespace with a single underscore - text = re.sub(r"\s+", "_", text) - - return text.lower() - - -def prep_stac_attrs(attrs: dict, prefix: str = None) -> dict: - """ - Converts an unformatted HDF attributes dictionary to STAC format by converting values to snake case - and adding a prefix if one is given. - - Parameters: - attrs (dict): Unformatted attribute dictionary. - prefix (str): Optional prefix to be added to each key of formatted dictionary. - - Returns: - results (dict): The new attribute dictionary snake case values and prefix. - """ - results = {} - for k, value in attrs.items(): - if prefix: - key = f"{to_snake_case(prefix)}:{to_snake_case(k)}" - else: - key = k - results[key] = value - - return results - - -def get_stac_plan_attrs(ras_hdf: RasPlanHdf, include_results: bool = False) -> dict: - """ - This function retrieves the attributes of a plan from a HEC-RAS plan HDF file, converting them to STAC format. - - Parameters: - ras_hdf (RasPlanHdf): An instance of RasPlanHdf which the attributes will be retrieved from. - include_results (bool, optional): Whether to include the results attributes in the returned dictionary. - Defaults to False. - - Returns: - stac_plan_attrs (dict): A dictionary with the attributes of the plan. - """ - stac_plan_attrs = ras_hdf.get_root_attrs() - if stac_plan_attrs is not None: - stac_plan_attrs = prep_stac_attrs(stac_plan_attrs) - else: - stac_plan_attrs = {} - logging.warning("No root attributes found.") - - plan_info_attrs = ras_hdf.get_plan_info_attrs() - if plan_info_attrs is not None: - plan_info_stac_attrs = prep_stac_attrs( - plan_info_attrs, prefix="Plan Information" - ) - stac_plan_attrs.update(plan_info_stac_attrs) - else: - logging.warning("No plan information attributes found.") - - plan_params_attrs = ras_hdf.get_plan_param_attrs() - if plan_params_attrs is not None: - plan_params_stac_attrs = prep_stac_attrs( - plan_params_attrs, prefix="Plan Parameters" - ) - stac_plan_attrs.update(plan_params_stac_attrs) - else: - logging.warning("No plan parameters attributes found.") - - precip_attrs = ras_hdf.get_meteorology_precip_attrs() - if precip_attrs is not None: - precip_stac_attrs = prep_stac_attrs(precip_attrs, prefix="Meteorology") - precip_stac_attrs.pop("meteorology:projection", None) - stac_plan_attrs.update(precip_stac_attrs) - else: - logging.warning("No meteorology precipitation attributes found.") - - if include_results: - stac_plan_attrs.update(get_stac_plan_results_attrs(ras_hdf)) - return stac_plan_attrs - - -def get_stac_plan_results_attrs(ras_hdf: RasPlanHdf): - """ - This function retrieves the results attributes of a plan from a HEC-RAS plan HDF file, converting - them to STAC format. For summary atrributes, it retrieves the total computation time, the run time window, - and the solution from it, and calculates the total computation time in minutes if it exists. - - Parameters: - ras_hdf (RasPlanHdf): An instance of RasPlanHdf which the results attributes will be retrieved from. - - Returns: - results_attrs (dict): A dictionary with the results attributes of the plan. - """ - results_attrs = {} - - unsteady_results_attrs = ras_hdf.get_results_unsteady_attrs() - if unsteady_results_attrs is not None: - unsteady_results_stac_attrs = prep_stac_attrs( - unsteady_results_attrs, prefix="Unsteady Results" - ) - results_attrs.update(unsteady_results_stac_attrs) - else: - logging.warning("No unsteady results attributes found.") - - summary_attrs = ras_hdf.get_results_unsteady_summary_attrs() - if summary_attrs is not None: - summary_stac_attrs = prep_stac_attrs(summary_attrs, prefix="Results Summary") - computation_time_total = summary_stac_attrs.get( - "results_summary:computation_time_total" - ) - results_summary = { - "results_summary:computation_time_total": computation_time_total, - "results_summary:run_time_window": summary_stac_attrs.get( - "results_summary:run_time_window" - ), - "results_summary:solution": summary_stac_attrs.get( - "results_summary:solution" - ), - } - if computation_time_total is not None: - computation_time_total_minutes = ( - parse_duration(computation_time_total).total_seconds() / 60 - ) - results_summary["results_summary:computation_time_total_minutes"] = ( - computation_time_total_minutes - ) - results_attrs.update(results_summary) - else: - logging.warning("No unsteady results summary attributes found.") - - volume_accounting_attrs = ras_hdf.get_results_volume_accounting_attrs() - if volume_accounting_attrs is not None: - volume_accounting_stac_attrs = prep_stac_attrs( - volume_accounting_attrs, prefix="Volume Accounting" - ) - results_attrs.update(volume_accounting_stac_attrs) - else: - logging.warning("No results volume accounting attributes found.") - - return results_attrs - - -def get_stac_geom_attrs(ras_hdf: RasGeomHdf): - """ - This function retrieves the geometry attributes of a HEC-RAS HDF file, converting them to STAC format. - - Parameters: - ras_hdf (RasGeomHdf): An instance of RasGeomHdf which the geometry attributes will be retrieved from. - - Returns: - stac_geom_attrs (dict): A dictionary with the organized geometry attributes. - """ - - stac_geom_attrs = ras_hdf.get_root_attrs() - if stac_geom_attrs is not None: - stac_geom_attrs = prep_stac_attrs(stac_geom_attrs) - else: - stac_geom_attrs = {} - logging.warning("No root attributes found.") - - geom_attrs = ras_hdf.get_geom_attrs() - if geom_attrs is not None: - geom_stac_attrs = prep_stac_attrs(geom_attrs, prefix="Geometry") - stac_geom_attrs.update(geom_stac_attrs) - else: - logging.warning("No base geometry attributes found.") - - structures_attrs = ras_hdf.get_geom_structures_attrs() - if structures_attrs is not None: - structures_stac_attrs = prep_stac_attrs(structures_attrs, prefix="Structures") - stac_geom_attrs.update(structures_stac_attrs) - else: - logging.warning("No geometry structures attributes found.") - - d2_flow_area_attrs = ras_hdf.get_geom_2d_flow_area_attrs() - if d2_flow_area_attrs is not None: - d2_flow_area_stac_attrs = prep_stac_attrs( - d2_flow_area_attrs, prefix="2D Flow Areas" - ) - cell_average_size = d2_flow_area_stac_attrs.get( - "2d_flow_area:cell_average_size", None - ) - if cell_average_size is not None: - d2_flow_area_stac_attrs["2d_flow_area:cell_average_length"] = ( - cell_average_size**0.5 - ) - else: - logging.warning("Unable to add cell average size to attributes.") - stac_geom_attrs.update(d2_flow_area_stac_attrs) - else: - logging.warning("No flow area attributes found.") - - return stac_geom_attrs diff --git a/ras_stac/utils/ras_stac.py b/ras_stac/utils/ras_stac.py deleted file mode 100644 index f2dbd55..0000000 --- a/ras_stac/utils/ras_stac.py +++ /dev/null @@ -1,355 +0,0 @@ -import pystac -import json -from typing import List -import os -import fsspec -import shapely -from rashdf import RasPlanHdf, RasGeomHdf -from ras_hdf import ( - get_stac_geom_attrs, - get_stac_plan_attrs, - get_stac_plan_results_attrs, -) -from pathlib import Path -import re - -import logging - -logging.getLogger("boto3").setLevel(logging.WARNING) -logging.getLogger("botocore").setLevel(logging.WARNING) - - -def create_model_item( - ras_geom_hdf_url: str, - props_to_remove: List, - simplify: float = None, - minio_mode: bool = False, -) -> pystac.Item: - """ - This function creates a STAC (SpatioTemporal Asset Catalog) item from a given HDF (Hierarchical Data Format) - file URL. - - Parameters: - ras_geom_hdf_url (str): The URL of the HDF file. - props_to_remove (List): List of properties to be removed from item. - - Returns: - pystac.Item: The created STAC item. - - Raises: - ValueError: If the provided URL does not have a '.hdf' suffix. - - The function performs the following steps: - 1. Checks if the provided URL has a '.hdf' suffix. If not, it raises a ValueError. - 2. Extracts the model name from the URL by removing the '.hdf' suffix and getting the stem of the path. - 3. Logs the creation of the STAC item for the model. - 4. Opens the HDF file from the URL using the `RasGeomHdf.open_uri` method. - 5. Gets the perimeter of the 2D flow area from the HDF file and simplifies it using the provided `simplify` - parameter. - 6. Gets the attributes of the geometry from the HDF file. - 7. Extracts the geometry time from the properties. - 8. Removes unwanted properties. - 9. Creates a new STAC item with the model ID, the geometry converted to GeoJSON, the bounding box of the - perimeter, and the properties. - 10. Returns the created STAC item. - """ - if Path(ras_geom_hdf_url).suffix != ".hdf": - raise ValueError( - f"Expected pattern: `s3://bucket/prefix/ras-model-name.g**.hdf`, got {ras_geom_hdf_url}" - ) - - ras_model_name = Path(ras_geom_hdf_url.replace(".hdf", "")).stem - - logging.info(f"Creating STAC item for model {ras_model_name}") - if minio_mode: - ras_hdf = RasGeomHdf.open_uri( - ras_geom_hdf_url, - fsspec_kwargs={"endpoint_url": os.environ.get("MINIO_S3_ENDPOINT")}, - ) - else: - ras_hdf = RasGeomHdf.open_uri(ras_geom_hdf_url) - perimeter = ras_hdf.mesh_areas() - perimeter = perimeter.to_crs("EPSG:4326") - if simplify: - perimeter_polygon = perimeter.geometry.unary_union.simplify(tolerance=simplify) - else: - perimeter_polygon = perimeter.geometry.unary_union - properties = get_stac_geom_attrs(ras_hdf) - if not properties: - raise AttributeError(f"Could not find properties from: {ras_geom_hdf_url}") - geometry_time = properties.get("geometry:geometry_time") - - # Remove unwanted properties - for prop in props_to_remove: - try: - del properties[prop] - except KeyError: - logging.warning(f"Failed removing {prop}, property not found") - - model_id = ras_model_name - - item = pystac.Item( - id=model_id, - geometry=json.loads(shapely.to_geojson(perimeter_polygon)), - bbox=perimeter_polygon.bounds, - datetime=geometry_time, - properties=properties, - ) - return item - - -def new_geom_assets( - topo_assets: list = None, - lulc_assets: list = None, - mannings_assets: list = None, - other_assets: list = None, -): - """ - This function creates a dictionary of geometric assets. - - Parameters: - topo_assets (list): The topographic assets. Default is None. - lulc_assets (list): The land use and land cover assets. Default is None. - mannings_assets (list): The Manning's roughness coefficient assets. Default is None. - other_assets (list): Any other assets. Default is None. - - Returns: - dict: A dictionary with keys "topo", "lulc", "mannings", and "other", and values being the corresponding input - parameters. - """ - geom_assets = { - "topo": topo_assets, - "lulc": lulc_assets, - "mannings": mannings_assets, - "other": other_assets, - } - return geom_assets - - -def ras_geom_asset_info(s3_key: str, asset_type: str) -> dict: - """ - This function generates information about a geometric asset used in a HEC-RAS model. - - Parameters: - asset_type (str): The type of the asset. Must be one of: "mannings", "lulc", "topo", "other". - s3_key (str): The S3 key of the asset. - - Returns: - dict: A dictionary with the roles, the description, and the title of the asset. - - Raises: - ValueError: If the provided asset_type is not one of: "mannings", "lulc", "topo", "other". - """ - - if asset_type not in ["mannings", "lulc", "topo", "other"]: - raise ValueError("asset_type must be one of: mannings, lulc, topo, other") - - file_extension = Path(s3_key).suffix - title = Path(s3_key).name - - if asset_type == "mannings": - description = "Friction surface used in HEC-RAS model geometry" - - elif asset_type == "lulc": - description = "Land Use / Land Cover data used in HEC-RAS model geometry" - - elif asset_type == "topo": - description = "Topo data used in HEC-RAS model geometry" - - elif asset_type == "other": - description = "Other data used in HEC-RAS model geometry" - - else: - description = "" - - if file_extension == ".hdf": - roles = [pystac.MediaType.HDF, f"ras-{asset_type}"] - elif file_extension == ".tif": - roles = [pystac.MediaType.GEOTIFF, f"ras-{asset_type}"] - else: - roles = [f"ras-{asset_type}"] - - return {"roles": roles, "description": description, "title": title} - - -def ras_plan_asset_info(s3_key: str) -> dict: - """ - This function generates information about a plan asset used in a HEC-RAS model. - - Parameters: - s3_key (str): The S3 key of the asset. - - Returns: - dict: A dictionary with the roles, the description, and the title of the asset. - - The function performs the following steps: - 1. Extracts the file extension and the file name from the provided `s3_key`. - 2. If the file extension is ".hdf", it sets the `ras_extension` to the extension of the file name without the - ".hdf" suffix and adds `pystac.MediaType.HDF5` to the roles. Otherwise, it sets the `ras_extension` to the file - extension. - 3. Removes the leading dot from the `ras_extension`. - 4. Depending on the `ras_extension`, it sets the roles and the description for the asset. The `ras_extension` is - expected to match one of the following patterns: "g[0-9]{2}", "p[0-9]{2}", "u[0-9]{2}", "s[0-9]{2}", "prj", - "dss", "log". If it doesn't match any of these patterns, it adds "ras-file" to the roles. - 5. Returns a dictionary with the roles, the description, and the title of the asset. - """ - file_extension = Path(s3_key).suffix - title = Path(s3_key).name - description = "" - roles = [] - - if file_extension == ".hdf": - ras_extension = Path(s3_key.replace(".hdf", "")).suffix - roles.append(pystac.MediaType.HDF5) - else: - ras_extension = file_extension - - ras_extension = ras_extension.lstrip(".") - - if re.match("g[0-9]{2}", ras_extension): - roles.append("ras-geometry") - if file_extension != ".hdf": - roles.append(pystac.MediaType.TEXT) - description = """The geometry file contains the 2D flow area perimeter and other geometry information.""" - - elif re.match("p[0-9]{2}", ras_extension): - roles.append("ras-plan") - if file_extension != ".hdf": - roles.append(pystac.MediaType.TEXT) - description = """The plan file contains the simulation plan and other simulation information.""" - - elif re.match("u[0-9]{2}", ras_extension): - roles.extend(["ras-unsteady", pystac.MediaType.TEXT]) - description = """The unsteady file contains the unsteady flow results and other simulation information.""" - - elif re.match("s[0-9]{2}", ras_extension): - roles.extend(["ras-steady", pystac.MediaType.TEXT]) - description = """The steady file contains the steady flow results and other simulation information.""" - - elif ras_extension == "prj": - roles.extend(["ras-project", pystac.MediaType.TEXT]) - description = """The project file contains the project information and other simulation information.""" - - elif ras_extension == "dss": - roles.extend(["ras-dss"]) - description = """The dss file contains the dss results and other simulation information.""" - - elif ras_extension == "log": - roles.extend(["ras-log", pystac.MediaType.TEXT]) - description = """The log file contains the log information and other simulation information.""" - - else: - roles.extend(["ras-file"]) - - return {"roles": roles, "description": description, "title": title} - - -def get_simulation_metadata( - ras_plan_hdf_url: str, simulation: str, minio_mode: bool = False -) -> dict: - """ - This function retrieves the metadata of a simulation from a HEC-RAS plan HDF file. - - Parameters: - ras_plan_hdf_url (str): The URL of the HEC-RAS plan HDF file. - simulation (str): The name of the simulation. - - Returns: - dict: A dictionary with the metadata of the simulation. - - The function performs the following steps: - 1. Opens the HEC-RAS plan HDF file from the provided URL in read-binary mode. - 2. Initializes a dictionary `metadata` with the key "ras:simulation" and the value being the provided - `simulation` such as the plan id. - 3. Tries to open the HEC-RAS plan HDF file in read mode. If the file is not found, it logs an error - and returns. - 4. Tries to get the plan attributes from the HDF file and update the `metadata` dictionary with them. - If an exception occurs, it logs an error and returns. - 5. Tries to get the plan results attributes from the HDF file and update the `metadata` dictionary with - them. If an exception occurs, it logs an error and returns. - 6. Returns the `metadata` dictionary. - """ - if minio_mode: - s3f = fsspec.open( - ras_plan_hdf_url, - client_kwargs={"endpoint_url": os.environ.get("MINIO_S3_ENDPOINT")}, - mode="rb", - ) - else: - s3f = fsspec.open(ras_plan_hdf_url, mode="rb") - metadata = { - "ras:simulation": simulation, - } - - try: - plan_hdf = RasPlanHdf(s3f.open()) - except FileNotFoundError: - return logging.error(f"file not found: {ras_plan_hdf_url}") - - try: - plan_attrs = get_stac_plan_attrs(plan_hdf) - metadata.update(plan_attrs) - except Exception as e: - return logging.error( - f"unable to extract plan_attrs from {ras_plan_hdf_url}: {e}" - ) - - try: - results_attrs = get_stac_plan_results_attrs(plan_hdf) - metadata.update(results_attrs) - except Exception as e: - return logging.error( - f"unable to extract results_attrs from {ras_plan_hdf_url}: {e}" - ) - - return metadata - - -def create_model_simulation_item( - ras_item: pystac.Item, - results_meta: dict, - model_sim_id: str, - item_props_to_remove: List, -) -> pystac.Item: - """ - This function creates a PySTAC Item for a model simulation. - - Parameters: - ras_item (pystac.Item): The PySTAC Item of the RAS model. - results_meta (dict): The metadata of the simulation results. - model_sim_id (str): The ID of the model simulation. - item_props_to_remove (List): List of properties to be removed from the item. - - Returns: - pystac.Item: A PySTAC Item for the model simulation. - - The function performs the following steps: - 1. Retrieves the runtime window from the `results_meta` dictionary. - 2. Removes unwanted properties. - 3. Creates a PySTAC Item with the ID being the `model_sim_id`, the geometry and the bounding box being those of - the `ras_item`, the start and end datetimes being the converted start and end times of the runtime window, - the datetime being the start datetime, and the properties being the `results_meta` with unwanted properties removed. - 4. Returns the created PySTAC Item. - """ - runtime_window = results_meta.get("results_summary:run_time_window") - start_datetime = runtime_window[0] - end_datetime = runtime_window[1] - - for prop in item_props_to_remove: - try: - del results_meta[prop] - except KeyError: - logging.warning( - f"Failed to remove property:{prop} not found in simulation results metadata." - ) - - item = pystac.Item( - id=model_sim_id, - geometry=ras_item.geometry, - bbox=ras_item.bbox, - start_datetime=start_datetime, - end_datetime=end_datetime, - datetime=start_datetime, - properties=results_meta, - ) - return item diff --git a/ras_stac/utils/ras_utils.py b/ras_stac/utils/ras_utils.py new file mode 100644 index 0000000..327cc0f --- /dev/null +++ b/ras_stac/utils/ras_utils.py @@ -0,0 +1,573 @@ +import logging +from dotenv import load_dotenv, find_dotenv +import pystac +import json +from typing import List +import shapely +from pathlib import Path +import re +from datetime import datetime + +from rashdf import RasPlanHdf, RasGeomHdf +from rashdf.utils import parse_duration + +logging.getLogger("boto3").setLevel(logging.WARNING) +logging.getLogger("botocore").setLevel(logging.WARNING) + +load_dotenv(find_dotenv()) + + +class RasStacGeom: + def __init__(self, rg: RasGeomHdf): + self.rg = rg + + def get_stac_geom_attrs(self) -> dict: + """ + This function retrieves the geometry attributes of a HEC-RAS HDF file, converting them to STAC format. + + Returns: + stac_geom_attrs (dict): A dictionary with the organized geometry attributes. + """ + + stac_geom_attrs = self.rg.get_root_attrs() + if stac_geom_attrs is not None: + stac_geom_attrs = prep_stac_attrs(stac_geom_attrs) + else: + stac_geom_attrs = {} + logging.warning("No root attributes found.") + + geom_attrs = self.rg.get_geom_attrs() + if geom_attrs is not None: + geom_stac_attrs = prep_stac_attrs(geom_attrs, prefix="Geometry") + stac_geom_attrs.update(geom_stac_attrs) + else: + logging.warning("No base geometry attributes found.") + + structures_attrs = self.rg.get_geom_structures_attrs() + if structures_attrs is not None: + structures_stac_attrs = prep_stac_attrs( + structures_attrs, prefix="Structures" + ) + stac_geom_attrs.update(structures_stac_attrs) + else: + logging.warning("No geometry structures attributes found.") + + d2_flow_area_attrs = self.rg.get_geom_2d_flow_area_attrs() + if d2_flow_area_attrs is not None: + d2_flow_area_stac_attrs = prep_stac_attrs( + d2_flow_area_attrs, prefix="2D Flow Areas" + ) + cell_average_size = d2_flow_area_stac_attrs.get( + "2d_flow_area:cell_average_size", None + ) + if cell_average_size is not None: + d2_flow_area_stac_attrs["2d_flow_area:cell_average_length"] = ( + cell_average_size**0.5 + ) + else: + logging.warning("Unable to add cell average size to attributes.") + stac_geom_attrs.update(d2_flow_area_stac_attrs) + else: + logging.warning("No flow area attributes found.") + + return stac_geom_attrs + + def get_perimeter(self, simplify: float = None, crs: str = "EPSG:4326"): + return ras_perimeter(self.rg, simplify, crs) + + def to_item( + self, + props_to_remove: List, + ras_model_name: str, + simplify: float = None, + stac_item_id: str = None, + ) -> pystac.Item: + """ + Creates a STAC (SpatioTemporal Asset Catalog) item from a given RasGeomHdf object. + + Parameters: + props_to_remove (List): List of properties to be removed from the item. + ras_model_name (str): The name of the RAS model. + simplify (float, optional): Tolerance for simplifying the perimeter polygon. Defaults to None. + + Returns: + pystac.Item: The created STAC item. + + Raises: + AttributeError: If the properties cannot be extracted from the RasGeomHdf object. + + The function performs the following steps: + 1. Gets the perimeter of the 2D flow area from the RasGeomHdf object. + 2. Extracts the attributes of the geometry from the RasGeomHdf object. + 3. Extracts the geometry time from the properties. + 4. Removes unwanted properties specified in `props_to_remove`. + 5. Creates a new STAC item with the model ID, the geometry converted to GeoJSON, the bounding box of the perimeter, and the properties. + 6. Returns the created STAC item. + """ + + perimeter_polygon = self.get_perimeter(simplify) + + properties = self.get_stac_geom_attrs() + if not properties: + raise AttributeError( + f"Could not find properties while creating model item for {ras_model_name}." + ) + + geometry_time = properties.get("geometry:geometry_time") + if not geometry_time: + raise AttributeError( + f"Could not find data for 'geometry:geometry_time' while creating model item for {ras_model_name}." + ) + + for prop in props_to_remove: + try: + del properties[prop] + except KeyError: + logging.warning(f"Failed removing {prop}, property not found") + + iso_properties = properties_to_isoformat(properties) + + if not stac_item_id: + stac_item_id = ras_model_name + + item = pystac.Item( + id=stac_item_id, + geometry=json.loads(shapely.to_geojson(perimeter_polygon)), + bbox=perimeter_polygon.bounds, + datetime=geometry_time, + properties=iso_properties, + ) + return item + + +class RasStacPlan(RasStacGeom): + def __init__(self, rp: RasPlanHdf): + super().__init__(rp) + self.rp = rp + + def to_item( + self, + ras_item: pystac.Item, + results_meta: dict, + model_sim_id: str, + item_props_to_remove: List, + ) -> pystac.Item: + """ + This function creates a PySTAC Item for a model simulation. + + Parameters: + ras_item (pystac.Item): The PySTAC Item of the RAS model. + results_meta (dict): The metadata of the simulation results. + model_sim_id (str): The ID of the model simulation. + item_props_to_remove (List): List of properties to be removed from the item. + + Returns: + pystac.Item: A PySTAC Item for the model simulation. + + The function performs the following steps: + 1. Retrieves the runtime window from the `results_meta` dictionary. + 2. Removes unwanted properties. + 3. Creates a PySTAC Item with the ID being the `model_sim_id`, the geometry and the bounding box being those of + the `ras_item`, the start and end datetimes being the converted start and end times of the runtime window, + the datetime being the start datetime, and the properties being the `results_meta` with unwanted properties removed. + 4. Returns the created PySTAC Item. + """ + runtime_window = results_meta.get("results_summary:run_time_window") + if not runtime_window: + raise AttributeError( + f"Could not find data for 'results_summary:run_time_window' while creating model item for model id:{model_sim_id}." + ) + start_datetime = runtime_window[0] + end_datetime = runtime_window[1] + + for prop in item_props_to_remove: + try: + del results_meta[prop] + except KeyError: + logging.warning( + f"Failed to remove property:{prop} not found in simulation results metadata." + ) + + properties = properties_to_isoformat(results_meta) + + item = pystac.Item( + id=model_sim_id, + geometry=ras_item.geometry, + bbox=ras_item.bbox, + start_datetime=start_datetime, + end_datetime=end_datetime, + datetime=start_datetime, + properties=properties, + ) + return item + + def get_stac_plan_attrs(self, include_results: bool = False) -> dict: + """ + This function retrieves the attributes of a plan from a HEC-RAS plan HDF file, converting them to STAC format. + + Parameters: + include_results (bool, optional): Whether to include the results attributes in the returned dictionary. + Defaults to False. + + Returns: + stac_plan_attrs (dict): A dictionary with the attributes of the plan. + """ + stac_plan_attrs = self.rp.get_root_attrs() + if stac_plan_attrs is not None: + stac_plan_attrs = prep_stac_attrs(stac_plan_attrs) + else: + stac_plan_attrs = {} + logging.warning("No root attributes found.") + + plan_info_attrs = self.rp.get_plan_info_attrs() + if plan_info_attrs is not None: + plan_info_stac_attrs = prep_stac_attrs( + plan_info_attrs, prefix="Plan Information" + ) + stac_plan_attrs.update(plan_info_stac_attrs) + else: + logging.warning("No plan information attributes found.") + + plan_params_attrs = self.rp.get_plan_param_attrs() + if plan_params_attrs is not None: + plan_params_stac_attrs = prep_stac_attrs( + plan_params_attrs, prefix="Plan Parameters" + ) + stac_plan_attrs.update(plan_params_stac_attrs) + else: + logging.warning("No plan parameters attributes found.") + + precip_attrs = self.rp.get_meteorology_precip_attrs() + if precip_attrs is not None: + precip_stac_attrs = prep_stac_attrs(precip_attrs, prefix="Meteorology") + precip_stac_attrs.pop("meteorology:projection", None) + stac_plan_attrs.update(precip_stac_attrs) + else: + logging.warning("No meteorology precipitation attributes found.") + + if include_results: + stac_plan_attrs.update(self.rp.get_stac_plan_results_attrs()) + return stac_plan_attrs + + def get_stac_plan_results_attrs(self): + """ + This function retrieves the results attributes of a plan from a HEC-RAS plan HDF file, converting + them to STAC format. For summary atrributes, it retrieves the total computation time, the run time window, + and the solution from it, and calculates the total computation time in minutes if it exists. + + Returns: + results_attrs (dict): A dictionary with the results attributes of the plan. + """ + results_attrs = {} + + unsteady_results_attrs = self.rp.get_results_unsteady_attrs() + if unsteady_results_attrs is not None: + unsteady_results_stac_attrs = prep_stac_attrs( + unsteady_results_attrs, prefix="Unsteady Results" + ) + results_attrs.update(unsteady_results_stac_attrs) + else: + logging.warning("No unsteady results attributes found.") + + summary_attrs = self.rp.get_results_unsteady_summary_attrs() + if summary_attrs is not None: + summary_stac_attrs = prep_stac_attrs( + summary_attrs, prefix="Results Summary" + ) + computation_time_total = str( + summary_stac_attrs.get("results_summary:computation_time_total") + ) + results_summary = { + "results_summary:computation_time_total": computation_time_total, + "results_summary:run_time_window": summary_stac_attrs.get( + "results_summary:run_time_window" + ), + "results_summary:solution": summary_stac_attrs.get( + "results_summary:solution" + ), + } + if computation_time_total is not None: + computation_time_total_minutes = ( + parse_duration(computation_time_total).total_seconds() / 60 + ) + results_summary["results_summary:computation_time_total_minutes"] = ( + computation_time_total_minutes + ) + results_attrs.update(results_summary) + else: + logging.warning("No unsteady results summary attributes found.") + + volume_accounting_attrs = self.rp.get_results_volume_accounting_attrs() + if volume_accounting_attrs is not None: + volume_accounting_stac_attrs = prep_stac_attrs( + volume_accounting_attrs, prefix="Volume Accounting" + ) + results_attrs.update(volume_accounting_stac_attrs) + else: + logging.warning("No results volume accounting attributes found.") + + return results_attrs + + def get_simulation_metadata(self, simulation: str) -> dict: + """ + This function retrieves the metadata of a simulation from a HEC-RAS plan HDF file. + + Parameters: + simulation (str): The name of the simulation. + + Returns: + dict: A dictionary with the metadata of the simulation. + + The function performs the following steps: + 1. Initializes a metadata dictionary with the key "ras:simulation" and the value being the provided simulation. + 2. Tries to get the plan attributes from the RasPlanHdf object and update the `metadata` dictionary with them. + 3. Tries to get the plan results attributes from the RasPlanHdf object and update the `metadata` dictionary with them. + 4. Returns the `metadata` dictionary. + """ + metadata = {"ras:simulation": simulation} + + try: + plan_attrs = self.get_stac_plan_attrs() + metadata.update(plan_attrs) + except Exception as e: + return logging.error(f"unable to extract plan_attrs from plan: {e}") + + try: + results_attrs = self.get_stac_plan_results_attrs() + metadata.update(results_attrs) + except Exception as e: + return logging.error(f"unable to extract results_attrs from plan: {e}") + + return metadata + + +def new_geom_assets( + topo_assets: list = None, + lulc_assets: list = None, + mannings_assets: list = None, + other_assets: list = None, +): + """ + This function creates a dictionary of geometric assets. + + Parameters: + topo_assets (list): The topographic assets. Default is None. + lulc_assets (list): The land use and land cover assets. Default is None. + mannings_assets (list): The Manning's roughness coefficient assets. Default is None. + other_assets (list): Any other assets. Default is None. + + Returns: + dict: A dictionary with keys "topo", "lulc", "mannings", and "other", and values being the corresponding input + parameters. + """ + geom_assets = { + "topo": topo_assets, + "lulc": lulc_assets, + "mannings": mannings_assets, + "other": other_assets, + } + return geom_assets + + +def ras_geom_asset_info(s3_key: str, asset_type: str) -> dict: + """ + This function generates information about a geometric asset used in a HEC-RAS model. + + Parameters: + asset_type (str): The type of the asset. Must be one of: "mannings", "lulc", "topo", "other". + s3_key (str): The S3 key of the asset. + + Returns: + dict: A dictionary with the roles, the description, and the title of the asset. + + Raises: + ValueError: If the provided asset_type is not one of: "mannings", "lulc", "topo", "other". + """ + + if asset_type not in ["mannings", "lulc", "topo", "other"]: + raise ValueError("asset_type must be one of: mannings, lulc, topo, other") + + file_extension = Path(s3_key).suffix + title = Path(s3_key).name + + if asset_type == "mannings": + description = "Friction surface used in HEC-RAS model geometry" + + elif asset_type == "lulc": + description = "Land Use / Land Cover data used in HEC-RAS model geometry" + + elif asset_type == "topo": + description = "Topo data used in HEC-RAS model geometry" + + elif asset_type == "other": + description = "Other data used in HEC-RAS model geometry" + + else: + description = "" + + if file_extension == ".hdf": + roles = [pystac.MediaType.HDF, f"ras-{asset_type}"] + elif file_extension == ".tif": + roles = [pystac.MediaType.GEOTIFF, f"ras-{asset_type}"] + else: + roles = [f"ras-{asset_type}"] + + return {"roles": roles, "description": description, "title": title} + + +def ras_plan_asset_info(s3_key: str) -> dict: + """ + This function generates information about a plan asset used in a HEC-RAS model. + + Parameters: + s3_key (str): The S3 key of the asset. + + Returns: + dict: A dictionary with the roles, the description, and the title of the asset. + + The function performs the following steps: + 1. Extracts the file extension and the file name from the provided `s3_key`. + 2. If the file extension is ".hdf", it sets the `ras_extension` to the extension of the file name without the + ".hdf" suffix and adds `pystac.MediaType.HDF5` to the roles. Otherwise, it sets the `ras_extension` to the file + extension. + 3. Removes the leading dot from the `ras_extension`. + 4. Depending on the `ras_extension`, it sets the roles and the description for the asset. The `ras_extension` is + expected to match one of the following patterns: "g[0-9]{2}", "p[0-9]{2}", "u[0-9]{2}", "s[0-9]{2}", "prj", + "dss", "log". If it doesn't match any of these patterns, it adds "ras-file" to the roles. + 5. Returns a dictionary with the roles, the description, and the title of the asset. + """ + file_extension = Path(s3_key).suffix + title = Path(s3_key).name + description = "" + roles = [] + + if file_extension == ".hdf": + ras_extension = Path(s3_key.replace(".hdf", "")).suffix + roles.append(pystac.MediaType.HDF5) + else: + ras_extension = file_extension + + ras_extension = ras_extension.lstrip(".") + + if re.match("g[0-9]{2}", ras_extension): + roles.append("ras-geometry") + if file_extension != ".hdf": + roles.append(pystac.MediaType.TEXT) + description = """The geometry file contains the 2D flow area perimeter and other geometry information.""" + + elif re.match("p[0-9]{2}", ras_extension): + roles.append("ras-plan") + if file_extension != ".hdf": + roles.append(pystac.MediaType.TEXT) + description = """The plan file contains the simulation plan and other simulation information.""" + + elif re.match("u[0-9]{2}", ras_extension): + roles.extend(["ras-unsteady", pystac.MediaType.TEXT]) + description = """The unsteady file contains the unsteady flow results and other simulation information.""" + + elif re.match("s[0-9]{2}", ras_extension): + roles.extend(["ras-steady", pystac.MediaType.TEXT]) + description = """The steady file contains the steady flow results and other simulation information.""" + + elif ras_extension == "prj": + roles.extend(["ras-project", pystac.MediaType.TEXT]) + description = """The project file contains the project information and other simulation information.""" + + elif ras_extension == "dss": + roles.extend(["ras-dss"]) + description = """The dss file contains the dss results and other simulation information.""" + + elif ras_extension == "log": + roles.extend(["ras-log", pystac.MediaType.TEXT]) + description = """The log file contains the log information and other simulation information.""" + + else: + roles.extend(["ras-file"]) + + return {"roles": roles, "description": description, "title": title} + + +def to_snake_case(text): + """ + Convert a string to snake case, removing punctuation and other symbols. + + Parameters: + text (str): The string to be converted. + + Returns: + str: The snake case version of the string. + """ + import re + + # Remove all non-word characters (everything except numbers and letters) + text = re.sub(r"[^\w\s]", "", text) + + # Replace all runs of whitespace with a single underscore + text = re.sub(r"\s+", "_", text) + + return text.lower() + + +def prep_stac_attrs(attrs: dict, prefix: str = None) -> dict: + """ + Converts an unformatted HDF attributes dictionary to STAC format by converting values to snake case + and adding a prefix if one is given. + + Parameters: + attrs (dict): Unformatted attribute dictionary. + prefix (str): Optional prefix to be added to each key of formatted dictionary. + + Returns: + results (dict): The new attribute dictionary snake case values and prefix. + """ + results = {} + for k, value in attrs.items(): + if prefix: + key = f"{to_snake_case(prefix)}:{to_snake_case(k)}" + else: + key = to_snake_case(k) + results[key] = value + + return results + + +def ras_perimeter(rg: RasGeomHdf, simplify: float = None, crs: str = "EPSG:4326"): + """ + Calculate the perimeter of a HEC-RAS geometry as a GeoDataFrame in the specified coordinate reference system. + + Parameters: + rg (RasGeomHdf): A HEC-RAS geometry HDF file object which provides mesh areas. + simplify (float, optional): A tolerance level to simplify the perimeter geometry to reduce complexity. + If None, the geometry will not be simplified. Defaults to None. + crs (str): The coordinate reference system which the perimeter geometry will be converted to. Defaults to "EPSG:4326". + + Returns: + gpd.GeoDataFrame: A GeoDataFrame containing the calculated perimeter polygon in the specified CRS. + """ + + perimeter = rg.mesh_areas() + perimeter = perimeter.to_crs(crs) + if simplify: + perimeter_polygon = perimeter.geometry.unary_union.simplify(tolerance=simplify) + else: + perimeter_polygon = perimeter.geometry.unary_union + return perimeter_polygon + + +def properties_to_isoformat(properties: dict): + """Converts datetime objects in properties to isoformat + + Parameters: + properties (dict): Properties dictionary with datetime object values + + Returns: + properties (dict): Properties dictionary with datetime objects converted to isoformat + """ + for k, v in properties.items(): + if isinstance(v, list): + properties[k] = [ + item.isoformat() if isinstance(item, datetime) else item for item in v + ] + elif isinstance(v, datetime): + properties[k] = v.isoformat() + return properties diff --git a/ras_stac/utils/s3_utils.py b/ras_stac/utils/s3_utils.py index 01bac8d..7a50256 100644 --- a/ras_stac/utils/s3_utils.py +++ b/ras_stac/utils/s3_utils.py @@ -4,8 +4,9 @@ import logging import re import os +from rashdf import RasPlanHdf, RasGeomHdf +from pathlib import Path -from datetime import datetime from dotenv import load_dotenv, find_dotenv from mypy_boto3_s3.service_resource import ObjectSummary @@ -15,6 +16,73 @@ load_dotenv(find_dotenv()) +def read_ras_geom_from_s3(ras_geom_hdf_url: str, minio_mode: bool = False): + """ + Reads a RAS geometry HDF file from an S3 URL. + + Parameters: + ras_geom_hdf_url (str): The URL of the RAS geometry HDF file. + minio_mode (bool, optional): If True, uses MinIO endpoint for S3. Defaults to False. + + Returns: + geom_hdf_obj (RasGeomHdf): The RasGeomHdf object. + ras_model_name (str): The RAS model name. + + Raises: + ValueError: If the provided URL does not have a '.hdf' suffix. + """ + pattern = r".*\.g[0-9]{2}\.hdf$" + if not re.fullmatch(pattern, ras_geom_hdf_url): + raise ValueError( + f"RAS geom URL does not match pattern {pattern}: {ras_geom_hdf_url}" + ) + + ras_model_name = Path(ras_geom_hdf_url.replace(".hdf", "")).stem + + logging.info(f"Reading hdf file from {ras_geom_hdf_url}") + if minio_mode: + geom_hdf_obj = RasGeomHdf.open_uri( + ras_geom_hdf_url, + fsspec_kwargs={"endpoint_url": os.environ.get("MINIO_S3_ENDPOINT")}, + ) + else: + geom_hdf_obj = RasGeomHdf.open_uri(ras_geom_hdf_url) + + return geom_hdf_obj, ras_model_name + + +def read_ras_plan_from_s3(ras_plan_hdf_url: str, minio_mode: bool = False): + """ + Reads a RAS plan HDF file from an S3 URL. + + Parameters: + ras_plan_hdf_url (str): The URL of the RAS plan HDF file. + minio_mode (bool, optional): If True, uses MinIO endpoint for S3. Defaults to False. + + Returns: + plan_hdf_obj (RasPlanHdf): The RasPlanHdf object. + + Raises: + ValueError: If the provided URL does not have a '.hdf' suffix. + """ + pattern = r".*\.p[0-9]{2}\.hdf$" + if not re.fullmatch(pattern, ras_plan_hdf_url): + raise ValueError( + f"RAS plan URL does not match pattern {pattern}: {ras_plan_hdf_url}" + ) + + logging.info(f"Reading hdf file from {ras_plan_hdf_url}") + if minio_mode: + plan_hdf_obj = RasPlanHdf.open_uri( + ras_plan_hdf_url, + fsspec_kwargs={"endpoint_url": os.environ.get("MINIO_S3_ENDPOINT")}, + ) + else: + plan_hdf_obj = RasPlanHdf.open_uri(ras_plan_hdf_url) + + return plan_hdf_obj + + def get_basic_object_metadata(obj: ObjectSummary) -> dict: """ This function retrieves basic metadata of an AWS S3 object. @@ -42,15 +110,6 @@ def get_basic_object_metadata(obj: ObjectSummary) -> dict: ) -class DateTimeEncoder(json.JSONEncoder): - """JSON encoder for handling datetime objects.""" - - def default(self, obj): - if isinstance(obj, datetime): - return obj.isoformat() - return json.JSONEncoder.default(self, obj) - - def copy_item_to_s3(item, s3_key, s3client): """ This function copies an item to an AWS S3 bucket. @@ -67,31 +126,33 @@ def copy_item_to_s3(item, s3_key, s3client): # s3 = boto3.client("s3") bucket, key = split_s3_key(s3_key) - item_json = json.dumps(item.to_dict(), cls=DateTimeEncoder).encode("utf-8") + item_json = json.dumps(item.to_dict()).encode("utf-8") s3client.put_object(Body=item_json, Bucket=bucket, Key=key) -def split_s3_key(s3_key: str) -> tuple[str, str]: +def split_s3_key(s3_path: str) -> tuple[str, str]: """ - This function splits an S3 key into the bucket name and the key. + This function splits an S3 path into the bucket name and the key. Parameters: - s3_key (str): The S3 key to split. It should be in the format 's3://bucket/key'. + s3_path (str): The S3 path to split. It should be in the format 's3://bucket/key'. Returns: - tuple: A tuple containing the bucket name and the key. If the S3 key does not contain a key, the second element + tuple: A tuple containing the bucket name and the key. If the S3 path does not contain a key, the second element of the tuple will be None. The function performs the following steps: - 1. Removes the 's3://' prefix from the S3 key. + 1. Removes the 's3://' prefix from the S3 path. 2. Splits the remaining string on the first '/' character. 3. Returns the first part as the bucket name and the second part as the key. If there is no '/', the key will be None. """ - parts = s3_key.replace("s3://", "").split("/", 1) - bucket = parts[0] - key = parts[1] if len(parts) > 1 else None + if not s3_path.startswith("s3://"): + raise ValueError(f"s3_path does not start with s3://: {s3_path}") + bucket, _, key = s3_path[5:].partition("/") + if not key: + raise ValueError(f"s3_path contains bucket only, no key: {s3_path}") return bucket, key diff --git a/requirements.txt b/requirements.txt index 0b9aa4c..e2cc008 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ boto3==1.34.34 botocore==1.34.34 -fsspec==2024.2.0 mypy-boto3-s3==1.34.14 -numpy==1.26.4 papipyplug==2024.3.4 python-dotenv==1.0.1 pystac==1.9.0 shapely==2.0.3 rasterio==1.3.9 -rashdf==0.1.0 \ No newline at end of file +rashdf==0.2.2 +s3fs==2024.6.0 +pytest==8.2.2 +jsonschema==4.22.0 \ No newline at end of file diff --git a/tests/data/json/test_geom_item.json b/tests/data/json/test_geom_item.json new file mode 100644 index 0000000..585efc4 --- /dev/null +++ b/tests/data/json/test_geom_item.json @@ -0,0 +1,944 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "id": "test-1", + "properties": { + "file_type": "HEC-RAS Results", + "file_version": "HEC-RAS 6.5 February 2024", + "projection": "PROJCS[\"NAD_1983_StatePlane_Indiana_East_FIPS_1301_Feet\",GEOGCS[\"GCS_North_American_1983\",DATUM[\"D_North_American_1983\",SPHEROID[\"GRS_1980\",6378137.0,298.257222101]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"False_Easting\",328083.333],PARAMETER[\"False_Northing\",820208.333],PARAMETER[\"Central_Meridian\",-85.6666666666667],PARAMETER[\"Scale_Factor\",0.999966667],PARAMETER[\"Latitude_Of_Origin\",37.5],UNIT[\"US survey foot\",0.304800609601219]]", + "units_system": "US Customary", + "geometry:complete_geometry": true, + "geometry:extents": [ + 401571.15, + 414302.31, + 1799648.8455, + 1806670.07 + ], + "geometry:geometry_time": "2024-04-17T10:18:18", + "geometry:si_units": false, + "geometry:terrain_file_date": "2014-05-23T13:58:50", + "geometry:terrain_filename": ".\\Terrain\\Terrain.hdf", + "geometry:terrain_layername": "Terrain", + "geometry:title": "Muncie_rashdf_edit", + "geometry:version": "1.0.19 (25Jan2024)", + "structures:bridgeculvert_count": 0, + "structures:connection_count": 0, + "structures:inline_structure_count": 0, + "structures:lateral_structure_count": 3, + "2d_flow_areas:cell_average_size": 2600.986328125, + "2d_flow_areas:cell_maximum_index": 2305, + "2d_flow_areas:cell_maximum_size": 4863.12109375, + "2d_flow_areas:cell_minimum_area_fraction": 0.009999999776482582, + "2d_flow_areas:cell_minimum_size": 1297.7921142578125, + "2d_flow_areas:cell_volume_tolerance": 0.009999999776482582, + "2d_flow_areas:composite_lc": 0, + "2d_flow_areas:connection_profile_hash": [ + 227, + 176, + 196, + 66, + 152, + 252, + 28, + 20, + 154, + 251, + 244, + 200, + 153, + 111, + 185, + 36, + 39, + 174, + 65, + 228, + 100, + 155, + 147, + 76, + 164, + 149, + 153, + 27, + 120, + 82, + 184, + 85 + ], + "2d_flow_areas:data_date": "2024-04-17T10:18:18", + "2d_flow_areas:extents": [ + 404980.15, + 412236.63, + 1801018.71, + 1805022.84 + ], + "2d_flow_areas:face_area_conveyance_ratio": 0.019999999552965164, + "2d_flow_areas:face_area_elevation_tolerance": 0.009999999776482582, + "2d_flow_areas:face_profile_tolerance": 0.009999999776482582, + "2d_flow_areas:infiltration_override_table_hash": [ + 227, + 176, + 196, + 66, + 152, + 252, + 28, + 20, + 154, + 251, + 244, + 200, + 153, + 111, + 185, + 36, + 39, + 174, + 65, + 228, + 100, + 155, + 147, + 76, + 164, + 149, + 153, + 27, + 120, + 82, + 184, + 85 + ], + "2d_flow_areas:laminar_depth": 0.20000000298023224, + "2d_flow_areas:mannings_n": 0.05999999865889549, + "2d_flow_areas:multiple_face_mann_n": 0, + "2d_flow_areas:property_tables_last_computed": "2024-04-17T10:18:30", + "2d_flow_areas:terrain_file_date": "2014-05-23T13:58:50", + "2d_flow_areas:terrain_filename": ".\\Terrain\\Terrain.hdf", + "2d_flow_areas:version": "1.0", + "datetime": "2024-04-17T10:18:18Z" + }, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -85.3746937595796, + 40.19699035666282 + ], + [ + -85.37436195080397, + 40.19701535059264 + ], + [ + -85.37427393700763, + 40.19702904591547 + ], + [ + -85.37397159376525, + 40.197094658075336 + ], + [ + -85.37391906525218, + 40.19710605461664 + ], + [ + -85.37355076743945, + 40.197208255580335 + ], + [ + -85.37321125957673, + 40.197316540093 + ], + [ + -85.37286004499836, + 40.197441346767725 + ], + [ + -85.37247746806234, + 40.19759975488705 + ], + [ + -85.37209304803012, + 40.197770043269635 + ], + [ + -85.37177994396083, + 40.19790545730298 + ], + [ + -85.37146165078326, + 40.19804846114124 + ], + [ + -85.37112803956913, + 40.198198562218174 + ], + [ + -85.37075403443713, + 40.19836785703592 + ], + [ + -85.3702242548352, + 40.19859774765116 + ], + [ + -85.3700862476448, + 40.198656139003155 + ], + [ + -85.36976864757946, + 40.19879455564308 + ], + [ + -85.36956196380478, + 40.198847938861846 + ], + [ + -85.36918434401802, + 40.198876039550846 + ], + [ + -85.36861653220924, + 40.19877995592773 + ], + [ + -85.36846204250924, + 40.19871065641814 + ], + [ + -85.36843943504985, + 40.19869854725343 + ], + [ + -85.36835224245748, + 40.19863845227802 + ], + [ + -85.36830974560547, + 40.198516845263335 + ], + [ + -85.36828684913937, + 40.19843185329535 + ], + [ + -85.36820574714672, + 40.19814214746937 + ], + [ + -85.36813333779165, + 40.19787036195103 + ], + [ + -85.36810613840905, + 40.19777893534028 + ], + [ + -85.36809334896475, + 40.19774845934986 + ], + [ + -85.36806415206941, + 40.19767174123747 + ], + [ + -85.36800023626246, + 40.197414361775024 + ], + [ + -85.36798344771768, + 40.196761509450766 + ], + [ + -85.36795943302091, + 40.1965970991509 + ], + [ + -85.36783787093819, + 40.19612270893301 + ], + [ + -85.36780598436624, + 40.1958760903588 + ], + [ + -85.36775759580061, + 40.19562942915831 + ], + [ + -85.3672192438496, + 40.194636841733725 + ], + [ + -85.36702636565927, + 40.194426947380464 + ], + [ + -85.36698514599699, + 40.19438226045971 + ], + [ + -85.36694047521587, + 40.194332788136286 + ], + [ + -85.36679235631553, + 40.194168852225594 + ], + [ + -85.36662206542982, + 40.194024540938166 + ], + [ + -85.366330751013, + 40.1937823550626 + ], + [ + -85.36600555164863, + 40.193569947005685 + ], + [ + -85.36584043964221, + 40.19346136160298 + ], + [ + -85.36543378415428, + 40.193157273897235 + ], + [ + -85.36612743902114, + 40.19279905841117 + ], + [ + -85.36635654007902, + 40.192715653237045 + ], + [ + -85.36671043859991, + 40.192653159190996 + ], + [ + -85.36721015803316, + 40.19267405282558 + ], + [ + -85.36746005760627, + 40.19273654581515 + ], + [ + -85.36758504398628, + 40.1928613585424 + ], + [ + -85.3675850554511, + 40.193111353771 + ], + [ + -85.3680430404241, + 40.193548538419456 + ], + [ + -85.36850116292489, + 40.19346525288526 + ], + [ + -85.36891766389769, + 40.19348606204334 + ], + [ + -85.36901141377402, + 40.19400174831589 + ], + [ + -85.36920777582195, + 40.19411998984939 + ], + [ + -85.36962553797078, + 40.19413165856048 + ], + [ + -85.3700420462026, + 40.19411073835733 + ], + [ + -85.37062496770024, + 40.19413163825747 + ], + [ + -85.37143705718188, + 40.194152461054614 + ], + [ + -85.37207634164089, + 40.1941524425457 + ], + [ + -85.37235855888497, + 40.19415244663331 + ], + [ + -85.3725135564014, + 40.194152456259495 + ], + [ + -85.37401275567328, + 40.19417313817422 + ], + [ + -85.37464543689737, + 40.194118653687724 + ], + [ + -85.37497846306499, + 40.19413203831805 + ], + [ + -85.37528504540403, + 40.19425194720721 + ], + [ + -85.37544494782549, + 40.1946116541381 + ], + [ + -85.3758024572669, + 40.19464274890237 + ], + [ + -85.37625264024238, + 40.19466636138525 + ], + [ + -85.37651324502646, + 40.194690046137836 + ], + [ + -85.37741355589039, + 40.19504545470521 + ], + [ + -85.37769785853814, + 40.19518753673482 + ], + [ + -85.37800573250534, + 40.19514015598286 + ], + [ + -85.37859803915869, + 40.19506916105463 + ], + [ + -85.37904813225195, + 40.19502174702151 + ], + [ + -85.37997215700005, + 40.19492694528983 + ], + [ + -85.38028006403688, + 40.19487955856626 + ], + [ + -85.38039856006944, + 40.19469005625643 + ], + [ + -85.38101454906084, + 40.193742353736546 + ], + [ + -85.38110934310407, + 40.19333974583567 + ], + [ + -85.3820569579436, + 40.19291324159792 + ], + [ + -85.38243593388131, + 40.19293695691165 + ], + [ + -85.38293116726526, + 40.19318564151163 + ], + [ + -85.38359455935178, + 40.19344623720858 + ], + [ + -85.38454205636664, + 40.19339894637612 + ], + [ + -85.38501596651633, + 40.19342263750257 + ], + [ + -85.38620045404011, + 40.19351733640859 + ], + [ + -85.38665054733858, + 40.193849044460656 + ], + [ + -85.3874086410036, + 40.194109655159956 + ], + [ + -85.38762184313146, + 40.1941096476921 + ], + [ + -85.38759934194462, + 40.19573034847753 + ], + [ + -85.38759813432189, + 40.19581536093803 + ], + [ + -85.3877403676867, + 40.1961707533761 + ], + [ + -85.38833263370317, + 40.19737894675642 + ], + [ + -85.38847476032818, + 40.197639550092134 + ], + [ + -85.38861695269281, + 40.197805447901416 + ], + [ + -85.38928026320289, + 40.198445052258485 + ], + [ + -85.38989695541851, + 40.19910735297772 + ], + [ + -85.39002274759073, + 40.19930233517913 + ], + [ + -85.39018855386341, + 40.19956145567172 + ], + [ + -85.39030673949831, + 40.19975224685299 + ], + [ + -85.39040733872906, + 40.19991894898722 + ], + [ + -85.39046885757355, + 40.20002093847127 + ], + [ + -85.3906383330768, + 40.20031185513928 + ], + [ + -85.39080004557974, + 40.20056643556261 + ], + [ + -85.39084604711317, + 40.20063734093737 + ], + [ + -85.39098843526033, + 40.200890447595526 + ], + [ + -85.39114943780005, + 40.201153041466725 + ], + [ + -85.39134576692736, + 40.20164803569543 + ], + [ + -85.39135815734375, + 40.20173566083265 + ], + [ + -85.39137303402903, + 40.20201404784417 + ], + [ + -85.39136124380799, + 40.20230906154038 + ], + [ + -85.39127406364857, + 40.20259365672818 + ], + [ + -85.39108934027095, + 40.20285693741823 + ], + [ + -85.39086504289489, + 40.20307913940943 + ], + [ + -85.39060626378455, + 40.20318755733298 + ], + [ + -85.38919613416498, + 40.203521556808425 + ], + [ + -85.38881864676151, + 40.2035654511558 + ], + [ + -85.38844454918717, + 40.20362074451804 + ], + [ + -85.38808474269743, + 40.20363934183705 + ], + [ + -85.38771105849163, + 40.20367215158293 + ], + [ + -85.38721414644513, + 40.203696152947806 + ], + [ + -85.38567596267032, + 40.20354485374499 + ], + [ + -85.38565386158395, + 40.20353903541933 + ], + [ + -85.38529533848092, + 40.203454961037544 + ], + [ + -85.38500113353561, + 40.20337504988276 + ], + [ + -85.38464943650224, + 40.203294256758745 + ], + [ + -85.3845463409417, + 40.20327424094452 + ], + [ + -85.3844604660026, + 40.20325415721749 + ], + [ + -85.38411963805851, + 40.203142644036525 + ], + [ + -85.38360935551276, + 40.20291306131339 + ], + [ + -85.38334064171734, + 40.202773749735435 + ], + [ + -85.3832147423516, + 40.20267634789212 + ], + [ + -85.38313393329425, + 40.20260066017331 + ], + [ + -85.38290705392708, + 40.20238854059522 + ], + [ + -85.38273595395479, + 40.20224485509642 + ], + [ + -85.38263906765394, + 40.202195837267 + ], + [ + -85.3824785558348, + 40.20216596112165 + ], + [ + -85.38230013355337, + 40.202061456807954 + ], + [ + -85.38206155662357, + 40.20183964412352 + ], + [ + -85.3818216607919, + 40.201616839464556 + ], + [ + -85.38158344451378, + 40.201395136471625 + ], + [ + -85.381376048972, + 40.20120013626701 + ], + [ + -85.38135983661132, + 40.20119894337552 + ], + [ + -85.38133808405672, + 40.20118711334736 + ], + [ + -85.3799919371536, + 40.20045584587313 + ], + [ + -85.37979974365945, + 40.20021465322657 + ], + [ + -85.37967173140203, + 40.2000857835808 + ], + [ + -85.37957134670783, + 40.19998473512655 + ], + [ + -85.3793393369977, + 40.199771442820484 + ], + [ + -85.37909894802377, + 40.199559035106674 + ], + [ + -85.37882274067398, + 40.19936004365336 + ], + [ + -85.37872694018745, + 40.19929293511624 + ], + [ + -85.37827516009467, + 40.199055842933575 + ], + [ + -85.37799566072965, + 40.19887443729431 + ], + [ + -85.37775634665442, + 40.19866353924469 + ], + [ + -85.37754263481656, + 40.198434641972106 + ], + [ + -85.3773250603347, + 40.198205240527585 + ], + [ + -85.37715194819457, + 40.19802645946064 + ], + [ + -85.37703456322056, + 40.197900852638355 + ], + [ + -85.37687919004422, + 40.19775162521564 + ], + [ + -85.37680731488742, + 40.197682625952645 + ], + [ + -85.37652213354625, + 40.19740874797734 + ], + [ + -85.37587415965231, + 40.197122840948666 + ], + [ + -85.37547744979491, + 40.19705565972209 + ], + [ + -85.37508015620055, + 40.19700793833109 + ], + [ + -85.3746937595796, + 40.19699035666282 + ] + ] + ], + [ + [ + [ + -85.39414339064994, + 40.203511540813324 + ], + [ + -85.39240697084784, + 40.20227492434461 + ], + [ + -85.3922399111087, + 40.20117756491884 + ], + [ + -85.39210159278188, + 40.200934845883985 + ], + [ + -85.39156941912684, + 40.200033825679974 + ], + [ + -85.39098895823302, + 40.19916145642786 + ], + [ + -85.39054168268484, + 40.198474283496196 + ], + [ + -85.39006201931952, + 40.1978486585847 + ], + [ + -85.38983761252982, + 40.19769609925633 + ], + [ + -85.38948495614483, + 40.197461072027984 + ], + [ + -85.38976542391921, + 40.197005701878986 + ], + [ + -85.38988438797294, + 40.1967307364524 + ], + [ + -85.39024478056079, + 40.196382367505755 + ], + [ + -85.39110382492233, + 40.195936609345125 + ], + [ + -85.39192532417573, + 40.19548661058371 + ], + [ + -85.3929129432786, + 40.195037026342256 + ], + [ + -85.3937616931092, + 40.19509689674137 + ], + [ + -85.39506269657274, + 40.19542692386787 + ], + [ + -85.39623249357749, + 40.19701525274974 + ], + [ + -85.39624077517962, + 40.199706445573995 + ], + [ + -85.39720666136184, + 40.200115986331184 + ], + [ + -85.3980968153132, + 40.20049627287583 + ], + [ + -85.39940652079106, + 40.20031019500278 + ], + [ + -85.40096389660307, + 40.19986284468457 + ], + [ + -85.40151186496831, + 40.20041687913585 + ], + [ + -85.40143396584708, + 40.20094038075375 + ], + [ + -85.40082256041948, + 40.20204453410351 + ], + [ + -85.3984259997691, + 40.20361007266954 + ], + [ + -85.39518198542314, + 40.20357341228795 + ], + [ + -85.39414339064994, + 40.203511540813324 + ] + ] + ] + ] + }, + "links": [], + "assets": {}, + "bbox": [ + -85.40151186496831, + 40.192653159190996, + -85.36543378415428, + 40.203696152947806 + ], + "stac_extensions": [] +} \ No newline at end of file diff --git a/tests/data/json/test_geom_properties.json b/tests/data/json/test_geom_properties.json new file mode 100644 index 0000000..636243a --- /dev/null +++ b/tests/data/json/test_geom_properties.json @@ -0,0 +1,116 @@ +{ + "file_type": "HEC-RAS Results", + "file_version": "HEC-RAS 6.5 February 2024", + "projection": "PROJCS[\"NAD_1983_StatePlane_Indiana_East_FIPS_1301_Feet\",GEOGCS[\"GCS_North_American_1983\",DATUM[\"D_North_American_1983\",SPHEROID[\"GRS_1980\",6378137.0,298.257222101]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"False_Easting\",328083.333],PARAMETER[\"False_Northing\",820208.333],PARAMETER[\"Central_Meridian\",-85.6666666666667],PARAMETER[\"Scale_Factor\",0.999966667],PARAMETER[\"Latitude_Of_Origin\",37.5],UNIT[\"US survey foot\",0.304800609601219]]", + "units_system": "US Customary", + "geometry:complete_geometry": true, + "geometry:extents": [ + 401571.15, + 414302.31, + 1799648.8455, + 1806670.07 + ], + "geometry:geometry_time": "2024-04-17T10:18:18", + "geometry:si_units": false, + "geometry:terrain_file_date": "2014-05-23T13:58:50", + "geometry:terrain_filename": ".\\Terrain\\Terrain.hdf", + "geometry:terrain_layername": "Terrain", + "geometry:title": "Muncie_rashdf_edit", + "geometry:version": "1.0.19 (25Jan2024)", + "structures:bridgeculvert_count": 0, + "structures:connection_count": 0, + "structures:inline_structure_count": 0, + "structures:lateral_structure_count": 3, + "2d_flow_areas:cell_average_size": 2600.986328125, + "2d_flow_areas:cell_maximum_index": 2305, + "2d_flow_areas:cell_maximum_size": 4863.12109375, + "2d_flow_areas:cell_minimum_area_fraction": 0.009999999776482582, + "2d_flow_areas:cell_minimum_size": 1297.7921142578125, + "2d_flow_areas:cell_volume_tolerance": 0.009999999776482582, + "2d_flow_areas:composite_lc": 0, + "2d_flow_areas:connection_profile_hash": [ + 227, + 176, + 196, + 66, + 152, + 252, + 28, + 20, + 154, + 251, + 244, + 200, + 153, + 111, + 185, + 36, + 39, + 174, + 65, + 228, + 100, + 155, + 147, + 76, + 164, + 149, + 153, + 27, + 120, + 82, + 184, + 85 + ], + "2d_flow_areas:data_date": "2024-04-17T10:18:18", + "2d_flow_areas:extents": [ + 404980.15, + 412236.63, + 1801018.71, + 1805022.84 + ], + "2d_flow_areas:face_area_conveyance_ratio": 0.019999999552965164, + "2d_flow_areas:face_area_elevation_tolerance": 0.009999999776482582, + "2d_flow_areas:face_profile_tolerance": 0.009999999776482582, + "2d_flow_areas:infiltration_override_table_hash": [ + 227, + 176, + 196, + 66, + 152, + 252, + 28, + 20, + 154, + 251, + 244, + 200, + 153, + 111, + 185, + 36, + 39, + 174, + 65, + 228, + 100, + 155, + 147, + 76, + 164, + 149, + 153, + 27, + 120, + 82, + 184, + 85 + ], + "2d_flow_areas:laminar_depth": 0.20000000298023224, + "2d_flow_areas:mannings_n": 0.05999999865889549, + "2d_flow_areas:multiple_face_mann_n": 0, + "2d_flow_areas:property_tables_last_computed": "2024-04-17T10:18:30", + "2d_flow_areas:terrain_file_date": "2014-05-23T13:58:50", + "2d_flow_areas:terrain_filename": ".\\Terrain\\Terrain.hdf", + "2d_flow_areas:version": "1.0" +} \ No newline at end of file diff --git a/tests/data/json/test_perimeter.json b/tests/data/json/test_perimeter.json new file mode 100644 index 0000000..eb5f16d --- /dev/null +++ b/tests/data/json/test_perimeter.json @@ -0,0 +1,821 @@ +{ + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -85.3746937595796, + 40.19699035666282 + ], + [ + -85.37436195080397, + 40.19701535059264 + ], + [ + -85.37427393700763, + 40.19702904591547 + ], + [ + -85.37397159376525, + 40.197094658075336 + ], + [ + -85.37391906525218, + 40.19710605461664 + ], + [ + -85.37355076743945, + 40.197208255580335 + ], + [ + -85.37321125957673, + 40.197316540093 + ], + [ + -85.37286004499836, + 40.197441346767725 + ], + [ + -85.37247746806234, + 40.19759975488705 + ], + [ + -85.37209304803012, + 40.197770043269635 + ], + [ + -85.37177994396083, + 40.19790545730298 + ], + [ + -85.37146165078326, + 40.19804846114124 + ], + [ + -85.37112803956913, + 40.198198562218174 + ], + [ + -85.37075403443713, + 40.19836785703592 + ], + [ + -85.3702242548352, + 40.19859774765116 + ], + [ + -85.3700862476448, + 40.198656139003155 + ], + [ + -85.36976864757946, + 40.19879455564308 + ], + [ + -85.36956196380478, + 40.198847938861846 + ], + [ + -85.36918434401802, + 40.198876039550846 + ], + [ + -85.36861653220924, + 40.19877995592773 + ], + [ + -85.36846204250924, + 40.19871065641814 + ], + [ + -85.36843943504985, + 40.19869854725343 + ], + [ + -85.36835224245748, + 40.19863845227802 + ], + [ + -85.36830974560547, + 40.198516845263335 + ], + [ + -85.36828684913937, + 40.19843185329535 + ], + [ + -85.36820574714672, + 40.19814214746937 + ], + [ + -85.36813333779165, + 40.19787036195103 + ], + [ + -85.36810613840905, + 40.19777893534028 + ], + [ + -85.36809334896475, + 40.19774845934986 + ], + [ + -85.36806415206941, + 40.19767174123747 + ], + [ + -85.36800023626246, + 40.197414361775024 + ], + [ + -85.36798344771768, + 40.196761509450766 + ], + [ + -85.36795943302091, + 40.1965970991509 + ], + [ + -85.36783787093819, + 40.19612270893301 + ], + [ + -85.36780598436624, + 40.1958760903588 + ], + [ + -85.36775759580061, + 40.19562942915831 + ], + [ + -85.3672192438496, + 40.194636841733725 + ], + [ + -85.36702636565927, + 40.194426947380464 + ], + [ + -85.36698514599699, + 40.19438226045971 + ], + [ + -85.36694047521587, + 40.194332788136286 + ], + [ + -85.36679235631553, + 40.194168852225594 + ], + [ + -85.36662206542982, + 40.194024540938166 + ], + [ + -85.366330751013, + 40.1937823550626 + ], + [ + -85.36600555164863, + 40.193569947005685 + ], + [ + -85.36584043964221, + 40.19346136160298 + ], + [ + -85.36543378415428, + 40.193157273897235 + ], + [ + -85.36612743902114, + 40.19279905841117 + ], + [ + -85.36635654007902, + 40.192715653237045 + ], + [ + -85.36671043859991, + 40.192653159190996 + ], + [ + -85.36721015803316, + 40.19267405282558 + ], + [ + -85.36746005760627, + 40.19273654581515 + ], + [ + -85.36758504398628, + 40.1928613585424 + ], + [ + -85.3675850554511, + 40.193111353771 + ], + [ + -85.3680430404241, + 40.193548538419456 + ], + [ + -85.36850116292489, + 40.19346525288526 + ], + [ + -85.36891766389769, + 40.19348606204334 + ], + [ + -85.36901141377402, + 40.19400174831589 + ], + [ + -85.36920777582195, + 40.19411998984939 + ], + [ + -85.36962553797078, + 40.19413165856048 + ], + [ + -85.3700420462026, + 40.19411073835733 + ], + [ + -85.37062496770024, + 40.19413163825747 + ], + [ + -85.37143705718188, + 40.194152461054614 + ], + [ + -85.37207634164089, + 40.1941524425457 + ], + [ + -85.37235855888497, + 40.19415244663331 + ], + [ + -85.3725135564014, + 40.194152456259495 + ], + [ + -85.37401275567328, + 40.19417313817422 + ], + [ + -85.37464543689737, + 40.194118653687724 + ], + [ + -85.37497846306499, + 40.19413203831805 + ], + [ + -85.37528504540403, + 40.19425194720721 + ], + [ + -85.37544494782549, + 40.1946116541381 + ], + [ + -85.3758024572669, + 40.19464274890237 + ], + [ + -85.37625264024238, + 40.19466636138525 + ], + [ + -85.37651324502646, + 40.194690046137836 + ], + [ + -85.37741355589039, + 40.19504545470521 + ], + [ + -85.37769785853814, + 40.19518753673482 + ], + [ + -85.37800573250534, + 40.19514015598286 + ], + [ + -85.37859803915869, + 40.19506916105463 + ], + [ + -85.37904813225195, + 40.19502174702151 + ], + [ + -85.37997215700005, + 40.19492694528983 + ], + [ + -85.38028006403688, + 40.19487955856626 + ], + [ + -85.38039856006944, + 40.19469005625643 + ], + [ + -85.38101454906084, + 40.193742353736546 + ], + [ + -85.38110934310407, + 40.19333974583567 + ], + [ + -85.3820569579436, + 40.19291324159792 + ], + [ + -85.38243593388131, + 40.19293695691165 + ], + [ + -85.38293116726526, + 40.19318564151163 + ], + [ + -85.38359455935178, + 40.19344623720858 + ], + [ + -85.38454205636664, + 40.19339894637612 + ], + [ + -85.38501596651633, + 40.19342263750257 + ], + [ + -85.38620045404011, + 40.19351733640859 + ], + [ + -85.38665054733858, + 40.193849044460656 + ], + [ + -85.3874086410036, + 40.194109655159956 + ], + [ + -85.38762184313146, + 40.1941096476921 + ], + [ + -85.38759934194462, + 40.19573034847753 + ], + [ + -85.38759813432189, + 40.19581536093803 + ], + [ + -85.3877403676867, + 40.1961707533761 + ], + [ + -85.38833263370317, + 40.19737894675642 + ], + [ + -85.38847476032818, + 40.197639550092134 + ], + [ + -85.38861695269281, + 40.197805447901416 + ], + [ + -85.38928026320289, + 40.198445052258485 + ], + [ + -85.38989695541851, + 40.19910735297772 + ], + [ + -85.39002274759073, + 40.19930233517913 + ], + [ + -85.39018855386341, + 40.19956145567172 + ], + [ + -85.39030673949831, + 40.19975224685299 + ], + [ + -85.39040733872906, + 40.19991894898722 + ], + [ + -85.39046885757355, + 40.20002093847127 + ], + [ + -85.3906383330768, + 40.20031185513928 + ], + [ + -85.39080004557974, + 40.20056643556261 + ], + [ + -85.39084604711317, + 40.20063734093737 + ], + [ + -85.39098843526033, + 40.200890447595526 + ], + [ + -85.39114943780005, + 40.201153041466725 + ], + [ + -85.39134576692736, + 40.20164803569543 + ], + [ + -85.39135815734375, + 40.20173566083265 + ], + [ + -85.39137303402903, + 40.20201404784417 + ], + [ + -85.39136124380799, + 40.20230906154038 + ], + [ + -85.39127406364857, + 40.20259365672818 + ], + [ + -85.39108934027095, + 40.20285693741823 + ], + [ + -85.39086504289489, + 40.20307913940943 + ], + [ + -85.39060626378455, + 40.20318755733298 + ], + [ + -85.38919613416498, + 40.203521556808425 + ], + [ + -85.38881864676151, + 40.2035654511558 + ], + [ + -85.38844454918717, + 40.20362074451804 + ], + [ + -85.38808474269743, + 40.20363934183705 + ], + [ + -85.38771105849163, + 40.20367215158293 + ], + [ + -85.38721414644513, + 40.203696152947806 + ], + [ + -85.38567596267032, + 40.20354485374499 + ], + [ + -85.38565386158395, + 40.20353903541933 + ], + [ + -85.38529533848092, + 40.203454961037544 + ], + [ + -85.38500113353561, + 40.20337504988276 + ], + [ + -85.38464943650224, + 40.203294256758745 + ], + [ + -85.3845463409417, + 40.20327424094452 + ], + [ + -85.3844604660026, + 40.20325415721749 + ], + [ + -85.38411963805851, + 40.203142644036525 + ], + [ + -85.38360935551276, + 40.20291306131339 + ], + [ + -85.38334064171734, + 40.202773749735435 + ], + [ + -85.3832147423516, + 40.20267634789212 + ], + [ + -85.38313393329425, + 40.20260066017331 + ], + [ + -85.38290705392708, + 40.20238854059522 + ], + [ + -85.38273595395479, + 40.20224485509642 + ], + [ + -85.38263906765394, + 40.202195837267 + ], + [ + -85.3824785558348, + 40.20216596112165 + ], + [ + -85.38230013355337, + 40.202061456807954 + ], + [ + -85.38206155662357, + 40.20183964412352 + ], + [ + -85.3818216607919, + 40.201616839464556 + ], + [ + -85.38158344451378, + 40.201395136471625 + ], + [ + -85.381376048972, + 40.20120013626701 + ], + [ + -85.38135983661132, + 40.20119894337552 + ], + [ + -85.38133808405672, + 40.20118711334736 + ], + [ + -85.3799919371536, + 40.20045584587313 + ], + [ + -85.37979974365945, + 40.20021465322657 + ], + [ + -85.37967173140203, + 40.2000857835808 + ], + [ + -85.37957134670783, + 40.19998473512655 + ], + [ + -85.3793393369977, + 40.199771442820484 + ], + [ + -85.37909894802377, + 40.199559035106674 + ], + [ + -85.37882274067398, + 40.19936004365336 + ], + [ + -85.37872694018745, + 40.19929293511624 + ], + [ + -85.37827516009467, + 40.199055842933575 + ], + [ + -85.37799566072965, + 40.19887443729431 + ], + [ + -85.37775634665442, + 40.19866353924469 + ], + [ + -85.37754263481656, + 40.198434641972106 + ], + [ + -85.3773250603347, + 40.198205240527585 + ], + [ + -85.37715194819457, + 40.19802645946064 + ], + [ + -85.37703456322056, + 40.197900852638355 + ], + [ + -85.37687919004422, + 40.19775162521564 + ], + [ + -85.37680731488742, + 40.197682625952645 + ], + [ + -85.37652213354625, + 40.19740874797734 + ], + [ + -85.37587415965231, + 40.197122840948666 + ], + [ + -85.37547744979491, + 40.19705565972209 + ], + [ + -85.37508015620055, + 40.19700793833109 + ], + [ + -85.3746937595796, + 40.19699035666282 + ] + ] + ], + [ + [ + [ + -85.39414339064994, + 40.203511540813324 + ], + [ + -85.39240697084784, + 40.20227492434461 + ], + [ + -85.3922399111087, + 40.20117756491884 + ], + [ + -85.39210159278188, + 40.200934845883985 + ], + [ + -85.39156941912684, + 40.200033825679974 + ], + [ + -85.39098895823302, + 40.19916145642786 + ], + [ + -85.39054168268484, + 40.198474283496196 + ], + [ + -85.39006201931952, + 40.1978486585847 + ], + [ + -85.38983761252982, + 40.19769609925633 + ], + [ + -85.38948495614483, + 40.197461072027984 + ], + [ + -85.38976542391921, + 40.197005701878986 + ], + [ + -85.38988438797294, + 40.1967307364524 + ], + [ + -85.39024478056079, + 40.196382367505755 + ], + [ + -85.39110382492233, + 40.195936609345125 + ], + [ + -85.39192532417573, + 40.19548661058371 + ], + [ + -85.3929129432786, + 40.195037026342256 + ], + [ + -85.3937616931092, + 40.19509689674137 + ], + [ + -85.39506269657274, + 40.19542692386787 + ], + [ + -85.39623249357749, + 40.19701525274974 + ], + [ + -85.39624077517962, + 40.199706445573995 + ], + [ + -85.39720666136184, + 40.200115986331184 + ], + [ + -85.3980968153132, + 40.20049627287583 + ], + [ + -85.39940652079106, + 40.20031019500278 + ], + [ + -85.40096389660307, + 40.19986284468457 + ], + [ + -85.40151186496831, + 40.20041687913585 + ], + [ + -85.40143396584708, + 40.20094038075375 + ], + [ + -85.40082256041948, + 40.20204453410351 + ], + [ + -85.3984259997691, + 40.20361007266954 + ], + [ + -85.39518198542314, + 40.20357341228795 + ], + [ + -85.39414339064994, + 40.203511540813324 + ] + ] + ] + ] + }, + "bounds": [ + -85.40151186496831, + 40.192653159190996, + -85.36543378415428, + 40.203696152947806 + ] +} \ No newline at end of file diff --git a/tests/data/json/test_plan_attrs.json b/tests/data/json/test_plan_attrs.json new file mode 100644 index 0000000..c82b049 --- /dev/null +++ b/tests/data/json/test_plan_attrs.json @@ -0,0 +1,78 @@ +{ + "file_type": "HEC-RAS Results", + "file_version": "HEC-RAS 6.3.1 September 2022", + "projection": "PROJCS[\"NAD_1983_StatePlane_Indiana_East_FIPS_1301_Feet\",GEOGCS[\"GCS_North_American_1983\",DATUM[\"D_North_American_1983\",SPHEROID[\"GRS_1980\",6378137.0,298.257222101]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"False_Easting\",328083.3333333333],PARAMETER[\"False_Northing\",820208.3333333333],PARAMETER[\"Central_Meridian\",-85.66666666666667],PARAMETER[\"Scale_Factor\",0.9999666666666667],PARAMETER[\"Latitude_Of_Origin\",37.5],UNIT[\"Foot_US\",0.3048006096012192]]", + "units_system": "US Customary", + "plan_information:base_output_interval": "1HOUR", + "plan_information:computation_time_step_base": "10SEC", + "plan_information:flow_filename": "Muncie.u01", + "plan_information:flow_title": "Flow Boundary Conditions", + "plan_information:geometry_filename": "Muncie.g04", + "plan_information:geometry_title": "Muncie Geometry - 50ft User n Value Regi", + "plan_information:plan_filename": "Muncie.p04", + "plan_information:plan_name": "Unsteady Run with 2D 50ft User n Value R", + "plan_information:plan_shortid": "50ft User n Regions", + "plan_information:plan_title": "Unsteady Run with 2D 50ft User n Value R", + "plan_information:project_filename": "C:\\Users\\slawler\\Downloads\\Example_Projects_6_5\\Example_Projects\\2D Unsteady Flow Hydraulics\\Muncie\\Muncie.prj", + "plan_information:project_title": "Muncie 2D Flow Area", + "plan_information:simulation_end_time": "1900-01-03T00:00:00", + "plan_information:simulation_start_time": "1900-01-02T00:00:00", + "plan_information:time_window": [ + "1900-01-02T00:00:00", + "1900-01-03T00:00:00" + ], + "plan_parameters:1d_cores": 0, + "plan_parameters:1d_flow_tolerance": null, + "plan_parameters:1d_maximum_iterations": 20, + "plan_parameters:1d_maximum_iterations_without_improvement": 0, + "plan_parameters:1d_maximum_water_surface_error_to_abort": 100.0, + "plan_parameters:1d_methodology": "Finite Difference", + "plan_parameters:1d_storage_area_elevation_tolerance": 0.009999999776482582, + "plan_parameters:1d_theta": 1.0, + "plan_parameters:1d_theta_warmup": 1.0, + "plan_parameters:1d_water_surface_elevation_tolerance": 0.009999999776482582, + "plan_parameters:1d2d_flow_tolerance": 0.10000000149011612, + "plan_parameters:1d2d_gate_flow_submergence_decay_exponent": 1.0, + "plan_parameters:1d2d_is_stablity_factor": 1.0, + "plan_parameters:1d2d_ls_stablity_factor": 2.0, + "plan_parameters:1d2d_maximum_iterations": 0, + "plan_parameters:1d2d_maximum_number_of_time_slices": 20, + "plan_parameters:1d2d_minimum_flow_tolerance": 1.0, + "plan_parameters:1d2d_minimum_time_step_for_slicinghours": 0.0, + "plan_parameters:1d2d_number_of_warmup_steps": 20, + "plan_parameters:1d2d_warmup_time_step_hours": 0.0, + "plan_parameters:1d2d_water_surface_tolerance": 0.009999999776482582, + "plan_parameters:1d2d_weir_flow_submergence_decay_exponent": 3.0, + "plan_parameters:1d2d_maxiter": 0, + "plan_parameters:1d2d_ws_tolerance": 0.009999999776482582, + "plan_parameters:2d_boundary_condition_ramp_up_fraction": 0.5, + "plan_parameters:2d_boundary_condition_volume_check": false, + "plan_parameters:2d_cores_per_mesh": 6, + "plan_parameters:2d_coriolis": false, + "plan_parameters:2d_equation_set": "Diffusion Wave", + "plan_parameters:2d_initial_conditions_ramp_up_time_hrs": 0.0, + "plan_parameters:2d_latitude_for_coriolis": 3.4028234663852886e+38, + "plan_parameters:2d_longitudinal_mixing_coefficient": 0.0, + "plan_parameters:2d_matrix_solver": "Pardiso", + "plan_parameters:2d_maximum_iterations": 20, + "plan_parameters:2d_names": "2D Interior Area", + "plan_parameters:2d_number_of_time_slices": 1, + "plan_parameters:2d_only": false, + "plan_parameters:2d_smagorinsky_mixing_coefficient": 0.0, + "plan_parameters:2d_theta": 1.0, + "plan_parameters:2d_theta_warmup": 1.0, + "plan_parameters:2d_transverse_mixing_coefficient": 0.0, + "plan_parameters:2d_turbulence_formulation": "None", + "plan_parameters:2d_volume_tolerance": 0.009999999776482582, + "plan_parameters:2d_water_surface_tolerance": 0.009999999776482582, + "plan_parameters:gravity": 32.174049377441406, + "plan_parameters:hdf_chunk_size": 1.0, + "plan_parameters:hdf_compression": 1, + "plan_parameters:hdf_fixed_rows": 1, + "plan_parameters:hdf_flush_buffer": false, + "plan_parameters:hdf_spatial_parts": 1, + "plan_parameters:hdf_use_max_rows": 0, + "plan_parameters:hdf_write_time_slices": false, + "plan_parameters:hdf_write_warmup": false, + "plan_parameters:pardiso_solver": false +} \ No newline at end of file diff --git a/tests/data/json/test_plan_item.json b/tests/data/json/test_plan_item.json new file mode 100644 index 0000000..41038dd --- /dev/null +++ b/tests/data/json/test_plan_item.json @@ -0,0 +1,927 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "id": "test-1", + "properties": { + "ras:simulation": "test-1", + "file_type": "HEC-RAS Results", + "file_version": "HEC-RAS 6.3.1 September 2022", + "projection": "PROJCS[\"NAD_1983_StatePlane_Indiana_East_FIPS_1301_Feet\",GEOGCS[\"GCS_North_American_1983\",DATUM[\"D_North_American_1983\",SPHEROID[\"GRS_1980\",6378137.0,298.257222101]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"False_Easting\",328083.3333333333],PARAMETER[\"False_Northing\",820208.3333333333],PARAMETER[\"Central_Meridian\",-85.66666666666667],PARAMETER[\"Scale_Factor\",0.9999666666666667],PARAMETER[\"Latitude_Of_Origin\",37.5],UNIT[\"Foot_US\",0.3048006096012192]]", + "units_system": "US Customary", + "plan_information:base_output_interval": "1HOUR", + "plan_information:computation_time_step_base": "10SEC", + "plan_information:flow_filename": "Muncie.u01", + "plan_information:flow_title": "Flow Boundary Conditions", + "plan_information:geometry_filename": "Muncie.g04", + "plan_information:geometry_title": "Muncie Geometry - 50ft User n Value Regi", + "plan_information:plan_filename": "Muncie.p04", + "plan_information:plan_name": "Unsteady Run with 2D 50ft User n Value R", + "plan_information:plan_shortid": "50ft User n Regions", + "plan_information:plan_title": "Unsteady Run with 2D 50ft User n Value R", + "plan_information:project_filename": "C:\\Users\\slawler\\Downloads\\Example_Projects_6_5\\Example_Projects\\2D Unsteady Flow Hydraulics\\Muncie\\Muncie.prj", + "plan_information:project_title": "Muncie 2D Flow Area", + "plan_information:simulation_end_time": "1900-01-03T00:00:00", + "plan_information:simulation_start_time": "1900-01-02T00:00:00", + "plan_information:time_window": [ + "1900-01-02T00:00:00", + "1900-01-03T00:00:00" + ], + "plan_parameters:1d_cores": 0, + "plan_parameters:1d_flow_tolerance": null, + "plan_parameters:1d_maximum_iterations": 20, + "plan_parameters:1d_maximum_iterations_without_improvement": 0, + "plan_parameters:1d_maximum_water_surface_error_to_abort": 100.0, + "plan_parameters:1d_methodology": "Finite Difference", + "plan_parameters:1d_storage_area_elevation_tolerance": 0.009999999776482582, + "plan_parameters:1d_theta": 1.0, + "plan_parameters:1d_theta_warmup": 1.0, + "plan_parameters:1d_water_surface_elevation_tolerance": 0.009999999776482582, + "plan_parameters:1d2d_flow_tolerance": 0.10000000149011612, + "plan_parameters:1d2d_gate_flow_submergence_decay_exponent": 1.0, + "plan_parameters:1d2d_is_stablity_factor": 1.0, + "plan_parameters:1d2d_ls_stablity_factor": 2.0, + "plan_parameters:1d2d_maximum_iterations": 0, + "plan_parameters:1d2d_maximum_number_of_time_slices": 20, + "plan_parameters:1d2d_minimum_flow_tolerance": 1.0, + "plan_parameters:1d2d_minimum_time_step_for_slicinghours": 0.0, + "plan_parameters:1d2d_number_of_warmup_steps": 20, + "plan_parameters:1d2d_warmup_time_step_hours": 0.0, + "plan_parameters:1d2d_water_surface_tolerance": 0.009999999776482582, + "plan_parameters:1d2d_weir_flow_submergence_decay_exponent": 3.0, + "plan_parameters:1d2d_maxiter": 0, + "plan_parameters:1d2d_ws_tolerance": 0.009999999776482582, + "plan_parameters:2d_boundary_condition_ramp_up_fraction": 0.5, + "plan_parameters:2d_boundary_condition_volume_check": false, + "plan_parameters:2d_cores_per_mesh": 6, + "plan_parameters:2d_coriolis": false, + "plan_parameters:2d_equation_set": "Diffusion Wave", + "plan_parameters:2d_initial_conditions_ramp_up_time_hrs": 0.0, + "plan_parameters:2d_latitude_for_coriolis": 3.4028234663852886e+38, + "plan_parameters:2d_longitudinal_mixing_coefficient": 0.0, + "plan_parameters:2d_matrix_solver": "Pardiso", + "plan_parameters:2d_maximum_iterations": 20, + "plan_parameters:2d_names": "2D Interior Area", + "plan_parameters:2d_number_of_time_slices": 1, + "plan_parameters:2d_only": false, + "plan_parameters:2d_smagorinsky_mixing_coefficient": 0.0, + "plan_parameters:2d_theta": 1.0, + "plan_parameters:2d_theta_warmup": 1.0, + "plan_parameters:2d_transverse_mixing_coefficient": 0.0, + "plan_parameters:2d_turbulence_formulation": "None", + "plan_parameters:2d_volume_tolerance": 0.009999999776482582, + "plan_parameters:2d_water_surface_tolerance": 0.009999999776482582, + "plan_parameters:gravity": 32.174049377441406, + "plan_parameters:hdf_chunk_size": 1.0, + "plan_parameters:hdf_compression": 1, + "plan_parameters:hdf_fixed_rows": 1, + "plan_parameters:hdf_flush_buffer": false, + "plan_parameters:hdf_spatial_parts": 1, + "plan_parameters:hdf_use_max_rows": 0, + "plan_parameters:hdf_write_time_slices": false, + "plan_parameters:hdf_write_warmup": false, + "plan_parameters:pardiso_solver": false, + "unsteady_results:plan_title": "Unsteady Run with 2D 50ft User n Value R", + "unsteady_results:program_name": "HEC-RAS - River Analysis System", + "unsteady_results:program_version": "HEC-RAS 6.3.1 October 2022", + "unsteady_results:project_file_name": "C:\\Users\\slawler\\Downloads\\Example_Projects_6_5\\Example_Projects\\2D Unsteady Flow Hydraulics\\Muncie\\Muncie.prj", + "unsteady_results:project_title": "Muncie 2D Flow Area", + "unsteady_results:short_id": "50ft User n Regions", + "unsteady_results:simulation_time_window": [ + "1900-01-02T00:00:00", + "1900-01-03T00:00:00" + ], + "unsteady_results:type_of_run": "Unsteady Flow Analysis", + "results_summary:computation_time_total": "0:00:27", + "results_summary:run_time_window": [ + "2024-05-14T15:20:01", + "2024-05-14T15:20:28" + ], + "results_summary:solution": "Unsteady Finished Successfully", + "results_summary:computation_time_total_minutes": 0.45, + "start_datetime": "2024-05-14T15:20:01Z", + "end_datetime": "2024-05-14T15:20:28Z", + "datetime": "2024-05-14T15:20:01Z" + }, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -85.3746937595796, + 40.19699035666282 + ], + [ + -85.37436195080397, + 40.19701535059264 + ], + [ + -85.37427393700763, + 40.19702904591547 + ], + [ + -85.37397159376525, + 40.197094658075336 + ], + [ + -85.37391906525218, + 40.19710605461664 + ], + [ + -85.37355076743945, + 40.197208255580335 + ], + [ + -85.37321125957673, + 40.197316540093 + ], + [ + -85.37286004499836, + 40.197441346767725 + ], + [ + -85.37247746806234, + 40.19759975488705 + ], + [ + -85.37209304803012, + 40.197770043269635 + ], + [ + -85.37177994396083, + 40.19790545730298 + ], + [ + -85.37146165078326, + 40.19804846114124 + ], + [ + -85.37112803956913, + 40.198198562218174 + ], + [ + -85.37075403443713, + 40.19836785703592 + ], + [ + -85.3702242548352, + 40.19859774765116 + ], + [ + -85.3700862476448, + 40.198656139003155 + ], + [ + -85.36976864757946, + 40.19879455564308 + ], + [ + -85.36956196380478, + 40.198847938861846 + ], + [ + -85.36918434401802, + 40.198876039550846 + ], + [ + -85.36861653220924, + 40.19877995592773 + ], + [ + -85.36846204250924, + 40.19871065641814 + ], + [ + -85.36843943504985, + 40.19869854725343 + ], + [ + -85.36835224245748, + 40.19863845227802 + ], + [ + -85.36830974560547, + 40.198516845263335 + ], + [ + -85.36828684913937, + 40.19843185329535 + ], + [ + -85.36820574714672, + 40.19814214746937 + ], + [ + -85.36813333779165, + 40.19787036195103 + ], + [ + -85.36810613840905, + 40.19777893534028 + ], + [ + -85.36809334896475, + 40.19774845934986 + ], + [ + -85.36806415206941, + 40.19767174123747 + ], + [ + -85.36800023626246, + 40.197414361775024 + ], + [ + -85.36798344771768, + 40.196761509450766 + ], + [ + -85.36795943302091, + 40.1965970991509 + ], + [ + -85.36783787093819, + 40.19612270893301 + ], + [ + -85.36780598436624, + 40.1958760903588 + ], + [ + -85.36775759580061, + 40.19562942915831 + ], + [ + -85.3672192438496, + 40.194636841733725 + ], + [ + -85.36702636565927, + 40.194426947380464 + ], + [ + -85.36698514599699, + 40.19438226045971 + ], + [ + -85.36694047521587, + 40.194332788136286 + ], + [ + -85.36679235631553, + 40.194168852225594 + ], + [ + -85.36662206542982, + 40.194024540938166 + ], + [ + -85.366330751013, + 40.1937823550626 + ], + [ + -85.36600555164863, + 40.193569947005685 + ], + [ + -85.36584043964221, + 40.19346136160298 + ], + [ + -85.36543378415428, + 40.193157273897235 + ], + [ + -85.36612743902114, + 40.19279905841117 + ], + [ + -85.36635654007902, + 40.192715653237045 + ], + [ + -85.36671043859991, + 40.192653159190996 + ], + [ + -85.36721015803316, + 40.19267405282558 + ], + [ + -85.36746005760627, + 40.19273654581515 + ], + [ + -85.36758504398628, + 40.1928613585424 + ], + [ + -85.3675850554511, + 40.193111353771 + ], + [ + -85.3680430404241, + 40.193548538419456 + ], + [ + -85.36850116292489, + 40.19346525288526 + ], + [ + -85.36891766389769, + 40.19348606204334 + ], + [ + -85.36901141377402, + 40.19400174831589 + ], + [ + -85.36920777582195, + 40.19411998984939 + ], + [ + -85.36962553797078, + 40.19413165856048 + ], + [ + -85.3700420462026, + 40.19411073835733 + ], + [ + -85.37062496770024, + 40.19413163825747 + ], + [ + -85.37143705718188, + 40.194152461054614 + ], + [ + -85.37207634164089, + 40.1941524425457 + ], + [ + -85.37235855888497, + 40.19415244663331 + ], + [ + -85.3725135564014, + 40.194152456259495 + ], + [ + -85.37401275567328, + 40.19417313817422 + ], + [ + -85.37464543689737, + 40.194118653687724 + ], + [ + -85.37497846306499, + 40.19413203831805 + ], + [ + -85.37528504540403, + 40.19425194720721 + ], + [ + -85.37544494782549, + 40.1946116541381 + ], + [ + -85.3758024572669, + 40.19464274890237 + ], + [ + -85.37625264024238, + 40.19466636138525 + ], + [ + -85.37651324502646, + 40.194690046137836 + ], + [ + -85.37741355589039, + 40.19504545470521 + ], + [ + -85.37769785853814, + 40.19518753673482 + ], + [ + -85.37800573250534, + 40.19514015598286 + ], + [ + -85.37859803915869, + 40.19506916105463 + ], + [ + -85.37904813225195, + 40.19502174702151 + ], + [ + -85.37997215700005, + 40.19492694528983 + ], + [ + -85.38028006403688, + 40.19487955856626 + ], + [ + -85.38039856006944, + 40.19469005625643 + ], + [ + -85.38101454906084, + 40.193742353736546 + ], + [ + -85.38110934310407, + 40.19333974583567 + ], + [ + -85.3820569579436, + 40.19291324159792 + ], + [ + -85.38243593388131, + 40.19293695691165 + ], + [ + -85.38293116726526, + 40.19318564151163 + ], + [ + -85.38359455935178, + 40.19344623720858 + ], + [ + -85.38454205636664, + 40.19339894637612 + ], + [ + -85.38501596651633, + 40.19342263750257 + ], + [ + -85.38620045404011, + 40.19351733640859 + ], + [ + -85.38665054733858, + 40.193849044460656 + ], + [ + -85.3874086410036, + 40.194109655159956 + ], + [ + -85.38762184313146, + 40.1941096476921 + ], + [ + -85.38759934194462, + 40.19573034847753 + ], + [ + -85.38759813432189, + 40.19581536093803 + ], + [ + -85.3877403676867, + 40.1961707533761 + ], + [ + -85.38833263370317, + 40.19737894675642 + ], + [ + -85.38847476032818, + 40.197639550092134 + ], + [ + -85.38861695269281, + 40.197805447901416 + ], + [ + -85.38928026320289, + 40.198445052258485 + ], + [ + -85.38989695541851, + 40.19910735297772 + ], + [ + -85.39002274759073, + 40.19930233517913 + ], + [ + -85.39018855386341, + 40.19956145567172 + ], + [ + -85.39030673949831, + 40.19975224685299 + ], + [ + -85.39040733872906, + 40.19991894898722 + ], + [ + -85.39046885757355, + 40.20002093847127 + ], + [ + -85.3906383330768, + 40.20031185513928 + ], + [ + -85.39080004557974, + 40.20056643556261 + ], + [ + -85.39084604711317, + 40.20063734093737 + ], + [ + -85.39098843526033, + 40.200890447595526 + ], + [ + -85.39114943780005, + 40.201153041466725 + ], + [ + -85.39134576692736, + 40.20164803569543 + ], + [ + -85.39135815734375, + 40.20173566083265 + ], + [ + -85.39137303402903, + 40.20201404784417 + ], + [ + -85.39136124380799, + 40.20230906154038 + ], + [ + -85.39127406364857, + 40.20259365672818 + ], + [ + -85.39108934027095, + 40.20285693741823 + ], + [ + -85.39086504289489, + 40.20307913940943 + ], + [ + -85.39060626378455, + 40.20318755733298 + ], + [ + -85.38919613416498, + 40.203521556808425 + ], + [ + -85.38881864676151, + 40.2035654511558 + ], + [ + -85.38844454918717, + 40.20362074451804 + ], + [ + -85.38808474269743, + 40.20363934183705 + ], + [ + -85.38771105849163, + 40.20367215158293 + ], + [ + -85.38721414644513, + 40.203696152947806 + ], + [ + -85.38567596267032, + 40.20354485374499 + ], + [ + -85.38565386158395, + 40.20353903541933 + ], + [ + -85.38529533848092, + 40.203454961037544 + ], + [ + -85.38500113353561, + 40.20337504988276 + ], + [ + -85.38464943650224, + 40.203294256758745 + ], + [ + -85.3845463409417, + 40.20327424094452 + ], + [ + -85.3844604660026, + 40.20325415721749 + ], + [ + -85.38411963805851, + 40.203142644036525 + ], + [ + -85.38360935551276, + 40.20291306131339 + ], + [ + -85.38334064171734, + 40.202773749735435 + ], + [ + -85.3832147423516, + 40.20267634789212 + ], + [ + -85.38313393329425, + 40.20260066017331 + ], + [ + -85.38290705392708, + 40.20238854059522 + ], + [ + -85.38273595395479, + 40.20224485509642 + ], + [ + -85.38263906765394, + 40.202195837267 + ], + [ + -85.3824785558348, + 40.20216596112165 + ], + [ + -85.38230013355337, + 40.202061456807954 + ], + [ + -85.38206155662357, + 40.20183964412352 + ], + [ + -85.3818216607919, + 40.201616839464556 + ], + [ + -85.38158344451378, + 40.201395136471625 + ], + [ + -85.381376048972, + 40.20120013626701 + ], + [ + -85.38135983661132, + 40.20119894337552 + ], + [ + -85.38133808405672, + 40.20118711334736 + ], + [ + -85.3799919371536, + 40.20045584587313 + ], + [ + -85.37979974365945, + 40.20021465322657 + ], + [ + -85.37967173140203, + 40.2000857835808 + ], + [ + -85.37957134670783, + 40.19998473512655 + ], + [ + -85.3793393369977, + 40.199771442820484 + ], + [ + -85.37909894802377, + 40.199559035106674 + ], + [ + -85.37882274067398, + 40.19936004365336 + ], + [ + -85.37872694018745, + 40.19929293511624 + ], + [ + -85.37827516009467, + 40.199055842933575 + ], + [ + -85.37799566072965, + 40.19887443729431 + ], + [ + -85.37775634665442, + 40.19866353924469 + ], + [ + -85.37754263481656, + 40.198434641972106 + ], + [ + -85.3773250603347, + 40.198205240527585 + ], + [ + -85.37715194819457, + 40.19802645946064 + ], + [ + -85.37703456322056, + 40.197900852638355 + ], + [ + -85.37687919004422, + 40.19775162521564 + ], + [ + -85.37680731488742, + 40.197682625952645 + ], + [ + -85.37652213354625, + 40.19740874797734 + ], + [ + -85.37587415965231, + 40.197122840948666 + ], + [ + -85.37547744979491, + 40.19705565972209 + ], + [ + -85.37508015620055, + 40.19700793833109 + ], + [ + -85.3746937595796, + 40.19699035666282 + ] + ] + ], + [ + [ + [ + -85.39414339064994, + 40.203511540813324 + ], + [ + -85.39240697084784, + 40.20227492434461 + ], + [ + -85.3922399111087, + 40.20117756491884 + ], + [ + -85.39210159278188, + 40.200934845883985 + ], + [ + -85.39156941912684, + 40.200033825679974 + ], + [ + -85.39098895823302, + 40.19916145642786 + ], + [ + -85.39054168268484, + 40.198474283496196 + ], + [ + -85.39006201931952, + 40.1978486585847 + ], + [ + -85.38983761252982, + 40.19769609925633 + ], + [ + -85.38948495614483, + 40.197461072027984 + ], + [ + -85.38976542391921, + 40.197005701878986 + ], + [ + -85.38988438797294, + 40.1967307364524 + ], + [ + -85.39024478056079, + 40.196382367505755 + ], + [ + -85.39110382492233, + 40.195936609345125 + ], + [ + -85.39192532417573, + 40.19548661058371 + ], + [ + -85.3929129432786, + 40.195037026342256 + ], + [ + -85.3937616931092, + 40.19509689674137 + ], + [ + -85.39506269657274, + 40.19542692386787 + ], + [ + -85.39623249357749, + 40.19701525274974 + ], + [ + -85.39624077517962, + 40.199706445573995 + ], + [ + -85.39720666136184, + 40.200115986331184 + ], + [ + -85.3980968153132, + 40.20049627287583 + ], + [ + -85.39940652079106, + 40.20031019500278 + ], + [ + -85.40096389660307, + 40.19986284468457 + ], + [ + -85.40151186496831, + 40.20041687913585 + ], + [ + -85.40143396584708, + 40.20094038075375 + ], + [ + -85.40082256041948, + 40.20204453410351 + ], + [ + -85.3984259997691, + 40.20361007266954 + ], + [ + -85.39518198542314, + 40.20357341228795 + ], + [ + -85.39414339064994, + 40.203511540813324 + ] + ] + ] + ] + }, + "links": [], + "assets": {}, + "bbox": [ + -85.40151186496831, + 40.192653159190996, + -85.36543378415428, + 40.203696152947806 + ], + "stac_extensions": [] +} \ No newline at end of file diff --git a/tests/data/json/test_plan_results_attrs.json b/tests/data/json/test_plan_results_attrs.json new file mode 100644 index 0000000..206528c --- /dev/null +++ b/tests/data/json/test_plan_results_attrs.json @@ -0,0 +1,20 @@ +{ + "unsteady_results:plan_title": "Unsteady Run with 2D 50ft User n Value R", + "unsteady_results:program_name": "HEC-RAS - River Analysis System", + "unsteady_results:program_version": "HEC-RAS 6.3.1 October 2022", + "unsteady_results:project_file_name": "C:\\Users\\slawler\\Downloads\\Example_Projects_6_5\\Example_Projects\\2D Unsteady Flow Hydraulics\\Muncie\\Muncie.prj", + "unsteady_results:project_title": "Muncie 2D Flow Area", + "unsteady_results:short_id": "50ft User n Regions", + "unsteady_results:simulation_time_window": [ + "1900-01-02T00:00:00", + "1900-01-03T00:00:00" + ], + "unsteady_results:type_of_run": "Unsteady Flow Analysis", + "results_summary:computation_time_total": "0:00:27", + "results_summary:run_time_window": [ + "2024-05-14T15:20:01", + "2024-05-14T15:20:28" + ], + "results_summary:solution": "Unsteady Finished Successfully", + "results_summary:computation_time_total_minutes": 0.45 +} \ No newline at end of file diff --git a/tests/data/ras/Muncie.g05.hdf b/tests/data/ras/Muncie.g05.hdf new file mode 100644 index 0000000..020e3b4 Binary files /dev/null and b/tests/data/ras/Muncie.g05.hdf differ diff --git a/tests/data/ras/Muncie.p04.hdf b/tests/data/ras/Muncie.p04.hdf new file mode 100644 index 0000000..161e136 Binary files /dev/null and b/tests/data/ras/Muncie.p04.hdf differ diff --git a/tests/data/ras/projection.prj b/tests/data/ras/projection.prj new file mode 100644 index 0000000..f086c02 --- /dev/null +++ b/tests/data/ras/projection.prj @@ -0,0 +1 @@ +PROJCS["NAD_1983_StatePlane_Indiana_East_FIPS_1301_Feet",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",328083.333],PARAMETER["False_Northing",820208.333],PARAMETER["Central_Meridian",-85.6666666666667],PARAMETER["Scale_Factor",0.999966667],PARAMETER["Latitude_Of_Origin",37.5],UNIT["US survey foot",0.304800609601219]] \ No newline at end of file diff --git a/tests/test_ras_geom.py b/tests/test_ras_geom.py new file mode 100644 index 0000000..e2a1f3d --- /dev/null +++ b/tests/test_ras_geom.py @@ -0,0 +1,84 @@ +from pathlib import Path +from rashdf import RasGeomHdf +import shapely +import json + +import sys + +sys.path.append("../") +from ras_stac.utils.ras_utils import ( + RasStacGeom, + to_snake_case, + prep_stac_attrs, + properties_to_isoformat, +) + +TEST_DATA = Path("data") +TEST_JSON = TEST_DATA / "json" +TEST_RAS = TEST_DATA / "ras" +TEST_GEOM = TEST_RAS / "Muncie.g05.hdf" +TEST_GEOM_ITEM = TEST_JSON / "test_geom_item.json" +TEST_GEOM_PERIMETER = TEST_JSON / "test_perimeter.json" +TEST_GEOM_PROPERTIES = TEST_JSON / "test_geom_properties.json" + + +def test_geom_stac_item(): + ghdf = RasGeomHdf(TEST_GEOM) + ras_stac_geom = RasStacGeom(ghdf) + item = ras_stac_geom.to_item(props_to_remove=[], ras_model_name="test-1") + item.validate() + + with open(TEST_GEOM_ITEM, "r") as f: + test_item_content = json.load(f) + + item_dict = json.loads(json.dumps(item.to_dict())) + + assert item_dict == test_item_content + + +def test_geom_properties(): + ghdf = RasGeomHdf(TEST_GEOM) + ras_stac_geom = RasStacGeom(ghdf) + test_properties = properties_to_isoformat(ras_stac_geom.get_stac_geom_attrs()) + + with open(TEST_GEOM_PROPERTIES, "r") as f: + properties_json = json.load(f) + + assert test_properties == properties_json + + +def test_geom_perimeter(): + ghdf = RasGeomHdf(TEST_GEOM) + ras_stac_geom = RasStacGeom(ghdf) + + perimeter = ras_stac_geom.get_perimeter() + + test_geom = json.loads(shapely.to_geojson(perimeter)) + test_bounds = list(perimeter.bounds) + + with open(TEST_GEOM_PERIMETER, "r") as f: + perimeter_json = json.load(f) + + json_geometry = perimeter_json["geometry"] + json_bounds = perimeter_json["bounds"] + + assert test_geom == json_geometry + assert test_bounds == json_bounds + + +def test_to_snake_case(): + assert to_snake_case("Hello World") == "hello_world" + assert to_snake_case("Hello, World!") == "hello_world" + assert to_snake_case("Hello World") == "hello_world" + + +def test_prep_stac_attrs(): + attrs = {"Attribute One": "Value1", "Attribute Two": "Value2"} + expected_result = {"attribute_one": "Value1", "attribute_two": "Value2"} + assert prep_stac_attrs(attrs) == expected_result + + expected_result_with_prefix = { + "prefix:attribute_one": "Value1", + "prefix:attribute_two": "Value2", + } + assert prep_stac_attrs(attrs, prefix="prefix") == expected_result_with_prefix diff --git a/tests/test_ras_plan.py b/tests/test_ras_plan.py new file mode 100644 index 0000000..138510d --- /dev/null +++ b/tests/test_ras_plan.py @@ -0,0 +1,58 @@ +from pathlib import Path +from rashdf import RasPlanHdf +import pystac +import json + +import sys + +sys.path.append("../") +from ras_stac.utils.ras_utils import RasStacPlan, properties_to_isoformat + +TEST_DATA = Path("data") +TEST_JSON = TEST_DATA / "json" +TEST_RAS = TEST_DATA / "ras" +TEST_PLAN = TEST_RAS / "Muncie.p04.hdf" +TEST_PLAN_ATTRS = TEST_JSON / "test_plan_attrs.json" +TEST_PLAN_RESULTS_ATTRS = TEST_JSON / "test_plan_results_attrs.json" +TEST_GEOM_ITEM = TEST_JSON / "test_geom_item.json" +TEST_PLAN_ITEM = TEST_JSON / "test_plan_item.json" + + +def test_plan_stac_item(): + phdf = RasPlanHdf(TEST_PLAN) + ras_stac_plan = RasStacPlan(phdf) + geom_item = pystac.Item.from_file(TEST_GEOM_ITEM) + plan_meta = ras_stac_plan.get_simulation_metadata(simulation="test-1") + plan_item = ras_stac_plan.to_item( + geom_item, plan_meta, model_sim_id="test-1", item_props_to_remove=[] + ) + plan_item.validate() + + with open(TEST_PLAN_ITEM, "r") as f: + test_item_content = json.load(f) + + item_dict = json.loads(json.dumps(plan_item.to_dict())) + + assert item_dict == test_item_content + + +def test_plan_attrs(): + phdf = RasPlanHdf(TEST_PLAN) + ras_stac_plan = RasStacPlan(phdf) + test_attrs = properties_to_isoformat(ras_stac_plan.get_stac_plan_attrs()) + + with open(TEST_PLAN_ATTRS, "r") as f: + attrs_json = json.load(f) + + assert test_attrs == attrs_json + + +def test_plan_results_attrs(): + phdf = RasPlanHdf(TEST_PLAN) + ras_stac_plan = RasStacPlan(phdf) + test_attrs = properties_to_isoformat(ras_stac_plan.get_stac_plan_results_attrs()) + + with open(TEST_PLAN_RESULTS_ATTRS, "r") as f: + attrs_json = json.load(f) + + assert test_attrs == attrs_json diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..9f284a4 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,57 @@ +import json +import shapely +import pystac +import sys + +sys.path.append("../") +from ras_stac.utils.ras_utils import RasStacGeom, properties_to_isoformat, RasStacPlan + +### Functions for test items creation + + +def stac_item_to_json(item: pystac.Item, filename: str): + """Writes a STAC item to a JSON file.""" + item_json = json.dumps(item.to_dict(), indent=4) + with open(filename, "w") as f: + f.write(item_json) + + +def create_perimeter_json( + ras_stac_geom: RasStacGeom, output_json_fn: str = "test_perimeter.json" +): + perimeter = ras_stac_geom.get_perimeter() + geometry = json.loads(shapely.to_geojson(perimeter)) + bounds = perimeter.bounds + test_perimeter = {"geometry": geometry, "bounds": bounds} + with open(output_json_fn, "w") as f: + json.dump(test_perimeter, f) + + +def geom_properties_to_json( + ras_stac_geom: RasStacGeom, output_json_fn: str = "test_geom_properties.json" +): + properties = ras_stac_geom.get_stac_geom_attrs() + iso_properties = properties_to_isoformat(properties) + + with open(output_json_fn, "w") as f: + json.dump(iso_properties, f) + + +def plan_attrs_to_json( + ras_stac_plan: RasStacPlan, output_json_fn: str = "test_plan_attrs.json" +): + plan_attrs = ras_stac_plan.get_stac_plan_attrs() + iso_attrs = properties_to_isoformat(plan_attrs) + + with open(output_json_fn, "w") as f: + json.dump(iso_attrs, f) + + +def plan_results_attrs_to_json( + ras_stac_plan: RasStacPlan, output_json_fn: str = "test_plan_results_attrs.json" +): + plan_results_attrs = ras_stac_plan.get_stac_plan_results_attrs() + iso_attrs = properties_to_isoformat(plan_results_attrs) + + with open(output_json_fn, "w") as f: + json.dump(iso_attrs, f)