Merge pull request #20 from fema-ffrd/feature/add_unit_tests

Feature/add unit tests

.dockerignore

@@ -0,0 +1,10 @@
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+ras_stac_venv/

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.0
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.5
 
 RUN apt-get update && \
     apt-get install jq -y && \
@@ -11,12 +11,11 @@ RUN pip3 install -r requirements.txt
 WORKDIR /plugins
 
 # Copy plugin utils
-COPY ras_stac/utils utils
+COPY ras_stac/utils ras_stac/utils
 
 # Copy plugin functions
-COPY ras_stac/ras_geom_hdf.py .
-COPY ras_stac/ras_plan_hdf.py .
-COPY ras_stac/ras_plan_dg.py .
+COPY ras_stac/ras_geom_hdf.py ras_stac/
+COPY ras_stac/ras_plan_hdf.py ras_stac/
+COPY ras_stac/ras_plan_dg.py ras_stac/
 
-# Copy plugin main functions
-COPY ras_stac/plugins .
+COPY tests tests

README.md

@@ -19,4 +19,9 @@ This repository contains code for developing STAC items from HEC-RAS models. Cur
 3. Run the `populate-sample-data.sh` script to test set-up, connetivity, and view a sample stac catalog created using this library.
 
 
-**NOTE** It is recommended that ras-stac not be run in a container for testing and development due to networking issues that complicate use of these tools, when using minio. 
+**NOTE** It is recommended that ras-stac not be run in a container for testing and development of its full codebase due to networking issues that complicate use of these tools, when using minio. 
+
+
+## Core tests (can run locally in Docker)
+
+When `docker-test.sh` is executed, the Docker image is built and `pytest` is invoked to run the Python test scripts. This leverages test data that is included in this repository. This does not use cloud storage, s3, minio, nor other forms of network data.

docker-test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+set -x
+
+docker build -t ras-stac .
+docker run --rm -it -w /plugins/tests ras-stac pytest

ras_stac/ras_geom_hdf.py

@@ -8,14 +8,15 @@
 from typing import List
 
 from .utils.common import check_params, GEOM_HDF_IGNORE_PROPERTIES
-from .utils.ras_stac import create_model_item, new_geom_assets, ras_geom_asset_info
+from .utils.ras_utils import RasStacGeom, new_geom_assets, ras_geom_asset_info
 from .utils.s3_utils import (
     verify_safe_prefix,
     s3_key_public_url_converter,
     split_s3_key,
     init_s3_resources,
     get_basic_object_metadata,
     copy_item_to_s3,
+    read_ras_geom_from_s3,
 )
 
 logging.getLogger("boto3").setLevel(logging.WARNING)
@@ -56,12 +57,12 @@ def new_geom_item(
     _, s3_client, s3_resource = init_s3_resources(minio_mode=minio_mode)
     bucket = s3_resource.Bucket(bucket_name)
     # Create geometry item
+    geom_hdf_obj, ras_model_name = read_ras_geom_from_s3(geom_hdf, minio_mode)
+    ras_stac_geom = RasStacGeom(geom_hdf_obj)
     if item_props_to_remove:
-        item = create_model_item(geom_hdf, item_props_to_remove, minio_mode=minio_mode)
+        item = ras_stac_geom.to_item(item_props_to_remove, ras_model_name)
     else:
-        item = create_model_item(
-            geom_hdf, GEOM_HDF_IGNORE_PROPERTIES, minio_mode=minio_mode
-        )
+        item = ras_stac_geom.to_item(GEOM_HDF_IGNORE_PROPERTIES, ras_model_name)
 
     if item_props_to_add:
         item.properties.update(item_props_to_add)

ras_stac/ras_plan_dg.py

@@ -8,7 +8,7 @@
 
 from .utils.common import check_params
 from .utils.dg_utils import create_depth_grid_item
-from utils.ras_stac import ras_plan_asset_info
+from .utils.ras_utils import ras_plan_asset_info
 from .utils.s3_utils import (
     verify_safe_prefix,
     s3_key_public_url_converter,

ras_stac/ras_plan_hdf.py

@@ -7,18 +7,15 @@
 from typing import List
 
 from .utils.common import check_params, PLAN_HDF_IGNORE_PROPERTIES
-from .utils.ras_stac import (
-    get_simulation_metadata,
-    create_model_simulation_item,
-    ras_plan_asset_info,
-)
+from .utils.ras_utils import ras_plan_asset_info, RasStacPlan
 from .utils.s3_utils import (
     verify_safe_prefix,
     s3_key_public_url_converter,
     split_s3_key,
     init_s3_resources,
     get_basic_object_metadata,
     copy_item_to_s3,
+    read_ras_plan_from_s3,
 )
 
 
@@ -66,16 +63,18 @@ def new_plan_item(
     geom_item = pystac.Item.from_file(geom_item_public_url)
 
     logging.info("fetching plan metadata")
-    plan_meta = get_simulation_metadata(plan_hdf, sim_id, minio_mode=minio_mode)
+    plan_hdf_obj = read_ras_plan_from_s3(plan_hdf, minio_mode)
+    ras_stac_plan = RasStacPlan(plan_hdf_obj)
+    plan_meta = ras_stac_plan.get_simulation_metadata(sim_id)
     if plan_meta:
         try:
             logging.info("creating plan item")
             if item_props_to_remove:
-                plan_item = create_model_simulation_item(
+                plan_item = ras_stac_plan.to_item(
                     geom_item, plan_meta, sim_id, item_props_to_remove
                 )
             else:
-                plan_item = create_model_simulation_item(
+                plan_item = ras_stac_plan.to_item(
                     geom_item, plan_meta, sim_id, PLAN_HDF_IGNORE_PROPERTIES
                 )
         except TypeError:

ras_stac/utils/dg_utils.py

@@ -3,8 +3,9 @@
 import os
 import pystac
 import rasterio
+import rasterio.warp
 
-from datetime import datetime
+from datetime import datetime, timezone
 from mypy_boto3_s3.service_resource import Object
 from pathlib import Path
 from rasterio.session import AWSSession
@@ -91,9 +92,9 @@ def bbox_to_polygon(bbox) -> Polygon:
     return Polygon(
         [
             [min_x, min_y],
-            [min_x, max_y],
-            [max_x, max_y],
             [max_x, min_y],
+            [max_x, max_y],
+            [min_x, max_y],
         ]
     )
 
@@ -125,7 +126,7 @@ def create_depth_grid_item(
         id=item_id,
         properties={},
         bbox=bbox,
-        datetime=datetime.now(),
+        datetime=datetime.now(timezone.utc),
         geometry=json.loads(to_geojson(geometry)),
     )
     # non_null = not raster_is_all_null(depth_grid.key)