WIP - Probabilistic Flood Inundation Mapping

Pipfile

@@ -10,8 +10,8 @@ ipython = "==8.24.0"
 certifi = "==2023.7.22"
 fiona = "==1.8.22"
 geopandas = "==0.14.3"
-numba = "==0.56.4"
-numpy = "==1.23.5"
+numba = "==0.60.0"
+numpy = "==1.26.0"
 pandas = "==2.0.2"
 rasterio = "==1.3.6"
 rasterstats = "==0.18.0"
@@ -20,7 +20,7 @@ tqdm = "==4.66.3"
 seaborn = "==0.12.2"
 python-dotenv = "==1.0.0"
 natsort = "==8.3.1"
-xarray = "==2023.1.0"
+xarray = ">=2024.6.0,<2025"
 netcdf4 = "==1.6.3"
 tables = "==3.8.0"
 pyproj = "==3.5.0"
@@ -30,13 +30,14 @@ jupyter = "==1.0.0"
 jupyterlab = "==3.6.7"
 ipympl = "==0.9.3"
 pytest = "==7.3.0"
-whitebox = "2.3.4"
+whitebox = "==2.3.4"
 shapely = "==2.0.1"
 pyarrow = "==14.0.1"
 rtree = "==1.0.1"
 py7zr = "==0.20.4"
-scipy = "==1.10.1"
-gval = "==0.2.3"
+scipy = "==1.12.0"
+gval = "==0.2.7"
+distributed = "==2024.6.2"
 flake8 = "==6.0.0"
 black = "==24.3.0"
 flake8-pyproject = "==1.2.3"
@@ -52,6 +53,7 @@ lmoments3 = "==1.0.6"
 zarr = "==2.18.0"
 requests = "==2.32.3"
 aiohttp = "==3.9.5"
+monaco = "==0.13.1"
 
 [requires]
 python_version = "3.10"

src/utils/shared_functions.py

@@ -455,7 +455,7 @@ def print_date_time_duration(start_dt, end_dt):
         total_hours, rem_seconds = divmod(rem_seconds, 60 * 60)
         total_mins, seconds = divmod(rem_seconds, 60)
 
-        time_fmt = f"{total_hours:02d} hours {total_mins:02d} mins {seconds:02d} secs"
+        time_fmt = f"{total_days:02d} days {total_hours:02d} hours {total_mins:02d} mins {seconds:02d} secs"
 
         duration_msg = "Duration: " + time_fmt
         print(duration_msg)

tools/inundate_gms.py

@@ -15,6 +15,7 @@ def Inundate_gms(
     hydrofabric_dir,
     forecast,
     num_workers=1,
+    hydro_table_df=None,
     hucs=None,
     inundation_raster=None,
     inundation_polygon=None,
@@ -67,6 +68,7 @@ def Inundate_gms(
         inundation_polygon,
         depths_raster,
         forecast,
+        hydro_table_df,
         verbose=False,
     )
 
@@ -160,8 +162,16 @@ def __inundate_gms_generator(
     inundation_polygon,
     depths_raster,
     forecast,
+    hydro_table_df=None,
     verbose=False,
 ):
+    """
+    Generator for use in parallelizing inundation
+
+    Parameters
+    ----------
+
+    """
     # Iterate over branches
     for idx, row in hucs_branches.iterrows():
         huc = str(row[0])
@@ -176,36 +186,41 @@ def __inundate_gms_generator(
         catchments_file_name = f"gw_catchments_reaches_filtered_addedAttributes_{branch_id}.tif"
         catchments_branch = os.path.join(branch_dir, catchments_file_name)
 
-        # FIM versions > 4.3.5 use an aggregated hydrotable file rather than individual branch hydrotables
-        hydroTable_huc = os.path.join(huc_dir, "hydrotable.csv")
-        if os.path.isfile(hydroTable_huc):
-            htable_req_cols = [
-                "HUC",
-                "branch_id",
-                "feature_id",
-                "HydroID",
-                "stage",
-                "discharge_cms",
-                "LakeID",
-            ]
-            hydroTable_all = pd.read_csv(
-                hydroTable_huc,
-                dtype={
-                    "HUC": str,
-                    "branch_id": int,
-                    "feature_id": str,
-                    "HydroID": str,
-                    "stage": float,
-                    "discharge_cms": float,
-                    "LakeID": int,
-                },
-                usecols=htable_req_cols,
-            )
-            hydroTable_all.set_index(["HUC", "feature_id", "HydroID"], inplace=True)
-            hydroTable_branch = hydroTable_all.loc[hydroTable_all["branch_id"] == int(branch_id)]
+        if hydro_table_df is not None:
+            if 'HUC' not in hydro_table_df.index.names:
+                hydro_table_df.set_index(["HUC", "feature_id", "HydroID"], inplace=True)
+            hydro_table_branch = hydro_table_df.loc[hydro_table_df["branch_id"] == int(branch_id)]
         else:
-            # Earlier FIM4 versions only have branch level hydrotables
-            hydroTable_branch = os.path.join(branch_dir, f"hydroTable_{branch_id}.csv")
+            # FIM versions > 4.3.5 use an aggregated hydrotable file rather than individual branch hydrotables
+            hydro_table_huc = os.path.join(huc_dir, "hydrotable.csv")
+            if os.path.isfile(hydro_table_huc):
+                htable_req_cols = [
+                    "HUC",
+                    "branch_id",
+                    "feature_id",
+                    "HydroID",
+                    "stage",
+                    "discharge_cms",
+                    "LakeID",
+                ]
+                hydro_table_all = pd.read_csv(
+                    hydro_table_huc,
+                    dtype={
+                        "HUC": str,
+                        "branch_id": int,
+                        "feature_id": str,
+                        "HydroID": str,
+                        "stage": float,
+                        "discharge_cms": float,
+                        "LakeID": int,
+                    },
+                    usecols=htable_req_cols,
+                )
+                hydro_table_all.set_index(["HUC", "feature_id", "HydroID"], inplace=True)
+                hydro_table_branch = hydro_table_all.loc[hydro_table_all["branch_id"] == int(branch_id)]
+            else:
+                # Earlier FIM4 versions only have branch level hydrotables
+                hydro_table_branch = os.path.join(branch_dir, f"hydroTable_{branch_id}.csv")
 
         xwalked_file_name = f"gw_catchments_reaches_filtered_addedAttributes_crosswalked_{branch_id}.gpkg"
         catchment_poly = os.path.join(branch_dir, xwalked_file_name)
@@ -237,7 +252,7 @@ def __inundate_gms_generator(
             "rem": rem_branch,
             "catchments": catchments_branch,
             "catchment_poly": catchment_poly,
-            "hydro_table": hydroTable_branch,
+            "hydro_table": hydro_table_branch,
             "forecast": forecast,
             "mask_type": "filter",
             "hucs": None,