Merge branch 'dev' into dev-jdk-17

config/deny_unit.lst

@@ -35,3 +35,7 @@ nwm_headwaters.gpkg
 # wbd8_clp.gpkg
 wbd_buffered.gpkg
 wbd_buffered_streams.gpkg
+HUC6_dem_domain.gpkg
+usgs_gages.gpkg
+reformat_ras_rating_curve_points_rel_101.gpkg
+nws_lid.gpkg

docs/CHANGELOG.md

@@ -1,7 +1,7 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-## v4.4.x.x - 2023-12-08 - [PR#1047](https://github.com/NOAA-OWP/inundation-mapping/pull/1047)
+## v4.4.8.1 - 2023-12-08 - [PR#1047](https://github.com/NOAA-OWP/inundation-mapping/pull/1047)
 
 Upgrades JDK to v.17.0.9 in Docker image to address security vulnerabilities.
 
@@ -11,6 +11,33 @@ Upgrades JDK to v.17.0.9 in Docker image to address security vulnerabilities.
 
 <br/><br/>
 
+## v4.4.8.0 - 2023-12-08 - [PR#1045](https://github.com/NOAA-OWP/inundation-mapping/pull/1045)
+
+In order to avoid file system collisions on AWS, and keep the reads/writes from the same file on disk to a minimum, three files (`HUC6_dem_domain.gpkg`, `nws_lid.gpkg`, `reformat_ras_rating_curve_points_rel_101.gpkg`, & `usgs_gages.gpkg`) are now copied from disk into a scratch directory (temporary working directory), and removed after processing steps are completed.
+
+### Changes
+
+- `config`/`deny_unit.lst`: Add files to remove list - repetitive copies needed for processing step (`run_unit_wb.sh`)
+- `src`
+    - `bash_variables.env`: Add a new variable for the ras rating curve filename. It will be easier to track the filename in the `.env`, and pull into `run_unit_wb.sh`, rather than hardcode it.
+    - `run_unit_wb.sh`: Copy files and update references from `$inputsDir` to `$tempHucDataDir`.
+
+<br/><br/>
+
+## v4.4.7.2 - 2023-12-08 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026)
+
+A couple of directly related issues were fixed in this PR.
+The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundation-mapping/issues/1025) which was about a pathing issue for the outputs directory. In testing that fix, it exposed a few other pathing and file cleanup issues which are now fixed. We also added more console output to help view variables and pathing.
+
+### Changes
+
+- `config`/`params_template.env`:  Updated for a newer mannings global file. Changed and tested by Ryan Spies.
+- `tools`
+    - `inundate_mosiac_wrapper.py`:  Took out a misleading and non-required print statement.
+    - `inundate_nation.py`: As mentioned above.
+
+<br/><br/>
+
 ## v4.4.7.1 - 2023-12-01 - [PR#1036](https://github.com/NOAA-OWP/inundation-mapping/pull/1036)
 
 Quick update to match incoming ras2fim calibration output files being feed into FIM was the initial change.
@@ -100,6 +127,7 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1
 
 <br/><br/>
 
+
 ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
 
 During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times.

src/bash_variables.env

@@ -1,43 +1,43 @@
 ## Define inputs
 
 # NOTE: $inputsDir is defined in Dockerfile
-export      DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
-export                       input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
-export                input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
-export             input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
-export                       input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
-export       input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg
-export input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
-export            input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg
-export                 input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
-export            input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
-export                 input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
-export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
-export          input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
-export                pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
-
-export                 bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
+export        DEFAULT_FIM_PROJECTION_CRS=EPSG:5070
+export                         input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt
+export                  input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg
+export               input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg
+export                         input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg
+export         input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg
+export   input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg
+export              input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg
+export                   input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg
+export              input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg
+export                   input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg
+export                     input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg
+export            input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
+export                  pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
+export                   bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
-export             bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
+export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
 
 # input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export                vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv
+export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv
 
 # input file location with nwm feature_id and recurrence flow values
-export                  nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
+export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
 
 # input file location with usgs rating curve database
-export           usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
+export             usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
 
 # input file locations for ras2fim locations and rating curve data
-export            ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
-export    ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
+export              ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
+export      ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
+export    ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points_rel_101.gpkg
 
-export                fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
+export                  fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
 
 # Input file location with HUC, nwm feature_id and manual calibration coefficients
-export                   man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv
+export                     man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv
 
 
 # Styling

src/run_unit_wb.sh

@@ -26,6 +26,13 @@ huc_start_time=`date +%s`
 echo -e $startDiv"Copying staged wbd and .gpkg files from $pre_clip_huc_dir/$hucNumber"
 cp -a $pre_clip_huc_dir/$hucNumber/. $tempHucDataDir
 
+# Copy necessary files from $inputsDir into $tempHucDataDir to avoid File System Collisions
+# For buffer_stream_branches.py
+cp $input_DEM_domain $tempHucDataDir
+# For usgs_gage_unit_setup.py
+cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir
+cp $ras_rating_curve_points_gpkg $tempHucDataDir
+cp $inputsDir/ahps_sites/nws_lid.gpkg $tempHucDataDir
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"
@@ -77,7 +84,7 @@ Tcount
 echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber"
 date -u
 Tstart
-$srcDir/buffer_stream_branches.py -a $input_DEM_domain \
+$srcDir/buffer_stream_branches.py -a $tempHucDataDir/HUC6_dem_domain.gpkg \
     -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \
     -i $branch_id_attribute \
     -d $branch_buffer_distance_meters \
@@ -274,12 +281,12 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     date -u
     Tstart
     python3 $srcDir/usgs_gage_unit_setup.py \
-        -gages $inputsDir/usgs_gages/usgs_gages.gpkg \
+        -gages $tempHucDataDir/usgs_gages.gpkg \
         -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
-        -ras $ras_rating_curve_points_gpkg \
+        -ras $tempHucDataDir/$ras_rating_curve_gpkg_filename \
         -o $tempHucDataDir/usgs_subset_gages.gpkg \
         -huc $hucNumber \
-        -ahps $inputsDir/ahps_sites/nws_lid.gpkg \
+        -ahps $tempHucDataDir/nws_lid.gpkg \
         -bzero_id $branch_zero_id
     Tcount
 fi

tools/inundate_mosaic_wrapper.py

@@ -93,8 +93,6 @@ def produce_mosaicked_inundation(
             "Please lower the num_workers.".format(num_workers, total_cpus_available)
         )
 
-    fh.vprint("Running inundate for " + huc + "...", verbose)
-
     # Call Inundate_gms
     map_file = Inundate_gms(
         hydrofabric_dir=hydrofabric_dir,