[2pt] PR: CatFIM 2.1: Add Alaska, reconcile sites, fix CatFIM V2.0 bugs

.gitignore

@@ -14,3 +14,5 @@ config/*.swp
 .vscode/
 **/.DS_Store
 .private/
+!tools/catfim/*.csv
+!tools/catfim/catfim.env.template

.pre-commit-config.yaml

@@ -2,14 +2,13 @@
 # Additional hooks: https://pre-commit.com/hooks.html
 
 repos:
+
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.0.1
     hooks:
         -   id: trailing-whitespace
             # Below is python regex to exclude all .md files
             exclude: .*md$
-        -   id: end-of-file-fixer
-            exclude:  Pipfile.lock
         -   id: check-added-large-files
             args: ['--maxkb=5000']
         -   id: check-json

docs/CHANGELOG.md

@@ -1,6 +1,14 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+
+## v4.5.13.0 - 2024-12-10 - [PR#1285](https://github.com/NOAA-OWP/inundation-mapping/pull/1285)
+
+Major upgrades and bug fixes to the CatFIM product, informally called CatFIM 2.1. See the PR for all details
+
+<br/><br/>
+
+
 ## v4.5.12.2 - 2024-12-10 - [PR#1346](https://github.com/NOAA-OWP/inundation-mapping/pull/1346)
 
 This PR updates deny lists to avoid saving unnecessary files.
@@ -44,6 +52,8 @@ As part of this PR, small modification was applied to bridge_inundation.py.
 <br/><br/>
 
 
+
+
 ## v4.5.11.3 - 2024-10-25 - [PR#1320](https://github.com/NOAA-OWP/inundation-mapping/pull/1320)
 
 The fix: During the post processing scan for the word "error" or "warning", it was only finding records which had either of those two words as stand alone words and not part of bigger phrases.  ie); "error" was found, but not "fielderror". Added wildcards and it is now fixed.

pyproject.toml

@@ -20,7 +20,6 @@ maintainers = [
     {name = "Hamideh Safa", email = "[email protected]"},
     {name = "James Coll", email = "[email protected]"},
     {name = "Matt Luck", email = "[email protected]"},
-    {name = "Nick Chadwick", email = "[email protected]"},
     {name = "Riley McDermott", email = "[email protected]"},
     {name = "Robert Hanna", email = "[email protected]"},
     {name = "Ryan Spies", email = "[email protected]"}
@@ -39,17 +38,24 @@ Wiki = "https://github.com/NOAA-OWP/inundation-mapping/wiki"
 # - Tools -
 
 [tool.black]
-line_length = 110
 skip-string-normalization = true
 skip-magic-trailing-comma = true
-
+line-length = 110
+exclude = '''
+/(
+   \.csv$
+)/
+'''
 
 [tool.isort]
 profile = 'black'
 line_length = 110
 multi_line_output = 3
 lines_after_imports = 2
+# extend-skip = [".csv"]
+skip_glob = ["*.csv"]
 
+# yes, the exclue format is different from black
 [tool.flake8]
 count = true
 doctests = true
@@ -59,26 +65,21 @@ extend-ignore = """
     E203,
     E266,
     E501,
+    E712,
     W503,
-    W391
-    F403,
+    W391,
     F401,
+    F403,
     """
+exclude = ["*.csv"]
+
 per-file-ignores = """
-    src/subdiv_chan_obank_src.py: E712
-    src/src_roughness_optimization.py: E712, F841
-    src/agreedem.py: E712
+    src/src_roughness_optimization.py: F841
     src/build_stream_traversal.py: E722
-
-    tools/check_deep_flooding.py: E712
     tools/eval_alt_catfim.py: F841
-    tools/generate_categorical_fim.py: C901, E712
-    tools/generate_categorical_fim_mapping.py: E712
     tools/inundation.py: F821
     tools/rating_curve_comparison.py: F821, F841
     tools/run_test_case.py: E711
     tools/tools_shared_functions.py: F821, F841, E711
-    tools/vary_mannings_n_composite.py: E712
-
     data/usgs/rating_curve_get_usgs_curves.py: F841
     """

src/utils/fim_logger.py

@@ -2,8 +2,6 @@
 
 import datetime as dt
 import os
-import random
-import traceback
 from pathlib import Path
 
 
@@ -147,7 +145,8 @@ def MP_Log_setup(self, parent_log_output_file, file_prefix):
             using a defined file path.
 
             As this is an MP file, the parent_log_output_file may have a date in it
-            The file name is calculated as such {file_prefix}-{random 12 digit key}.log()
+            The file name is calculated as such
+            {file_prefix}-{currernt datetime with milli}.log()
             ie) catfim_2024_07_09-16_30_02__012345678901.log
 
             The extra file portion is added as in MultiProc, you can have dozens of processes
@@ -165,8 +164,10 @@ def MP_Log_setup(self, parent_log_output_file, file_prefix):
         # -----------------
         log_folder = os.path.dirname(parent_log_output_file)
 
-        random_id = random.randrange(1000000000, 99999999999)
-        log_file_name = f"{file_prefix}___{random_id}.log"
+        # random_id = random.randrange(1000000000, 99999999999)
+        # this is an epoch time
+        dt_str = dt.datetime.now().strftime('%H%M%S%f')
+        log_file_name = f"{file_prefix}___{dt_str}.log"
         log_file_path = os.path.join(log_folder, log_file_name)
 
         self.setup(log_file_path)
@@ -232,63 +233,68 @@ def merge_log_files(self, parent_log_output_file, file_prefix, remove_old_files=
         folder_path = os.path.dirname(parent_log_output_file)
         os.makedirs(folder_path, exist_ok=True)
 
-        log_file_list_paths = list(Path(folder_path).rglob(f"{file_prefix}*"))
+        log_file_list_paths = list(Path(folder_path).glob(f"*{file_prefix}*"))
         log_file_list = [str(x) for x in log_file_list_paths]
 
         if len(log_file_list) > 0:
             log_file_list.sort()
 
             # we are merging them in order (reg files, then warnings, then errors)
 
-            # open and write to the parent log
-            # This will write all logs including errors and warning
-            with open(parent_log_output_file, 'a') as main_log:
-                # Iterate through list
-                for temp_log_file in log_file_list:
-                    # Open each file in read mode
-                    with open(temp_log_file) as infile:
-                        main_log.write(infile.read())
-                    if "warning" not in temp_log_file and "error" not in temp_log_file:
-                        if remove_old_files:
-                            os.remove(temp_log_file)
-
-            # now the warning files if there are any
-            log_warning_file_list = list(Path(folder_path).rglob(f"{file_prefix}*_warnings*"))
-            if len(log_warning_file_list) > 0:
-                log_warning_file_list.sort()
-                parent_warning_file = parent_log_output_file.replace(".log", "_warnings.log")
-                with open(parent_warning_file, 'a') as warning_log:
+            # It is ok if it fails with a file not found. Sometimes during multi proc
+            # merging, we get some anomylies. Rare but it happens.
+            try:
+                # open and write to the parent log
+                # This will write all logs including errors and warning
+                with open(parent_log_output_file, 'a') as main_log:
                     # Iterate through list
-                    for temp_log_file in log_warning_file_list:
+                    for temp_log_file in log_file_list:
                         # Open each file in read mode
                         with open(temp_log_file) as infile:
-                            warning_log.write(infile.read())
-
-                        if remove_old_files:
-                            os.remove(temp_log_file)
-
-            # now the warning files if there are any
-            log_error_file_list = list(Path(folder_path).rglob(f"{file_prefix}*_errors*"))
-            if len(log_error_file_list) > 0:
-                log_error_file_list.sort()
-                parent_error_file = parent_log_output_file.replace(".log", "_errors.log")
-                # doesn't yet exist, then create a blank one
-                with open(parent_error_file, 'a') as error_log:
-                    # Iterate through list
-                    for temp_log_file in log_error_file_list:
-                        # Open each file in read mode
-                        with open(temp_log_file) as infile:
-                            error_log.write(infile.read())
-
-                        if remove_old_files:
-                            os.remove(temp_log_file)
+                            main_log.write(infile.read())
+                        if "warning" not in temp_log_file and "error" not in temp_log_file:
+                            if remove_old_files:
+                                os.remove(temp_log_file)
+
+                # now the warning files if there are any
+                log_warning_file_list = list(Path(folder_path).rglob(f"{file_prefix}*_warnings*"))
+                if len(log_warning_file_list) > 0:
+                    log_warning_file_list.sort()
+                    parent_warning_file = parent_log_output_file.replace(".log", "_warnings.log")
+                    with open(parent_warning_file, 'a') as warning_log:
+                        # Iterate through list
+                        for temp_log_file in log_warning_file_list:
+                            # Open each file in read mode
+                            with open(temp_log_file) as infile:
+                                warning_log.write(infile.read())
+
+                            if remove_old_files:
+                                os.remove(temp_log_file)
+
+                # now the warning files if there are any
+                log_error_file_list = list(Path(folder_path).rglob(f"{file_prefix}*_errors*"))
+                if len(log_error_file_list) > 0:
+                    log_error_file_list.sort()
+                    parent_error_file = parent_log_output_file.replace(".log", "_errors.log")
+                    # doesn't yet exist, then create a blank one
+                    with open(parent_error_file, 'a') as error_log:
+                        # Iterate through list
+                        for temp_log_file in log_error_file_list:
+                            # Open each file in read mode
+                            with open(temp_log_file) as infile:
+                                error_log.write(infile.read())
+
+                            if remove_old_files:
+                                os.remove(temp_log_file)
+            except FileNotFoundError as ex:
+                print(f"Merge file not found. Details: {ex}. Program continuing")
 
         return
 
     # -------------------------------------------------
     def trace(self, msg):
         # goes to file only, not console
-        level = "TRACE   "  # keeps spacing the same
+        level = "TRACE".ljust(9)  # keeps spacing the same  (9 chars wide)
         if self.LOG_FILE_PATH == "":
             print(self.LOG_SYS_NOT_SETUP_MSG)
             return
@@ -301,7 +307,7 @@ def trace(self, msg):
     # -------------------------------------------------
     def lprint(self, msg):
         # goes to console and log file
-        level = "LPRINT  "  # keeps spacing the same
+        level = "LPRINT".ljust(9)  # keeps spacing the same  (9 chars wide)
         print(f"{msg} ")
 
         if self.LOG_FILE_PATH == "":
@@ -316,7 +322,7 @@ def lprint(self, msg):
     # -------------------------------------------------
     def notice(self, msg):
         # goes to console and log file
-        level = "NOTICE  "  # keeps spacing the same
+        level = "NOTICE".ljust(9)  # keeps spacing the same  (9 chars wide)
         # print(f"{cl.fore.TURQUOISE_2}{msg}{cl.style.RESET}")
         print(f"{level}: {msg}")
 
@@ -332,7 +338,7 @@ def notice(self, msg):
     # -------------------------------------------------
     def success(self, msg):
         # goes to console and log file
-        level = "SUCCESS "  # keeps spacing the same
+        level = "SUCCESS".ljust(9)  # keeps spacing the same  (9 chars wide)
 
         # c_msg_type = f"{cl.fore.SPRING_GREEN_2B}<{level}>{cl.style.RESET}"
         # print(f"{self.__get_clog_dt()} {c_msg_type} : {msg}")
@@ -350,7 +356,7 @@ def success(self, msg):
     # -------------------------------------------------
     def warning(self, msg):
         # goes to console and log file and warning log file
-        level = "WARNING "  # keeps spacing the same
+        level = "WARNING".ljust(9)  # keeps spacing the same  (9 chars wide)
 
         # c_msg_type = f"{cl.fore.LIGHT_YELLOW}<{level}>{cl.style.RESET}"
         # print(f"{self.__get_clog_dt()} {c_msg_type} : {msg}")
@@ -372,7 +378,7 @@ def warning(self, msg):
     # -------------------------------------------------
     def error(self, msg):
         # goes to console and log file and error log file
-        level = "ERROR   "  # keeps spacing the same
+        level = "ERROR".ljust(9)  # keeps spacing the same  (9 chars wide)
 
         # c_msg_type = f"{cl.fore.RED_1}<{level}>{cl.style.RESET}"
         # print(f"{self.__get_clog_dt()} {c_msg_type} : {msg}")
@@ -393,7 +399,7 @@ def error(self, msg):
 
     # -------------------------------------------------
     def critical(self, msg):
-        level = "CRITICAL".ljust(9)
+        level = "CRITICAL".ljust(9)  # keeps spacing the same (9 chars wide)
 
         # c_msg_type = f"{cl.style.BOLD}{cl.fore.RED_3A}{cl.back.WHITE}{self.__get_dt()}"
         # c_msg_type += f" <{level}>"

tools/catfim/README.md

@@ -0,0 +1,76 @@
+##  About CatFIM
+CatFIM is short for Categorical Flood Inundation Mapping. CatFIM is a tool that is run on HAND FIM outputs to inundate a bunch of AHPS sites to specific stages or flow levels. This is useful for quality controlling HAND FIM as well as for preparing for hypothetical flood events. CatFIM models floods at up to 5 different magnitudes: action, minor, moderate, major, and record. The flows or stages for these magnitudes is pulled from the WRDS API.
+
+### Stage-based vs Flow-based
+There are two modes for running CatFIM: stage-based and flow-based. Stage-based CatFIM inundates the HAND FIM relative elevation model (REM) based on different magnitudes of stage, which is the height of the water surface. Flow-based CatFIM inundates the REM based on different magnitudes of flow, which is measured as the volume over time of water flowing past a certain point of the river. A rating curve is used to get the water height from the flow volume at a given location.
+
+### How are CatFIM sites selected?
+
+Our goal is to produce good CatFIM for as many APHS sites as possible. In order to meet this goal, a site must meet a variety of acceptance critera and data availability checks for us to produce CatFIM at the site. 
+
+If you have a question about why a specific point is being excluded, you can check the "Status" attribute of the CatFIM point to see what issue might be occurring. If there is a question or concern about a specific point, feel free to reach out to GID via Slack or VLAB.
+
+
+CatFIM sites must: 
+- have site-specific stage- or flow- thresholds available
+- be an NWM forecast point (for sites in CONUS*)
+- not be on the stage-based AHPS restricted sites list (for stage-based CatFIM)
+- have an accurate vertical datum (for stage-based CatFIM)
+- meet the USGS Gages Acceptance Criteria (detailed below)
+
+USGS Gages Acceptance Criteria:
+- [Lat/Long Coordinate Method](https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html) must be one of the following: "C", "D", "W", "X", "Y", "Z", "N", "M", "L", "G", "R", "F", "S"
+- [Acceptable Altitute Accuracy Threshold](https://help.waterdata.usgs.gov/codes-and-parameters/codes#SI) must be 1 or lower
+- [Altitute Method Type](https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html) must be one of the following: "A", "D", "F", "I", "J", "L", "N", "R", "W", "X", "Y", "Z"
+- [Site Type](https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html) must be: "ST" (stream)
+
+*Note: Previous versions of CatFIM also restricted sites based on [Lat/Long Coordinate Accuracy](https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html), but that criteria was removed in Fall 2024 to increase availability of CatFIM sites.*
+
+*For sites outside of CONUS: As of 12/4/2024, these criteria are currently being workshopped to account for the unique challenges of producing CatFIM in non-CONUS locations. Check back for updates or reach out to GID via Slack if you have specific questions! 
+
+
+## Running CatFIM
+### Who can run CatFIM?
+CatFIM can only be run by systems that can access the WRDS API, which is restricted to computers on the NOAA network. If you are outside the NOAA network and would like to run code from inundation-mapping, see the README in NOAA-OWP/inundation-mapping.
+
+
+### Commands
+Stage-based example with step system and pre-downloaded metadata: 
+
+`python /foss_fim/tools/generate_categorical_fim.py -f /outputs/Rob_catfim_test_1 -jh 1 -jn 10 -ji 8 -e /data/config/catfim.env -t /data/docker_test_1 -me '/data/nwm_metafile.pkl' -sb -step 2`
+
+Flow-based example with HUC list:
+
+`python /foss_fim/tools/catfim/generate_categorical_fim.py -f /data/previous_fim/fim_4_5_2_11/ -jh 4 -jn 2 -e /data/config/catfim.env -t /data/hand_4_5_11_1_catfim_datavis -o -lh '06010105 17110004 10300101 19020401 19020302'`
+
+
+### Arguments
+- `-f`, `--fim_run_dir`: Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_5_2_11
+- `-e`, `--env_file`: Docker mount path to the catfim environment file. ie) data/config/catfim.env
+- `-jh`, `--job_number_huc`: OPTIONAL: Number of processes to use for HUC scale operations. HUC and inundation job numbers should multiply to no more than one less than the CPU count of the machine. CatFIM sites generally only have 2-3 branches overlapping a site, so this number can be kept low (2-4). Defaults to 1.
+- `-jn`, `--job_number_inundate`: OPTIONAL: Number of processes to use for inundating HUC and inundation job numbers should multiply to no more than one less than the CPU count of the machine. Defaults to 1.
+- `-ji`, `--job_number_intervals`: OPTIONAL: Number of processes to use for inundating multiple intervals in stage-based inundation and interval job numbers should multiply to no more than one less than the CPU count of the machine. Defaults to 1.
+- `-sb`, `--is_stage_based`: Run stage-based CatFIM instead of flow-based? Add this -sb param to make it stage based, leave it off for flow based.
+- `-t`, `--output_folder`: OPTIONAL: Target location, Where the output folder will be. Defaults to /data/catfim/
+- `-s`, `--search`: OPTIONAL: Upstream and downstream search in miles. How far up and downstream do you want to go? Defaults to 5.
+- `-lh`, `--lst_hucs`: OPTIONAL: Space-delimited list of HUCs to produce CatFIM for. Defaults to all HUCs.
+- `-mc`, `--past_major_interval_cap`: OPTIONAL: Stage-Based Only. How many feet past major do you want to go for the interval FIMs? of the machine. Defaults to 5.
+- `-step`: 'OPTIONAL: By adding a number here, you may be able to skip levels of processing. The number you submit means it will start at that step. e.g. step of 2 means start at step 2 which for flow based is the creating of tifs and gpkgs. Note: This assumes those previous steps have already been processed and the files are present. Defaults to 0 which means all steps processed.
+- `-me`, `--nwm_metafile`: OPTIONAL: If you have a pre-existing nwm metadata pickle file, you can path to it here.  NOTE: This parameter is for quick debugging only and should not be used in a production mode.
+- `-o`, `--overwrite`: OPTIONAL: Overwrite files.
+
+## Visualization Tips & Tricks
+
+### Changing Symbol Drawing Order
+
+- **ArcGIS Pro:** Go to the Symbology pane and navgate to the Symbol layer drawing tab. Make sure the "Enable symbol layer drawing" option is switched "ON", and then adjust the Drawing Order of the magnitudes so they are in this order: action, minor, moderate, major, record.
+
+![screenshot of Symbology pane in ArcGIS Pro](https://github.com/NOAA-OWP/inundation-mapping/blob/b527e762478fef2c1ffc5f0ff4d494f1746663bb/tools/catfim/images/screenshot_vis_settings.JPG)
+
+- **QGIS**: To change the symbol drawing order in QGIS...
+
+## Glossary
+- AHPS: Advanced Hydrologic Prediction Services (NOAA)
+- WRDS: Water Resources Data System (NOAA)
+- REM: relative elevation model
+- HAND: height above nearest drainage