From 5e794f55f2f2ac909795f66bfba06dfec8d28e0f Mon Sep 17 00:00:00 2001 From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com> Date: Fri, 8 Dec 2023 13:40:41 -0700 Subject: [PATCH 1/2] v4.4.7.2 Fix inundation nation pathing errors (#1026) * Fixes for inundate_nation to create boolean rasters * fixed bug with default huc lists * took out debug line, added more prints * fix some output print errors * debugging * rolled most changes back to Ryan's orig fix * minor adj to temp dir * update notes in_nation * Update inundate_nation.py * Update CHANGELOG.md * Update params_template.env * Added vrt raster and final mosaic step; fixed logging * More fixes for logging info and adding log printing to screen * fix linting error * Added multi-threading to gdal.translate * linting fix * Revert "linting fix" This reverts commit 15aad0427198d5bece4f33b6b7629f45f1c86417. * fixing linting errors * more linting fixes --------- Co-authored-by: RyanSpies-NOAA Co-authored-by: Carson Pruitt <90792257+CarsonPruitt-NOAA@users.noreply.github.com> --- docs/CHANGELOG.md | 15 ++++ tools/inundate_mosaic_wrapper.py | 2 - tools/inundate_nation.py | 124 +++++++++++++++++++++++-------- 3 files changed, 107 insertions(+), 34 deletions(-) mode change 100644 => 100755 tools/inundate_nation.py diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 09d892805..a1314d95f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,20 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.4.7.2 - 2023-12-08 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026) + +A couple of directly related issues were fixed in this PR. +The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundation-mapping/issues/1025) which was about a pathing issue for the outputs directory. In testing that fix, it exposed a few other pathing and file cleanup issues which are now fixed. We also added more console output to help view variables and pathing. + +### Changes + +- `config`/`params_template.env`: Updated for a newer mannings global file. Changed and tested by Ryan Spies. +- `tools` + - `inundate_mosiac_wrapper.py`: Took out a misleading and non-required print statement. + - `inundate_nation.py`: As mentioned above. + +

+ ## v4.4.7.1 - 2023-12-01 - [PR#1036](https://github.com/NOAA-OWP/inundation-mapping/pull/1036) Quick update to match incoming ras2fim calibration output files being feed into FIM was the initial change. @@ -90,6 +104,7 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1

+ ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018) During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times. diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py index c75c4ea2e..5b274122a 100644 --- a/tools/inundate_mosaic_wrapper.py +++ b/tools/inundate_mosaic_wrapper.py @@ -93,8 +93,6 @@ def produce_mosaicked_inundation( "Please lower the num_workers.".format(num_workers, total_cpus_available) ) - fh.vprint("Running inundate for " + huc + "...", verbose) - # Call Inundate_gms map_file = Inundate_gms( hydrofabric_dir=hydrofabric_dir, diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py old mode 100644 new mode 100755 index fe8680418..050436670 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -6,17 +6,14 @@ import os import re import shutil -import sys from datetime import datetime from multiprocessing import Pool import rasterio from inundate_mosaic_wrapper import produce_mosaicked_inundation -from osgeo import gdal, ogr -from rasterio.merge import merge +from osgeo import gdal from utils.shared_functions import FIM_Helpers as fh -from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv # INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/' @@ -26,6 +23,9 @@ # DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/' +# TODO: Nov 2023, Logging system appears to be not working correctly. + + def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, inc_mosaic, job_number): assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}" @@ -37,12 +37,14 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, + " max jobs will be used instead." ) + print() + print("Inundation Nation script starting...") + fim_version = os.path.basename(os.path.normpath(fim_run_dir)) - logging.info(f"Using fim version: {fim_version}") output_base_file_name = magnitude_key + "_" + fim_version - # print(output_base_file_name) __setup_logger(output_dir, output_base_file_name) + logging.info(f"Using fim version: {fim_version}") start_dt = datetime.now() @@ -52,48 +54,42 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(f"flow_file: {flow_file}") logging.info(f"inc_mosaic: {str(inc_mosaic)}") - print("Preparing to generate inundation outputs for magnitude: " + magnitude_key) - print("Input flow file: " + flow_file) - magnitude_output_dir = os.path.join(output_dir, output_base_file_name) if not os.path.exists(magnitude_output_dir): - print("Creating new output directory for raw mosaic files: " + magnitude_output_dir) + logging.info( + "Removing previous output dir and creating new output dir for inunation wrapper files: " + + magnitude_output_dir + ) os.mkdir(magnitude_output_dir) else: # we need to empty it. we will kill it and remake it (using rmtree to force it) shutil.rmtree(magnitude_output_dir, ignore_errors=True) os.mkdir(magnitude_output_dir) - if huc_list is None: + if huc_list == 'all' or len(huc_list) == 0: huc_list = [] for huc in os.listdir(fim_run_dir): - # if ( - # huc != 'logs' - # and huc != 'branch_errors' - # and huc != 'unit_errors' - # and os.path.isdir(os.path.join(fim_run_dir, huc)) - # ): if re.match(r'\d{8}', huc): huc_list.append(huc) else: for huc in huc_list: - assert os.path.isdir( - fim_run_dir + os.sep + huc - ), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}' + huc_path = os.path.join(fim_run_dir, huc) + assert os.path.isdir(huc_path), f'ERROR: could not find the input fim_dir location: {huc_path}' - print("Inundation raw mosaic outputs here: " + magnitude_output_dir) + huc_list.sort() + logging.info(f"Inundation mosaic wrapper outputs will saved here: {magnitude_output_dir}") run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number]) # Perform mosaic operation if inc_mosaic: fh.print_current_date_time() logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S")) - print("Performing bool mosaic process...") logging.info("Performing bool mosaic process...") - output_bool_dir = os.path.join(output_dir, "bool_temp") + logging.info(f"output_bool_dir is {output_bool_dir}") + if not os.path.exists(output_bool_dir): os.mkdir(output_bool_dir) else: @@ -105,7 +101,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, for rasfile in os.listdir(magnitude_output_dir): if rasfile.endswith(".tif") and "extent" in rasfile: # p = magnitude_output_dir + rasfile - procs_list.append([magnitude_output_dir, rasfile, output_bool_dir]) + procs_list.append([magnitude_output_dir, rasfile, output_bool_dir, fim_version]) # Multiprocess --> create boolean inundation rasters for all hucs if len(procs_list) > 0: @@ -116,10 +112,17 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print(msg) logging.info(msg) - # now cleanup the raw mosiac directories + # Perform VRT creation and mosaic all of the huc rasters using boolean rasters + vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name, job_number) + + # now cleanup the temp bool directory shutil.rmtree(output_bool_dir, ignore_errors=True) + else: + print("Skipping mosiaking") + # now cleanup the raw mosiac directories + # comment this out if you want to see the individual huc rasters shutil.rmtree(magnitude_output_dir, ignore_errors=True) fh.print_current_date_time() @@ -149,7 +152,16 @@ def run_inundation(args): inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif") - print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude)) + logging.info( + "Running inundation wrapper for the NWM recurrence intervals for each huc using magnitude: " + + str(magnitude) + ) + print( + "This will take a long time depending on the number of HUCs. Progress bar may not appear." + " Once it gets to boolean/mosiacing (if applicable), screen output will exist. To see if the script has frozen," + " you should be able to watch the file system for some changes." + ) + print() produce_mosaicked_inundation( fim_run_dir, @@ -167,6 +179,7 @@ def create_bool_rasters(args): in_raster_dir = args[0] rasfile = args[1] output_bool_dir = args[2] + fim_version = args[3] print("Calculating boolean inundate raster: " + rasfile) p = in_raster_dir + os.sep + rasfile @@ -189,23 +202,70 @@ def create_bool_rasters(args): dtype="int8", compress="lzw", ) - with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, "w", **profile) as dst: + with rasterio.open( + output_bool_dir + os.sep + rasfile[:-4] + '_' + fim_version + '.tif', "w", **profile + ) as dst: dst.write(array.astype(rasterio.int8)) -def __setup_logger(output_folder_path, log_file_name_key): +def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads): + rasters_to_mosaic = [] + for rasfile in os.listdir(output_bool_dir): + if rasfile.endswith('.tif') and "extent" in rasfile: + p = output_bool_dir + os.sep + rasfile + rasters_to_mosaic.append(p) + + output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt") + logging.info("Creating virtual raster: " + output_mosiac_vrt) + vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic) + + output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif") + logging.info("Building raster mosaic: " + output_mosiac_raster) + logging.info("Using " + str(threads) + " threads for parallizing") + print("Note: This step can take a number of hours if processing 100s of hucs") + gdal.Translate( + output_mosiac_raster, + vrt, + xRes=10, + yRes=-10, + creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS=' + str(threads)], + ) + vrt = None + + +def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO): start_time = datetime.now() file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S") log_file_name = f"{log_file_name_key}-{file_dt_string}.log" log_file_path = os.path.join(output_folder_path, log_file_name) + print('Log file created here:' + str(log_file_path)) + + # Clear previous logging configuration + logging.getLogger().handlers = [] + + # Create a StreamHandler and set the level + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + + # Create a FileHandler and set the level + file_handler = logging.FileHandler(log_file_path) + file_handler.setLevel(log_level) - logging.basicConfig(filename=log_file_path, level=logging.DEBUG, format="%(message)s") + # Create a formatter and set the formatter for the handlers + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + console_handler.setFormatter(formatter) + file_handler.setFormatter(formatter) - # yes.. this can do console logs as well, but it can be a bit unstable and ugly + # Add the handlers to the logger + logger = logging.getLogger() + logger.setLevel(log_level) + logger.addHandler(console_handler) + logger.addHandler(file_handler) - logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}') - logging.info("----------------") + # Log the start time + logger.info(f'Started: {start_time.strftime("%m/%d/%Y %H:%M:%S")}') + logger.info("----------------") if __name__ == "__main__": From 9ed73c6a91ac5668a15e3fadcfefa43027e3eeea Mon Sep 17 00:00:00 2001 From: Rob Gonzalez-Pita Date: Fri, 8 Dec 2023 13:45:53 -0700 Subject: [PATCH 2/2] v4.4.8.0 Copy files to scratch directory for processing, and remove upon completion (#1045) --- Pipfile.lock | 340 +++++++++++++++++++++-------------------- config/deny_unit.lst | 4 + docs/CHANGELOG.md | 13 ++ src/bash_variables.env | 48 +++--- src/run_unit_wb.sh | 15 +- 5 files changed, 223 insertions(+), 197 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 61853dec6..a0e867a67 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "196cd7e33d0458f7b032017fcd2e1940761158c53a4130adb8915f036ed61b49" + "sha256": "05bec99d8be9c133d86cc5a13bbb8dd751b89eaa4368df49170d3527a84bb127" }, "pipfile-spec": 6, "requires": { @@ -42,11 +42,11 @@ }, "anyio": { "hashes": [ - "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", - "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a" + "sha256:56a415fbc462291813a94528a779597226619c8e78af7de0507333f700011e5f", + "sha256:5a0bec7085176715be77df87fc66d6c9d70626bd752fcc85f57cdbee5b3760da" ], "markers": "python_version >= '3.8'", - "version": "==4.0.0" + "version": "==4.1.0" }, "appdirs": { "hashes": [ @@ -133,7 +133,7 @@ "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da", "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.12.2" }, "black": { @@ -773,11 +773,11 @@ }, "exceptiongroup": { "hashes": [ - "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9", - "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3" + "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", + "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68" ], "markers": "python_version < '3.11'", - "version": "==1.1.3" + "version": "==1.2.0" }, "executing": { "hashes": [ @@ -789,10 +789,10 @@ }, "fastjsonschema": { "hashes": [ - "sha256:06dc8680d937628e993fa0cd278f196d20449a1adc087640710846b324d422ea", - "sha256:aec6a19e9f66e9810ab371cc913ad5f4e9e479b63a7072a2cd060a9369e329a8" + "sha256:b9fd1a2dd6971dbc7fee280a95bd199ae0dd9ce22beb91cc75e9c1c528a5170e", + "sha256:e25df6647e1bc4a26070b700897b07b542ec898dd4f1f6ea013e7f6a88417225" ], - "version": "==2.18.1" + "version": "==2.19.0" }, "filelock": { "hashes": [ @@ -963,19 +963,19 @@ }, "idna": { "hashes": [ - "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", - "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", + "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" ], "markers": "python_version >= '3.5'", - "version": "==3.4" + "version": "==3.6" }, "importlib-metadata": { "hashes": [ - "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", - "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743" + "sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7", + "sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67" ], "markers": "python_version < '3.9'", - "version": "==6.8.0" + "version": "==7.0.0" }, "importlib-resources": { "hashes": [ @@ -1157,19 +1157,19 @@ "format-nongpl" ], "hashes": [ - "sha256:c9ff4d7447eed9592c23a12ccee508baf0dd0d59650615e847feb6cdca74f392", - "sha256:eee9e502c788e89cb166d4d37f43084e3b64ab405c795c03d343a4dbc2c810fc" + "sha256:4f614fd46d8d61258610998997743ec5492a648b33cf478c1ddc23ed4598a5fa", + "sha256:ed6231f0429ecf966f5bc8dfef245998220549cbbcf140f913b7464c52c3b6b3" ], "markers": "python_version >= '3.8'", - "version": "==4.19.2" + "version": "==4.20.0" }, "jsonschema-specifications": { "hashes": [ - "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1", - "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb" + "sha256:9472fc4fea474cd74bea4a2b190daeccb5a9e4db2ea80efcf7a1b582fc9a81b8", + "sha256:e74ba7c0a65e8cb49dc26837d6cfe576557084a8b423ed16a420984228104f93" ], "markers": "python_version >= '3.8'", - "version": "==2023.7.1" + "version": "==2023.11.2" }, "jupyter": { "hashes": [ @@ -1214,11 +1214,12 @@ }, "jupyter-server": { "hashes": [ - "sha256:47b8f5e63440125cb1bb8957bf12b18453ee5ed9efe42d2f7b2ca66a7019a278", - "sha256:dde56c9bc3cb52d7b72cc0f696d15d7163603526f1a758eb4a27405b73eab2a5" + "sha256:0c548151b54bcb516ca466ec628f7f021545be137d01b5467877e87f6fff4374", + "sha256:0c99f9367b0f24141e527544522430176613f9249849be80504c6d2b955004bb" ], + "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==2.10.0" + "version": "==2.11.2" }, "jupyter-server-fileid": { "hashes": [ @@ -1262,11 +1263,11 @@ }, "jupyterlab-pygments": { "hashes": [ - "sha256:2405800db07c9f770863bcf8049a529c3dd4d3e28536638bd7c1c01d2748309f", - "sha256:7405d7fde60819d905a9fa8ce89e4cd830e318cdad22a0030f7a901da705585d" + "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d", + "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780" ], - "markers": "python_version >= '3.7'", - "version": "==0.2.2" + "markers": "python_version >= '3.8'", + "version": "==0.3.0" }, "jupyterlab-server": { "hashes": [ @@ -1710,16 +1711,16 @@ "sha256:4b28c207877cf33ef3a9838cdc7a54c5ceff981194a82eac59d558f05487295e", "sha256:a3a1ddfb34d4a9d17fc744d655962714a866639acd30130e9be84191cd97cd15" ], - "markers": "python_version >= '3.8'", + "markers": "python_full_version >= '3.8.0'", "version": "==0.9.0" }, "nbconvert": { "hashes": [ - "sha256:abedc01cf543177ffde0bfc2a69726d5a478f6af10a332fc1bf29fcb4f0cf000", - "sha256:d1d417b7f34a4e38887f8da5bdfd12372adf3b80f995d57556cb0972c68909fe" + "sha256:5b6c848194d270cc55fb691169202620d7b52a12fec259508d142ecbe4219310", + "sha256:b1564bd89f69a74cd6398b0362da94db07aafb991b7857216a766204a71612c0" ], "markers": "python_version >= '3.8'", - "version": "==7.11.0" + "version": "==7.12.0" }, "nbformat": { "hashes": [ @@ -2093,11 +2094,11 @@ }, "platformdirs": { "hashes": [ - "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3", - "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e" + "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380", + "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420" ], - "markers": "python_version >= '3.7'", - "version": "==3.11.0" + "markers": "python_version >= '3.8'", + "version": "==4.1.0" }, "pluggy": { "hashes": [ @@ -2117,11 +2118,11 @@ }, "prometheus-client": { "hashes": [ - "sha256:35f7a8c22139e2bb7ca5a698e92d38145bc8dc74c1c0bf56f25cca886a764e17", - "sha256:8de3ae2755f890826f4b6479e5571d4f74ac17a81345fe69a6778fdb92579184" + "sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1", + "sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92" ], "markers": "python_version >= '3.8'", - "version": "==0.18.0" + "version": "==0.19.0" }, "prompt-toolkit": { "hashes": [ @@ -2226,6 +2227,7 @@ "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220" ], + "markers": "os_name != 'nt'", "version": "==0.7.0" }, "pure-eval": { @@ -2453,11 +2455,11 @@ }, "pygments": { "hashes": [ - "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692", - "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29" + "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c", + "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367" ], "markers": "python_version >= '3.7'", - "version": "==2.16.1" + "version": "==2.17.2" }, "pyparsing": { "hashes": [ @@ -2946,11 +2948,11 @@ }, "referencing": { "hashes": [ - "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf", - "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0" + "sha256:81a1471c68c9d5e3831c30ad1dd9815c45b558e596653db751a2bfdd17b3b9ec", + "sha256:c19c4d006f1757e3dd75c4f784d38f8698d87b649c54f9ace14e5e8c9667c01d" ], "markers": "python_version >= '3.8'", - "version": "==0.30.2" + "version": "==0.31.1" }, "requests": { "hashes": [ @@ -3013,108 +3015,108 @@ }, "rpds-py": { "hashes": [ - "sha256:0525847f83f506aa1e28eb2057b696fe38217e12931c8b1b02198cfe6975e142", - "sha256:05942656cb2cb4989cd50ced52df16be94d344eae5097e8583966a1d27da73a5", - "sha256:0831d3ecdea22e4559cc1793f22e77067c9d8c451d55ae6a75bf1d116a8e7f42", - "sha256:0853da3d5e9bc6a07b2486054a410b7b03f34046c123c6561b535bb48cc509e1", - "sha256:08e6e7ff286254016b945e1ab632ee843e43d45e40683b66dd12b73791366dd1", - "sha256:0a38612d07a36138507d69646c470aedbfe2b75b43a4643f7bd8e51e52779624", - "sha256:0bedd91ae1dd142a4dc15970ed2c729ff6c73f33a40fa84ed0cdbf55de87c777", - "sha256:0c5441b7626c29dbd54a3f6f3713ec8e956b009f419ffdaaa3c80eaf98ddb523", - "sha256:0e9e976e0dbed4f51c56db10831c9623d0fd67aac02853fe5476262e5a22acb7", - "sha256:0fadfdda275c838cba5102c7f90a20f2abd7727bf8f4a2b654a5b617529c5c18", - "sha256:1096ca0bf2d3426cbe79d4ccc91dc5aaa73629b08ea2d8467375fad8447ce11a", - "sha256:171d9a159f1b2f42a42a64a985e4ba46fc7268c78299272ceba970743a67ee50", - "sha256:188912b22b6c8225f4c4ffa020a2baa6ad8fabb3c141a12dbe6edbb34e7f1425", - "sha256:1b4cf9ab9a0ae0cb122685209806d3f1dcb63b9fccdf1424fb42a129dc8c2faa", - "sha256:1e04581c6117ad9479b6cfae313e212fe0dfa226ac727755f0d539cd54792963", - "sha256:1fa73ed22c40a1bec98d7c93b5659cd35abcfa5a0a95ce876b91adbda170537c", - "sha256:2124f9e645a94ab7c853bc0a3644e0ca8ffbe5bb2d72db49aef8f9ec1c285733", - "sha256:240687b5be0f91fbde4936a329c9b7589d9259742766f74de575e1b2046575e4", - "sha256:25740fb56e8bd37692ed380e15ec734be44d7c71974d8993f452b4527814601e", - "sha256:27ccc93c7457ef890b0dd31564d2a05e1aca330623c942b7e818e9e7c2669ee4", - "sha256:281c8b219d4f4b3581b918b816764098d04964915b2f272d1476654143801aa2", - "sha256:2d34a5450a402b00d20aeb7632489ffa2556ca7b26f4a63c35f6fccae1977427", - "sha256:301bd744a1adaa2f6a5e06c98f1ac2b6f8dc31a5c23b838f862d65e32fca0d4b", - "sha256:30e5ce9f501fb1f970e4a59098028cf20676dee64fc496d55c33e04bbbee097d", - "sha256:33ab498f9ac30598b6406e2be1b45fd231195b83d948ebd4bd77f337cb6a2bff", - "sha256:35585a8cb5917161f42c2104567bb83a1d96194095fc54a543113ed5df9fa436", - "sha256:389c0e38358fdc4e38e9995e7291269a3aead7acfcf8942010ee7bc5baee091c", - "sha256:3acadbab8b59f63b87b518e09c4c64b142e7286b9ca7a208107d6f9f4c393c5c", - "sha256:3b7a64d43e2a1fa2dd46b678e00cabd9a49ebb123b339ce799204c44a593ae1c", - "sha256:3c8c0226c71bd0ce9892eaf6afa77ae8f43a3d9313124a03df0b389c01f832de", - "sha256:429349a510da82c85431f0f3e66212d83efe9fd2850f50f339341b6532c62fe4", - "sha256:466030a42724780794dea71eb32db83cc51214d66ab3fb3156edd88b9c8f0d78", - "sha256:47aeceb4363851d17f63069318ba5721ae695d9da55d599b4d6fb31508595278", - "sha256:48aa98987d54a46e13e6954880056c204700c65616af4395d1f0639eba11764b", - "sha256:4b2416ed743ec5debcf61e1242e012652a4348de14ecc7df3512da072b074440", - "sha256:4d0a675a7acbbc16179188d8c6d0afb8628604fc1241faf41007255957335a0b", - "sha256:4eb74d44776b0fb0782560ea84d986dffec8ddd94947f383eba2284b0f32e35e", - "sha256:4f8a1d990dc198a6c68ec3d9a637ba1ce489b38cbfb65440a27901afbc5df575", - "sha256:513ccbf7420c30e283c25c82d5a8f439d625a838d3ba69e79a110c260c46813f", - "sha256:5210a0018c7e09c75fa788648617ebba861ae242944111d3079034e14498223f", - "sha256:54cdfcda59251b9c2f87a05d038c2ae02121219a04d4a1e6fc345794295bdc07", - "sha256:56dd500411d03c5e9927a1eb55621e906837a83b02350a9dc401247d0353717c", - "sha256:57ec6baec231bb19bb5fd5fc7bae21231860a1605174b11585660236627e390e", - "sha256:5f1519b080d8ce0a814f17ad9fb49fb3a1d4d7ce5891f5c85fc38631ca3a8dc4", - "sha256:6174d6ad6b58a6bcf67afbbf1723420a53d06c4b89f4c50763d6fa0a6ac9afd2", - "sha256:68172622a5a57deb079a2c78511c40f91193548e8ab342c31e8cb0764d362459", - "sha256:6915fc9fa6b3ec3569566832e1bb03bd801c12cea030200e68663b9a87974e76", - "sha256:6b75b912a0baa033350367a8a07a8b2d44fd5b90c890bfbd063a8a5f945f644b", - "sha256:6f5dcb658d597410bb7c967c1d24eaf9377b0d621358cbe9d2ff804e5dd12e81", - "sha256:6f8d7fe73d1816eeb5378409adc658f9525ecbfaf9e1ede1e2d67a338b0c7348", - "sha256:7036316cc26b93e401cedd781a579be606dad174829e6ad9e9c5a0da6e036f80", - "sha256:7188ddc1a8887194f984fa4110d5a3d5b9b5cd35f6bafdff1b649049cbc0ce29", - "sha256:761531076df51309075133a6bc1db02d98ec7f66e22b064b1d513bc909f29743", - "sha256:7979d90ee2190d000129598c2b0c82f13053dba432b94e45e68253b09bb1f0f6", - "sha256:8015835494b21aa7abd3b43fdea0614ee35ef6b03db7ecba9beb58eadf01c24f", - "sha256:81c4d1a3a564775c44732b94135d06e33417e829ff25226c164664f4a1046213", - "sha256:81cf9d306c04df1b45971c13167dc3bad625808aa01281d55f3cf852dde0e206", - "sha256:88857060b690a57d2ea8569bca58758143c8faa4639fb17d745ce60ff84c867e", - "sha256:8c567c664fc2f44130a20edac73e0a867f8e012bf7370276f15c6adc3586c37c", - "sha256:91bd2b7cf0f4d252eec8b7046fa6a43cee17e8acdfc00eaa8b3dbf2f9a59d061", - "sha256:9620650c364c01ed5b497dcae7c3d4b948daeae6e1883ae185fef1c927b6b534", - "sha256:9b007c2444705a2dc4a525964fd4dd28c3320b19b3410da6517cab28716f27d3", - "sha256:9bf9acce44e967a5103fcd820fc7580c7b0ab8583eec4e2051aec560f7b31a63", - "sha256:a239303acb0315091d54c7ff36712dba24554993b9a93941cf301391d8a997ee", - "sha256:a2baa6be130e8a00b6cbb9f18a33611ec150b4537f8563bddadb54c1b74b8193", - "sha256:a54917b7e9cd3a67e429a630e237a90b096e0ba18897bfb99ee8bd1068a5fea0", - "sha256:a689e1ded7137552bea36305a7a16ad2b40be511740b80748d3140614993db98", - "sha256:a952ae3eb460c6712388ac2ec706d24b0e651b9396d90c9a9e0a69eb27737fdc", - "sha256:aa32205358a76bf578854bf31698a86dc8b2cb591fd1d79a833283f4a403f04b", - "sha256:b2287c09482949e0ca0c0eb68b2aca6cf57f8af8c6dfd29dcd3bc45f17b57978", - "sha256:b6b0e17d39d21698185097652c611f9cf30f7c56ccec189789920e3e7f1cee56", - "sha256:b710bf7e7ae61957d5c4026b486be593ed3ec3dca3e5be15e0f6d8cf5d0a4990", - "sha256:b8e11715178f3608874508f08e990d3771e0b8c66c73eb4e183038d600a9b274", - "sha256:b92aafcfab3d41580d54aca35a8057341f1cfc7c9af9e8bdfc652f83a20ced31", - "sha256:bec29b801b4adbf388314c0d050e851d53762ab424af22657021ce4b6eb41543", - "sha256:c694bee70ece3b232df4678448fdda245fd3b1bb4ba481fb6cd20e13bb784c46", - "sha256:c6b52b7028b547866c2413f614ee306c2d4eafdd444b1ff656bf3295bf1484aa", - "sha256:cb41ad20064e18a900dd427d7cf41cfaec83bcd1184001f3d91a1f76b3fcea4e", - "sha256:cd316dbcc74c76266ba94eb021b0cc090b97cca122f50bd7a845f587ff4bf03f", - "sha256:ced40cdbb6dd47a032725a038896cceae9ce267d340f59508b23537f05455431", - "sha256:d1c562a9bb72244fa767d1c1ab55ca1d92dd5f7c4d77878fee5483a22ffac808", - "sha256:d389ff1e95b6e46ebedccf7fd1fadd10559add595ac6a7c2ea730268325f832c", - "sha256:d56b1cd606ba4cedd64bb43479d56580e147c6ef3f5d1c5e64203a1adab784a2", - "sha256:d72a4315514e5a0b9837a086cb433b004eea630afb0cc129de76d77654a9606f", - "sha256:d9e7f29c00577aff6b318681e730a519b235af292732a149337f6aaa4d1c5e31", - "sha256:dbc25baa6abb205766fb8606f8263b02c3503a55957fcb4576a6bb0a59d37d10", - "sha256:e57919c32ee295a2fca458bb73e4b20b05c115627f96f95a10f9f5acbd61172d", - "sha256:e5bbe011a2cea9060fef1bb3d668a2fd8432b8888e6d92e74c9c794d3c101595", - "sha256:e6aea5c0eb5b0faf52c7b5c4a47c8bb64437173be97227c819ffa31801fa4e34", - "sha256:e888be685fa42d8b8a3d3911d5604d14db87538aa7d0b29b1a7ea80d354c732d", - "sha256:eebaf8c76c39604d52852366249ab807fe6f7a3ffb0dd5484b9944917244cdbe", - "sha256:efbe0b5e0fd078ed7b005faa0170da4f72666360f66f0bb2d7f73526ecfd99f9", - "sha256:efddca2d02254a52078c35cadad34762adbae3ff01c6b0c7787b59d038b63e0d", - "sha256:f05450fa1cd7c525c0b9d1a7916e595d3041ac0afbed2ff6926e5afb6a781b7f", - "sha256:f12d69d568f5647ec503b64932874dade5a20255736c89936bf690951a5e79f5", - "sha256:f45321224144c25a62052035ce96cbcf264667bcb0d81823b1bbc22c4addd194", - "sha256:f62581d7e884dd01ee1707b7c21148f61f2febb7de092ae2f108743fcbef5985", - "sha256:f8832a4f83d4782a8f5a7b831c47e8ffe164e43c2c148c8160ed9a6d630bc02a", - "sha256:fa35ad36440aaf1ac8332b4a4a433d4acd28f1613f0d480995f5cfd3580e90b7" + "sha256:06d218e4464d31301e943b65b2c6919318ea6f69703a351961e1baaf60347276", + "sha256:12ecf89bd54734c3c2c79898ae2021dca42750c7bcfb67f8fb3315453738ac8f", + "sha256:15253fff410873ebf3cfba1cc686a37711efcd9b8cb30ea21bb14a973e393f60", + "sha256:188435794405c7f0573311747c85a96b63c954a5f2111b1df8018979eca0f2f0", + "sha256:1ceebd0ae4f3e9b2b6b553b51971921853ae4eebf3f54086be0565d59291e53d", + "sha256:244e173bb6d8f3b2f0c4d7370a1aa341f35da3e57ffd1798e5b2917b91731fd3", + "sha256:25b28b3d33ec0a78e944aaaed7e5e2a94ac811bcd68b557ca48a0c30f87497d2", + "sha256:25ea41635d22b2eb6326f58e608550e55d01df51b8a580ea7e75396bafbb28e9", + "sha256:29d311e44dd16d2434d5506d57ef4d7036544fc3c25c14b6992ef41f541b10fb", + "sha256:2a1472956c5bcc49fb0252b965239bffe801acc9394f8b7c1014ae9258e4572b", + "sha256:2a7bef6977043673750a88da064fd513f89505111014b4e00fbdd13329cd4e9a", + "sha256:2ac26f50736324beb0282c819668328d53fc38543fa61eeea2c32ea8ea6eab8d", + "sha256:2e72f750048b32d39e87fc85c225c50b2a6715034848dbb196bf3348aa761fa1", + "sha256:31e220a040b89a01505128c2f8a59ee74732f666439a03e65ccbf3824cdddae7", + "sha256:35f53c76a712e323c779ca39b9a81b13f219a8e3bc15f106ed1e1462d56fcfe9", + "sha256:38d4f822ee2f338febcc85aaa2547eb5ba31ba6ff68d10b8ec988929d23bb6b4", + "sha256:38f9bf2ad754b4a45b8210a6c732fe876b8a14e14d5992a8c4b7c1ef78740f53", + "sha256:3a44c8440183b43167fd1a0819e8356692bf5db1ad14ce140dbd40a1485f2dea", + "sha256:3ab96754d23372009638a402a1ed12a27711598dd49d8316a22597141962fe66", + "sha256:3c55d7f2d817183d43220738270efd3ce4e7a7b7cbdaefa6d551ed3d6ed89190", + "sha256:46e1ed994a0920f350a4547a38471217eb86f57377e9314fbaaa329b71b7dfe3", + "sha256:4a5375c5fff13f209527cd886dc75394f040c7d1ecad0a2cb0627f13ebe78a12", + "sha256:4c2d26aa03d877c9730bf005621c92da263523a1e99247590abbbe252ccb7824", + "sha256:4c4e314d36d4f31236a545696a480aa04ea170a0b021e9a59ab1ed94d4c3ef27", + "sha256:4d0c10d803549427f427085ed7aebc39832f6e818a011dcd8785e9c6a1ba9b3e", + "sha256:4dcc5ee1d0275cb78d443fdebd0241e58772a354a6d518b1d7af1580bbd2c4e8", + "sha256:51967a67ea0d7b9b5cd86036878e2d82c0b6183616961c26d825b8c994d4f2c8", + "sha256:530190eb0cd778363bbb7596612ded0bb9fef662daa98e9d92a0419ab27ae914", + "sha256:5379e49d7e80dca9811b36894493d1c1ecb4c57de05c36f5d0dd09982af20211", + "sha256:5493569f861fb7b05af6d048d00d773c6162415ae521b7010197c98810a14cab", + "sha256:5a4c1058cdae6237d97af272b326e5f78ee7ee3bbffa6b24b09db4d828810468", + "sha256:5d75d6d220d55cdced2f32cc22f599475dbe881229aeddba6c79c2e9df35a2b3", + "sha256:5d97e9ae94fb96df1ee3cb09ca376c34e8a122f36927230f4c8a97f469994bff", + "sha256:5feae2f9aa7270e2c071f488fab256d768e88e01b958f123a690f1cc3061a09c", + "sha256:603d5868f7419081d616dab7ac3cfa285296735e7350f7b1e4f548f6f953ee7d", + "sha256:61d42d2b08430854485135504f672c14d4fc644dd243a9c17e7c4e0faf5ed07e", + "sha256:61dbc1e01dc0c5875da2f7ae36d6e918dc1b8d2ce04e871793976594aad8a57a", + "sha256:65cfed9c807c27dee76407e8bb29e6f4e391e436774bcc769a037ff25ad8646e", + "sha256:67a429520e97621a763cf9b3ba27574779c4e96e49a27ff8a1aa99ee70beb28a", + "sha256:6aadae3042f8e6db3376d9e91f194c606c9a45273c170621d46128f35aef7cd0", + "sha256:6ba8858933f0c1a979781272a5f65646fca8c18c93c99c6ddb5513ad96fa54b1", + "sha256:6bc568b05e02cd612be53900c88aaa55012e744930ba2eeb56279db4c6676eb3", + "sha256:729408136ef8d45a28ee9a7411917c9e3459cf266c7e23c2f7d4bb8ef9e0da42", + "sha256:751758d9dd04d548ec679224cc00e3591f5ebf1ff159ed0d4aba6a0746352452", + "sha256:76d59d4d451ba77f08cb4cd9268dec07be5bc65f73666302dbb5061989b17198", + "sha256:79bf58c08f0756adba691d480b5a20e4ad23f33e1ae121584cf3a21717c36dfa", + "sha256:7de12b69d95072394998c622cfd7e8cea8f560db5fca6a62a148f902a1029f8b", + "sha256:7f55cd9cf1564b7b03f238e4c017ca4794c05b01a783e9291065cb2858d86ce4", + "sha256:80e5acb81cb49fd9f2d5c08f8b74ffff14ee73b10ca88297ab4619e946bcb1e1", + "sha256:87a90f5545fd61f6964e65eebde4dc3fa8660bb7d87adb01d4cf17e0a2b484ad", + "sha256:881df98f0a8404d32b6de0fd33e91c1b90ed1516a80d4d6dc69d414b8850474c", + "sha256:8a776a29b77fe0cc28fedfd87277b0d0f7aa930174b7e504d764e0b43a05f381", + "sha256:8c2a61c0e4811012b0ba9f6cdcb4437865df5d29eab5d6018ba13cee1c3064a0", + "sha256:8fa6bd071ec6d90f6e7baa66ae25820d57a8ab1b0a3c6d3edf1834d4b26fafa2", + "sha256:96f2975fb14f39c5fe75203f33dd3010fe37d1c4e33177feef1107b5ced750e3", + "sha256:96fb0899bb2ab353f42e5374c8f0789f54e0a94ef2f02b9ac7149c56622eaf31", + "sha256:97163a1ab265a1073a6372eca9f4eeb9f8c6327457a0b22ddfc4a17dcd613e74", + "sha256:9c95a1a290f9acf7a8f2ebbdd183e99215d491beea52d61aa2a7a7d2c618ddc6", + "sha256:9d94d78418203904730585efa71002286ac4c8ac0689d0eb61e3c465f9e608ff", + "sha256:a6ba2cb7d676e9415b9e9ac7e2aae401dc1b1e666943d1f7bc66223d3d73467b", + "sha256:aa0379c1935c44053c98826bc99ac95f3a5355675a297ac9ce0dfad0ce2d50ca", + "sha256:ac96d67b37f28e4b6ecf507c3405f52a40658c0a806dffde624a8fcb0314d5fd", + "sha256:ade2ccb937060c299ab0dfb2dea3d2ddf7e098ed63ee3d651ebfc2c8d1e8632a", + "sha256:aefbdc934115d2f9278f153952003ac52cd2650e7313750390b334518c589568", + "sha256:b07501b720cf060c5856f7b5626e75b8e353b5f98b9b354a21eb4bfa47e421b1", + "sha256:b5267feb19070bef34b8dea27e2b504ebd9d31748e3ecacb3a4101da6fcb255c", + "sha256:b5f6328e8e2ae8238fc767703ab7b95785521c42bb2b8790984e3477d7fa71ad", + "sha256:b8996ffb60c69f677245f5abdbcc623e9442bcc91ed81b6cd6187129ad1fa3e7", + "sha256:b981a370f8f41c4024c170b42fbe9e691ae2dbc19d1d99151a69e2c84a0d194d", + "sha256:b9d121be0217787a7d59a5c6195b0842d3f701007333426e5154bf72346aa658", + "sha256:bcef4f2d3dc603150421de85c916da19471f24d838c3c62a4f04c1eb511642c1", + "sha256:bed0252c85e21cf73d2d033643c945b460d6a02fc4a7d644e3b2d6f5f2956c64", + "sha256:bfdfbe6a36bc3059fff845d64c42f2644cf875c65f5005db54f90cdfdf1df815", + "sha256:c0095b8aa3e432e32d372e9a7737e65b58d5ed23b9620fea7cb81f17672f1fa1", + "sha256:c1f41d32a2ddc5a94df4b829b395916a4b7f103350fa76ba6de625fcb9e773ac", + "sha256:c45008ca79bad237cbc03c72bc5205e8c6f66403773929b1b50f7d84ef9e4d07", + "sha256:c82bbf7e03748417c3a88c1b0b291288ce3e4887a795a3addaa7a1cfd9e7153e", + "sha256:c918621ee0a3d1fe61c313f2489464f2ae3d13633e60f520a8002a5e910982ee", + "sha256:d204957169f0b3511fb95395a9da7d4490fb361763a9f8b32b345a7fe119cb45", + "sha256:d329896c40d9e1e5c7715c98529e4a188a1f2df51212fd65102b32465612b5dc", + "sha256:d3a61e928feddc458a55110f42f626a2a20bea942ccedb6fb4cee70b4830ed41", + "sha256:d48db29bd47814671afdd76c7652aefacc25cf96aad6daefa82d738ee87461e2", + "sha256:d5593855b5b2b73dd8413c3fdfa5d95b99d657658f947ba2c4318591e745d083", + "sha256:d79c159adea0f1f4617f54aa156568ac69968f9ef4d1e5fefffc0a180830308e", + "sha256:db09b98c7540df69d4b47218da3fbd7cb466db0fb932e971c321f1c76f155266", + "sha256:ddf23960cb42b69bce13045d5bc66f18c7d53774c66c13f24cf1b9c144ba3141", + "sha256:e06cfea0ece444571d24c18ed465bc93afb8c8d8d74422eb7026662f3d3f779b", + "sha256:e7c564c58cf8f248fe859a4f0fe501b050663f3d7fbc342172f259124fb59933", + "sha256:e86593bf8637659e6a6ed58854b6c87ec4e9e45ee8a4adfd936831cef55c2d21", + "sha256:eaffbd8814bb1b5dc3ea156a4c5928081ba50419f9175f4fc95269e040eff8f0", + "sha256:ee353bb51f648924926ed05e0122b6a0b1ae709396a80eb583449d5d477fcdf7", + "sha256:ee6faebb265e28920a6f23a7d4c362414b3f4bb30607141d718b991669e49ddc", + "sha256:efe093acc43e869348f6f2224df7f452eab63a2c60a6c6cd6b50fd35c4e075ba", + "sha256:f03a1b3a4c03e3e0161642ac5367f08479ab29972ea0ffcd4fa18f729cd2be0a", + "sha256:f0d320e70b6b2300ff6029e234e79fe44e9dbbfc7b98597ba28e054bd6606a57", + "sha256:f252dfb4852a527987a9156cbcae3022a30f86c9d26f4f17b8c967d7580d65d2", + "sha256:f5f4424cb87a20b016bfdc157ff48757b89d2cc426256961643d443c6c277007", + "sha256:f8eae66a1304de7368932b42d801c67969fd090ddb1a7a24f27b435ed4bed68f", + "sha256:fdb82eb60d31b0c033a8e8ee9f3fc7dfbaa042211131c29da29aea8531b4f18f" ], "markers": "python_version >= '3.8'", - "version": "==0.12.0" + "version": "==0.13.2" }, "rtree": { "hashes": [ @@ -3437,11 +3439,11 @@ }, "terminado": { "hashes": [ - "sha256:6ccbbcd3a4f8a25a5ec04991f39a0b8db52dfcd487ea0e578d977e6752380333", - "sha256:8650d44334eba354dd591129ca3124a6ba42c3d5b70df5051b6921d506fdaeae" + "sha256:1ea08a89b835dd1b8c0c900d92848147cef2537243361b2e3f4dc15df9b6fded", + "sha256:87b0d96642d0fe5f5abd7783857b9cab167f221a39ff98e3b9619a788a3c0f2e" ], - "markers": "python_version >= '3.7'", - "version": "==0.17.1" + "markers": "python_version >= '3.8'", + "version": "==0.18.0" }, "texttable": { "hashes": [ @@ -3476,20 +3478,20 @@ }, "tornado": { "hashes": [ - "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f", - "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5", - "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d", - "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3", - "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2", - "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a", - "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16", - "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a", - "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17", - "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0", - "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe" + "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0", + "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63", + "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263", + "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052", + "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f", + "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee", + "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78", + "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579", + "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212", + "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e", + "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2" ], "markers": "python_version >= '3.8'", - "version": "==6.3.3" + "version": "==6.4" }, "tqdm": { "hashes": [ @@ -3501,11 +3503,11 @@ }, "traitlets": { "hashes": [ - "sha256:9b232b9430c8f57288c1024b34a8f0251ddcc47268927367a0dd3eeaca40deb5", - "sha256:baf991e61542da48fe8aef8b779a9ea0aa38d8a54166ee250d5af5ecf4486619" + "sha256:f14949d23829023013c47df20b4a76ccd1a85effb786dc060f34de7948361b33", + "sha256:fcdaa8ac49c04dfa0ed3ee3384ef6dfdb5d6f3741502be247279407679296772" ], "markers": "python_version >= '3.8'", - "version": "==5.13.0" + "version": "==5.14.0" }, "typeguard": { "hashes": [ @@ -3591,11 +3593,11 @@ }, "websocket-client": { "hashes": [ - "sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24", - "sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df" + "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6", + "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588" ], "markers": "python_version >= '3.8'", - "version": "==1.6.4" + "version": "==1.7.0" }, "whitebox": { "hashes": [ diff --git a/config/deny_unit.lst b/config/deny_unit.lst index 9cf751d6a..f46240f89 100644 --- a/config/deny_unit.lst +++ b/config/deny_unit.lst @@ -35,3 +35,7 @@ nwm_headwaters.gpkg # wbd8_clp.gpkg wbd_buffered.gpkg wbd_buffered_streams.gpkg +HUC6_dem_domain.gpkg +usgs_gages.gpkg +reformat_ras_rating_curve_points_rel_101.gpkg +nws_lid.gpkg diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a1314d95f..8c4b441f5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,19 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.4.8.0 - 2023-12-08 - [PR#1045](https://github.com/NOAA-OWP/inundation-mapping/pull/1045) + +In order to avoid file system collisions on AWS, and keep the reads/writes from the same file on disk to a minimum, three files (`HUC6_dem_domain.gpkg`, `nws_lid.gpkg`, `reformat_ras_rating_curve_points_rel_101.gpkg`, & `usgs_gages.gpkg`) are now copied from disk into a scratch directory (temporary working directory), and removed after processing steps are completed. + +### Changes + +- `config`/`deny_unit.lst`: Add files to remove list - repetitive copies needed for processing step (`run_unit_wb.sh`) +- `src` + - `bash_variables.env`: Add a new variable for the ras rating curve filename. It will be easier to track the filename in the `.env`, and pull into `run_unit_wb.sh`, rather than hardcode it. + - `run_unit_wb.sh`: Copy files and update references from `$inputsDir` to `$tempHucDataDir`. + +

+ ## v4.4.7.2 - 2023-12-08 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026) A couple of directly related issues were fixed in this PR. diff --git a/src/bash_variables.env b/src/bash_variables.env index fe348a63c..67b20ea57 100644 --- a/src/bash_variables.env +++ b/src/bash_variables.env @@ -1,43 +1,43 @@ ## Define inputs # NOTE: $inputsDir is defined in Dockerfile -export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070 -export input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt -export input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg -export input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg -export input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg -export input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg -export input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg -export input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg -export input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg -export input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg -export input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg -export input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg -export input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/ -export pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17 - -export bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg +export DEFAULT_FIM_PROJECTION_CRS=EPSG:5070 +export input_DEM=${inputsDir}/3dep_dems/10m_5070/fim_seamless_3dep_dem_10m_5070.vrt +export input_DEM_domain=${inputsDir}/3dep_dems/10m_5070/HUC6_dem_domain.gpkg +export input_GL_boundaries=${inputsDir}/landsea/gl_water_polygons.gpkg +export input_NLD=${inputsDir}/nld_vectors/System_Routes_NLDFS_5070_230314.gpkg +export input_levees_preprocessed=${inputsDir}/nld_vectors/3d_nld_preprocessed_230314.gpkg +export input_nld_levee_protected_areas=${inputsDir}/nld_vectors/Leveed_Areas_NLDFS_5070_230314.gpkg +export input_nwm_catchments=${inputsDir}/nwm_hydrofabric/nwm_catchments.gpkg +export input_nwm_flows=${inputsDir}/nwm_hydrofabric/nwm_flows.gpkg +export input_nwm_headwaters=${inputsDir}/nwm_hydrofabric/nwm_headwaters.gpkg +export input_nwm_lakes=${inputsDir}/nwm_hydrofabric/nwm_lakes.gpkg +export input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_WBDHU8_clip_dem_domain.gpkg +export input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/ +export pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17 +export bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg # input file location with nwm feature_id and recurrence flow values -export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv +export bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv # input file location with nwm feature_id and channel roughness and overbank roughness attributes -export vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv +export vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv # input file location with nwm feature_id and recurrence flow values -export nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv +export nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv # input file location with usgs rating curve database -export usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv +export usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv # input file locations for ras2fim locations and rating curve data -export ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv -export ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg +export ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv +export ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg +export ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points_rel_101.gpkg -export fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg +export fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg # Input file location with HUC, nwm feature_id and manual calibration coefficients -export man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv +export man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv # Styling diff --git a/src/run_unit_wb.sh b/src/run_unit_wb.sh index 701246900..6c17078fa 100755 --- a/src/run_unit_wb.sh +++ b/src/run_unit_wb.sh @@ -26,6 +26,13 @@ huc_start_time=`date +%s` echo -e $startDiv"Copying staged wbd and .gpkg files from $pre_clip_huc_dir/$hucNumber" cp -a $pre_clip_huc_dir/$hucNumber/. $tempHucDataDir +# Copy necessary files from $inputsDir into $tempHucDataDir to avoid File System Collisions +# For buffer_stream_branches.py +cp $input_DEM_domain $tempHucDataDir +# For usgs_gage_unit_setup.py +cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir +cp $ras_rating_curve_points_gpkg $tempHucDataDir +cp $inputsDir/ahps_sites/nws_lid.gpkg $tempHucDataDir ## DERIVE LEVELPATH ## echo -e $startDiv"Generating Level Paths for $hucNumber" @@ -77,7 +84,7 @@ Tcount echo -e $startDiv"Generating Stream Branch Polygons for $hucNumber" date -u Tstart -$srcDir/buffer_stream_branches.py -a $input_DEM_domain \ +$srcDir/buffer_stream_branches.py -a $tempHucDataDir/HUC6_dem_domain.gpkg \ -s $tempHucDataDir/nwm_subset_streams_levelPaths_dissolved.gpkg \ -i $branch_id_attribute \ -d $branch_buffer_distance_meters \ @@ -274,12 +281,12 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then date -u Tstart python3 $srcDir/usgs_gage_unit_setup.py \ - -gages $inputsDir/usgs_gages/usgs_gages.gpkg \ + -gages $tempHucDataDir/usgs_gages.gpkg \ -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \ - -ras $ras_rating_curve_points_gpkg \ + -ras $tempHucDataDir/$ras_rating_curve_gpkg_filename \ -o $tempHucDataDir/usgs_subset_gages.gpkg \ -huc $hucNumber \ - -ahps $inputsDir/ahps_sites/nws_lid.gpkg \ + -ahps $tempHucDataDir/nws_lid.gpkg \ -bzero_id $branch_zero_id Tcount fi