Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Inverse distance tuning #45

Merged
merged 2 commits into from
Nov 23, 2023
Merged

Inverse distance tuning #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f72b6fa
Create cross validation and tune interpolation
gnrgomes Nov 23, 2023
2d13e4f
Avoid cdd config when using other interpolation
gnrgomes Nov 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 50 additions & 45 deletions src/lisfloodutilities/gridding/generate_grids.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
from pathlib import Path
from argparse import ArgumentParser, ArgumentTypeError
from datetime import datetime, timedelta
from lisfloodutilities.gridding.lib.utils import Printable, Dem, Config, FileUtils, GriddingUtils
from lisfloodutilities.gridding.lib.writers import NetCDFWriter, GDALWriter
from src.lisfloodutilities.gridding.lib.utils import Printable, Dem, Config, FileUtils, GriddingUtils
from src.lisfloodutilities.gridding.lib.writers import NetCDFWriter, GDALWriter


END_DATE_DEFAULT = datetime.now().strftime('%Y%m%d060000')
Expand All @@ -35,8 +35,14 @@ def print_msg(msg: str = ''):
if not quiet_mode:
print(msg)

def run(config_filename: str, infolder: str, outfolder_or_file: str, processing_dates_file: str, file_utils: FileUtils,
output_tiff: bool, overwrite_output: bool, start_date: datetime = None, end_date: datetime = None):
def interpolation_mode_type(mode: str) -> str:
if not mode or mode not in Config.INTERPOLATION_MODES:
raise ArgumentTypeError(f'You must select a mode out of {list(Config.INTERPOLATION_MODES.keys())}.')
return mode

def run(config_filename: str, infolder: str, output_file: str, processing_dates_file: str, file_utils: FileUtils,
output_tiff: bool, overwrite_output: bool, start_date: datetime = None, end_date: datetime = None,
interpolation_mode: str = 'adw', use_broadcasting: bool = False):
"""
Interpolate text files containing (x, y, value) using inverse distance interpolation.
Produces as output, either a netCDF file containing all the grids or one TIFF file per grid.
Expand All @@ -47,13 +53,13 @@ def run(config_filename: str, infolder: str, outfolder_or_file: str, processing_
"""
global quiet_mode

conf = Config(config_filename, start_date, end_date, quiet_mode)
conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode)

if conf.start_date > conf.end_date:
raise ArgumentTypeError("Start date is greater than End date.")

dates_to_process = file_utils.read_processing_dates_file(processing_dates_file)
grid_utils = GriddingUtils(conf, quiet_mode)
grid_utils = GriddingUtils(conf, quiet_mode, use_broadcasting)

size_lons = len(conf.dem_lons)
size_lats = len(conf.dem_lats)
Expand All @@ -69,34 +75,25 @@ def run(config_filename: str, infolder: str, outfolder_or_file: str, processing_

netcdf_offset_file_date = int(conf.get_config_field('VAR_TIME','OFFSET_FILE_DATE'))

outfile = output_file
if output_tiff:
outfolder = outfolder_or_file
output_writer = GDALWriter(conf, overwrite_output, quiet_mode)
for filename in sorted(Path(infolder).rglob(inwildcard)):
file_timestamp = file_utils.get_timestamp_from_filename(filename) + timedelta(days=netcdf_offset_file_date)
if not file_utils.processable_file(file_timestamp, dates_to_process, conf.start_date, conf.end_date):
continue # Skip processing file
print_msg(f'Processing file: {filename}')
outfile = str(filename).replace(infolder, outfolder)
outfilepath = Path(outfile).with_suffix('.tiff')
# Create the output parent folders if not exist yet
Path(outfilepath.parent).mkdir(parents=True, exist_ok=True)
output_writer.open(Path(outfilepath))
grid_data = grid_utils.generate_grid(filename)
output_writer.write(grid_data, file_timestamp)
output_writer.close()
else: # NetCDF
outfile = outfolder_or_file
output_writer = NetCDFWriter(conf, overwrite_output, quiet_mode)
output_writer.open(Path(outfile))
for filename in sorted(Path(infolder).rglob(inwildcard)):
file_timestamp = file_utils.get_timestamp_from_filename(filename) + timedelta(days=netcdf_offset_file_date)
if not file_utils.processable_file(file_timestamp, dates_to_process, conf.start_date, conf.end_date):
continue # Skip processing file
print_msg(f'Processing file: {filename}')
grid_data = grid_utils.generate_grid(filename)
output_writer.write(grid_data, file_timestamp)
output_writer.close()
output_writer_tiff = GDALWriter(conf, overwrite_output, quiet_mode)
output_writer_netcdf = NetCDFWriter(conf, overwrite_output, quiet_mode)
output_writer_netcdf.open(Path(outfile))
for filename in sorted(Path(infolder).rglob(inwildcard)):
file_timestamp = file_utils.get_timestamp_from_filename(filename) + timedelta(days=netcdf_offset_file_date)
if not file_utils.processable_file(file_timestamp, dates_to_process, conf.start_date, conf.end_date):
continue # Skip processing file
print_msg(f'Processing file: {filename}')
if output_tiff:
outfilepath = filename.with_suffix('.tiff')
output_writer_tiff.open(outfilepath)
grid_data = grid_utils.generate_grid(filename)
output_writer_netcdf.write(grid_data, file_timestamp)
if output_tiff:
output_writer_tiff.write(grid_data, file_timestamp)
output_writer_tiff.close()
output_writer_netcdf.close()
print_msg('Finished writing files')


Expand All @@ -111,7 +108,7 @@ def main(argv):

program_version_string = 'version %s (%s)\n' % (program_version, program_build_date)
program_longdesc = '''
This script interpolates meteo input variables data into either a single NETCDF4 file or one GEOTIFF file per timestep.
This script interpolates meteo input variables data into a single NETCDF4 file and, if selected, generates also a GEOTIFF file per timestep.
The resulting netCDF is CF-1.6 compliant.
'''
program_license = """
Expand All @@ -134,14 +131,16 @@ def main(argv):
out_tiff=False,
overwrite_output=False,
start_date='',
end_date=END_DATE_DEFAULT)
end_date=END_DATE_DEFAULT,
interpolation_mode='adw',
use_broadcasting=False)

parser.add_argument("-i", "--in", dest="infolder", required=True, type=FileUtils.folder_type,
help="Set input folder path with kiwis/point files",
metavar="input_folder")
parser.add_argument("-o", "--out", dest="outfolder_or_file", required=True, type=FileUtils.file_or_folder,
help="Set output folder base path for the tiff files or the netCDF file path.",
metavar="{output_folder, netcdf_file}")
parser.add_argument("-o", "--out", dest="output_file", required=True, type=FileUtils.file_or_folder,
help="Set the output netCDF file path containing all the timesteps between start and end dates.",
metavar="output_netcdf_file")
parser.add_argument("-c", "--conf", dest="config_type", required=True,
help="Set the grid configuration type to use.",
metavar="{5x5km, 1arcmin,...}")
Expand All @@ -159,9 +158,14 @@ def main(argv):
metavar="YYYYMMDDHHMISS")
parser.add_argument("-q", "--quiet", dest="quiet", action="store_true", help="Set script output into quiet mode [default: %(default)s]")
parser.add_argument("-t", "--tiff", dest="out_tiff", action="store_true",
help="Outputs a tiff file per timestep instead of the default single netCDF [default: %(default)s]")
help="Outputs a tiff file per timestep and also the single netCDF with all the timesteps [default: %(default)s]")
parser.add_argument("-f", "--force", dest="overwrite_output", action="store_true",
help="Force write to existing file. TIFF files will be overwritten and netCDF file will be appended. [default: %(default)s]")
parser.add_argument("-m", "--mode", dest="interpolation_mode", required=False, type=interpolation_mode_type,
help="Set interpolation mode. [default: %(default)s]",
metavar=f"{list(Config.INTERPOLATION_MODES.keys())}")
parser.add_argument("-b", "--broadcast", dest="use_broadcasting", action="store_true",
help="When set, computations will run faster in full broadcasting mode but require more memory. [default: %(default)s]")

# process options
args = parser.parse_args(argv)
Expand Down Expand Up @@ -194,17 +198,18 @@ def main(argv):

print_msg(f"Input Folder: {args.infolder}")
print_msg(f"Overwrite Output: {args.overwrite_output}")
print_msg(f"Interpolation Mode: {args.interpolation_mode}")
print_msg(f"Broadcasting: {args.use_broadcasting}")
if args.out_tiff:
print_msg("Output Type: TIFF")
print_msg(f"Output Folder: {args.outfolder_or_file}")
else:
print_msg("Output Type: netCDF")
print_msg(f"Output File: {args.outfolder_or_file}")
print_msg(f"Output Folder: {args.infolder}")
print_msg("Output Type: netCDF")
print_msg(f"Output File: {args.output_file}")
print_msg(f"Processing Dates File: {args.processing_dates_file}")
print_msg(f"Config File: {config_filename}")

run(config_filename, args.infolder, args.outfolder_or_file, args.processing_dates_file,
file_utils, args.out_tiff, args.overwrite_output, start_date, end_date)
run(config_filename, args.infolder, args.output_file, args.processing_dates_file,
file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode, args.use_broadcasting)
except Exception as e:
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
Expand Down
Loading
Loading