From e4d79fa70d6e4dd99dd81667ca57537c274cd65c Mon Sep 17 00:00:00 2001 From: gnrgomes Date: Fri, 15 Dec 2023 16:34:17 +0100 Subject: [PATCH] Create RAM usage save mode. --- .../gridding/generate_grids.py | 19 +++++++++++---- src/lisfloodutilities/gridding/lib/utils.py | 24 +++++++++++++------ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/lisfloodutilities/gridding/generate_grids.py b/src/lisfloodutilities/gridding/generate_grids.py index 728e090..9e27258 100755 --- a/src/lisfloodutilities/gridding/generate_grids.py +++ b/src/lisfloodutilities/gridding/generate_grids.py @@ -40,9 +40,14 @@ def interpolation_mode_type(mode: str) -> str: raise ArgumentTypeError(f'You must select a mode out of {list(Config.INTERPOLATION_MODES.keys())}.') return mode +def memory_save_mode_type(mode: str) -> str: + if mode not in Config.MEMORY_SAVE_MODES: + raise ArgumentTypeError(f'You must select a RAM save mode out of {list(Config.MEMORY_SAVE_MODES.keys())}.') + return mode + def run(config_filename: str, infolder: str, output_file: str, processing_dates_file: str, file_utils: FileUtils, output_tiff: bool, overwrite_output: bool, start_date: datetime = None, end_date: datetime = None, - interpolation_mode: str = 'adw', use_broadcasting: bool = False): + interpolation_mode: str = 'adw', use_broadcasting: bool = False, memory_save_mode: str = None): """ Interpolate text files containing (x, y, value) using inverse distance interpolation. Produces as output, either a netCDF file containing all the grids or one TIFF file per grid. @@ -53,7 +58,7 @@ def run(config_filename: str, infolder: str, output_file: str, processing_dates_ """ global quiet_mode - conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode) + conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode, memory_save_mode) if conf.start_date > conf.end_date: raise ArgumentTypeError("Start date is greater than End date.") @@ -135,7 +140,8 @@ def main(argv): start_date='', end_date=END_DATE_DEFAULT, interpolation_mode='adw', - use_broadcasting=False) + use_broadcasting=False, + memory_save_mode='0') parser.add_argument("-i", "--in", dest="infolder", required=True, type=FileUtils.folder_type, help="Set input folder path with kiwis/point files", @@ -169,6 +175,9 @@ def main(argv): parser.add_argument("-m", "--mode", dest="interpolation_mode", required=False, type=interpolation_mode_type, help="Set interpolation mode. [default: %(default)s]", metavar=f"{list(Config.INTERPOLATION_MODES.keys())}") + parser.add_argument("-r", "--ramsavemode", dest="memory_save_mode", required=False, type=memory_save_mode_type, + help="Set memory save mode level. Used to reduce memory usage [default: %(default)s]", + metavar=f"{list(Config.MEMORY_SAVE_MODES.keys())}") parser.add_argument("-b", "--broadcast", dest="use_broadcasting", action="store_true", help="When set, computations will run faster in full broadcasting mode but require more memory. [default: %(default)s]") @@ -206,6 +215,7 @@ def main(argv): print_msg(f"Input Folder: {args.infolder}") print_msg(f"Overwrite Output: {args.overwrite_output}") print_msg(f"Interpolation Mode: {args.interpolation_mode}") + print_msg(f"RAM Save Mode: {args.memory_save_mode}") print_msg(f"Broadcasting: {args.use_broadcasting}") if args.out_tiff: print_msg("Output Type: TIFF") @@ -216,7 +226,8 @@ def main(argv): print_msg(f"Config File: {config_filename}") run(config_filename, args.infolder, args.output_file, args.processing_dates_file, - file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode, args.use_broadcasting) + file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode, + args.use_broadcasting, args.memory_save_mode) except Exception as e: indent = len(program_name) * " " sys.stderr.write(program_name + ": " + repr(e) + "\n") diff --git a/src/lisfloodutilities/gridding/lib/utils.py b/src/lisfloodutilities/gridding/lib/utils.py index 183e7fa..0e3d0d3 100755 --- a/src/lisfloodutilities/gridding/lib/utils.py +++ b/src/lisfloodutilities/gridding/lib/utils.py @@ -157,8 +157,10 @@ class Config(Printable): # Each interpolation is paired with the number of neighbours used by the interpolation algorithm. INTERPOLATION_MODES = {'nearest': 1, 'invdist': 20, 'adw': 11, 'cdd': 11, 'bilinear': 4, 'triangulation': 3, 'bilinear_delaunay': 4} - def __init__(self, config_filename: str, start_date: datetime = None, - end_date: datetime = None, quiet_mode: bool = False, interpolation_mode: str = 'adw'): + # Allows splitting the grid to interpolate into several parts and run isolated interpolations for each + MEMORY_SAVE_MODES = {'0': None, '1': 2, '2': 4, '3': 10, '4': 20, '5': 50} + def __init__(self, config_filename: str, start_date: datetime = None, end_date: datetime = None, + quiet_mode: bool = False, interpolation_mode: str = 'adw', memory_save_mode: str = '0'): super().__init__(quiet_mode) self.COLUMN_HEIGHT = "height" self.COLUMN_VALUE = "value" @@ -166,6 +168,7 @@ def __init__(self, config_filename: str, start_date: datetime = None, self.COLUMN_LAT = "lat" self.VALUE_NAN = -9999.0 self.interpolation_mode = interpolation_mode + self.memory_save_mode = memory_save_mode self.__setup_variable_config(config_filename) self.__HEIGHT_CORRECTION_FACTOR = {"tx": 0.006, "tn": 0.006, "ta": 0.006, "ta6": 0.006, "pd": 0.00025} @@ -330,6 +333,10 @@ def height_correction_factor(self) -> float: def neighbours_near(self) -> int: return self.scipy_modes_nnear[self.interpolation_mode] + @property + def num_of_splits(self) -> int: + return Config.MEMORY_SAVE_MODES[self.memory_save_mode] + class GriddingUtils(Printable): @@ -407,6 +414,7 @@ def generate_grid(self, filename: Path) -> np.ndarray: xp = np.array(x) yp = np.array(y) values = np.array(z) + df = None if self.conf.interpolation_mode == 'cdd': scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values, self.conf.neighbours_near, mv_target, mv_source, @@ -415,13 +423,15 @@ def generate_grid(self, filename: Path) -> np.ndarray: cdd_map=self.conf.cdd_map, cdd_mode=self.conf.cdd_mode, cdd_options=self.conf.cdd_options, - use_broadcasting=self.use_broadcasting) + use_broadcasting=self.use_broadcasting, + num_of_splits=self.conf.num_of_splits) else: scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values, self.conf.neighbours_near, mv_target, mv_source, target_is_rotated=False, parallel=False, mode=self.conf.interpolation_mode, - use_broadcasting=self.use_broadcasting) + use_broadcasting=self.use_broadcasting, + num_of_splits=self.conf.num_of_splits) # reset search radius to a fixed value scipy_interpolation.min_upper_bound = self.conf.min_upper_bound @@ -440,7 +450,7 @@ def generate_grid(self, filename: Path) -> np.ndarray: # CSV_DELIMITER = '\t' # FILES_WILDCARD = '??????????00_all.kiwis' # -# def __init__(self, conf: Config, overwrite_file: bool = False, infolder: Path, quiet_mode: bool = False): +# def __init__(self, conf: Config, infolder: Path, overwrite_file: bool = False, quiet_mode: bool = False): # super().__init__(quiet_mode) # self.conf = conf # self.overwrite_file = overwrite_file @@ -469,7 +479,7 @@ def generate_grid(self, filename: Path) -> np.ndarray: # self.file_group_read_idx += 1 # raise StopIteration # -# def __get_filter_classes(self) -> array: +# def __get_filter_classes(self) -> list: # ''' # TODO: Implement the class. # ''' @@ -534,4 +544,4 @@ def generate_grid(self, filename: Path) -> np.ndarray: # # def __processable_file(self, file_timestamp: datetime, start_date: datetime = None, end_date: datetime = None) -> bool: # return (start_date is not None and start_date <= file_timestamp and -# end_date is not None and file_timestamp <= end_date) +# end_date is not None and file_timestamp <= end_date) \ No newline at end of file