Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create RAM usage save mode. #49

Merged
merged 1 commit into from
Dec 15, 2023
Merged

Create RAM usage save mode. #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions src/lisfloodutilities/gridding/generate_grids.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,14 @@ def interpolation_mode_type(mode: str) -> str:
raise ArgumentTypeError(f'You must select a mode out of {list(Config.INTERPOLATION_MODES.keys())}.')
return mode

def memory_save_mode_type(mode: str) -> str:
if mode not in Config.MEMORY_SAVE_MODES:
raise ArgumentTypeError(f'You must select a RAM save mode out of {list(Config.MEMORY_SAVE_MODES.keys())}.')
return mode

def run(config_filename: str, infolder: str, output_file: str, processing_dates_file: str, file_utils: FileUtils,
output_tiff: bool, overwrite_output: bool, start_date: datetime = None, end_date: datetime = None,
interpolation_mode: str = 'adw', use_broadcasting: bool = False):
interpolation_mode: str = 'adw', use_broadcasting: bool = False, memory_save_mode: str = None):
"""
Interpolate text files containing (x, y, value) using inverse distance interpolation.
Produces as output, either a netCDF file containing all the grids or one TIFF file per grid.
Expand All @@ -53,7 +58,7 @@ def run(config_filename: str, infolder: str, output_file: str, processing_dates_
"""
global quiet_mode

conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode)
conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode, memory_save_mode)

if conf.start_date > conf.end_date:
raise ArgumentTypeError("Start date is greater than End date.")
Expand Down Expand Up @@ -135,7 +140,8 @@ def main(argv):
start_date='',
end_date=END_DATE_DEFAULT,
interpolation_mode='adw',
use_broadcasting=False)
use_broadcasting=False,
memory_save_mode='0')

parser.add_argument("-i", "--in", dest="infolder", required=True, type=FileUtils.folder_type,
help="Set input folder path with kiwis/point files",
Expand Down Expand Up @@ -169,6 +175,9 @@ def main(argv):
parser.add_argument("-m", "--mode", dest="interpolation_mode", required=False, type=interpolation_mode_type,
help="Set interpolation mode. [default: %(default)s]",
metavar=f"{list(Config.INTERPOLATION_MODES.keys())}")
parser.add_argument("-r", "--ramsavemode", dest="memory_save_mode", required=False, type=memory_save_mode_type,
help="Set memory save mode level. Used to reduce memory usage [default: %(default)s]",
metavar=f"{list(Config.MEMORY_SAVE_MODES.keys())}")
parser.add_argument("-b", "--broadcast", dest="use_broadcasting", action="store_true",
help="When set, computations will run faster in full broadcasting mode but require more memory. [default: %(default)s]")

Expand Down Expand Up @@ -206,6 +215,7 @@ def main(argv):
print_msg(f"Input Folder: {args.infolder}")
print_msg(f"Overwrite Output: {args.overwrite_output}")
print_msg(f"Interpolation Mode: {args.interpolation_mode}")
print_msg(f"RAM Save Mode: {args.memory_save_mode}")
print_msg(f"Broadcasting: {args.use_broadcasting}")
if args.out_tiff:
print_msg("Output Type: TIFF")
Expand All @@ -216,7 +226,8 @@ def main(argv):
print_msg(f"Config File: {config_filename}")

run(config_filename, args.infolder, args.output_file, args.processing_dates_file,
file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode, args.use_broadcasting)
file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode,
args.use_broadcasting, args.memory_save_mode)
except Exception as e:
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
Expand Down
24 changes: 17 additions & 7 deletions src/lisfloodutilities/gridding/lib/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,15 +157,18 @@ class Config(Printable):
# Each interpolation is paired with the number of neighbours used by the interpolation algorithm.
INTERPOLATION_MODES = {'nearest': 1, 'invdist': 20, 'adw': 11, 'cdd': 11,
'bilinear': 4, 'triangulation': 3, 'bilinear_delaunay': 4}
def __init__(self, config_filename: str, start_date: datetime = None,
end_date: datetime = None, quiet_mode: bool = False, interpolation_mode: str = 'adw'):
# Allows splitting the grid to interpolate into several parts and run isolated interpolations for each
MEMORY_SAVE_MODES = {'0': None, '1': 2, '2': 4, '3': 10, '4': 20, '5': 50}
def __init__(self, config_filename: str, start_date: datetime = None, end_date: datetime = None,
quiet_mode: bool = False, interpolation_mode: str = 'adw', memory_save_mode: str = '0'):
super().__init__(quiet_mode)
self.COLUMN_HEIGHT = "height"
self.COLUMN_VALUE = "value"
self.COLUMN_LON = "lon"
self.COLUMN_LAT = "lat"
self.VALUE_NAN = -9999.0
self.interpolation_mode = interpolation_mode
self.memory_save_mode = memory_save_mode
self.__setup_variable_config(config_filename)
self.__HEIGHT_CORRECTION_FACTOR = {"tx": 0.006, "tn": 0.006, "ta": 0.006, "ta6": 0.006, "pd": 0.00025}

Expand Down Expand Up @@ -330,6 +333,10 @@ def height_correction_factor(self) -> float:
def neighbours_near(self) -> int:
return self.scipy_modes_nnear[self.interpolation_mode]

@property
def num_of_splits(self) -> int:
return Config.MEMORY_SAVE_MODES[self.memory_save_mode]


class GriddingUtils(Printable):

Expand Down Expand Up @@ -407,6 +414,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
xp = np.array(x)
yp = np.array(y)
values = np.array(z)
df = None
if self.conf.interpolation_mode == 'cdd':
scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values,
self.conf.neighbours_near, mv_target, mv_source,
Expand All @@ -415,13 +423,15 @@ def generate_grid(self, filename: Path) -> np.ndarray:
cdd_map=self.conf.cdd_map,
cdd_mode=self.conf.cdd_mode,
cdd_options=self.conf.cdd_options,
use_broadcasting=self.use_broadcasting)
use_broadcasting=self.use_broadcasting,
num_of_splits=self.conf.num_of_splits)
else:
scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values,
self.conf.neighbours_near, mv_target, mv_source,
target_is_rotated=False, parallel=False,
mode=self.conf.interpolation_mode,
use_broadcasting=self.use_broadcasting)
use_broadcasting=self.use_broadcasting,
num_of_splits=self.conf.num_of_splits)

# reset search radius to a fixed value
scipy_interpolation.min_upper_bound = self.conf.min_upper_bound
Expand All @@ -440,7 +450,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
# CSV_DELIMITER = '\t'
# FILES_WILDCARD = '??????????00_all.kiwis'
#
# def __init__(self, conf: Config, overwrite_file: bool = False, infolder: Path, quiet_mode: bool = False):
# def __init__(self, conf: Config, infolder: Path, overwrite_file: bool = False, quiet_mode: bool = False):
# super().__init__(quiet_mode)
# self.conf = conf
# self.overwrite_file = overwrite_file
Expand Down Expand Up @@ -469,7 +479,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
# self.file_group_read_idx += 1
# raise StopIteration
#
# def __get_filter_classes(self) -> array:
# def __get_filter_classes(self) -> list:
# '''
# TODO: Implement the class.
# '''
Expand Down Expand Up @@ -534,4 +544,4 @@ def generate_grid(self, filename: Path) -> np.ndarray:
#
# def __processable_file(self, file_timestamp: datetime, start_date: datetime = None, end_date: datetime = None) -> bool:
# return (start_date is not None and start_date <= file_timestamp and
# end_date is not None and file_timestamp <= end_date)
# end_date is not None and file_timestamp <= end_date)
Loading