Merge pull request #49 from ec-jrc/module_kiwis_interpreter

Create RAM usage save mode.

src/lisfloodutilities/gridding/generate_grids.py

@@ -40,9 +40,14 @@ def interpolation_mode_type(mode: str) -> str:
         raise ArgumentTypeError(f'You must select a mode out of {list(Config.INTERPOLATION_MODES.keys())}.')
     return mode
 
+def memory_save_mode_type(mode: str) -> str:
+    if mode not in Config.MEMORY_SAVE_MODES:
+        raise ArgumentTypeError(f'You must select a RAM save mode out of {list(Config.MEMORY_SAVE_MODES.keys())}.')
+    return mode
+
 def run(config_filename: str, infolder: str, output_file: str, processing_dates_file: str, file_utils: FileUtils,
         output_tiff: bool, overwrite_output: bool, start_date: datetime = None, end_date: datetime = None,
-        interpolation_mode: str = 'adw', use_broadcasting: bool = False):
+        interpolation_mode: str = 'adw', use_broadcasting: bool = False, memory_save_mode: str = None):
     """
     Interpolate text files containing (x, y, value) using inverse distance interpolation.
     Produces as output, either a netCDF file containing all the grids or one TIFF file per grid.
@@ -53,7 +58,7 @@ def run(config_filename: str, infolder: str, output_file: str, processing_dates_
     """
     global quiet_mode
 
-    conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode)
+    conf = Config(config_filename, start_date, end_date, quiet_mode, interpolation_mode, memory_save_mode)
 
     if conf.start_date > conf.end_date:
         raise ArgumentTypeError("Start date is greater than End date.")
@@ -135,7 +140,8 @@ def main(argv):
                             start_date='',
                             end_date=END_DATE_DEFAULT,
                             interpolation_mode='adw',
-                            use_broadcasting=False)
+                            use_broadcasting=False,
+                            memory_save_mode='0')
 
         parser.add_argument("-i", "--in", dest="infolder", required=True, type=FileUtils.folder_type,
                             help="Set input folder path with kiwis/point files",
@@ -169,6 +175,9 @@ def main(argv):
         parser.add_argument("-m", "--mode", dest="interpolation_mode", required=False, type=interpolation_mode_type,
                             help="Set interpolation mode. [default: %(default)s]",
                             metavar=f"{list(Config.INTERPOLATION_MODES.keys())}")
+        parser.add_argument("-r", "--ramsavemode", dest="memory_save_mode", required=False, type=memory_save_mode_type,
+                            help="Set memory save mode level. Used to reduce memory usage [default: %(default)s]",
+                            metavar=f"{list(Config.MEMORY_SAVE_MODES.keys())}")
         parser.add_argument("-b", "--broadcast", dest="use_broadcasting", action="store_true",
                             help="When set, computations will run faster in full broadcasting mode but require more memory. [default: %(default)s]")
 
@@ -206,6 +215,7 @@ def main(argv):
         print_msg(f"Input Folder:  {args.infolder}")
         print_msg(f"Overwrite Output: {args.overwrite_output}")
         print_msg(f"Interpolation Mode: {args.interpolation_mode}")
+        print_msg(f"RAM Save Mode: {args.memory_save_mode}")
         print_msg(f"Broadcasting: {args.use_broadcasting}")
         if args.out_tiff:
             print_msg("Output Type: TIFF")
@@ -216,7 +226,8 @@ def main(argv):
         print_msg(f"Config File: {config_filename}")
 
         run(config_filename, args.infolder, args.output_file, args.processing_dates_file, 
-            file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode, args.use_broadcasting)
+            file_utils, args.out_tiff, args.overwrite_output, start_date, end_date, args.interpolation_mode,
+            args.use_broadcasting, args.memory_save_mode)
     except Exception as e:
         indent = len(program_name) * " "
         sys.stderr.write(program_name + ": " + repr(e) + "\n")

src/lisfloodutilities/gridding/lib/utils.py

@@ -157,15 +157,18 @@ class Config(Printable):
     # Each interpolation is paired with the number of neighbours used by the interpolation algorithm.
     INTERPOLATION_MODES = {'nearest': 1, 'invdist': 20, 'adw': 11, 'cdd': 11,
                            'bilinear': 4, 'triangulation': 3, 'bilinear_delaunay': 4}
-    def __init__(self, config_filename: str, start_date: datetime = None,
-                 end_date: datetime = None, quiet_mode: bool = False, interpolation_mode: str = 'adw'):
+    # Allows splitting the grid to interpolate into several parts and run isolated interpolations for each
+    MEMORY_SAVE_MODES = {'0': None, '1': 2, '2': 4, '3': 10, '4': 20, '5': 50}
+    def __init__(self, config_filename: str, start_date: datetime = None, end_date: datetime = None,
+                 quiet_mode: bool = False, interpolation_mode: str = 'adw', memory_save_mode: str = '0'):
         super().__init__(quiet_mode)
         self.COLUMN_HEIGHT = "height"
         self.COLUMN_VALUE = "value"
         self.COLUMN_LON = "lon"
         self.COLUMN_LAT = "lat"
         self.VALUE_NAN = -9999.0
         self.interpolation_mode = interpolation_mode
+        self.memory_save_mode = memory_save_mode
         self.__setup_variable_config(config_filename)
         self.__HEIGHT_CORRECTION_FACTOR = {"tx": 0.006, "tn": 0.006, "ta": 0.006, "ta6": 0.006, "pd": 0.00025}
 
@@ -330,6 +333,10 @@ def height_correction_factor(self) -> float:
     def neighbours_near(self) -> int:
         return self.scipy_modes_nnear[self.interpolation_mode]
 
+    @property
+    def num_of_splits(self) -> int:
+        return Config.MEMORY_SAVE_MODES[self.memory_save_mode]
+
 
 class GriddingUtils(Printable):
 
@@ -407,6 +414,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
         xp = np.array(x)
         yp = np.array(y)
         values = np.array(z)
+        df = None
         if self.conf.interpolation_mode == 'cdd':
             scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values,
                                                      self.conf.neighbours_near, mv_target, mv_source,
@@ -415,13 +423,15 @@ def generate_grid(self, filename: Path) -> np.ndarray:
                                                      cdd_map=self.conf.cdd_map,
                                                      cdd_mode=self.conf.cdd_mode,
                                                      cdd_options=self.conf.cdd_options,
-                                                     use_broadcasting=self.use_broadcasting)
+                                                     use_broadcasting=self.use_broadcasting,
+                                                     num_of_splits=self.conf.num_of_splits)
         else:
             scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values,
                                                      self.conf.neighbours_near, mv_target, mv_source,
                                                      target_is_rotated=False, parallel=False,
                                                      mode=self.conf.interpolation_mode,
-                                                     use_broadcasting=self.use_broadcasting)
+                                                     use_broadcasting=self.use_broadcasting,
+                                                     num_of_splits=self.conf.num_of_splits)
 
         # reset search radius to a fixed value
         scipy_interpolation.min_upper_bound = self.conf.min_upper_bound
@@ -440,7 +450,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
 #     CSV_DELIMITER = '\t'
 #     FILES_WILDCARD = '??????????00_all.kiwis'
 #
-#     def __init__(self, conf: Config, overwrite_file: bool = False, infolder: Path, quiet_mode: bool = False):
+#     def __init__(self, conf: Config, infolder: Path, overwrite_file: bool = False, quiet_mode: bool = False):
 #         super().__init__(quiet_mode)
 #         self.conf = conf
 #         self.overwrite_file = overwrite_file
@@ -469,7 +479,7 @@ def generate_grid(self, filename: Path) -> np.ndarray:
 #                 self.file_group_read_idx += 1
 #         raise StopIteration
 #
-#     def __get_filter_classes(self) -> array:
+#     def __get_filter_classes(self) -> list:
 #         '''
 #         TODO: Implement the class.
 #         '''
@@ -534,4 +544,4 @@ def generate_grid(self, filename: Path) -> np.ndarray:
 #
 #     def __processable_file(self, file_timestamp: datetime, start_date: datetime = None, end_date: datetime = None) -> bool:
 #         return (start_date is not None and start_date <= file_timestamp and
-#                 end_date is not None and file_timestamp <= end_date)
+#                 end_date is not None and file_timestamp <= end_date)