Add configuration to allow forcing a time on the input files.

src/lisfloodutilities/gridding/configuration/1arcmin/config_pd.txt

@@ -17,6 +17,9 @@ LONG_NAME = Daily Average Vapor Pressure
 UNIT = days since 1990-01-02 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 1
 

src/lisfloodutilities/gridding/configuration/1arcmin/config_pr.txt

@@ -12,13 +12,17 @@ DATA_TYPE_PACKED = i2
 STANDARD_NAME = precipitation_amount
 LONG_NAME = Daily Accumulated Precipitation
 # 1303 - ERAinterim
+# 1329 - ERA5-land
 KIWIS_FILTER_PLUGIN_CLASSES = {'ObservationsKiwisFilter': {'1303': 100.0}}
 
 [VAR_TIME]
 
 UNIT = days since 1990-01-02 06:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0600
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 0
 

src/lisfloodutilities/gridding/configuration/1arcmin/config_pr6.txt

@@ -15,6 +15,7 @@ LONG_NAME = 6 Hourly Accumulated Precipitation
 # 1302 - CarpatClim
 # 1295 - MARS
 # 1303 - ERAinterim
+# 1329 - ERA5-land
 KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0}}
 
 

src/lisfloodutilities/gridding/configuration/1arcmin/config_rg.txt

@@ -17,6 +17,9 @@ LONG_NAME = Daily Calculated Radiation
 UNIT = days since 1990-01-02 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 1
 

src/lisfloodutilities/gridding/configuration/1arcmin/config_ta.txt

@@ -17,3 +17,6 @@ LONG_NAME = Daily Average Temperature
 UNIT = days since 1990-01-01 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+

src/lisfloodutilities/gridding/configuration/1arcmin/config_tn.txt

@@ -17,3 +17,5 @@ LONG_NAME = Daily Minimum Temperature
 UNIT = days since 1990-01-01 06:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0600

src/lisfloodutilities/gridding/configuration/1arcmin/config_tx.txt

@@ -1,8 +1,3 @@
-[GENERIC]
-
-INPUT_WILDCARD = ??????????.kiwis
-INPUT_TIMESTAMP_PATTERN = %%Y%%m%%d%%H.kiwis
-
 [PROPERTIES]
 
 VAR_CODE = tx
@@ -17,11 +12,10 @@ DATA_TYPE_PACKED = i2
 STANDARD_NAME = air_temperature
 LONG_NAME = Daily Maximum Temperature
 
-# KIWIS_FILTER_COLUMNS = {'COL_LAT': 'station_local_y', 'COL_LON': 'station_local_x', 'COL_IS_IN_DOMAIN': 'EFAS-ADDATTR-ISINARCMINDOMAIN'}
-KIWIS_FILTER_COLUMNS = {'COL_LAT': 'lat', 'COL_LON': 'lon', 'COL_IS_IN_DOMAIN': 'EFAS-ADDATTR-ISINARCMINDOMAIN'}
-
 [VAR_TIME]
 
 UNIT = days since 1990-01-01 18:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 1800

src/lisfloodutilities/gridding/configuration/1arcmin/config_ws.txt

@@ -17,6 +17,9 @@ LONG_NAME = Daily Average Wind Speed
 UNIT = days since 1990-01-02 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 1
 

src/lisfloodutilities/gridding/configuration/1arcmin/default.txt

@@ -1,7 +1,11 @@
 [GENERIC]
 
+# Wildcard used to load the input kiwis files
 INPUT_WILDCARD = ??????????00_all.kiwis
+# Datetime pattern used to extract the date and from the kiwis file name
 INPUT_TIMESTAMP_PATTERN = %%Y%%m%%d%%H%%M_all.kiwis
+# Datetime pattern used to extract the date and time from the point file name
+POINTS_TIMESTAMP_PATTERN = %%Y%%m%%d%%H%%M
 
 NETCDF_REFERENCE=A European daily high-resolution gridded meteorological data set for 1990 - 2022
 NETCDF_TITLE = Lisflood meteo maps 1990-2023 for European setting Feb. 2023
@@ -25,6 +29,7 @@ DATA_TYPE_PACKED = i2
 STANDARD_NAME = DUMMY_STANDARD_NAME
 LONG_NAME = DUMMY LONG NAME
 KIWIS_FILTER_COLUMNS = {'COL_LAT': 'station_local_y', 'COL_LON': 'station_local_x', 'COL_IS_IN_DOMAIN': 'EFAS_ADDATTR_ISINNEWDOMAIN'}
+# KIWIS_FILTER_COLUMNS = {}
 KIWIS_FILTER_PLUGIN_CLASSES = {'KiwisFilter': {}}
 
 [DIMENSION]
@@ -81,6 +86,9 @@ UNIT_PATTERN = days since %%Y-%%m-%%d %%H:%%M:%%S.%%f
 UNIT = days since 1990-01-02 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0600
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 0
 

src/lisfloodutilities/gridding/configuration/5x5km/config_pd.txt

@@ -17,6 +17,9 @@ LONG_NAME = Daily Average Vapor Pressure
 UNIT = days since 1990-01-02 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 1
 

src/lisfloodutilities/gridding/configuration/5x5km/config_pr.txt

@@ -19,6 +19,8 @@ KIWIS_FILTER_PLUGIN_CLASSES = {'ObservationsKiwisFilter': {'1303': 100.0}}
 # UNIT = days since 1990-01-02 06:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0600
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 0
-

src/lisfloodutilities/gridding/configuration/5x5km/config_rg.txt

@@ -17,6 +17,9 @@ LONG_NAME = Daily Calculated Radiation
 UNIT = days since 1990-01-02 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 1
 

src/lisfloodutilities/gridding/configuration/5x5km/config_ta.txt

@@ -17,3 +17,6 @@ LONG_NAME = Daily Average Temperature
 UNIT = hours since 1990-01-01 00:00:00.0
 FREQUENCY = 6
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+

src/lisfloodutilities/gridding/configuration/5x5km/config_tn.txt

@@ -17,3 +17,5 @@ LONG_NAME = Daily Minimum Temperature
 UNIT = days since 1990-01-01 06:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0600

src/lisfloodutilities/gridding/configuration/5x5km/config_tx.txt

@@ -17,3 +17,5 @@ LONG_NAME = Daily Maximum Temperature
 UNIT = days since 1990-01-01 18:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0600

src/lisfloodutilities/gridding/configuration/5x5km/config_ws.txt

@@ -20,3 +20,6 @@ FREQUENCY = 1
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0000
+

src/lisfloodutilities/gridding/configuration/5x5km/default.txt

@@ -1,7 +1,11 @@
 [GENERIC]
 
+# Wildcard used to load the input kiwis files
 INPUT_WILDCARD = ??????????00_all.kiwis
+# Datetime pattern used to extract the date and from the kiwis file name
 INPUT_TIMESTAMP_PATTERN = %%Y%%m%%d%%H%%M_all.kiwis
+# Datetime pattern used to extract the date and time from the point file name
+POINTS_TIMESTAMP_PATTERN = %%Y%%m%%d%%H%%M
 
 NETCDF_REFERENCE=A European daily high-resolution gridded meteorological data set for 1990 - 2022
 NETCDF_TITLE = Lisflood meteo maps 1990-2023 for European setting Feb. 2023
@@ -82,6 +86,9 @@ UNIT_PATTERN = days since %%Y-%%m-%%d %%H:%%M:%%S.%%f
 UNIT = days since 1990-01-02 00:00:00.0
 FREQUENCY = 1
 
+# Used to force the time value when it is not defined in the filename. Format: HHMM
+# FORCE_TIME = 0600
+
 # Used to allow moving the files 1 day forward for variables (WS, RG, PD, PR?)
 OFFSET_FILE_DATE = 0
 

src/lisfloodutilities/gridding/generate_grids.py

@@ -77,7 +77,7 @@ def run(config_filename: str, infolder: str, output_file: str, processing_dates_
         str(conf.dem_max_y)))
 
     print_msg('Start reading files')
-    inwildcard = conf.var_code + FileUtils.FILES_WILDCARD
+    # inwildcard = conf.var_code + FileUtils.FILES_WILDCARD
 
     netcdf_offset_file_date = int(conf.get_config_field('VAR_TIME','OFFSET_FILE_DATE'))
 
@@ -88,8 +88,10 @@ def run(config_filename: str, infolder: str, output_file: str, processing_dates_
         output_writer_netcdf = NetCDFWriter(conf, overwrite_output, quiet_mode)
         output_writer_netcdf.open(Path(outfile))
     file_loader = KiwisLoader(conf, Path(infolder), dates_to_process, overwrite_output, use_existing_file, quiet_mode)
-    for filename in file_loader:
-        file_timestamp = file_utils.get_timestamp_from_filename(filename) + timedelta(days=netcdf_offset_file_date)
+    for filename, kiwis_timestamp_str in file_loader:
+        # file_timestamp = file_utils.get_timestamp_from_filename(filename) + timedelta(days=netcdf_offset_file_date)
+        kiwis_timestamp = datetime.strptime(kiwis_timestamp_str, FileUtils.DATE_PATTERN_CONDENSED_SHORT)
+        file_timestamp = kiwis_timestamp + timedelta(days=netcdf_offset_file_date)
         print_msg(f'Processing file: {filename}')
         if output_tiff:
             outfilepath = filename.with_suffix('.tiff')

src/lisfloodutilities/gridding/lib/utils.py

@@ -376,6 +376,21 @@ def input_timestamp_pattern(self) -> str:
         default_pattern = self.get_config_field('GENERIC', 'INPUT_TIMESTAMP_PATTERN')
         return f'{self.var_code}{default_pattern}'
 
+    @property
+    def points_timestamp_pattern(self) -> str:
+        return self.get_config_field('GENERIC', 'POINTS_TIMESTAMP_PATTERN')
+
+    @property
+    def force_time(self) -> str:
+        """
+        Gets the value of time to force when reading the filenames in case the input files cannot have the time component on their names.
+        This is a workaround only to allow operating with daily files provided without the time component. 
+        """
+        try:
+            return self.get_config_field('VAR_TIME', 'FORCE_TIME')
+        except Exception as e:
+            return None
+
 
 class GriddingUtils(Printable):
 
@@ -518,7 +533,7 @@ def __next__(self):
             self.__process_next_batch_of_files()
         # Get next file
         filepath_kiwis, filepath_points, kiwis_timestamps = self.files_cache.pop()
-        return filepath_points
+        return filepath_points, kiwis_timestamps
 
     def __load_next_batch_of_files(self):
         """
@@ -605,18 +620,19 @@ def __get_filter_classes(self) -> list:
             print(f"Error: Could not find class '{class_name}' in module '{module_name}'")
         return plugins_array
 
-    def __get_points_filename(self, kiwis_timestamp: str, filename_kiwis: Path) -> Path:
+    def __get_points_filename(self, kiwis_timestamp: datetime, filename_kiwis: Path) -> Path:
         '''
         Returns the points file path.
         If the mode is overwrite tries to get the first pointfile path it finds and if it does not find generates a new file path.
         Otherwise generates a new file path.
         '''
+        points_input_timestamp = kiwis_timestamp.strftime(self.conf.points_timestamp_pattern)
         if self.use_existing_file:
-            for points_path in sorted(filename_kiwis.parent.rglob(f'{self.var_code}{kiwis_timestamp}_??????????????.txt')):
+            for points_path in sorted(filename_kiwis.parent.rglob(f'{self.var_code}{points_input_timestamp}_??????????????.txt')):
                 if points_path.is_file():
                     return points_path
         pointfile_timestamp = datetime.now().strftime(FileUtils.DATE_PATTERN_CONDENSED)
-        return Path(filename_kiwis.parent, f'{self.var_code}{kiwis_timestamp}_{pointfile_timestamp}.txt')
+        return Path(filename_kiwis.parent, f'{self.var_code}{points_input_timestamp}_{pointfile_timestamp}.txt')
 
     def __load_kiwis_paths(self):
         netcdf_offset_file_date = int(self.conf.get_config_field('VAR_TIME','OFFSET_FILE_DATE'))
@@ -625,11 +641,15 @@ def __load_kiwis_paths(self):
             file_timestamp = kiwis_timestamp + timedelta(days=netcdf_offset_file_date)
             if self.__processable_file(file_timestamp, self.conf.start_date, self.conf.end_date):
                 kiwis_timestamp_str = kiwis_timestamp.strftime(FileUtils.DATE_PATTERN_CONDENSED_SHORT)
-                filename_points = self.__get_points_filename(kiwis_timestamp_str, filename_kiwis)
+                filename_points = self.__get_points_filename(kiwis_timestamp, filename_kiwis)
                 self.files_list_dic[kiwis_timestamp_str] = (filename_kiwis, filename_points, kiwis_timestamp_str)
 
     def __get_timestamp_from_filename(self, filename: Path) -> datetime:
-        return datetime.strptime(filename.name, self.conf.input_timestamp_pattern)
+        file_timestamp = datetime.strptime(filename.name, self.conf.input_timestamp_pattern)
+        if self.conf.force_time is not None:
+            new_time = datetime.strptime(self.conf.force_time, "%H%M").time()
+            file_timestamp = datetime.combine(file_timestamp.date(), new_time)
+        return file_timestamp
 
     def __processable_file(self, file_timestamp: datetime, start_date: datetime = None, end_date: datetime = None) -> bool:
         is_processable_date = True