Merge pull request #57 from ec-jrc/module_kiwis_interpreter

Correct time axis write in future and generate hourly files

src/lisfloodutilities/gridding/configuration/1arcmin/config_pd1.txt

@@ -0,0 +1,22 @@
+[PROPERTIES]
+
+VAR_CODE = pd1
+CELL_METHODS = time: mean
+UNIT = hPa
+UNIT_CONVERSION = 1.0
+HEIGHT_CORRECTION_FACTOR = 0.00025
+# After the height correction allows removing negative values by setting them to zero
+TRUNCATE_NEGATIVE_VALUES = True
+VALUE_MIN = 0
+VALUE_MAX = 50
+VALUE_SCALE = 0.1
+VALUE_OFFSET = 0.0
+DATA_TYPE_PACKED = i2
+STANDARD_NAME = water_vapor_partial_pressure_in_air
+LONG_NAME = Hourly Average Vapor Pressure
+
+[VAR_TIME]
+
+UNIT_PATTERN = hours since %%Y-%%m-%%d %%H:%%M:%%S.%%f
+UNIT = hours since 1990-01-01 01:00:00.0
+FREQUENCY = 1

src/lisfloodutilities/gridding/configuration/1arcmin/config_pr1.txt

@@ -0,0 +1,27 @@
+[PROPERTIES]
+
+VAR_CODE = pr1
+CELL_METHODS = time: sum
+UNIT = mm
+UNIT_CONVERSION = 1.0
+VALUE_MIN = 0
+VALUE_MAX = 600
+VALUE_SCALE = 0.1
+VALUE_OFFSET = 0.0
+DATA_TYPE_PACKED = i2
+STANDARD_NAME = precipitation_amount
+LONG_NAME = Hourly Accumulated Precipitation
+# 1304 - EURO4M-APGD
+# 1302 - CarpatClim
+# 1295 - MARS
+# 1303 - ERAinterim
+# 1329 - ERA5-land
+KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0, '1329': 100.0}}
+
+
+[VAR_TIME]
+
+UNIT_PATTERN = hours since %%Y-%%m-%%d %%H:%%M:%%S.%%f
+UNIT = hours since 1990-01-01 12:00:00.0
+FREQUENCY = 1
+

src/lisfloodutilities/gridding/configuration/1arcmin/config_pr6x4.txt

@@ -15,7 +15,8 @@ LONG_NAME = 6 Hourly Accumulated Precipitation per Day
 # 1302 - CarpatClim
 # 1295 - MARS
 # 1303 - ERAinterim
-KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0}}
+# 1329 - ERA5-land
+KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0, '1329': 100.0}}
 
 [VAR_TIME]
 

src/lisfloodutilities/gridding/configuration/1arcmin/config_rg1.txt

@@ -0,0 +1,19 @@
+[PROPERTIES]
+
+VAR_CODE = rg1
+CELL_METHODS = time: sum
+UNIT = J/m2
+UNIT_CONVERSION = 1.0
+VALUE_MIN = 0
+VALUE_MAX = 115000000
+VALUE_SCALE = 10000.0
+VALUE_OFFSET = 0.0
+DATA_TYPE_PACKED = i2
+STANDARD_NAME = integral_wrt_time_of_surface_downwelling_shortwave_flux_in_air
+LONG_NAME = Hourly Calculated Radiation
+
+[VAR_TIME]
+
+UNIT_PATTERN = hours since %%Y-%%m-%%d %%H:%%M:%%S.%%f
+UNIT = hours since 1990-01-01 01:00:00.0
+FREQUENCY = 1

src/lisfloodutilities/gridding/configuration/1arcmin/config_tn1.txt

@@ -0,0 +1,20 @@
+[PROPERTIES]
+
+VAR_CODE = tn1
+CELL_METHODS = time: minimum
+UNIT = degree_Celsius
+UNIT_CONVERSION = 1.0
+HEIGHT_CORRECTION_FACTOR = 0.006
+VALUE_MIN = -55
+VALUE_MAX = 50
+VALUE_SCALE = 0.1
+VALUE_OFFSET = 0.0
+DATA_TYPE_PACKED = i2
+STANDARD_NAME = air_temperature
+LONG_NAME = Hourly Minimum Temperature
+
+[VAR_TIME]
+
+UNIT_PATTERN = hours since %%Y-%%m-%%d %%H:%%M:%%S.%%f
+UNIT = hours since 1990-01-01 01:00:00.0
+FREQUENCY = 1

src/lisfloodutilities/gridding/configuration/1arcmin/config_tx1.txt

@@ -0,0 +1,20 @@
+[PROPERTIES]
+
+VAR_CODE = tx1
+CELL_METHODS = time: maximum
+UNIT = degree_Celsius
+UNIT_CONVERSION = 1.0
+HEIGHT_CORRECTION_FACTOR = 0.006
+VALUE_MIN = -55
+VALUE_MAX = 55
+VALUE_SCALE = 0.1
+VALUE_OFFSET = 0.0
+DATA_TYPE_PACKED = i2
+STANDARD_NAME = air_temperature
+LONG_NAME = Hourly Maximum Temperature
+
+[VAR_TIME]
+
+UNIT_PATTERN = hours since %%Y-%%m-%%d %%H:%%M:%%S.%%f
+UNIT = hours since 1990-01-01 01:00:00.0
+FREQUENCY = 1

src/lisfloodutilities/gridding/configuration/1arcmin/config_ws1.txt

@@ -0,0 +1,19 @@
+[PROPERTIES]
+
+VAR_CODE = ws1
+CELL_METHODS = time: mean
+UNIT = m/s
+UNIT_CONVERSION = 1.0
+VALUE_MIN = 0
+VALUE_MAX = 45
+VALUE_SCALE = 0.1
+VALUE_OFFSET = 0.0
+DATA_TYPE_PACKED = i2
+STANDARD_NAME = wind_speed
+LONG_NAME = Hourly Average Wind Speed
+
+[VAR_TIME]
+
+UNIT_PATTERN = hours since %%Y-%%m-%%d %%H:%%M:%%S.%%f
+UNIT = hours since 1990-01-01 01:00:00.0
+FREQUENCY = 1

src/lisfloodutilities/gridding/configuration/5x5km/config_pr6x4.txt

@@ -15,7 +15,8 @@ LONG_NAME = 6 Hourly Accumulated Precipitation per Day
 # 1302 - CarpatClim
 # 1295 - MARS
 # 1303 - ERAinterim
-KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0}}
+# 1329 - ERA5-land
+KIWIS_FILTER_PLUGIN_CLASSES = {'DowgradedObservationsKiwisFilter': {'1304': 1.0, '1302': 1.0, '1295': 1.0}, 'ObservationsKiwisFilter': {'1303': 100.0, '1329': 100.0}}
 
 [VAR_TIME]
 

src/lisfloodutilities/gridding/lib/writers.py

@@ -82,12 +82,17 @@ def __init__(self, conf: Config, overwrite_file: bool = False, quiet_mode: bool
         self.time_frequency = int(self.conf.get_config_field('VAR_TIME','FREQUENCY'))
         self.calendar_type = self.NETCDF_VAR_TIME_CALENDAR_TYPE
         self.calendar_time_unit = self.conf.start_date.strftime(self.netcdf_var_time_unit_pattern)
+        # Write index not set yet
+        self.write_idx = -1
+        self.current_timestamp = None
+        self.is_new_file = False
 
     def open(self, out_filename: Path):
         super().open(out_filename)
         if not os.path.isfile(self.filepath):
             self.nf = Dataset(self.filepath, 'w', format=self.NETCDF_DATASET_FORMAT)
             self.__setup_netcdf_metadata()
+            self.is_new_file = True
         elif self.overwrite_file:
             self.nf = Dataset(self.filepath, 'r+', clobber=True, format=self.NETCDF_DATASET_FORMAT)
         else:
@@ -115,22 +120,35 @@ def setNaN(self, value, defaultNaN=np.nan):
     def write(self, grid: np.ndarray, timestamp: datetime = None):
         timestep = -1
         if timestamp is not None:
+            self.current_timestamp = timestamp
             timestep = date2num(timestamp, self.calendar_time_unit, self.calendar_type)
+        else:
+            self.current_timestamp = None
         self.write_timestep(grid, timestep)
 
     def write_timestep(self, grid: np.ndarray, timestep: int = -1):
         if timestep >= 0:
             if not self.opened():
                 raise Exception("netCDF Dataset was not initialized. If file already exists, use --force flag to append.")
-            timestep_idx = int(timestep / self.time_frequency)
-            self.nf.variables[self.netcdf_var_time][timestep_idx] = timestep
+            self.__set_write_index(timestep)
+            self.nf.variables[self.netcdf_var_time][self.write_idx] = timestep
             values = self.setNaN(copy.deepcopy(grid))
             values[values < self.conf.value_min_packed] = np.nan
             values[values > self.conf.value_max_packed] = np.nan
             values[values != self.conf.VALUE_NAN] *= self.conf.scale_factor
             values[values != self.conf.VALUE_NAN] += self.conf.add_offset
             values[np.isnan(values)] = self.conf.VALUE_NAN * self.conf.scale_factor + self.conf.add_offset
-            self.nf.variables[self.var_code][timestep_idx, :, :] = values
+            self.nf.variables[self.var_code][self.write_idx, :, :] = values
+
+    def __set_write_index(self, timestep: int):
+        if not self.is_new_file:
+            # Writing into an existing file need to calculate write index
+            time_array = self.nf.variables[self.netcdf_var_time][:].tolist()
+            self.write_idx = time_array.index(timestep)
+        elif self.write_idx >= 0:
+            self.write_idx += 1
+        else:
+            self.write_idx = 0
 
     def opened(self) -> bool:
         return not self.nf is None