Skip to content

Commit

Permalink
Reduce minor diffs in grids from diff PCs
Browse files Browse the repository at this point in the history
  • Loading branch information
@gnrgomes
gnrgomes committed Jul 11, 2024
1 parent 2ddc09f commit df668c3
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 29 deletions.
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ more-itertools>=8.2.0
netCDF4>=1.5.3,<=1.6.4
nine>=1.1.0
numexpr>=2.8.4
numpy>=1.21.6
numpy>=1.22.3
packaging>=23.1
pandas>=1.3.5
partd>=1.4.0
Expand Down
8 changes: 4 additions & 4 deletions src/lisfloodutilities/gridding/lib/filters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ class ObservationsKiwisFilter(KiwisFilter):
filter (as key) and the radius (in decimal degrees) to find the vicinity station from other providers (as value).
"""

CLUSTER_COLLAPSE_RADIUS = 0.011582073434000193 # decimal degrees (1287 m)
CLUSTER_COLLAPSE_RADIUS = np.float64(0.011582073434000193) # decimal degrees (1287 m)

def __init__(self, filter_columns: dict = {}, filter_args: dict = {}, var_code: str = '', quiet_mode: bool = False):
super().__init__(filter_columns, filter_args, var_code, quiet_mode)
Expand All @@ -207,9 +207,9 @@ def __init__(self, filter_columns: dict = {}, filter_args: dict = {}, var_code:
self.provider_radius[provider_id] = radius

@staticmethod
def kilometers2degrees(km: float) -> float:
def kilometers2degrees(km: np.float64) -> np.float64:
# Convert km to degrees of latitude
delta_lat = km * 0.00899928005
delta_lat = km * np.float64(0.00899928005)
return delta_lat

def apply_filter(self, df: pd.DataFrame) -> pd.DataFrame:
Expand All @@ -226,7 +226,7 @@ def apply_filter(self, df: pd.DataFrame) -> pd.DataFrame:
return df

def has_neighbor_within_radius_from_other_providers(self, row: pd.Series, tree: cKDTree = None, provider_id: int = 0,
radius: float = CLUSTER_COLLAPSE_RADIUS) -> bool:
radius: np.float64 = CLUSTER_COLLAPSE_RADIUS) -> bool:
cur_provider_id = row[self.COL_PROVIDER_ID]
if cur_provider_id == provider_id:
location = (row[self.COL_LON], row[self.COL_LAT])
Expand Down
34 changes: 17 additions & 17 deletions src/lisfloodutilities/gridding/lib/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,11 @@ def __init__(self, dem_map: Path, quiet_mode: bool = False):
reader = NetCDFReader(self._dem_map)
self.nrows = reader._rows
self.ncols = reader._cols
self.mv = reader.mv.astype(np.float32)
self.values = reader.values.astype(np.float32)
self.mv = reader.mv.astype(np.float64)
self.values = reader.values.astype(np.float64)
self.lats, self.lons = reader.get_lat_lon_values()
self.lats = self.lats.astype(np.float32)
self.lons = self.lons.astype(np.float32)
self.lats = self.lats.astype(np.float64)
self.lons = self.lons.astype(np.float64)
self.lat_values = reader.get_lat_values()
self.lon_values = reader.get_lon_values()
self.cell_size_x = reader._pxlW
Expand Down Expand Up @@ -319,12 +319,12 @@ def get_config_field(self, config_group: str = '', config_property: str = '') ->
return self.__configFile.get(config_group, config_property)

@property
def scale_factor(self) -> float:
return float(self.get_config_field('PROPERTIES', 'VALUE_SCALE'))
def scale_factor(self) -> np.float64:
return np.float64(self.get_config_field('PROPERTIES', 'VALUE_SCALE'))

@property
def add_offset(self) -> float:
return float(self.get_config_field('PROPERTIES', 'VALUE_OFFSET'))
def add_offset(self) -> np.float64:
return np.float64(self.get_config_field('PROPERTIES', 'VALUE_OFFSET'))

@property
def value_min(self) -> int:
Expand All @@ -343,8 +343,8 @@ def value_max_packed(self) -> int:
return int((self.value_max - self.add_offset) / self.scale_factor)

@property
def value_nan_packed(self) -> float:
return float((self.VALUE_NAN - self.add_offset) / self.scale_factor)
def value_nan_packed(self) -> np.float64:
return np.float64((self.VALUE_NAN - self.add_offset) / self.scale_factor)

@property
def var_code(self) -> str:
Expand All @@ -355,8 +355,8 @@ def do_height_correction(self) -> bool:
return self.height_correction_factor != 0.0

@property
def height_correction_factor(self) -> np.float32:
return np.float32(self.get_config_field('PROPERTIES', 'HEIGHT_CORRECTION_FACTOR'))
def height_correction_factor(self) -> np.float64:
return np.float64(self.get_config_field('PROPERTIES', 'HEIGHT_CORRECTION_FACTOR'))

@property
def truncate_negative_values(self) -> bool:
Expand Down Expand Up @@ -402,7 +402,7 @@ def __init__(self, conf: Config, quiet_mode: bool = False, use_broadcasting: boo
super().__init__(quiet_mode)
self.conf = conf
self.use_broadcasting = use_broadcasting
self.unit_conversion = np.float32(self.conf.get_config_field('PROPERTIES', 'UNIT_CONVERSION'))
self.unit_conversion = np.float64(self.conf.get_config_field('PROPERTIES', 'UNIT_CONVERSION'))

def correct_height(self, df: pd.DataFrame) -> pd.DataFrame:
if self.conf.do_height_correction:
Expand Down Expand Up @@ -437,7 +437,7 @@ def prepare_grid(self, result: np.ndarray, grid_shape: np.ndarray) -> np.ndarray
result[np.where(result == self.conf.VALUE_NAN)] = np.nan
result[np.where(result < self.conf.value_min)] = np.nan
result[np.where(result > self.conf.value_max)] = np.nan
result = result.astype(np.float32)
result = result.astype(np.float64)
result = np.round(result, 1)
result[~np.isnan(result)] -= self.conf.add_offset
result[~np.isnan(result)] /= self.conf.scale_factor
Expand Down Expand Up @@ -479,9 +479,9 @@ def generate_grid(self, filename: Path) -> np.ndarray:
x = df[self.conf.COLUMN_LON].values
y = df[self.conf.COLUMN_LAT].values
z = df[self.conf.COLUMN_VALUE].values
xp = np.array(x).astype(np.float32)
yp = np.array(y).astype(np.float32)
values = np.array(z).astype(np.float32)
xp = np.array(x).astype(np.float64)
yp = np.array(y).astype(np.float64)
values = np.array(z).astype(np.float64)
df = None
if self.conf.interpolation_mode == 'cdd':
scipy_interpolation = ScipyInterpolation(xp, yp, self.conf.grid_details, values,
Expand Down
14 changes: 7 additions & 7 deletions src/lisfloodutilities/gridding/lib/writers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,16 +231,16 @@ def __setup_netcdf_metadata(self, start_date: datetime = None):

proj = self.nf.createVariable(self.conf.get_config_field('PROJECTION','GRID_MAPPING'), self.NETCDF_COORDINATES_DATA_TYPE)
self.__set_property(proj, 'grid_mapping_name', 'PROJECTION', 'GRID_MAPPING')
self.__set_property(proj, 'false_easting', 'PROJECTION', 'FALSE_EASTING', float)
self.__set_property(proj, 'false_northing', 'PROJECTION', 'FALSE_NORTHING', float)
self.__set_property(proj, 'longitude_of_projection_origin', 'PROJECTION', 'ORIGIN_LONGITUDE', float)
self.__set_property(proj, 'latitude_of_projection_origin', 'PROJECTION', 'ORIGIN_LATITUDE', float)
self.__set_property(proj, 'semi_major_axis', 'PROJECTION', 'SEMI_MAJOR_AXIS', float)
self.__set_property(proj, 'inverse_flattening', 'PROJECTION', 'INVERSE_FLATTENING', float)
self.__set_property(proj, 'false_easting', 'PROJECTION', 'FALSE_EASTING', np.float64)
self.__set_property(proj, 'false_northing', 'PROJECTION', 'FALSE_NORTHING', np.float64)
self.__set_property(proj, 'longitude_of_projection_origin', 'PROJECTION', 'ORIGIN_LONGITUDE', np.float64)
self.__set_property(proj, 'latitude_of_projection_origin', 'PROJECTION', 'ORIGIN_LATITUDE', np.float64)
self.__set_property(proj, 'semi_major_axis', 'PROJECTION', 'SEMI_MAJOR_AXIS', np.float64)
self.__set_property(proj, 'inverse_flattening', 'PROJECTION', 'INVERSE_FLATTENING', np.float64)
self.__set_property(proj, 'proj4_params', 'PROJECTION', 'PARAMS')
self.__set_property(proj, 'EPSG_code', 'PROJECTION', 'EPSG_CODE')
self.__set_property(proj, 'spatial_ref', 'PROJECTION', 'STRING')
# self.__set_property(proj, 'longitude_of_prime_meridian', 'PROJECTION', 'LONGITUDE_PRIME_MERIDIAN', float)
# self.__set_property(proj, 'longitude_of_prime_meridian', 'PROJECTION', 'LONGITUDE_PRIME_MERIDIAN', np.float64)
# self.__set_property(proj, 'GeoTransform', 'PROJECTION', 'GEO_TRANSFORM', self.__get_tuple)

var_data_type_packed = self.conf.get_config_field('PROPERTIES', 'DATA_TYPE_PACKED')
Expand Down

0 comments on commit df668c3

Please sign in to comment.