From 384988d7f45f55af971faf49a7d1e5f2795f1b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20BRAUN?= Date: Tue, 7 Dec 2021 14:56:49 +0100 Subject: [PATCH] - FIX: Fixing flag type for `Sentinel-3` data - FIX: Do not multiply the flags values by the radiance adjustment factor for `Sentinel-3 SLSTR`! - FIX: Fixing flag exception threshold for `Sentinel-3 SLSTR` - FIX: Fixing preprocessed band filenames for `Sentinel-3 SLSTR` --- CHANGES.md | 38 ++++++++++++++----- eoreader/__init__.py | 2 +- eoreader/products/optical/s3_olci_product.py | 6 ++- eoreader/products/optical/s3_product.py | 6 +-- eoreader/products/optical/s3_slstr_product.py | 26 ++++++------- eoreader/products/product.py | 1 + 6 files changed, 50 insertions(+), 29 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a0937cf7..74f3728d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,14 @@ # Release History +## 0.9.2 (2021-12-07) + +- FIX: Fixing flag type for `Sentinel-3` data +- FIX: Do not multiply the flags values by the radiance adjustment factor for `Sentinel-3 SLSTR`! +- FIX: Fixing flag exception threshold for `Sentinel-3 SLSTR` +- FIX: Fixing preprocessed band filenames for `Sentinel-3 SLSTR` + ## 0.9.1 (2021-12-07) + - FIX: `Reader().valid_mtd` now correctly accepts strings instead of only `Platform` objects - FIX: Better handling of `Sentinel-2` product type - FIX: Save bands' new attributes in `str` (to pickle them) @@ -10,8 +18,10 @@ - **ENH: Adding the support of the ICEYE sensor** - **ENH: Adding the support of the COSMO-SkyMed 2nd Generation sensor** -- **ENH: Adding some attributes to bands and stack: `sensor`, `sensor_id`, `product_type`, `acquisition_date`, `condensed_name`** [#7](https://github.com/sertit/eoreader/issues/7) -- **ENH: Replace name by filename and read directly the true name of the product in the metadata** [#15](https://github.com/sertit/eoreader/issues/15) +- **ENH: Adding some attributes to bands and stack: `sensor`, `sensor_id`, `product_type`, `acquisition_date` + , `condensed_name`** [#7](https://github.com/sertit/eoreader/issues/7) +- **ENH: Replace name by filename and read directly the true name of the product in the + metadata** [#15](https://github.com/sertit/eoreader/issues/15) - FIX: `Sentinel-1` metadata file with archived products (discarding RFI folder in its search). - FIX: Add `Quickbird`, `GeoEye` and `WorldView` sensors in `reader` regexes. - FIX: Add scipy in `requirements.txt` and `setup.py` @@ -25,21 +35,29 @@ ## 0.8.0 (2021-10-25) - **BREAKING CHANGE: `crs`, `footprint`, `extent`, `wgs84_extent` are now properties !** -- **BREAKING CHANGE: Removing raw `gdaldem` CLI from EOReader (the `HILLSHADE` and `SLOPE` bands are now slightly different !)** [#10](https://github.com/sertit/eoreader/issues/10) +- **BREAKING CHANGE: Removing raw `gdaldem` CLI from EOReader (the `HILLSHADE` and `SLOPE` bands are now slightly + different !)** [#10](https://github.com/sertit/eoreader/issues/10) - **BREAKING CHANGE: `HILLSHADE` is given in `float32` instead of `uint8`** - **BREAKING CHANGE: `SLOPE` is given in degrees instead of percents** - **ENH: Adding the support of the PAZ SAR sensor** -- **ENH: Adding the support of the Sentinel-2 processed with the [processing baseline 4.0](https://sentinels.copernicus.eu/web/sentinel/-/copernicus-sentinel-2-major-products-upgrade-upcoming)** [#11](https://github.com/sertit/eoreader/issues/11) -- **ENH: Removing SNAP from Sentinel-3 pre-process -> Freeing optical data from SNAP dependency !** [#12](https://github.com/sertit/eoreader/issues/12) +- **ENH: Adding the support of the Sentinel-2 processed with + the [processing baseline 4.0](https://sentinels.copernicus.eu/web/sentinel/-/copernicus-sentinel-2-major-products-upgrade-upcoming)** [#11](https://github.com/sertit/eoreader/issues/11) +- **ENH: Removing SNAP from Sentinel-3 pre-process -> Freeing optical data from SNAP + dependency !** [#12](https://github.com/sertit/eoreader/issues/12) - **ENH: Enabling the use of other S3-SLSTR suffixes than `an` (stripe A at nadir position)** - **ENH: Thermal bands of Sentinel-3 SLSTR can now be used** -- **ENH: All bands of Sentinel-3 SLSTR/OLCI can now be used (`S7`, `F1`, `F2` for SLSTR, `Oaxx` for OLCI)** [#14](https://github.com/sertit/eoreader/issues/14) +- **ENH: All bands of Sentinel-3 SLSTR/OLCI can now be used (`S7`, `F1`, `F2` for SLSTR, `Oaxx` for + OLCI)** [#14](https://github.com/sertit/eoreader/issues/14) - **ENH: `YELLOW` band is mapped to `Oa07` band of Sentinel-3 OLCI** - **ENH: Zipped Sentinel-3 products can now be processed** -- **ENH: Allow the use of `kwargs` in `load`, mainly for `rasters.read` (and allowing ie. radiance adjustment in S3-SLSTR)** -- OPTIM: `crs`, `footprint`, `extent`, `default_transform`, `wgs84_extent` are cached (using `@cached_property`) [#13](https://github.com/sertit/eoreader/issues/13) -- OPTIM: `get_mean_sun_angles` and `default_transform` are now cached (using `@cache`) [#13](https://github.com/sertit/eoreader/issues/13) -- OPTIM: `get_datetime`: Look for the date only if `datetime` attribute is None [#13](https://github.com/sertit/eoreader/issues/13) +- **ENH: Allow the use of `kwargs` in `load`, mainly for `rasters.read` (and allowing ie. radiance adjustment in + S3-SLSTR)** +- OPTIM: `crs`, `footprint`, `extent`, `default_transform`, `wgs84_extent` are cached ( + using `@cached_property`) [#13](https://github.com/sertit/eoreader/issues/13) +- OPTIM: `get_mean_sun_angles` and `default_transform` are now cached ( + using `@cache`) [#13](https://github.com/sertit/eoreader/issues/13) +- OPTIM: `get_datetime`: Look for the date only if `datetime` attribute is + None [#13](https://github.com/sertit/eoreader/issues/13) - OPTIM: Better management of `fspath` for cloud-stored products (download the files only once) - OPTIM: Stop downloading/extracting files if not necessary - FIX: Bands are correctly ordered in stacks diff --git a/eoreader/__init__.py b/eoreader/__init__.py index 37ab985b..129eaf1e 100644 --- a/eoreader/__init__.py +++ b/eoreader/__init__.py @@ -50,7 +50,7 @@ def wrapper(*args, **kwargs): return wrapper -__version__ = "0.9.1" +__version__ = "0.9.2" __title__ = "eoreader" __description__ = ( "Remote-sensing opensource python library reading optical and SAR sensors, " diff --git a/eoreader/products/optical/s3_olci_product.py b/eoreader/products/optical/s3_olci_product.py index dd537b5f..5d85104f 100644 --- a/eoreader/products/optical/s3_olci_product.py +++ b/eoreader/products/optical/s3_olci_product.py @@ -281,7 +281,9 @@ def _preprocess( filename = band # Get raw band - band_arr = self._read_nc(filename, subdataset) + band_arr = self._read_nc( + filename, subdataset, dtype=kwargs.get("dtype", np.float32) + ) # Convert radiance to reflectances if needed # Convert first pixel by pixel before reprojection ! @@ -329,7 +331,7 @@ def _geocode( dst_crs=self.crs, resolution=resolution, gcps=self._gcps, - nodata=self.nodata, + nodata=self._mask_nodata if band_arr.dtype == np.uint8 else self.nodata, num_threads=MAX_CORES, **{"SRC_METHOD": "GCP_TPS"}, ) diff --git a/eoreader/products/optical/s3_product.py b/eoreader/products/optical/s3_product.py index 03f4bae3..8df07541 100644 --- a/eoreader/products/optical/s3_product.py +++ b/eoreader/products/optical/s3_product.py @@ -627,7 +627,7 @@ def _read_mtd(self) -> (etree._Element, dict): return mtd_el, {} def _read_nc( - self, filename: Union[str, obn], subdataset: str = None + self, filename: Union[str, obn], subdataset: str = None, dtype=np.float32 ) -> xr.DataArray: """ Read NetCDF file (as float32) and rescaled them to their true values @@ -701,8 +701,8 @@ def _read_nc( # BUT the xr.DataArray dimensions wont be correctly formatted ! # Align the NetCDF behaviour on rasterio's - # Read as float32 - nc = nc.astype(np.float32) + # Read as float32 (by default) or with given type + nc = nc.astype(dtype) # Add the band dimension if "band" not in nc.dims: diff --git a/eoreader/products/optical/s3_slstr_product.py b/eoreader/products/optical/s3_slstr_product.py index 241bfebd..3f271fbd 100644 --- a/eoreader/products/optical/s3_slstr_product.py +++ b/eoreader/products/optical/s3_slstr_product.py @@ -280,9 +280,9 @@ def _get_preprocessed_band_path( """ res_str = self._resolution_to_str(resolution) if filename.endswith(suffix): - pp_name = f"{self.condensed_name}_{filename}_{suffix}_{res_str}.tif" - else: pp_name = f"{self.condensed_name}_{filename}_{res_str}.tif" + else: + pp_name = f"{self.condensed_name}_{filename}_{suffix}_{res_str}.tif" return self._get_band_folder(writable=writable).joinpath(pp_name) @@ -412,10 +412,12 @@ def _preprocess( ) # Get raw band - band_arr = self._read_nc(filename, subdataset) + band_arr = self._read_nc( + filename, subdataset, dtype=kwargs.get("dtype", np.float32) + ) # Radiance pre process (BT bands are given in BT !) - if band_name in SLSTR_RAD_BANDS: + if not kwargs.get("flags", False) and band_name in SLSTR_RAD_BANDS: # Adjust radiance if needed # Get the user's radiance adjustment if existing rad_adjust = kwargs.get(SLSTR_RAD_ADJUST, self._rad_adjust) @@ -531,7 +533,7 @@ def _geocode( dst_crs=self.crs, resolution=resolution, gcps=self._gcps[suffix], - nodata=self.nodata, + nodata=self._mask_nodata if band_arr.dtype == np.uint8 else self.nodata, num_threads=MAX_CORES, resampling=Resampling.nearest, **{"SRC_METHOD": "GCP_TPS"}, @@ -695,7 +697,6 @@ def _radiance_adjustment( Returns: xr.DataArray: Adjusted band array """ - rad_coeff = None try: band_name = self.band_names[band] if band_name in SLSTR_RAD_BANDS: @@ -705,16 +706,13 @@ def _radiance_adjustment( else: rad_adjust_tuple = rad_adjust - # Get the band coefficient + # Get the band coefficient and multiply the band rad_coeff = getattr(rad_adjust_tuple, f"{band_name}_{view}") + band_arr *= rad_coeff except KeyError: # Not a band (ie Quality Flags) or Brilliance temperature: no adjust needed pass - # Correct if needed - if rad_coeff: - band_arr *= rad_coeff - return band_arr def _compute_sza_img_grid(self, suffix) -> np.ndarray: @@ -796,6 +794,8 @@ def _manage_invalid_pixels( subdataset=self._replace(self._exception_name, band=band, suffix=suffix), resolution=band_arr.rio.resolution(), to_reflectance=False, + flags=True, + dtype=np.uint8, ) # Open flag file @@ -806,8 +806,8 @@ def _manage_invalid_pixels( masked=False, ) - # Set no data for everything that caused an exception - exception = np.where(qual_arr > 2, self._mask_true, self._mask_false) + # Set no data for everything that caused an exception (3 and more) + exception = np.where(qual_arr >= 3, self._mask_true, self._mask_false) # Get nodata mask no_data = np.where(np.isnan(band_arr.data), self._mask_true, self._mask_false) diff --git a/eoreader/products/product.py b/eoreader/products/product.py index 915a147b..87406609 100644 --- a/eoreader/products/product.py +++ b/eoreader/products/product.py @@ -146,6 +146,7 @@ def __init__( # Mask values self._mask_true = 1 self._mask_false = 0 + self._mask_nodata = 255 self.platform = None """Product platform, such as Sentinel-2"""