Merge branch 'pytroll:main' into osi_saf

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 exclude: '^$'
 fail_fast: false
 repos:
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.247'
+    rev: 'v0.1.6'
     hooks:
       - id: ruff
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -19,7 +19,7 @@ repos:
       - id: bandit
         args: [--ini, .bandit]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.6.1'  # Use the sha / tag you want to point at
+    rev: 'v1.7.0'  # Use the sha / tag you want to point at
     hooks:
       - id: mypy
         additional_dependencies:

satpy/etc/readers/avhrr_l1b_eps.yaml

@@ -89,6 +89,7 @@ datasets:
             - latitude
         file_type: avhrr_eps
 
+
     latitude:
         name: latitude
         resolution: 1050
@@ -131,6 +132,13 @@ datasets:
         coordinates: [longitude, latitude]
         file_type: avhrr_eps
 
+    cloud_flags:
+        name: cloud_flags
+        sensor: avhrr-3
+        resolution: 1050
+        coordinates: [longitude, latitude]
+        file_type: avhrr_eps
+
 file_types:
     avhrr_eps:
         file_reader: !!python/name:satpy.readers.eps_l1b.EPSAVHRRFile

satpy/etc/readers/modis_l3.yaml

@@ -0,0 +1,16 @@
+reader:
+  name: modis_l3
+  short_name: MODIS l3
+  long_name: MODIS Level 3 (mcd43) data in HDF-EOS format
+  description: MODIS HDF-EOS L3 Reader
+  status: Beta
+  supports_fsspec: false
+  reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
+  sensors: [modis]
+
+file_types:
+  modis_l3_cmg_hdf:
+    file_patterns:
+    - 'MCD43C{prod_type}.A{start_time:%Y%j}.{collection:03d}.{production_time:%Y%j%H%M%S}.hdf'
+    - 'M{platform_indicator:1s}D09CMG.A{start_time:%Y%j}.{collection:03d}.{production_time:%Y%j%H%M%S}.hdf'
+    file_reader: !!python/name:satpy.readers.modis_l3.ModisL3GriddedHDFFileHandler

satpy/modifiers/angles.py

@@ -138,35 +138,43 @@ def _zarr_pattern(self, arg_hash, cache_version: Union[None, int, str] = None) -
 
     def __call__(self, *args, cache_dir: Optional[str] = None) -> Any:
         """Call the decorated function."""
-        new_args = self._sanitize_args_func(*args) if self._sanitize_args_func is not None else args
-        arg_hash = _hash_args(*new_args, unhashable_types=self._uncacheable_arg_types)
-        should_cache, cache_dir = self._get_should_cache_and_cache_dir(new_args, cache_dir)
-        zarr_fn = self._zarr_pattern(arg_hash)
-        zarr_format = os.path.join(cache_dir, zarr_fn)
-        zarr_paths = glob(zarr_format.format("*"))
-        if not should_cache or not zarr_paths:
-            # use sanitized arguments if we are caching, otherwise use original arguments
-            args_to_use = new_args if should_cache else args
-            res = self._func(*args_to_use)
-            if should_cache and not zarr_paths:
-                self._warn_if_irregular_input_chunks(args, args_to_use)
-                self._cache_results(res, zarr_format)
-        # if we did any caching, let's load from the zarr files
-        if should_cache:
-            # re-calculate the cached paths
-            zarr_paths = sorted(glob(zarr_format.format("*")))
-            if not zarr_paths:
-                raise RuntimeError("Data was cached to disk but no files were found")
-            new_chunks = _get_output_chunks_from_func_arguments(args)
-            res = tuple(da.from_zarr(zarr_path, chunks=new_chunks) for zarr_path in zarr_paths)
+        should_cache: bool = satpy.config.get(self._cache_config_key, False)
+        if not should_cache:
+            return self._func(*args)
+
+        try:
+            return self._cache_and_read(args, cache_dir)
+        except TypeError as err:
+            warnings.warn("Cannot cache function because of unhashable argument: " + str(err), stacklevel=2)
+            return self._func(*args)
+
+    def _cache_and_read(self, args, cache_dir):
+        sanitized_args = self._sanitize_args_func(*args) if self._sanitize_args_func is not None else args
+
+        zarr_file_pattern = self._get_zarr_file_pattern(sanitized_args, cache_dir)
+        zarr_paths = glob(zarr_file_pattern.format("*"))
+
+        if not zarr_paths:
+            # use sanitized arguments
+            self._warn_if_irregular_input_chunks(args, sanitized_args)
+            res_to_cache = self._func(*(sanitized_args))
+            self._cache_results(res_to_cache, zarr_file_pattern)
+
+        # if we did any caching, let's load from the zarr files, so that future calls have the same name
+        # re-calculate the cached paths
+        zarr_paths = sorted(glob(zarr_file_pattern.format("*")))
+        if not zarr_paths:
+            raise RuntimeError("Data was cached to disk but no files were found")
+
+        new_chunks = _get_output_chunks_from_func_arguments(args)
+        res = tuple(da.from_zarr(zarr_path, chunks=new_chunks) for zarr_path in zarr_paths)
         return res
 
-    def _get_should_cache_and_cache_dir(self, args, cache_dir: Optional[str]) -> tuple[bool, str]:
-        should_cache: bool = satpy.config.get(self._cache_config_key, False)
-        can_cache = not any(isinstance(arg, self._uncacheable_arg_types) for arg in args)
-        should_cache = should_cache and can_cache
+    def _get_zarr_file_pattern(self, sanitized_args, cache_dir):
+        arg_hash = _hash_args(*sanitized_args, unhashable_types=self._uncacheable_arg_types)
+        zarr_filename = self._zarr_pattern(arg_hash)
         cache_dir = self._get_cache_dir_from_config(cache_dir)
-        return should_cache, cache_dir
+        return os.path.join(cache_dir, zarr_filename)
 
     @staticmethod
     def _get_cache_dir_from_config(cache_dir: Optional[str]) -> str:
@@ -189,14 +197,14 @@ def _warn_if_irregular_input_chunks(args, modified_args):
                 stacklevel=3
             )
 
-    def _cache_results(self, res, zarr_format):
-        os.makedirs(os.path.dirname(zarr_format), exist_ok=True)
+    def _cache_results(self, res, zarr_file_pattern):
+        os.makedirs(os.path.dirname(zarr_file_pattern), exist_ok=True)
         new_res = []
         for idx, sub_res in enumerate(res):
             if not isinstance(sub_res, da.Array):
                 raise ValueError("Zarr caching currently only supports dask "
                                  f"arrays. Got {type(sub_res)}")
-            zarr_path = zarr_format.format(idx)
+            zarr_path = zarr_file_pattern.format(idx)
             # See https://github.com/dask/dask/issues/8380
             with dask.config.set({"optimization.fuse.active": False}):
                 new_sub_res = sub_res.to_zarr(zarr_path, compute=False)
@@ -252,7 +260,7 @@ def _hash_args(*args, unhashable_types=DEFAULT_UNCACHE_TYPES):
     hashable_args = []
     for arg in args:
         if isinstance(arg, unhashable_types):
-            continue
+            raise TypeError(f"Unhashable type ({type(arg)}).")
         if isinstance(arg, HASHABLE_GEOMETRIES):
             arg = hash(arg)
         elif isinstance(arg, datetime):

satpy/readers/eps_l1b.py

@@ -287,19 +287,9 @@ def get_dataset(self, key, info):
         if self.sections is None:
             self._read_all()
 
-        if key["name"] in ["longitude", "latitude"]:
-            lons, lats = self.get_full_lonlats()
-            if key["name"] == "longitude":
-                dataset = create_xarray(lons)
-            else:
-                dataset = create_xarray(lats)
-
-        elif key["name"] in ["solar_zenith_angle", "solar_azimuth_angle",
-                             "satellite_zenith_angle", "satellite_azimuth_angle"]:
-            dataset = self._get_angle_dataarray(key)
-        elif key["name"] in ["1", "2", "3a", "3A", "3b", "3B", "4", "5"]:
-            dataset = self._get_calibrated_dataarray(key)
-        else:
+        try:
+            dataset = self._get_data_array(key)
+        except KeyError:
             logger.info("Can't load channel in eps_l1b: " + str(key["name"]))
             return
 
@@ -311,18 +301,32 @@ def get_dataset(self, key, info):
         dataset.attrs.update(key.to_dict())
         return dataset
 
+    def _get_data_array(self, key):
+        name = key["name"]
+        if name in ["longitude", "latitude"]:
+            data = self.get_full_lonlats()[int(name == "latitude")]
+            dataset = create_xarray(data)
+        elif name in ["solar_zenith_angle", "solar_azimuth_angle", "satellite_zenith_angle", "satellite_azimuth_angle"]:
+            dataset = self._get_angle_dataarray(key)
+        elif name in ["1", "2", "3a", "3A", "3b", "3B", "4", "5"]:
+            dataset = self._get_calibrated_dataarray(key)
+        elif name == "cloud_flags":
+            array = self["CLOUD_INFORMATION"]
+            dataset = create_xarray(array)
+        else:
+            raise KeyError(f"Unknown channel: {name}")
+        return dataset
+
     def _get_angle_dataarray(self, key):
         """Get an angle dataarray."""
-        sun_azi, sun_zen, sat_azi, sat_zen = self.get_full_angles()
-        if key["name"] == "solar_zenith_angle":
-            dataset = create_xarray(sun_zen)
-        elif key["name"] == "solar_azimuth_angle":
-            dataset = create_xarray(sun_azi)
-        if key["name"] == "satellite_zenith_angle":
-            dataset = create_xarray(sat_zen)
-        elif key["name"] == "satellite_azimuth_angle":
-            dataset = create_xarray(sat_azi)
-        return dataset
+        arr_index = {
+            "solar_azimuth_angle": 0,
+            "solar_zenith_angle": 1,
+            "satellite_azimuth_angle": 2,
+            "satellite_zenith_angle": 3,
+        }[key["name"]]
+        data = self.get_full_angles()[arr_index]
+        return create_xarray(data)
 
     @cached_property
     def three_a_mask(self):

satpy/readers/fci_l1c_nc.py

@@ -330,13 +330,13 @@ def _get_dataset_measurand(self, key, info=None):
 
         fv = attrs.pop(
             "FillValue",
-            default_fillvals.get(data.dtype.str[1:], np.nan))
-        vr = attrs.get("valid_range", [-np.inf, np.inf])
+            default_fillvals.get(data.dtype.str[1:], np.float32(np.nan)))
+        vr = attrs.get("valid_range", [np.float32(-np.inf), np.float32(np.inf)])
         if key["calibration"] == "counts":
             attrs["_FillValue"] = fv
             nfv = data.dtype.type(fv)
         else:
-            nfv = np.nan
+            nfv = np.float32(np.nan)
         data = data.where((data >= vr[0]) & (data <= vr[1]), nfv)
 
         res = self.calibrate(data, key)
@@ -632,16 +632,15 @@ def calibrate_counts_to_rad(self, data, key):
     def calibrate_rad_to_bt(self, radiance, key):
         """IR channel calibration."""
         # using the method from PUG section Converting from Effective Radiance to Brightness Temperature for IR Channels
-
         measured = self.get_channel_measured_group_path(key["name"])
 
-        vc = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_wavenumber")
+        vc = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_wavenumber").astype(np.float32)
 
-        a = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_a")
-        b = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_b")
+        a = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_a").astype(np.float32)
+        b = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_b").astype(np.float32)
 
-        c1 = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_constant_c1")
-        c2 = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_constant_c2")
+        c1 = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_constant_c1").astype(np.float32)
+        c2 = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_constant_c2").astype(np.float32)
 
         for v in (vc, a, b, c1, c2):
             if v == v.attrs.get("FillValue",
@@ -652,26 +651,27 @@ def calibrate_rad_to_bt(self, radiance, key):
                         v.attrs.get("long_name",
                                     "at least one necessary coefficient"),
                         measured))
-                return radiance * np.nan
+                return radiance * np.float32(np.nan)
 
         nom = c2 * vc
-        denom = a * np.log(1 + (c1 * vc ** 3) / radiance)
+        denom = a * np.log(1 + (c1 * vc ** np.float32(3.)) / radiance)
 
         res = nom / denom - b / a
+
         return res
 
     def calibrate_rad_to_refl(self, radiance, key):
         """VIS channel calibration."""
         measured = self.get_channel_measured_group_path(key["name"])
 
-        cesi = self.get_and_cache_npxr(measured + "/channel_effective_solar_irradiance")
+        cesi = self.get_and_cache_npxr(measured + "/channel_effective_solar_irradiance").astype(np.float32)
 
         if cesi == cesi.attrs.get(
                 "FillValue", default_fillvals.get(cesi.dtype.str[1:])):
             logger.error(
                 "channel effective solar irradiance set to fill value, "
                 "cannot produce reflectance for {:s}.".format(measured))
-            return radiance * np.nan
+            return radiance * np.float32(np.nan)
 
         sun_earth_distance = np.mean(
             self.get_and_cache_npxr("state/celestial/earth_sun_distance")) / 149597870.7  # [AU]
@@ -683,7 +683,7 @@ def calibrate_rad_to_refl(self, radiance, key):
                         "".format(sun_earth_distance))
             sun_earth_distance = 1
 
-        res = 100 * radiance * np.pi * sun_earth_distance ** 2 / cesi
+        res = 100 * radiance * np.float32(np.pi) * np.float32(sun_earth_distance) ** np.float32(2) / cesi
         return res
 
 

satpy/readers/hdfeos_base.py

@@ -148,7 +148,7 @@ def _read_mda(cls, lines, element=None):
 
     @classmethod
     def _split_line(cls, line, lines):
-        key, val = line.split("=")
+        key, val = line.split("=", maxsplit=1)
         key = key.strip()
         val = val.strip()
         try: