Share FCI L1c metadata between segments

satpy/readers/fci_l1c_nc.py

@@ -158,6 +158,15 @@
                                        "grid_width": 5568},
                      }
 
+NONSHAREABLE_VARIABLE_ENDINGS = [
+    "index",
+    "time",
+    "measured/effective_radiance",
+    "measured/y",
+    "position_row",
+    "index_map",
+    "pixel_quality"]
+
 
 def _get_aux_data_name_from_dsname(dsname):
     aux_data_name = [key for key in AUX_DATA.keys() if key in dsname]
@@ -700,6 +709,31 @@
         res = 100 * radiance * np.float32(np.pi) * np.float32(sun_earth_distance) ** np.float32(2) / cesi
         return res
 
+    def _collect_listed_variables(self, file_handle, listed_variables, filetype_info):
+        listed_variables = self._collect_from_inter_segment_cache(listed_variables, filetype_info)
+        super()._collect_listed_variables(file_handle, listed_variables, filetype_info)
+        self._store_shared_info(filetype_info)
+
+    def _collect_from_inter_segment_cache(self, listed_variables, filetype_info):
+        if "shared_info" in filetype_info:
+            shared_info = filetype_info["shared_info"]
+            for key in shared_info:
+                self.file_content[key] = shared_info[key]
+                try:
+                    listed_variables.remove(key)
+                except ValueError:
+                    pass
+        return listed_variables
+
+    def _store_shared_info(self, filetype_info):
+        if "shared_info" not in filetype_info:
+            shared_info = {}
+            for key in self.file_content:
+                if any(key.endswith(k) for k in NONSHAREABLE_VARIABLE_ENDINGS):
+                    continue
+                shared_info[key] = self.file_content[key]
+            filetype_info["shared_info"] = shared_info
+
 
 def _ensure_dataarray(arr):
     if not isinstance(arr, xr.DataArray):

satpy/readers/netcdf_utils.py

@@ -111,7 +111,9 @@ def __init__(self, filename, filename_info, filetype_info,
 
         listed_variables = filetype_info.get("required_netcdf_variables")
         if listed_variables:
-            self._collect_listed_variables(file_handle, listed_variables)
+            variable_name_replacements = self.filetype_info.get("variable_name_replacements")
+            listed_variables = self._get_required_variable_names(listed_variables, variable_name_replacements)
+            self._collect_listed_variables(file_handle, listed_variables, filetype_info)
         else:
             self.collect_metadata("", file_handle)
             self.collect_dimensions("", file_handle)
@@ -168,9 +170,8 @@ def _collect_variable_info(self, var_name, var_obj):
         self.file_content[var_name + "/dimensions"] = var_obj.dimensions
         self._collect_attrs(var_name, var_obj)
 
-    def _collect_listed_variables(self, file_handle, listed_variables):
-        variable_name_replacements = self.filetype_info.get("variable_name_replacements")
-        for itm in self._get_required_variable_names(listed_variables, variable_name_replacements):
+    def _collect_listed_variables(self, file_handle, listed_variables, filetype_info):
+        for itm in listed_variables:
             parts = itm.split("/")
             grp = file_handle
             for p in parts[:-1]:

satpy/tests/reader_tests/test_fci_l1c_nc.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019 Satpy developers
 #
@@ -921,3 +921,81 @@
                                                  np.array([-5568000.227139, -5368000.221262,
                                                            5568000.100073, -5568000.227139]),
                                                  decimal=2)
+
+
+def _mock_np2str(val):
+    return val
+
+
+def parse_fdhsi_filename_info(filename):
+    """Parse filename info from the filename."""
+    from trollsift import parse
+
+    fmt = ("{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-"
+           "{coverage}-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_"
+           "{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_"
+           "{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc")
+    filename_info = parse(fmt, filename)
+    return filename_info
+
+
+def test_reusing_shared_segment_info():
+    """Test that certain data are reused after the first segment has been handled.
+
+    This is the case where something has been already handled and stored to filetype_info["shared_info"] dict.
+    """
+    filename = TEST_FILENAMES["fdhsi"][0]
+    filename_info = parse_fdhsi_filename_info(filename)
+    filetype_info = {
+        "shared_info": {
+            "attr/platform": "mock-satellite"
+        },
+        "required_netcdf_variables": ["attr/platform"]
+    }
+
+    with mock.patch("satpy.readers.netcdf_utils.NetCDF4FsspecFileHandler._get_file_handle"):
+        with mock.patch("satpy.readers.netcdf_utils.np2str", _mock_np2str):
+            fh_ = FCIL1cNCFileHandler(filename, filename_info, filetype_info)
+
+    assert fh_.file_content["attr/platform"] == filetype_info["shared_info"]["attr/platform"]
+
+
+def test_store_shared_segment_info():
+    """Test that certain data are collected to filetype_info["shared_info"] from the first segment."""
+    filename = TEST_FILENAMES["fdhsi"][0]
+    filename_info = parse_fdhsi_filename_info(filename)
+    filetype_info = {"required_netcdf_variables": ["attr/platform"]}
+
+    with mock.patch("satpy.readers.netcdf_utils.NetCDF4FsspecFileHandler._get_file_handle") as get_file_handle:
+        get_file_handle.return_value.platform = "mock-satellite"
+        with mock.patch("satpy.readers.netcdf_utils.np2str", _mock_np2str):
+            _ = FCIL1cNCFileHandler(filename, filename_info, filetype_info)
+
+    assert filetype_info["shared_info"]["attr/platform"] == "mock-satellite"
+
+
+def test_store_shared_segment_info_ignore_nonshareable():
+    """Test that nonshareable segment info are not collected for sharing."""
+    variables = [
+        "data/ir105/measured/effective_radiance",
+        "data/ir105/measured/y",
+        "data/ir105/measured/end_position_row",
+        "data/ir105/measured/start_position_row",
+        "data/ir105/measured/index_map",
+        "data/ir105/measured/pixel_quality",
+        "index",
+        "time",
+    ]
+
+    filename = TEST_FILENAMES["fdhsi"][0]
+    filename_info = parse_fdhsi_filename_info(filename)
+    filetype_info = {"required_netcdf_variables": variables}
+
+    with mock.patch("satpy.readers.netcdf_utils.NetCDF4FsspecFileHandler._get_file_handle") as get_file_handle:
+        for itm in variables:
+            setattr(get_file_handle.return_value, itm, "mock-value")
+        with mock.patch("satpy.readers.netcdf_utils.np2str", _mock_np2str):
+            _ = FCIL1cNCFileHandler(filename, filename_info, filetype_info)
+
+    for itm in variables:
+        assert itm not in filetype_info["shared_info"]