From e9732c8b0cad54c06486193e0e01d6b8f67cfd7e Mon Sep 17 00:00:00 2001 From: clement laplace Date: Fri, 22 Nov 2024 10:10:46 +0000 Subject: [PATCH 1/4] fix : Makes sure that the apply_fill_value method is not applyed if the _FillValue is not into the attributes --- satpy/readers/li_base_nc.py | 5 +++-- satpy/tests/reader_tests/test_li_l2_nc.py | 5 ----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/satpy/readers/li_base_nc.py b/satpy/readers/li_base_nc.py index cefbcc7e55..1d1e56a850 100644 --- a/satpy/readers/li_base_nc.py +++ b/satpy/readers/li_base_nc.py @@ -439,11 +439,12 @@ def get_measured_variable(self, var_paths, fill_value=np.nan): # Also handle fill value here (but only if it is not None, so that we can still bypass this # step if needed) arr = self.apply_fill_value(arr, fill_value) - return arr def apply_fill_value(self, arr, fill_value): - """Apply fill values, unless it is None.""" + """Apply fill values, unless it is None and when _FillValue is provided.""" + if arr.attrs.get("_FillValue") is None: + return arr if fill_value is not None: if np.isnan(fill_value): fill_value = np.float32(np.nan) diff --git a/satpy/tests/reader_tests/test_li_l2_nc.py b/satpy/tests/reader_tests/test_li_l2_nc.py index 36cc930683..04f11acf4c 100644 --- a/satpy/tests/reader_tests/test_li_l2_nc.py +++ b/satpy/tests/reader_tests/test_li_l2_nc.py @@ -601,7 +601,6 @@ def test_coords_generation(self, filetype_infos): products = ["li_l2_af_nc", "li_l2_afr_nc", "li_l2_afa_nc"] - for prod in products: handler = LIL2NCFileHandler("filename", {}, extract_filetype_info(filetype_infos, prod)) @@ -611,7 +610,6 @@ def test_coords_generation(self, filetype_infos): elevation = handler.get_measured_variable(handler.swath_coordinates["elevation"]) elevation = handler.apply_use_rescaling(elevation) - # Initialize proj_dict proj_var = handler.swath_coordinates["projection"] geos_proj = handler.get_measured_variable(proj_var, fill_value=None) @@ -634,9 +632,6 @@ def test_coords_generation(self, filetype_infos): elevation_vals = elevation.values * point_height azimuth_vals *= -1 lon_ref, lat_ref = projection(azimuth_vals, elevation_vals, inverse=True) - # Convert to float32: - lon_ref = lon_ref.astype(np.float32) - lat_ref = lat_ref.astype(np.float32) handler.generate_coords_from_scan_angles() lon = handler.internal_variables["longitude"].values From c48d54f4ecad1c8d311e6872f9ba3c74d9638e8e Mon Sep 17 00:00:00 2001 From: clement laplace Date: Thu, 5 Dec 2024 16:14:55 +0000 Subject: [PATCH 2/4] feat : Converts computed latitude and longitude to a float32 and add tests to check the dtype improve the mock datas --- satpy/readers/li_base_nc.py | 15 +- satpy/readers/li_l2_nc.py | 4 +- satpy/tests/reader_tests/_li_test_utils.py | 406 +++++++++++++++++---- satpy/tests/reader_tests/test_li_l2_nc.py | 38 +- 4 files changed, 362 insertions(+), 101 deletions(-) diff --git a/satpy/readers/li_base_nc.py b/satpy/readers/li_base_nc.py index 1d1e56a850..fe998a3b84 100644 --- a/satpy/readers/li_base_nc.py +++ b/satpy/readers/li_base_nc.py @@ -306,7 +306,7 @@ def get_projection_config(self): """Retrieve the projection configuration details.""" # We retrieve the projection variable name directly from our swath settings: proj_var = self.swath_coordinates["projection"] - + logger.error(proj_var) geos_proj = self.get_measured_variable(proj_var, fill_value=None) # cast projection attributes to float/str: major_axis = float(geos_proj.attrs["semi_major_axis"]) @@ -355,9 +355,9 @@ def generate_coords_from_scan_angles(self): # Finally, we should store those arrays as internal variables for later retrieval as # standard datasets: self.internal_variables[lon_name] = xr.DataArray( - da.asarray(lon), dims=["y"], attrs={"standard_name": "longitude"}) + da.asarray(lon), dims=["y"], attrs={"standard_name": "longitude"}).astype(np.float32) self.internal_variables[lat_name] = xr.DataArray( - da.asarray(lat), dims=["y"], attrs={"standard_name": "latitude"}) + da.asarray(lat), dims=["y"], attrs={"standard_name": "latitude"}).astype(np.float32) def inverse_projection(self, azimuth, elevation, proj_dict): """Compute inverse projection.""" @@ -442,10 +442,8 @@ def get_measured_variable(self, var_paths, fill_value=np.nan): return arr def apply_fill_value(self, arr, fill_value): - """Apply fill values, unless it is None and when _FillValue is provided.""" - if arr.attrs.get("_FillValue") is None: - return arr - if fill_value is not None: + """Apply fill values, unless it is None and when _FillValue is provided in the array attributes.""" + if fill_value is not None and arr.attrs.get("_FillValue") is not None: if np.isnan(fill_value): fill_value = np.float32(np.nan) arr = arr.where(arr != arr.attrs.get("_FillValue"), fill_value) @@ -598,9 +596,7 @@ def apply_use_rescaling(self, data_array, ds_info=None): # TODO remove scaling_factor fallback after issue in NetCDF is fixed scale_factor = attribs.setdefault("scale_factor", attribs.get("scaling_factor", 1)) add_offset = attribs.setdefault("add_offset", 0) - data_array = (data_array * scale_factor) + add_offset - # rescale the valid range accordingly if "valid_range" in attribs.keys(): attribs["valid_range"] = attribs["valid_range"] * scale_factor + add_offset @@ -743,7 +739,6 @@ def get_dataset(self, dataset_id, ds_info=None): # Retrieve default infos if missing: if ds_info is None: ds_info = self.get_dataset_infos(dataset_id["name"]) - # check for potential error: if ds_info is None: raise KeyError(f"No dataset registered for {dataset_id}") diff --git a/satpy/readers/li_l2_nc.py b/satpy/readers/li_l2_nc.py index 587039fa46..72722f034e 100644 --- a/satpy/readers/li_l2_nc.py +++ b/satpy/readers/li_l2_nc.py @@ -158,11 +158,9 @@ def get_array_on_fci_grid(self, data_array: xr.DataArray): data_2d = da.map_blocks(_np_add_at_wrapper, data_2d, (rows, cols), data_array, dtype=data_array.dtype, chunks=(LI_GRID_SHAPE[0], LI_GRID_SHAPE[1])) - data_2d = da.where(data_2d > 0, data_2d, np.nan) - + data_2d = da.where(data_2d > 0, data_2d, np.float32(np.nan)).astype(np.float32) xarr = xr.DataArray(da.asarray(data_2d, CHUNK_SIZE), dims=("y", "x")) xarr.attrs = attrs - return xarr diff --git a/satpy/tests/reader_tests/_li_test_utils.py b/satpy/tests/reader_tests/_li_test_utils.py index 5ac9730dee..05a2224d3a 100644 --- a/satpy/tests/reader_tests/_li_test_utils.py +++ b/satpy/tests/reader_tests/_li_test_utils.py @@ -36,6 +36,8 @@ "f8": np.float64, } +def rand_type(num,dtype): + return RANDOM_GEN.integers(low=np.iinfo(dtype).min, high=np.iinfo(dtype).max - 1, size=num, dtype=dtype) def l2_le_schema(settings=None): """Define schema for LI L2 LE product.""" @@ -44,9 +46,6 @@ def l2_le_schema(settings=None): nchunks = settings.get("num_chunks", 23) nfilters = settings.get("num_filters", 2) - def rand_u16(num): - return RANDOM_GEN.integers(low=0, high=np.iinfo(np.uint16).max - 1, size=num, dtype=np.uint16) - return { "providers": settings.get("providers", {}), "variable_path": settings.get("variable_path", "data/"), @@ -60,39 +59,34 @@ def rand_u16(num): "variables": {}, "sector_variables": { "event_id": { - "format": "u2", + "format": "u4", "shape": ("unfiltered_events",), - "fill_value": 65535, "long_name": "ID of LI L2 Event", - "default_data": lambda: rand_u16(nobs) + "default_data": lambda: rand_type(nobs,np.uint32) }, "group_id": { - "format": "u2", + "format": "u4", "shape": ("unfiltered_events",), - "fill_value": 65535, "long_name": "ID of associated LI L2 Group object", - "default_data": lambda: rand_u16(nobs) + "default_data": lambda: rand_type(nobs,np.uint32) }, "l1b_chunk_ids": { "format": "u4", "shape": ("l1b_chunks",), - "fill_value": 4294967295, "long_name": "Array of L1b event chunk IDs", - "default_data": lambda: np.arange(nchunks) + 10000 + "default_data": lambda: rand_type(nchunks,np.uint32) }, "l1b_chunk_offsets": { "format": "u4", "shape": ("l1b_offsets",), - "fill_value": 4294967295, "long_name": "Array offset for L1b event chunk boundaries", "default_data": lambda: np.arange(nchunks) }, "l1b_window": { "format": "u4", "shape": ("unfiltered_events",), - "fill_value": 4294967295, "long_name": "window index of associated L1b event", - "default_data": lambda: (np.arange(nobs) + 10000) + "default_data": lambda: rand_type(nobs,np.uint32) }, "filter_values": { "format": "u1", @@ -101,12 +95,11 @@ def rand_u16(num): "scale_factor": 0.004, "add_offset": 0.0, "long_name": "L2 filter results", - "default_data": lambda: RANDOM_GEN.integers(low=0, high=255, size=(nobs, nfilters), dtype=np.uint8) + "default_data": lambda: rand_type((nobs,nfilters),np.uint8) }, "epoch_time": { "format": "f8", "shape": ("scalar",), - "fill_value": 9.96920996886869e36, "long_name": "Start time of integration frame", "default_data": lambda: 1.234, "precision": "1 millisecond", @@ -122,6 +115,32 @@ def rand_u16(num): "default_data": lambda: np.linspace(0.0, 1000.0, nobs), "units": "seconds", }, + "detector": { + "format": "u1", + "fill_value": 255, + "shape": ("scalar",), + "long_name": "ID of detector for this group", + "default_data": lambda: 1, + "meanings": "1 = detector_1, 2 = detector_2, 3 = detector_3, 4 = detector_4", + }, + "l1b_filter_qa": { + "format": "u1", + "fill_value": 255, + "shape": ("unfiltered_events",), + "add_offset" : 0.0, + "scale_offset":0.004, + "long_name": "L1b event confidence", + "default_data": lambda : rand_type(nobs,np.uint8), + }, + "l2_group_filter_qa": { + "format": "u1", + "fill_value": 255, + "shape": ("unfiltered_events",), + "add_offset" : 0.0, + "scale_offset":0.004, + "long_name": "L2 group confidence", + "default_data": lambda: (np.arange(nobs) + 10000), + }, } } @@ -146,17 +165,20 @@ def l2_lef_schema(settings=None): "variables": { "l1b_geolocation_warning": { "format": "i1", - "shape": (), # test explicitly the scalar case + "shape": ("scalar",), # test explicitly the scalar case + "long_name": "L1b event geolocation warning", "default_data": lambda: 0 }, "l1b_missing_warning": { "format": "i1", "shape": ("scalar",), + "long_name": "Expected L1b inputs missing", "default_data": lambda: 0 }, "l1b_radiometric_warning": { "format": "i1", "shape": ("scalar",), + "long_name": "L1b event radiometric warning", "default_data": lambda: 0 }, }, @@ -164,29 +186,27 @@ def l2_lef_schema(settings=None): "event_id": { "format": "u4", "shape": ("events",), - "fill_value": 65535, "long_name": "ID of LI L2 Event", "default_data": lambda: np.arange(1, nobs + 1) }, "group_id": { "format": "u4", "shape": ("events",), - "fill_value": 65535, "long_name": "ID of associated LI L2 Group object", "default_data": lambda: np.arange(1, nobs + 1) }, "flash_id": { "format": "u4", "shape": ("events",), - "fill_value": 65535, "long_name": "ID of associated LI L2 Flash object", "default_data": lambda: np.arange(1, nobs + 1) }, "detector": { - "format": "u4", + "format": "u1", "shape": ("scalar",), - "fill_value": 65535, + "fill_value": 255, "long_name": "ID of detector for this group", + "meaning": "1 = detector_1, 2 = detector_2, 3 = detector_3, 4 = detector_4", "default_data": lambda: 1 }, "latitude": { @@ -216,8 +236,9 @@ def l2_lef_schema(settings=None): "default_data": lambda: np.clip(np.round(RANDOM_GEN.normal(500, 100, nobs)), 1, 2 ** 16 - 1) }, "event_filter_qa": { - "format": "u1", + "format": "u2", "shape": ("events",), + "fill_value": 255, "long_name": "L2 event pre-filtering quality assurance value", "default_data": lambda: RANDOM_GEN.integers(1, 2 ** 8 - 1, nobs) }, @@ -238,6 +259,7 @@ def l2_lef_schema(settings=None): "detector_row": { "format": "u2", "shape": ("events",), + "fill_value": 65535, "long_name": "Detector row position of event pixel", "units": "1", "default_data": lambda: RANDOM_GEN.integers(1, 1000, nobs) @@ -245,6 +267,7 @@ def l2_lef_schema(settings=None): "detector_column": { "format": "u2", "shape": ("events",), + "fill_value": 65535, "long_name": "Detector column position of event pixel", "units": "1", "default_data": lambda: RANDOM_GEN.integers(1, 1000, nobs) @@ -263,22 +286,84 @@ def l2_lgr_schema(settings=None): "variable_path": settings.get("variable_path", ""), "dimensions": { "groups": ngrps, + "scalar": 1, }, "variables": { + "group_time": { + "format": "f8", + "shape": ("groups",), + "long_name": "Start time of integration frame", + "standard_name": "time", + "units": "seconds since 2000-01-01 00:00:00.0", + "precision": "0.001", + "time_standard": "UTC", + "default_data": lambda: np.linspace(-90, 90, ngrps) + }, "latitude": { - "format": "f4", + "format": "i2", "shape": ("groups",), "long_name": "Latitude of group", "units": "degrees_north", + "fill_value":-32767, "default_data": lambda: np.linspace(-90, 90, ngrps) }, "longitude": { - "format": "f4", + "format": "i2", "shape": ("groups",), "long_name": "Longitude of group", + "fill_value":-32767, "units": "degrees_east", "default_data": lambda: np.linspace(-180, 80, ngrps) }, + "radiance": { + "format": "u2", + "shape": ("groups",), + "long_name": "Radiance of group", + "fill_value": 65535, + "units": "mW.m-2.sr-1", + "scale_factor":0.5, + "add_offset": 0.0, + "default_data": lambda: rand_type(ngrps,np.uint16) + }, + "group_id": { + "format": "u4", + "shape": ("groups",), + "long_name": "LI L2 group IDs", + "default_data": lambda: np.linspace(-180, 80, ngrps) + }, + "flash_id": { + "format": "u4", + "shape": ("groups",), + "long_name": "ID of associated LI L2 Flash object with each group", + "default_data": lambda: np.linspace(-180, 80, ngrps) + }, + "number_of_events": { + "format": "u2", + "shape": ("groups",), + "long_name": "Number of events in each group", + "default_data": lambda: np.linspace(-180, 80, ngrps) + }, + "group_filter_qa": { + "format": "u1", + "shape": ("groups",), + "fill_value": 255, + "long_name": "L2 filtered group quality assurance value", + "add_offset": 0.0, + "scale_factor": 0.004, + "default_data": lambda: np.linspace(-180, 80, ngrps) + }, + "l1b_geolocation_warning": { + "format": "i1", + "shape": ("scalar",), # test explicitly the scalar case + "long_name": "L1b event geolocation warning", + "default_data": lambda: 0 + }, + "l1b_radiometric_warning": { + "format": "i1", + "shape": ("scalar",), + "long_name": "L1b event radiometric warning", + "default_data": lambda: 0 + }, } } @@ -297,6 +382,7 @@ def l2_lfl_schema(settings=None): "variable_path": settings.get("variable_path", ""), "dimensions": { "flashes": nobs, + "scalar":1, }, "variables": { "latitude": { @@ -305,6 +391,7 @@ def l2_lfl_schema(settings=None): "long_name": "Latitude of Flash", "standard_name": "latitude", "units": "degrees_north", + "fill_value": -32767, "add_offset": 0.0, "scale_factor": 0.0027, # Note: using a default range of [-88.3deg, 88.3deg] to stay in @@ -317,6 +404,7 @@ def l2_lfl_schema(settings=None): "long_name": "Longitude of Flash", "standard_name": "longitude", "units": "degrees_east", + "fill_value": -32767, "add_offset": 0.0, "scale_factor": 0.0027, # Note: using a default range of [-88.3deg, 88.3deg] to stay in @@ -326,6 +414,7 @@ def l2_lfl_schema(settings=None): "radiance": { "format": "u2", "shape": ("flashes",), + "fill_value" : 65535, "long_name": "Radiance of Flash", "standard_name": "radiance", "units": "mW.m-2.sr-1", @@ -340,8 +429,9 @@ def l2_lfl_schema(settings=None): "default_data": lambda: np.linspace(0, 1000, nobs) }, "flash_filter_confidence": { - "format": "i1", + "format": "u1", "shape": ("flashes",), + "fill_value": 255, "long_name": "L2 filtered flash confidence", "standard_name": "flash_filter_confidence", "default_data": lambda: np.clip(np.round(RANDOM_GEN.normal(20, 10, nobs)), 1, 2 ** 7 - 1) @@ -372,13 +462,13 @@ def l2_lfl_schema(settings=None): }, "l1b_geolocation_warning": { "format": "i1", - "shape": ("flashes",), + "shape": ("scalar",), "long_name": "L1b geolocation warning", "default_data": lambda: -127 }, "l1b_radiometric_warning": { "format": "i1", - "shape": ("flashes",), + "shape": ("scalar",), "long_name": "L1b radiometric warning", "default_data": lambda: -127 }, @@ -386,13 +476,13 @@ def l2_lfl_schema(settings=None): "format": "u2", "shape": ("flashes",), "long_name": "Number of events in each flash", - "default_data": lambda: 1 + "default_data": lambda: rand_type(nobs,np.uint16) }, "number_of_groups": { - "format": "u4", + "format": "u2", "shape": ("flashes",), "long_name": "Number of flashes in each flash", - "default_data": lambda: 1 + "default_data": lambda: rand_type(nobs,np.uint16) }, } } @@ -401,76 +491,122 @@ def l2_lfl_schema(settings=None): def l2_af_schema(settings=None): """Define schema for LI L2 AF product.""" settings = settings or {} - nobs = settings.get("num_obs", 1234) - + nacc = settings.get("num_accumulations", 1) + npix = settings.get("num_pixels", 1234) return { "providers": settings.get("providers", {}), "variable_path": settings.get("variable_path", ""), - "dimensions": accumulation_dimensions(1, nobs), + "dimensions": accumulation_dimensions(nacc, npix), "variables": { "accumulation_offsets": { "format": "u4", "shape": ("accumulations",), - "default_data": lambda: 0 + "default_data": lambda: rand_type(nacc,np.uint32) }, "accumulation_start_times": { "format": "f8", "shape": ("accumulations",), - "default_data": lambda: 4.25055600161e8 + "long_name": "Accumulation start time", + "units": "seconds since 2000-01-01 00:00:00.0", + "precision" : "0.001", + "default_data": lambda: np.linspace(0.0, 1000.0, nacc) }, "l1b_geolocation_warning": { "format": "i1", "shape": ("accumulations",), "long_name": "L1b geolocation warning", - "default_data": lambda: -127 + "default_data": lambda: rand_type(nacc,np.int8) }, "l1b_radiometric_warning": { "format": "i1", "shape": ("accumulations",), "long_name": "L1b radiometric warning", - "default_data": lambda: -127 + "default_data": lambda: rand_type(nacc,np.int8) }, "average_flash_qa": { - "format": "i1", + "format": "u1", "shape": ("accumulations",), - "default_data": lambda: 23 + "default_data": lambda: rand_type(nacc,np.uint8), + "fill_value": 255, + "scale_factor": 0.004, + "add_offset": 0.0, + "long_name":"average flash confidence value", + }, "flash_accumulation": { "format": "u2", "shape": ("pixels",), - "default_data": lambda: np.clip(np.round(RANDOM_GEN.normal(1, 2, nobs)), 1, 2 ** 16 - 1) + "fill_value": 65535, + "scale_factor": 0.001, + "long_name": "Per area accumulation of flashes", + "grid_mapping": "mtg_geos_projection", + "units": "flashes/pixel", + "coordinate": "sparse: x y" , + "default_data": lambda: np.clip(np.round(RANDOM_GEN.normal(1, 2, npix)), 1, 2 ** 16 - 1) }, "mtg_geos_projection": mtg_geos_projection(), - "x": fci_grid_definition("X", nobs), - "y": fci_grid_definition("Y", nobs), + "x": fci_grid_definition("X", npix), + "y": fci_grid_definition("Y", npix), } } - -def l2_afa_schema(settings=None): - """Define schema for LI L2 AFA product.""" +def l2_afr_schema(settings=None): + """Define schema for LI L2 AFR product.""" settings = settings or {} - npix = settings.get("num_pixels", 120) - nacc = settings.get("num_accumulations", 20) + nacc = settings.get("num_accumulations", 1) + npix = settings.get("num_pixels", 1234) return { "providers": settings.get("providers", {}), "variable_path": settings.get("variable_path", ""), "dimensions": accumulation_dimensions(nacc, npix), "variables": { + "accumulation_offsets": { + "format": "u4", + "shape": ("accumulations",), + "default_data": lambda: rand_type(nacc,np.uint32) + }, "accumulation_start_times": { - "format": "f4", + "format": "f8", "shape": ("accumulations",), "long_name": "Accumulation start time", "units": "seconds since 2000-01-01 00:00:00.0", - "default_data": lambda: np.linspace(0.0, 1.0, nacc) + "precision" : "0.001", + "default_data": lambda: np.linspace(0.0, 1000.0, nacc) }, - "accumulated_flash_area": { - "format": "u4", + "l1b_geolocation_warning": { + "format": "i1", + "shape": ("accumulations",), + "long_name": "L1b geolocation warning", + "default_data": lambda: rand_type(nacc,np.int8) + }, + "l1b_radiometric_warning": { + "format": "i1", + "shape": ("accumulations",), + "long_name": "L1b radiometric warning", + "default_data": lambda: rand_type(nacc,np.int8) + }, + "average_flash_qa": { + "format": "u1", + "shape": ("accumulations",), + "default_data": lambda: rand_type(nacc,np.uint8), + "fill_value": 255, + "scale_factor": 0.004, + "add_offset": 0.0, + "long_name":"average flash confidence value", + + }, + "flash_radiance": { + "format": "u2", "shape": ("pixels",), - "fill_value": 4294967295, - "long_name": "Number of contributing unique flashes to each pixel", - "default_data": lambda: np.mod(np.arange(npix), 10) + 1 + "fill_value": 65535, + "scale_factor": 1.0, + "add_offset":0.0, + "long_name": "Area averaged flash radiance accumulation", + "grid_mapping": "mtg_geos_projection", + "units": "mW.m-2.sr-1", + "coordinate": "sparse: x y" , + "default_data": lambda: RANDOM_GEN.integers(low=1, high=6548, size=(npix), dtype=np.int16) }, "mtg_geos_projection": mtg_geos_projection(), "x": fci_grid_definition("X", npix), @@ -478,36 +614,63 @@ def l2_afa_schema(settings=None): } } - -def l2_afr_schema(settings=None): - """Define schema for LI L2 AFR product.""" +def l2_afa_schema(settings=None): + """Define schema for LI L2 AFA product.""" settings = settings or {} - nobs = settings.get("num_obs", 120) - nacc = settings.get("num_accumulations", 20) + nacc = settings.get("num_accumulations", 1) + npix = settings.get("num_pixels", 1234) return { "providers": settings.get("providers", {}), "variable_path": settings.get("variable_path", ""), - "dimensions": accumulation_dimensions(nacc, nobs), + "dimensions": accumulation_dimensions(nacc, npix), "variables": { - "flash_radiance": { - "format": "f4", - "shape": ("pixels",), - "long_name": "Area averaged flash radiance accumulation", - "grid_mapping": "mtg_geos_projection", - "coordinate": "sparse: x y", - "default_data": lambda: RANDOM_GEN.integers(low=1, high=6548, size=(120), dtype=np.int16) + "accumulation_offsets": { + "format": "u4", + "shape": ("accumulations",), + "default_data": lambda: rand_type(nacc,np.uint32) }, "accumulation_start_times": { - "format": "f4", + "format": "f8", "shape": ("accumulations",), "long_name": "Accumulation start time", "units": "seconds since 2000-01-01 00:00:00.0", - "default_data": lambda: 0 + "precision" : "0.001", + "default_data": lambda: np.linspace(0.0, 1000.0, nacc) + }, + "l1b_geolocation_warning": { + "format": "i1", + "shape": ("accumulations",), + "long_name": "L1b geolocation warning", + "default_data": lambda: rand_type(nacc,np.int8) + }, + "l1b_radiometric_warning": { + "format": "i1", + "shape": ("accumulations",), + "long_name": "L1b radiometric warning", + "default_data": lambda: rand_type(nacc,np.int8) + }, + "average_flash_qa": { + "format": "u1", + "shape": ("accumulations",), + "default_data": lambda: rand_type(nacc,np.uint8), + "fill_value": 255, + "scale_factor": 0.004, + "add_offset": 0.0, + "long_name":"average flash confidence value", + + }, + "accumulated_flash_area": { + "format": "u4", + "shape": ("pixels",), + "long_name": "Number of contributing unique flashes to each pixel", + "grid_mapping": "mtg_geos_projection", + "coordinate": "sparse: x y" , + "default_data": lambda: np.mod(np.arange(npix), 10) + 1 }, "mtg_geos_projection": mtg_geos_projection(), - "x": fci_grid_definition("X", nobs), - "y": fci_grid_definition("Y", nobs), + "x": fci_grid_definition("X", npix), + "y": fci_grid_definition("Y", npix), } } @@ -564,6 +727,105 @@ def mtg_geos_projection(): "default_data": lambda: -2147483647 } +#Dict containing the expecteded dtype output for each variable +expected_product_dtype = {"2-LE": + {"event_id":np.uint32, + "group_id":np.uint32, + "l1b_chunk_ids":np.uint32, + "l1b_chunk_offsets":np.uint32, + "l1b_window":np.uint32, + "filter_values":np.float32, + "flash_id":np.uint32, + "time_offset": np.dtype("timedelta64[ns]"), + "epoch_time":np.dtype("datetime64[ns]"), + "detector": np.float32, + "l1b_filter_qa": np.float32, + "l2_group_filter_qa":np.float32, + }, + "2-LEF": + {"l1b_geolocation_warning":np.int8, + "l1b_radiometric_warning":np.int8, + "l1b_missing_warning":np.int8, + "event_id":np.uint32, + "group_id":np.uint32, + "flash_id":np.uint32, + "detector":np.float32, + "latitude":np.float32, + "longitude":np.float32, + "radiance":np.uint16, + "event_filter_qa":np.float32, + "epoch_time":np.dtype("datetime64[ns]"), + "time_offset": np.dtype("timedelta64[ns]"), + "detector_row":np.float32, + "detector_column":np.float32}, + "2-LGR": + {"group_time":np.dtype("datetime64[ns]"), + "l1b_geolocation_warning":np.int8, + "l1b_radiometric_warning":np.int8, + "latitude":np.float32, + "longitude":np.float32, + "radiance":np.float32, + "group_id":np.uint32, + "flash_id":np.uint32, + "number_of_events": np.uint16, + "group_filter_qa": np.float32, + }, + "2-LFL": + {"latitude":np.float32, + "longitude":np.float32, + "radiance":np.float32, + "flash_duration":np.dtype("timedelta64[ns]"), + "flash_filter_confidence":np.float32, + "flash_footprint":np.uint16, + "flash_id":np.uint32, + "flash_time":np.dtype("datetime64[ns]"), + "l1b_geolocation_warning":np.int8, + "l1b_radiometric_warning":np.int8, + "l1b_missing_warning":np.int8, + "number_of_events":np.uint16, + "number_of_groups":np.uint16, + }, + "2-AF": + {"l1b_geolocation_warning":np.int8, + "l1b_radiometric_warning":np.int8, + "accumulation_offsets":np.uint32, + "accumulation_start_times": np.dtype("datetime64[ns]"), + "average_flash_qa":np.float32, + "mtg_geos_projection": np.int32, + "latitude":np.float32, + "longitude": np.float32, + "x": np.float64, + "y" : np.float64, + "flash_accumulation":np.float32, + }, + "2-AFA": + {"l1b_geolocation_warning":np.int8, + "l1b_radiometric_warning":np.int8, + "accumulation_offsets":np.uint32, + "accumulation_start_times": np.dtype("datetime64[ns]"), + "average_flash_qa":np.float32, + "mtg_geos_projection": np.int32, + "latitude":np.float32, + "longitude": np.float32, + "x": np.float64, + "y" : np.float64, + "accumulated_flash_area":np.uint32, + }, + "2-AFR": + {"l1b_geolocation_warning":np.int8, + "l1b_radiometric_warning":np.int8, + "l1b_missing_warning":np.int8, + "accumulation_offsets":np.uint32, + "accumulation_start_times": np.dtype("datetime64[ns]"), + "latitude":np.float32, + "longitude": np.float32, + "average_flash_qa":np.float32, + "mtg_geos_projection": np.int32, + "x": np.float64, + "y" : np.float64, + "flash_radiance":np.float32, + }, + } products_dict = { "2-LE": {"ftype": "li_l2_le_nc", "schema": l2_le_schema}, diff --git a/satpy/tests/reader_tests/test_li_l2_nc.py b/satpy/tests/reader_tests/test_li_l2_nc.py index 04f11acf4c..d239ecc475 100644 --- a/satpy/tests/reader_tests/test_li_l2_nc.py +++ b/satpy/tests/reader_tests/test_li_l2_nc.py @@ -30,6 +30,7 @@ from satpy.readers.yaml_reader import load_yaml_configs from satpy.tests.reader_tests._li_test_utils import ( FakeLIFileHandlerBase, + expected_product_dtype, extract_filetype_info, get_product_schema, products_dict, @@ -94,7 +95,6 @@ def _test_dataset_variables(self, settings, ds_desc, handler): """Check the loading of the non in sector variables.""" assert "variables" in ds_desc all_vars = ds_desc["variables"] - variables = settings.get("variables") for vname, desc in variables.items(): # variable should be in list of dataset: @@ -125,7 +125,10 @@ def _test_dataset_single_sector_variable(self, names, desc, settings, handler): def _test_dataset_variable(self, var_params, sname=""): """Test the validity of a given (sector) variable.""" dataset_info, desc, dname, handler, shape, var_path = var_params + product_type = handler.ds_desc["product_type"] res = self.get_variable_dataset(dataset_info, dname, handler) + resd = self.get_variable_dataset(None,dataset_info["name"], handler) + assert resd.dtype == expected_product_dtype[product_type][dname] assert res.shape == shape assert res.dims[0] == "y" # Should retrieve content with fullname key: @@ -175,9 +178,8 @@ def test_dataset_loading(self, filetype_infos): "start_time": "0000", "end_time": "1000" } - handler = LIL2NCFileHandler("filename", filename_info, extract_filetype_info(filetype_infos, ftype), - with_area_definition=False) + with_area_definition=False) ds_desc = handler.ds_desc # retrieve the schema that what used to generate the content for that product: @@ -275,10 +277,10 @@ def test_get_first_valid_variable(self, filetype_infos): handler = LIL2NCFileHandler("filename", filename_info, extract_filetype_info(filetype_infos, "li_l2_lef_nc")) # Check variable paths: - var1 = handler.get_first_valid_variable(["dummy/path", "data/north/event_id"]) - var2 = handler.get_first_valid_variable(["dummy/path", "data/east/event_id"]) - var3 = handler.get_first_valid_variable(["dummy/path", "data/south/group_id"]) - var4 = handler.get_first_valid_variable(["dummy/path", "data/west/group_id"]) + var1 = handler.get_first_valid_variable(["dummy/path", "data/north/detector_column"]) + var2 = handler.get_first_valid_variable(["dummy/path", "data/east/detector_column"]) + var3 = handler.get_first_valid_variable(["dummy/path", "data/south/detector_row"]) + var4 = handler.get_first_valid_variable(["dummy/path", "data/west/detector_row"]) assert isinstance(var1, xr.DataArray) assert isinstance(var2, xr.DataArray) @@ -290,15 +292,15 @@ def test_get_first_valid_variable(self, filetype_infos): assert id(var3) != id(var4) mix1 = handler.get_first_valid_variable(["dummy/path", - "data/north/event_id", - "data/east/event_id", - "data/south/group_id"]) + "data/north/detector_column", + "data/east/detector_column", + "data/south/detector_row"]) mix2 = handler.get_first_valid_variable(["dummy/path", - "data/west/group_id", - "data/north/event_id", - "data/east/event_id", - "data/south/group_id"]) + "data/west/detector_row", + "data/north/detector_column", + "data/east/detector_column", + "data/south/detector_row"]) # first mix should give us var1 and the second one var4: assert id(mix1) == id(var1) @@ -307,8 +309,8 @@ def test_get_first_valid_variable(self, filetype_infos): # get the measured variables now: # Note that we must specify fill_value==None below otherwise # a new array is generated filling the invalid values: - meas1 = handler.get_measured_variable("east/event_id", fill_value=None) - meas2 = handler.get_measured_variable("south/group_id", fill_value=None) + meas1 = handler.get_measured_variable("east/detector_column", fill_value=None) + meas2 = handler.get_measured_variable("south/detector_row", fill_value=None) assert id(meas1) == id(var2) assert id(meas2) == id(var3) @@ -633,6 +635,10 @@ def test_coords_generation(self, filetype_infos): azimuth_vals *= -1 lon_ref, lat_ref = projection(azimuth_vals, elevation_vals, inverse=True) + # Convert lon_ref, lat_ref to a np.float32 + lon_ref = lon_ref.astype(np.float32) + lat_ref = lat_ref.astype(np.float32) + handler.generate_coords_from_scan_angles() lon = handler.internal_variables["longitude"].values lat = handler.internal_variables["latitude"].values From a47690e148c5c748b9223ee5c64c2c5332eb282a Mon Sep 17 00:00:00 2001 From: clement laplace Date: Thu, 5 Dec 2024 16:36:41 +0000 Subject: [PATCH 3/4] typo: Erase whitespace as requested in codefactor --- satpy/tests/reader_tests/_li_test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/_li_test_utils.py b/satpy/tests/reader_tests/_li_test_utils.py index 05a2224d3a..0bd051a272 100644 --- a/satpy/tests/reader_tests/_li_test_utils.py +++ b/satpy/tests/reader_tests/_li_test_utils.py @@ -259,7 +259,7 @@ def l2_lef_schema(settings=None): "detector_row": { "format": "u2", "shape": ("events",), - "fill_value": 65535, + "fill_value": 65535, "long_name": "Detector row position of event pixel", "units": "1", "default_data": lambda: RANDOM_GEN.integers(1, 1000, nobs) @@ -267,7 +267,7 @@ def l2_lef_schema(settings=None): "detector_column": { "format": "u2", "shape": ("events",), - "fill_value": 65535, + "fill_value": 65535, "long_name": "Detector column position of event pixel", "units": "1", "default_data": lambda: RANDOM_GEN.integers(1, 1000, nobs) From 0821afb954c6f716250fce4f374538d84356c446 Mon Sep 17 00:00:00 2001 From: clement laplace Date: Thu, 5 Dec 2024 17:03:24 +0000 Subject: [PATCH 4/4] correction: Aplly corrections asked https://github.com/pytroll/satpy/pull/2998 --- satpy/readers/li_base_nc.py | 1 - satpy/readers/li_l2_nc.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/satpy/readers/li_base_nc.py b/satpy/readers/li_base_nc.py index fe998a3b84..c92741f715 100644 --- a/satpy/readers/li_base_nc.py +++ b/satpy/readers/li_base_nc.py @@ -306,7 +306,6 @@ def get_projection_config(self): """Retrieve the projection configuration details.""" # We retrieve the projection variable name directly from our swath settings: proj_var = self.swath_coordinates["projection"] - logger.error(proj_var) geos_proj = self.get_measured_variable(proj_var, fill_value=None) # cast projection attributes to float/str: major_axis = float(geos_proj.attrs["semi_major_axis"]) diff --git a/satpy/readers/li_l2_nc.py b/satpy/readers/li_l2_nc.py index 72722f034e..9d507e5a84 100644 --- a/satpy/readers/li_l2_nc.py +++ b/satpy/readers/li_l2_nc.py @@ -158,7 +158,7 @@ def get_array_on_fci_grid(self, data_array: xr.DataArray): data_2d = da.map_blocks(_np_add_at_wrapper, data_2d, (rows, cols), data_array, dtype=data_array.dtype, chunks=(LI_GRID_SHAPE[0], LI_GRID_SHAPE[1])) - data_2d = da.where(data_2d > 0, data_2d, np.float32(np.nan)).astype(np.float32) + data_2d = da.where(data_2d > 0, data_2d, np.nan).astype(np.float32) xarr = xr.DataArray(da.asarray(data_2d, CHUNK_SIZE), dims=("y", "x")) xarr.attrs = attrs return xarr