From 8bebfc138afe6a29bef97963fa82787062beaabc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?laurent=20p=C3=A9rier?= Date: Thu, 28 Nov 2024 13:42:28 +0100 Subject: [PATCH] Increasing the coverage of tested code. --- .../reader_tests/test_geos_netcdficare.py | 414 +++++++++++++++--- 1 file changed, 342 insertions(+), 72 deletions(-) diff --git a/satpy/tests/reader_tests/test_geos_netcdficare.py b/satpy/tests/reader_tests/test_geos_netcdficare.py index 27b95f900d..992971f44c 100644 --- a/satpy/tests/reader_tests/test_geos_netcdficare.py +++ b/satpy/tests/reader_tests/test_geos_netcdficare.py @@ -26,13 +26,13 @@ into the pytest directory pytest_projec. EXECUTION TIME : - 20 seconds. + 4 minutes. DATE OF CREATION : - 2024 16th october. + 2024 11th october. LAST VERSIONS : AUTHOR : - Meteo France. + Météo France. """ import os @@ -53,16 +53,49 @@ class TestGeosNetcdfIcareReader() : # Test of the geos_netcdficare reader. # This reader has been build for the Icare Meteo France netcdfs. - def test_geosNetcdficare(self, tmp_path) : + def test_mtg_netcdficare(self, tmp_path) : """ A dummy netcdf is built. - A scene self.scn for the convection product for the same date + A scene self.scn for the nir_16 product for the same date is built. We check that the scene parameters are the same as thoses in the dummy netcdf. This procedure is called by pytest. """ + self.initMtg(tmp_path) + self.scn.load(['nir_16']) + self.values = self.scn.values() + self.checkingSceneParameter("nir_16") + # test_mtg_netcdficare() + + def test_msg_netcdficare(self, tmp_path) : + self.initMsgHrv(tmp_path) + self.scn.load(['HRV']) + self.values = self.scn.values() + self.checkingSceneParameter("HRV") + + self.initMsg(tmp_path) + self.scn.load(['convection']) + self.values = self.scn.values() + self.checkingSceneParameter("convection") + # test_msg_netcdficare() - self.init(tmp_path) + def test_hima_netcdficare(self, tmp_path) : + self.initHima(tmp_path) + self.scn.load(['B10']) + self.values = self.scn.values() + self.checkingSceneParameter("B10") + # test_hima_netcdficare() + def test_goesr_netcdficare(self, tmp_path) : + self.initGoesr(tmp_path) + self.scn.load(['C02']) + self.values = self.scn.values() + self.checkingSceneParameter("C02") + # test_goesr_netcdficare() + + # ----------------------------------------------------- # + # typeImage : convection, airmass... + # ----------------------------------------------------- # + def checkingSceneParameter(self, typeImage) : startTime = self.scn.start_time startTimeString = startTime.strftime('%Y-%m-%dT%H:%M:%S') # 2024-06-28T10:00:40 @@ -84,7 +117,6 @@ def test_geosNetcdficare(self, tmp_path) : for data_arr in self.values : # values come from the scene. - # print("data_arr.attrs = ", data_arr.attrs) if "platform_name" in data_arr.attrs : platform = data_arr.attrs["platform_name"] if "orbital_parameters" in data_arr.attrs : @@ -100,9 +132,20 @@ def test_geosNetcdficare(self, tmp_path) : assert altitude == self.expectedAltitude xr = self.scn.to_xarray_dataset() - matrice = xr["convection"] - nblin = matrice.shape[1] - nbpix = matrice.shape[2] + matrice = xr[typeImage] + nbdim = len(matrice.shape) + if nbdim == 3 : + # RGB. + nblin = matrice.shape[1] + nbpix = matrice.shape[2] + elif nbdim == 2 : + # PGM. + nblin = matrice.shape[0] + nbpix = matrice.shape[1] + else : + print("Dimension of shape not expected : ", nbdim) + exit(1) + assert nblin == self.expectedNblin assert nbpix == self.expectedNbpix @@ -122,22 +165,29 @@ def test_geosNetcdficare(self, tmp_path) : resolution = satpyId.get("resolution") resolution = float(int(resolution * 10.)) / 10. assert resolution == self.expectedResolution + # checkingSceneParameter() - # A picture of convection composite will be displayed. - self.scn.show("convection") - logger.info("The picture should be pink.") - # test_geosNetcdficare(self, tmp_path) - - def init(self, tmp_path) : + def initMsg(self, tmp_path) : """ A fake netcdf is built. A scene is built with the reader to be tested, applied to this netcdf. - Called by test_geosNetcdficare(). + Called by test_msg_netcdficare(). """ self.netcdfName = tmp_path / "Mmultic3kmNC4_msg03_202406281000.nc" self.filepath = tmp_path - self.buildNetcdf(self.netcdfName) + listIR = { + "IR_039", "WV_062", "WV_073", "IR_087", "IR_097", + "IR_108", "IR_120", "IR_134" + } + + self.buildNetcdf( + self.netcdfName, 3712, "msg03", x0=-5570254, dx=3000.40604, + y0=5570254, cfac=1.3642337E7, coff=1857.0, + listVisible={"VIS006", "VIS008", "IR_016"}, listIR=listIR, + coordinateSystemName="GeosCoordinateSystem", + nomImageNavigation="ImageNavigation", + nomX="X", nomY="Y", time_coverage_end="2024-06-28T10:12:41Z365") # We will check that the parameters written in the dummy netcdf # can be read. @@ -168,32 +218,248 @@ def init(self, tmp_path) : # {'msg_netcdficare': ['/tmp/Mmultic3kmNC4_msg03_202406281000.nc']} self.scn = Scene(filenames=myfiles, reader=yaml_file) + # initMsg() - # logger.info(self.scn.available_dataset_names()) - # ['IR_016', 'IR_039', 'IR_087', 'IR_097', 'IR_108', 'IR_120', - # 'IR_134', 'VIS006', 'VIS008', 'WV_062', 'WV_073'] - - # logger.info(self.scn.available_composite_names()) - """ Static compositor {'_satpy_id': DataID(name='_night_background'), - 'standard_name': 'night_background', - 'prerequisites': [], - 'optional_prerequisites': []} - Static compositor {'_satpy_id': DataID(name='_night_background_hires'), - 'standard_name': 'night_background', 'prerequisites': [], - 'optional_prerequisites': []} - ['airmass', 'ash', 'cloud_phase_distinction', ... - 'cloudtop', 'cloudtop_daytime', 'colorized_ir_clouds', 'convection'... + def initMsgHrv(self, tmp_path) : + """ + A fake netcdf is built. + A scene is built with the reader to be tested, applied to this netcdf. + Called by test_msg_netcdficare(). """ + self.netcdfName = tmp_path / "Mmultic1kmNC4_msg03_202406281000.nc" + self.filepath = tmp_path - self.scn.load(['convection']) - self.values = self.scn.values() - # init() + self.buildNetcdf( + self.netcdfName, 11136, "msg03", x0=-5571254., dx=1000.135, + y0=5570254., cfac=40927014, coff=5566.0, listVisible={"HRV"}, + listIR={}, coordinateSystemName="GeosCoordinateSystem_h", + nomImageNavigation="ImageNavigation_h", nomX="X_h", nomY="Y_h", + time_coverage_end="2024-06-28T10:12:41Z365") - def buildNetcdf(self, ncName) : + # We will check that the parameters written in the dummy netcdf + # can be read. + self.expectedStartTime = "2024-06-28T10:00:09" + self.expectedEndTime = "2024-06-28T10:12:41" + actualAltitude = 35786691 + 6378169 # 42164860.0 + actualLongitude = 0.1 + self.expectedPlatform = "Meteosat-10" + self.expectedSensor = "seviri" + self.expectedAltitude = actualAltitude + self.expectedLongitude = actualLongitude + self.expectedCfac = 40927014 + self.expectedLfac = 40927014 + self.expectedCoff = 5566.0 + self.expectedLoff = 5566.0 + self.expectedResolution = 1000.1 + self.expectedNblin = 11136 + self.expectedNbpix = 11136 + + # To build a scene at the date 20240628_100000, + # a netcdf corresponding to msg_netcdficare + # is looked for, in the filepath directory. + yaml_file = 'msg_netcdficare' + myfiles = find_files_and_readers( + base_dir=self.filepath, start_time=datetime(2024, 6, 28, 10, 0), + end_time=datetime(2024, 6, 28, 10, 0), reader=yaml_file) + # logger.info("Found myfiles = ", myfiles) + # {'msg_netcdficare': ['/tmp/Mmultic3kmNC4_msg03_202406281000.nc']} + + self.scn = Scene(filenames=myfiles, reader=yaml_file) + # initMsgHrv() + + def initHima(self, tmp_path) : + """ + A fake netcdf is built. + A scene is built with the reader to be tested, applied to this netcdf. + Called by test_hima_netcdficare(). + """ + self.netcdfName = tmp_path / "Jmultic2kmNC4_hima09_202406281000.nc" + self.filepath = tmp_path + + listVisible = { + "VIS004", "VIS005", "VIS006", "VIS008", "IR_016", "IR_022" + } + listIR = { + "IR_038", "WV_062", "WV_069", "WV_073", "IR_085", "IR_096", + "IR_104", "IR_112", "IR_123", "IR_132" + } + self.buildNetcdf( + self.netcdfName, 5500, "hima09", x0=-5500000, dx=2000.0000047, + y0=5500000, cfac=20466275., coff=2750.5, + listVisible=listVisible, listIR=listIR, + coordinateSystemName="GeosCoordinateSystem2km", + nomImageNavigation="ImageNavigation2km", + nomX="X2km", nomY="Y2km", + time_coverage_end="2024-06-28T10:08:41Z365") + + # We will check that the parameters written in the dummy netcdf + # can be read. + self.expectedStartTime = "2024-06-28T10:00:09" + self.expectedEndTime = "2024-06-28T10:08:41" + actualAltitude = 35786691 + 6378169 # 42164860.0 + actualLongitude = 0.1 + self.expectedPlatform = "Himawari-9" + self.expectedSensor = "ahi" + self.expectedAltitude = actualAltitude + self.expectedLongitude = actualLongitude + self.expectedCfac = 20466275. + self.expectedLfac = 20466275. + self.expectedCoff = 2750.5 + self.expectedLoff = 2750.5 + self.expectedResolution = 2000. + self.expectedNblin = 5500 + self.expectedNbpix = 5500 + + # To build a scene at the date 20240628_100000, + # a netcdf corresponding to msg_netcdficare + # is looked for in the filepath directory. + yaml_file = 'hima_netcdficare' + myfiles = find_files_and_readers( + base_dir=self.filepath, start_time=datetime(2024, 6, 28, 10, 0), + end_time=datetime(2024, 6, 28, 10, 0), reader=yaml_file) + # logger.info("Found myfiles = ", myfiles) + # {'msg_netcdficare': ['/tmp/Mmultic3kmNC4_msg03_202406281000.nc']} + + self.scn = Scene(filenames=myfiles, reader=yaml_file) + # initHima() + + def initMtg(self, tmp_path) : + """ + A fake netcdf is built. + A scene is built with the reader to be tested, applied to this netcdf. + Called by test_mtg_netcdficare(). + """ + self.netcdfName = tmp_path / "Mmultic1kmNC4_mtgi1_202406281000.nc" + self.filepath = tmp_path + + # self.buildNetcdf( + # self.netcdfName, 5568, "mtgi1", + # x0 = -5568000, dx = 2000.0000047, y0 = 5568000, cfac = 13642337, coff = 2784.5, + # listVisible= {"VIS004", "VIS005", "VIS006", "VIS008", "VIS009", + # "IR_013", "IR_016", "IR_022"}, + # listIR={"IR_038", "WV_062", "WV_073", "IR_087", + # "IR_097", "IR_105", "IR_123", "IR_133"}, + # coordinateSystemName = "GeosCoordinateSystem2km", + # nomImageNavigation = "ImageNavigation2km", + # nomX = "X2km", nomY = "Y2km", + # time_coverage_end = "2024-06-28T10:08:41Z365") + + listVisible = { + "VIS004", "VIS005", "VIS006", "VIS008", + "VIS009", "IR_013", "IR_016", "IR_022" + } + + self.buildNetcdf( + self.netcdfName, 11136, "mtgi1", + x0=-5568000, dx=1000.0000047, y0=5568000, + cfac=4.093316350596011E7, coff=5568.5, + listVisible=listVisible, listIR={"IR_038", "IR_105"}, + coordinateSystemName="GeosCoordinateSystem1km", + nomImageNavigation="ImageNavigation1km", + nomX="X1km", nomY="Y1km", time_coverage_end="2024-06-28T10:08:41Z365") + + # We will check that the parameters written in the dummy netcdf + # can be read. + self.expectedStartTime = "2024-06-28T10:00:09" + self.expectedEndTime = "2024-06-28T10:08:41" + actualAltitude = 35786691 + 6378169 # 42164860.0 + actualLongitude = 0.1 + self.expectedPlatform = "Meteosat-12" + self.expectedSensor = "fci" + self.expectedAltitude = actualAltitude + self.expectedLongitude = actualLongitude + self.expectedCfac = 4.093316350596011E7 # 13642337. + self.expectedLfac = 4.093316350596011E7 # 13642337. + self.expectedCoff = 5568.5 # 2784.5 + self.expectedLoff = 5568.5 # 2784.5 + self.expectedResolution = 1000. + self.expectedNblin = 11136 # 5568 + self.expectedNbpix = 11136 # 5568 + + # To build a scene at the date 20240628_100000, + # a netcdf corresponding to msg_netcdficare + # is looked for in the filepath directory. + yaml_file = 'mtg_netcdficare' + myfiles = find_files_and_readers( + base_dir=self.filepath, start_time=datetime(2024, 6, 28, 10, 0), + end_time=datetime(2024, 6, 28, 10, 0), reader=yaml_file) + # logger.info("Found myfiles = ", myfiles) + # {'msg_netcdficare': ['/tmp/Mmultic3kmNC4_msg03_202406281000.nc']} + + self.scn = Scene(filenames=myfiles, reader=yaml_file) + # initMtg() + + def initGoesr(self, tmp_path) : + """ + A fake netcdf is built. + A scene is built with the reader to be tested, applied to this netcdf. + Called by test_goesr_netcdficare(). + """ + self.netcdfName = tmp_path / "Emultic2kmNC4_goes16_202406281000.nc" + self.filepath = tmp_path + + listVisible = { + "VIS_004", "VIS_006", "VIS_008", "VIS_014", "VIS_016", "VIS_022" + } + listIR = { + "IR_039", "IR_062", "IR_069", "IR_073", "IR_085", + "IR_096", "IR_103", "IR_114", "IR_123", "IR_133" + } + self.buildNetcdf( + self.netcdfName, 5424, "goes16", + x0=-5434894.8, dx=2004.017288, y0=5434894.8, + cfac=20425338.9, coff=2712.5, + listVisible=listVisible, listIR=listIR, + coordinateSystemName="GeosCoordinateSystem2km", + nomImageNavigation="ImageNavigation2km", + nomX="X2km", nomY="Y2km", + time_coverage_end="2024-06-28T10:08:41.1Z") + + # We will check that the parameters written in the dummy netcdf + # can be read. + self.expectedStartTime = "2024-06-28T10:00:09" + self.expectedEndTime = "2024-06-28T10:08:41" + actualAltitude = 35786691 + 6378169 # 42164860.0 + actualLongitude = 0.1 + self.expectedPlatform = "GOES-16" + self.expectedSensor = "abi" + self.expectedAltitude = actualAltitude + self.expectedLongitude = actualLongitude + self.expectedCfac = 20425338.9 + self.expectedLfac = 20425338.9 + self.expectedCoff = 2712.5 + self.expectedLoff = 2712.5 + self.expectedResolution = 2000. + self.expectedNblin = 5424 + self.expectedNbpix = 5424 + + # To build a scene at the date 20240628_100000, + # a netcdf corresponding to msg_netcdficare + # is looked for in the filepath directory. + yaml_file = 'goesr_netcdficare' + myfiles = find_files_and_readers( + base_dir=self.filepath, start_time=datetime(2024, 6, 28, 10, 0), + end_time=datetime(2024, 6, 28, 10, 0), reader=yaml_file) + + self.scn = Scene(filenames=myfiles, reader=yaml_file) + # initGoesr() + + def buildNetcdf( + self, ncName, nbpix=3712, nomSatellite="msg03", + x0=-5570254, dx=3000.40604, y0=5570254, + cfac=1.3642337E7, coff=1857.0, + listVisible={}, listIR={}, + coordinateSystemName="GeosCoordinateSystem", + nomImageNavigation="ImageNavigation", + nomX="X", nomY="Y", + time_coverage_end="2024-06-28T10:12:41Z365") : """ ncName : tmp_path / Mmultic3kmNC4_msg03_202406281000.nc A dummy icare Meteo France netcdf is built here. - Called by init(). + Called by initMsg... + listVisible = {"VIS006", "VIS008", "IR_016"} + listIR = {"IR_039", "WV_062", "WV_073", "IR_087", "IR_097", "IR_108", + "IR_120", "IR_134"} """ if os.path.exists(ncName) : os.remove(ncName) @@ -201,11 +467,11 @@ def buildNetcdf(self, ncName) : ncName, mode="w", clobber=True, diskless=False, persist=False, format='NETCDF4') - ncfileOut.createDimension(u'ny', 3712) - ncfileOut.createDimension(u'nx', 3712) + ncfileOut.createDimension(u'ny', nbpix) + ncfileOut.createDimension(u'nx', nbpix) ncfileOut.createDimension(u'numerical_count', 65536) ncfileOut.setncattr("time_coverage_start", "2024-06-28T10:00:09Z383") - ncfileOut.setncattr("time_coverage_end", "2024-06-28T10:12:41Z365") + ncfileOut.setncattr("time_coverage_end", time_coverage_end) ncfileOut.setncattr("Area_of_acquisition", "globe") fill_value = -32768 @@ -214,7 +480,7 @@ def buildNetcdf(self, ncName) : shuffle=True, fletcher32=False, contiguous=False, chunksizes=None, endian='native', least_significant_digit=None) - var.setncattr("id", "msg03") + var.setncattr("id", nomSatellite) var.setncattr("dst", 35786691.) var.setncattr("lon", float(0.1)) @@ -225,58 +491,61 @@ def buildNetcdf(self, ncName) : var.setncattr("longitude_of_projection_origin", 0.) var = ncfileOut.createVariable( - "GeosCoordinateSystem", "c", zlib=True, complevel=4, + coordinateSystemName, "c", zlib=True, complevel=4, shuffle=True, fletcher32=False, contiguous=False, chunksizes=None, endian='native', least_significant_digit=None) - var.setncattr( - "GeoTransform", - "-5570254, 3000.40604, 0, 5570254, 0, -3000.40604") + + stringGeotransform = str(x0) + ", " + str(dx) + ", 0, " + stringGeotransform += str(-x0) + ", 0, " + str(-dx) + # -5570254, 3000.40604, 0, 5570254, 0, -3000.40604 + var.setncattr("GeoTransform", stringGeotransform) var = ncfileOut.createVariable( - "ImageNavigation", "c", zlib=True, complevel=4, + nomImageNavigation, "c", zlib=True, complevel=4, shuffle=True, fletcher32=False, contiguous=False, chunksizes=None, endian='native', least_significant_digit=None) - var.setncattr("CFAC", 1.3642337E7) - var.setncattr("LFAC", 1.3642337E7) - var.setncattr("COFF", 1857.0) - var.setncattr("LOFF", 1857.0) + var.setncattr("CFAC", cfac) + var.setncattr("LFAC", cfac) + var.setncattr("COFF", coff) + var.setncattr("LOFF", coff) var = ncfileOut.createVariable( - "X", 'float32', u'nx', zlib=True, complevel=4, + nomX, 'float32', u'nx', zlib=True, complevel=4, shuffle=True, fletcher32=False, contiguous=False, chunksizes=None, endian='native', least_significant_digit=None) - x0 = -5570254. - dx = 3000.40604 - var[:] = np.array(([(x0 + dx * i) for i in range(3712)])) + var[:] = np.array(([(x0 + dx * i) for i in range(nbpix)])) var = ncfileOut.createVariable( - "Y", 'float32', u'ny', zlib=True, complevel=4, + nomY, 'float32', u'ny', zlib=True, complevel=4, shuffle=True, fletcher32=False, contiguous=False, chunksizes=None, endian='native', least_significant_digit=None) - y0 = 5570254. - dy = -3000.40604 - var[:] = np.array(([(y0 + dy * i) for i in range(3712)])) - - self.visibleChannelsCreation(ncfileOut, fill_value) - self.infrarougeChannelsCreation(ncfileOut, fill_value) - + y0 = -x0 + dy = -dx + var[:] = np.array(([(y0 + dy * i) for i in range(nbpix)])) + + self.visibleChannelsCreation( + ncfileOut, fill_value, listVisible, nbpix, coordinateSystemName) + self.infrarougeChannelsCreation( + ncfileOut, fill_value, listIR, nbpix, coordinateSystemName) ncfileOut.close # buildNetcdf() - def visibleChannelsCreation(self, ncfileOut, fill_value) : - for channel in {"VIS006", "VIS008", "IR_016"} : + def visibleChannelsCreation( + self, ncfileOut, fill_value, listVisible, nbpix, coordinateSystemName) : + + for channel in listVisible : var = ncfileOut.createVariable( channel, 'short', ('ny', 'nx'), zlib=True, complevel=4, shuffle=True, fletcher32=False, contiguous=False, chunksizes=None, endian='native', least_significant_digit=None, fill_value=fill_value) var[:] = np.array( - ([[i * 2 for i in range(3712)] for j in range(3712)])) + ([[i * 2 for i in range(nbpix)] for j in range(nbpix)])) # Hundredths of albedo between 0 and 10000. var.setncattr("scale_factor", 0.01) var.setncattr("add_offset", 0.) var.setncattr("bandfactor", 20.76) - var.setncattr("_CoordinateSystems", "GeosCoordinateSystem") + var.setncattr("_CoordinateSystems", coordinateSystemName) var = ncfileOut.createVariable( "Albedo_to_Native_count_" + channel, 'short', @@ -287,24 +556,25 @@ def visibleChannelsCreation(self, ncfileOut, fill_value) : var[:] = np.array(([-9999 for i in range(65536)])) # In order to come back to the native datas on 10, 12 or 16 bits. - def infrarougeChannelsCreation(self, ncfileOut, fill_value) : - for channel in { - "IR_039", "WV_062", "WV_073", "IR_087", "IR_097", - "IR_108", "IR_120", "IR_134"} : + def infrarougeChannelsCreation( + self, ncfileOut, fill_value, listIR, nbpix, coordinateSystemName) : + + for channel in listIR : var = ncfileOut.createVariable( channel, 'short', ('ny', 'nx'), zlib=True, complevel=4, shuffle=True, fletcher32=False, contiguous=False, chunksizes=None, endian='native', least_significant_digit=None, fill_value=fill_value) var[:] = np.array( - ([[-9000 + j * 4 for i in range(3712)] for j in range(3712)])) + ([[-9000 + j * 4 for i in range(nbpix)] for j in range(nbpix)]) + ) # Hundredths of celcius degrees. var.setncattr("scale_factor", 0.01) var.setncattr("add_offset", 273.15) var.setncattr("nuc", 1600.548) var.setncattr("alpha", 0.9963) var.setncattr("beta", 2.185) - var.setncattr("_CoordinateSystems", "GeosCoordinateSystem") + var.setncattr("_CoordinateSystems", coordinateSystemName) var = ncfileOut.createVariable( "Temp_to_Native_count_" + channel, 'short',