diff --git a/test/marine/CMakeLists.txt b/test/marine/CMakeLists.txt index 46f26cb86..048887772 100644 --- a/test/marine/CMakeLists.txt +++ b/test/marine/CMakeLists.txt @@ -35,7 +35,7 @@ set(PYIODACONV_DIR "${PROJECT_SOURCE_DIR}/build/lib/python${PYTHON_MAJOR_MINOR}/ set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine) set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine) -set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i) +set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/newb2i) set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput) set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref) @@ -54,7 +54,8 @@ function(CREATE_CONFIG_FILE string(REPLACE "__BUFRINPUTDIR__" "\"${bufr_input_dir}\"" temp_content "${file_content}") string(REPLACE "__IODAOUTPUTDIR__" "\"${ioda_output_dir}\"" temp_content2 "${temp_content}") string(REPLACE "__OCEANBASIN__" "\"${ocean_basin_file}\"" temp_content3 "${temp_content2}") - file(WRITE "${test_config_out}" "${temp_content3}") + string(REPLACE "__CONFIGDIR__" ${CONFIG_DIR} temp_content4 "${temp_content3}") + file(WRITE "${test_config_out}" "${temp_content4}") endfunction() @@ -121,15 +122,18 @@ function(ADD_INSITU_TEST testname testbufr) # set(CONFIG_TYPE "json") set(CONFIG_TYPE "yaml") - if (testbufr STREQUAL "dbuoy") - set(DATE "2019010700") - set(CYCLE "00") - else() - set(DATE "2021063006") - set(CYCLE "06") - endif() + set(DATE "2019010700") + set(CYCLE "00") + # if (testbufr STREQUAL "dbuoy") + # set(DATE "2019010700") + # set(CYCLE "00") + # else() + # set(DATE "2021063006") + # set(CYCLE "06") + # endif() - set(TEST "bufr2ioda_insitu_${testname}") + set(TESTEXEC "bufr2ioda_insitu_${testname}.py") + set(TEST "bufr2ioda_insitu_${testname}_${testbufr}") set(TESTREF_FILE "${TEST}_${DATE}.ref") @@ -158,7 +162,7 @@ function(ADD_INSITU_TEST testname testbufr) add_test( NAME test_gdasapp_${TEST} - COMMAND ${MARINE_BUFR2IODA_DIR}/${TEST}.py -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE} + COMMAND ${MARINE_BUFR2IODA_DIR}/${TESTEXEC} -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE} WORKING_DIRECTORY ${TEST_WORKING_DIR} ) set_property( @@ -176,7 +180,13 @@ if (GENERATE_BUFR2IODA_TESTS) ADD_INSITU_TEST("profile_glider" "subpfl") ADD_INSITU_TEST("profile_tesac" "tesac") ADD_INSITU_TEST("profile_tropical" "dbuoy") + ADD_INSITU_TEST("profile_tropical" "mbuoyb") ADD_INSITU_TEST("profile_xbtctd" "xbtctd") + ADD_INSITU_TEST("surface_altkob" "altkob") + ADD_INSITU_TEST("surface_cstgd" "cstgd") ADD_INSITU_TEST("surface_drifter" "dbuoy") + ADD_INSITU_TEST("surface_drifter" "dbuoyb") + ADD_INSITU_TEST("surface_lcman" "lcman") + ADD_INSITU_TEST("surface_shipsu" "shipsu") ADD_INSITU_TEST("surface_trkob" "trkob") endif() diff --git a/test/marine/testinput/altkob.yaml b/test/marine/testinput/altkob.yaml new file mode 100644 index 000000000..992d2f7db --- /dev/null +++ b/test/marine/testinput/altkob.yaml @@ -0,0 +1,76 @@ +--- + - class_name: "Temperature" + short_name: "seaSurfaceTemperature" + name: "seaSurfaceTemperature" + bufr_mnemonic: "*/SST0" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Salinity" + short_name: "salinity" + name: "salinity" + bufr_mnemonic: "*/SSS0" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "" + data_min: 0.0 + data_max: 45.0 + data_error: 0.01 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLONH" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLATH" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testinput/bathy.yaml b/test/marine/testinput/bathy.yaml new file mode 100644 index 000000000..0b2b5d46c --- /dev/null +++ b/test/marine/testinput/bathy.yaml @@ -0,0 +1,87 @@ +--- + - class_name: "Temperature" + short_name: "waterTemperature" + name: "waterTemperature" + bufr_mnemonic: "*/BTOCN/STMP" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Depth" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/BTOCN/DBSS" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/RPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testinput/bufr2ioda_insitu_profile_argo_2021063006.yaml.in b/test/marine/testinput/bufr2ioda_insitu_profile_argo_subpfl_2019010700.yaml.in similarity index 61% rename from test/marine/testinput/bufr2ioda_insitu_profile_argo_2021063006.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_profile_argo_subpfl_2019010700.yaml.in index 54ef72855..3ffb17557 100644 --- a/test/marine/testinput/bufr2ioda_insitu_profile_argo_2021063006.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_profile_argo_subpfl_2019010700.yaml.in @@ -1,13 +1,14 @@ --- data_format: subpfl -subsets: SUBPFL source: NCEP data tank data_type: argo cycle_type: gdas -cycle_datetime: '2021063006' +cycle_datetime: '2019010700' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ +platform_description: 'ARGO profiles from subpfl: temperature and salinity' data_description: 6-hrly in situ ARGO profiles +data_description_file: "__CONFIGDIR__/subpfl.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/bufr2ioda_insitu_profile_bathy_2021063006.yaml.in b/test/marine/testinput/bufr2ioda_insitu_profile_bathy_bathy_2019010700.yaml.in similarity index 64% rename from test/marine/testinput/bufr2ioda_insitu_profile_bathy_2021063006.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_profile_bathy_bathy_2019010700.yaml.in index 442d0a05a..0fb6d37c8 100644 --- a/test/marine/testinput/bufr2ioda_insitu_profile_bathy_2021063006.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_profile_bathy_bathy_2019010700.yaml.in @@ -1,13 +1,14 @@ --- data_format: bathy -subsets: BATHY source: NCEP data tank data_type: bathy cycle_type: gdas -cycle_datetime: '2021063006' +cycle_datetime: '2019010700' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ data_description: 6-hrly in situ Bathythermal profiles +platform_description: 'Profiles from BATHYthermal: temperature' +data_description_file: "__CONFIGDIR__/bathy.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/bufr2ioda_insitu_profile_glider_2021063006.yaml.in b/test/marine/testinput/bufr2ioda_insitu_profile_glider_subpfl_2019010700.yaml.in similarity index 61% rename from test/marine/testinput/bufr2ioda_insitu_profile_glider_2021063006.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_profile_glider_subpfl_2019010700.yaml.in index 8deeff58d..b76d1becf 100644 --- a/test/marine/testinput/bufr2ioda_insitu_profile_glider_2021063006.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_profile_glider_subpfl_2019010700.yaml.in @@ -1,13 +1,14 @@ --- data_format: subpfl -subsets: SUBPFL source: NCEP data tank data_type: glider cycle_type: gdas -cycle_datetime: '2021063006' +cycle_datetime: '2019010700' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ data_description: 6-hrly in situ GLIDER profiles +platform_description: 'GLIDER profiles from subpfl: temperature and salinity' +data_description_file: "__CONFIGDIR__/subpfl.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/bufr2ioda_insitu_profile_tesac_2021063006.yaml.in b/test/marine/testinput/bufr2ioda_insitu_profile_tesac_tesac_2019010700.yaml.in similarity index 62% rename from test/marine/testinput/bufr2ioda_insitu_profile_tesac_2021063006.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_profile_tesac_tesac_2019010700.yaml.in index 36e5e5ab1..37ed3bb94 100644 --- a/test/marine/testinput/bufr2ioda_insitu_profile_tesac_2021063006.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_profile_tesac_tesac_2019010700.yaml.in @@ -1,13 +1,14 @@ --- data_format: tesac -subsets: TESAC source: NCEP data tank data_type: tesac cycle_type: gdas -cycle_datetime: '2021063006' +cycle_datetime: '2019010700' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ data_description: 6-hrly in situ TESAC profiles +platform_description: 'Profiles from TESAC: temperature and salinity' +data_description_file: "__CONFIGDIR__/tesac.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/bufr2ioda_insitu_profile_tropical_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_profile_tropical_dbuoy_2019010700.yaml.in similarity index 68% rename from test/marine/testinput/bufr2ioda_insitu_profile_tropical_2019010700.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_profile_tropical_dbuoy_2019010700.yaml.in index 5657fe263..b4a4551c5 100644 --- a/test/marine/testinput/bufr2ioda_insitu_profile_tropical_2019010700.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_profile_tropical_dbuoy_2019010700.yaml.in @@ -1,6 +1,5 @@ --- data_format: dbuoy -subsets: dbuoy source: NCEP data tank data_type: tropical cycle_type: gdas @@ -9,5 +8,7 @@ dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ data_description: 6-hrly in situ tropical mooring profiles +platform_description: 'Tropical mooring profiles from dbuoy: temperature and salinity' +data_description_file: "__CONFIGDIR__/dbuoy.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/bufr2ioda_insitu_profile_tropical_mbuoyb_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_profile_tropical_mbuoyb_2019010700.yaml.in new file mode 100644 index 000000000..5986bd046 --- /dev/null +++ b/test/marine/testinput/bufr2ioda_insitu_profile_tropical_mbuoyb_2019010700.yaml.in @@ -0,0 +1,14 @@ +--- +data_format: mbuoyb +source: NCEP data tank +data_type: tropical +cycle_type: gdas +cycle_datetime: '2019010700' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ tropical mooring profiles +platform_description: 'Tropical mooring profiles from mbuoyb: temperature and salinity' +data_description_file: "__CONFIGDIR__/mbuoyb.yaml" +data_provider: U.S. NOAA + diff --git a/test/marine/testinput/bufr2ioda_insitu_profile_xbtctd_2021063006.yaml.in b/test/marine/testinput/bufr2ioda_insitu_profile_xbtctd_xbtctd_2019010700.yaml.in similarity index 62% rename from test/marine/testinput/bufr2ioda_insitu_profile_xbtctd_2021063006.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_profile_xbtctd_xbtctd_2019010700.yaml.in index 3d0df09e2..b095b9cd9 100644 --- a/test/marine/testinput/bufr2ioda_insitu_profile_xbtctd_2021063006.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_profile_xbtctd_xbtctd_2019010700.yaml.in @@ -1,13 +1,14 @@ --- data_format: xbtctd -subsets: XBTCTD source: NCEP data tank data_type: xbtctd cycle_type: gdas -cycle_datetime: '2021063006' +cycle_datetime: '2019010700' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ data_description: 6-hrly in situ XBT/XCTD profiles +platform_description: 'Profiles from XBT/CTD: temperature and salinity' +data_description_file: "__CONFIGDIR__/xbtctd.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/bufr2ioda_insitu_surface_altkob_altkob_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_surface_altkob_altkob_2019010700.yaml.in new file mode 100644 index 000000000..4f893ba0a --- /dev/null +++ b/test/marine/testinput/bufr2ioda_insitu_surface_altkob_altkob_2019010700.yaml.in @@ -0,0 +1,14 @@ +--- +data_format: altkob +source: NCEP data tank +data_type: altkob +cycle_type: gdas +cycle_datetime: '2019010700' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ ALTKOB surface +platform_description: 'Surface obs from altkob: temperature and salinity' +data_description_file: "__CONFIGDIR__/altkob.yaml" +data_provider: U.S. NOAA + diff --git a/test/marine/testinput/bufr2ioda_insitu_surface_cstgd_cstgd_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_surface_cstgd_cstgd_2019010700.yaml.in new file mode 100644 index 000000000..224db8337 --- /dev/null +++ b/test/marine/testinput/bufr2ioda_insitu_surface_cstgd_cstgd_2019010700.yaml.in @@ -0,0 +1,14 @@ +--- +data_format: cstgd +source: NCEP data tank +data_type: cstgd +cycle_type: gdas +cycle_datetime: '2019010700' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ CSTGD surface +platform_description: "sea surface temperature obs from cstgd" +data_description_file: "__CONFIGDIR__/cstgd.yaml" +data_provider: U.S. NOAA + diff --git a/test/marine/testinput/bufr2ioda_insitu_surface_drifter_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_surface_drifter_dbuoy_2019010700.yaml.in similarity index 68% rename from test/marine/testinput/bufr2ioda_insitu_surface_drifter_2019010700.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_surface_drifter_dbuoy_2019010700.yaml.in index 1fa5a1856..c054579ff 100644 --- a/test/marine/testinput/bufr2ioda_insitu_surface_drifter_2019010700.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_surface_drifter_dbuoy_2019010700.yaml.in @@ -1,6 +1,5 @@ --- data_format: dbuoy -subsets: dbuoy source: NCEP data tank data_type: drifter cycle_type: gdas @@ -9,5 +8,7 @@ dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ data_description: 6-hrly in situ drifter profiles +platform_description: "Lagrangian drifter drogue profiles from dbuy: temperature" +data_description_file: "__CONFIGDIR__/dbuoy.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/bufr2ioda_insitu_surface_drifter_dbuoyb_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_surface_drifter_dbuoyb_2019010700.yaml.in new file mode 100644 index 000000000..5be83733a --- /dev/null +++ b/test/marine/testinput/bufr2ioda_insitu_surface_drifter_dbuoyb_2019010700.yaml.in @@ -0,0 +1,14 @@ +--- +data_format: dbuoyb +source: NCEP data tank +data_type: drifter +cycle_type: gdas +cycle_datetime: '2019010700' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ drifter profiles +platform_description: "Lagrangian drifter drogue profiles from dbuyb: temperature" +data_description_file: "__CONFIGDIR__/dbuoyb.yaml" +data_provider: U.S. NOAA + diff --git a/test/marine/testinput/bufr2ioda_insitu_surface_lcman_lcman_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_surface_lcman_lcman_2019010700.yaml.in new file mode 100644 index 000000000..029be9c6f --- /dev/null +++ b/test/marine/testinput/bufr2ioda_insitu_surface_lcman_lcman_2019010700.yaml.in @@ -0,0 +1,14 @@ +--- +data_format: lcman +source: NCEP data tank +data_type: lcman +cycle_type: gdas +cycle_datetime: '2019010700' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ LCMAN surface +platform_description: 'Surface temperature obs from LCMAN' +data_description_file: "__CONFIGDIR__/lcman.yaml" +data_provider: U.S. NOAA + diff --git a/test/marine/testinput/bufr2ioda_insitu_surface_shipsu_shipsu_2019010700.yaml.in b/test/marine/testinput/bufr2ioda_insitu_surface_shipsu_shipsu_2019010700.yaml.in new file mode 100644 index 000000000..ea93e7d93 --- /dev/null +++ b/test/marine/testinput/bufr2ioda_insitu_surface_shipsu_shipsu_2019010700.yaml.in @@ -0,0 +1,14 @@ +--- +data_format: shipsu +source: NCEP data tank +data_type: shipsu +cycle_type: gdas +cycle_datetime: '2019010700' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ SHIPSU surface +platform_description: "temperature obs from shipsu" +data_description_file: "__CONFIGDIR__/shipsu.yaml" +data_provider: U.S. NOAA + diff --git a/test/marine/testinput/bufr2ioda_insitu_surface_trkob_2021063006.yaml.in b/test/marine/testinput/bufr2ioda_insitu_surface_trkob_trkob_2019010700.yaml.in similarity index 62% rename from test/marine/testinput/bufr2ioda_insitu_surface_trkob_2021063006.yaml.in rename to test/marine/testinput/bufr2ioda_insitu_surface_trkob_trkob_2019010700.yaml.in index 977f2bcc0..a317ec014 100644 --- a/test/marine/testinput/bufr2ioda_insitu_surface_trkob_2021063006.yaml.in +++ b/test/marine/testinput/bufr2ioda_insitu_surface_trkob_trkob_2019010700.yaml.in @@ -1,13 +1,14 @@ --- data_format: trkob -subsets: TRACKOB source: NCEP data tank data_type: trackob cycle_type: gdas -cycle_datetime: '2021063006' +cycle_datetime: '2019010700' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ data_description: 6-hrly in situ TRACKOB surface +platform_description: 'Surface obs from TRACKOB: temperature and salinity' +data_description_file: "__CONFIGDIR__/trkob.yaml" data_provider: U.S. NOAA diff --git a/test/marine/testinput/cstgd.yaml b/test/marine/testinput/cstgd.yaml new file mode 100644 index 000000000..02af51ccd --- /dev/null +++ b/test/marine/testinput/cstgd.yaml @@ -0,0 +1,76 @@ +--- + - class_name: "Temperature" + short_name: "seaSurfaceTemperature" + name: "seaSurfaceTemperature" + bufr_mnemonic: "*/SST1" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + +# - class_name: "StationID" +# short_name: "stationID" +# name: "BUOY/PLATFORM IDENTIFIER" +# bufr_mnemonic: "*/ID2SQ/BPID" +# descriptor: "MetaData" +# units: "" +# depth_profile_var_name: "" +# data_min: 0 +# data_max: 0 +# data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "RcptDateTime2D" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCPTIM/RCYR' + rmonth: '*/RCPTIM/RCMO' + rday: '*/RCPTIM/RCDY' + rhour: '*/RCPTIM/RCHR' + rminute: '*/RCPTIM/RCMI' diff --git a/test/marine/testinput/dbuoy.yaml b/test/marine/testinput/dbuoy.yaml new file mode 100644 index 000000000..2c9fcd746 --- /dev/null +++ b/test/marine/testinput/dbuoy.yaml @@ -0,0 +1,109 @@ +--- + - class_name: "Temperature" + short_name: "waterTemperature" + name: "waterTemperature" + bufr_mnemonic: "*/DTSCUR/STMP" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Salinity" + short_name: "salinity" + name: "salinity" + bufr_mnemonic: "*/DTSCUR/SALN" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 45.0 + data_error: 0.01 + + - class_name: "Depth" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/DTSCUR/DBSS" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/RPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "BuoyType" + short_name: "buoyType" + name: "Buoy Type" + bufr_mnemonic: "*/RPSEC4/BUYT" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "RcptDateTime2D" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCPTIM/RCYR' + rmonth: '*/RCPTIM/RCMO' + rday: '*/RCPTIM/RCDY' + rhour: '*/RCPTIM/RCHR' + rminute: '*/RCPTIM/RCMI' diff --git a/test/marine/testinput/dbuoyb.yaml b/test/marine/testinput/dbuoyb.yaml new file mode 100644 index 000000000..35d94fe23 --- /dev/null +++ b/test/marine/testinput/dbuoyb.yaml @@ -0,0 +1,109 @@ +--- + - class_name: "Temperature" + short_name: "waterTemperature" + name: "waterTemperature" + bufr_mnemonic: "*/IDMSMDBS/BBYSTSL/SST1" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Salinity" + short_name: "salinity" + name: "salinity" + bufr_mnemonic: "*/IDMSMDBS/BBYSTSL/SALN" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 45.0 + data_error: 0.01 + + - class_name: "Depth" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/IDMSMDBS/BBYSTSL/DBSS" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLONH" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLATH" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/WMOP" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "BuoyType" + short_name: "buoyType" + name: "Buoy Type" + bufr_mnemonic: "*/BUYT" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testinput/lcman.yaml b/test/marine/testinput/lcman.yaml new file mode 100644 index 000000000..457b0ddce --- /dev/null +++ b/test/marine/testinput/lcman.yaml @@ -0,0 +1,76 @@ +--- + - class_name: "Temperature" + short_name: "seaSurfaceTemperature" + name: "seaSurfaceTemperature" + bufr_mnemonic: "*/SST1" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/RPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "RcptDateTime2D" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCPTIM/RCYR' + rmonth: '*/RCPTIM/RCMO' + rday: '*/RCPTIM/RCDY' + rhour: '*/RCPTIM/RCHR' + rminute: '*/RCPTIM/RCMI' diff --git a/test/marine/testinput/mbuoy.yaml b/test/marine/testinput/mbuoy.yaml new file mode 100644 index 000000000..fba11012f --- /dev/null +++ b/test/marine/testinput/mbuoy.yaml @@ -0,0 +1,76 @@ +--- + - class_name: "Temperature" + short_name: "seaSurfaceTemperature" + name: "seaSurfaceTemperature" + bufr_mnemonic: "*/SST1" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "platformID" + name: "BUOY/PLATFORM IDENTIFIER" + bufr_mnemonic: "*/ID2SQ/BPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "RcptDateTime2D" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCPTIM/RCYR' + rmonth: '*/RCPTIM/RCMO' + rday: '*/RCPTIM/RCDY' + rhour: '*/RCPTIM/RCHR' + rminute: '*/RCPTIM/RCMI' diff --git a/test/marine/testinput/mbuoyb.yaml b/test/marine/testinput/mbuoyb.yaml new file mode 100644 index 000000000..05650fb0e --- /dev/null +++ b/test/marine/testinput/mbuoyb.yaml @@ -0,0 +1,109 @@ +--- + - class_name: "Temperature" + short_name: "waterTemperature" + name: "waterTemperature" + bufr_mnemonic: "*/IDMSMDBS/BBYSTSL/SST1" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Salinity" + short_name: "salinity" + name: "salinity" + bufr_mnemonic: "*/IDMSMDBS/BBYSTSL/SALN" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 45.0 + data_error: 0.01 + + - class_name: "Depth" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/IDMSMDBS/BBYSTSL/DBSS" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLONH" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLATH" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/RPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "BuoyType" + short_name: "buoyType" + name: "Buoy Type" + bufr_mnemonic: "*/BUYT" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testinput/shipsu.yaml b/test/marine/testinput/shipsu.yaml new file mode 100644 index 000000000..b7e794389 --- /dev/null +++ b/test/marine/testinput/shipsu.yaml @@ -0,0 +1,87 @@ +--- + - class_name: "Temperature" + short_name: "seaSurfaceTemperature" + name: "seaSurfaceTemperature" + bufr_mnemonic: "*/SST1" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "" + data_min: -10.0 + data_max: 50.0 + data_error: 0.24 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/RPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + +# - class_name: "StationID" +# short_name: "platformID" +# name: "BUOY/PLATFORM IDENTIFIER" +# bufr_mnemonic: "*/ID2SQ/BPID" +# descriptor: "MetaData" +# units: "" +# depth_profile_var_name: "" +# data_min: 0 +# data_max: 0 +# data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "RcptDateTime2D" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCPTIM/RCYR' + rmonth: '*/RCPTIM/RCMO' + rday: '*/RCPTIM/RCDY' + rhour: '*/RCPTIM/RCHR' + rminute: '*/RCPTIM/RCMI' diff --git a/test/marine/testinput/subpfl.yaml b/test/marine/testinput/subpfl.yaml new file mode 100644 index 000000000..c4bd147ec --- /dev/null +++ b/test/marine/testinput/subpfl.yaml @@ -0,0 +1,98 @@ +--- + - class_name: "Temperature" + short_name: "waterTemperature" + name: "waterTemperature" + bufr_mnemonic: "*/GLPFDATA/SSTH" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.02 + + - class_name: "Salinity" + short_name: "salinity" + name: "salinity" + bufr_mnemonic: "*/GLPFDATA/SALNH" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 45.0 + data_error: 0.01 + + - class_name: "DepthFromPressure" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/GLPFDATA/WPRES" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLONH" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLATH" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/WMOP" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testinput/tesac.yaml b/test/marine/testinput/tesac.yaml new file mode 100644 index 000000000..892fce5f8 --- /dev/null +++ b/test/marine/testinput/tesac.yaml @@ -0,0 +1,98 @@ +--- + - class_name: "Temperature" + short_name: "waterTemperature" + name: "waterTemperature" + bufr_mnemonic: "*/BTOCN/STMP" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.02 + + - class_name: "Salinity" + short_name: "salinity" + name: "salinity" + bufr_mnemonic: "*/BTOCN/SALN" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 45.0 + data_error: 0.01 + + - class_name: "Depth" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/BTOCN/DBSS" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/RPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testinput/trkob.yaml b/test/marine/testinput/trkob.yaml new file mode 100644 index 000000000..8f771d8d9 --- /dev/null +++ b/test/marine/testinput/trkob.yaml @@ -0,0 +1,98 @@ +--- + - class_name: "Temperature" + short_name: "seaSurfaceTemperature" + name: "seaSurfaceTemperature" + bufr_mnemonic: "*/BTOCN/STMP" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.3 + + - class_name: "Salinity" + short_name: "seaSurfaceSalinity" + name: "seaSurfaceSalinity" + bufr_mnemonic: "*/BTOCN/SALN" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 45.0 + data_error: 1.0 + + - class_name: "Depth" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/BTOCN/DBSS" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLON" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLAT" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + + - class_name: "StationID" + short_name: "stationID" + name: "Station Identification" + bufr_mnemonic: "*/RPID" + descriptor: "MetaData" + units: "" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testinput/xbtctd.yaml b/test/marine/testinput/xbtctd.yaml new file mode 100644 index 000000000..960da66a0 --- /dev/null +++ b/test/marine/testinput/xbtctd.yaml @@ -0,0 +1,98 @@ +--- + - class_name: "Temperature" + short_name: "waterTemperature" + name: "waterTemperature" + bufr_mnemonic: "*/TMSLPFSQ/SST1" + descriptor: "ObsValue" + units: "degC" + depth_profile_var_name: "depth" + data_min: -10.0 + data_max: 50.0 + data_error: 0.12 + + - class_name: "Salinity" + short_name: "salinity" + name: "salinity" + bufr_mnemonic: "*/TMSLPFSQ/SALNH" + descriptor: "ObsValue" + units: "psu" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 45.0 + data_error: 1.0 + + - class_name: "Depth" + short_name: "depth" + name: "Water depth" + bufr_mnemonic: "*/TMSLPFSQ/DBSS" + descriptor: "MetaData" + units: "m" + depth_profile_var_name: "depth" + data_min: 0.0 + data_max: 10000.0 + data_error: 0.0 + + - class_name: "Longitude" + short_name: "longitude" + name: "Longitude" + bufr_mnemonic: "*/CLONH" + descriptor: "MetaData" + units: "degrees_east" + depth_profile_var_name: "depth" + data_min: -180.0 + data_max: 180.0 + data_error: 0.0 + + - class_name: "Latitude" + short_name: "latitude" + name: "Latitude" + bufr_mnemonic: "*/CLATH" + descriptor: "MetaData" + units: "degrees_north" + depth_profile_var_name: "depth" + data_min: -90.0 + data_max: 90.0 + data_error: 0.0 + +# - class_name: "StationID" +# short_name: "stationID" +# name: "Station Identification" +# bufr_mnemonic: "*/WMOP" +# descriptor: "MetaData" +# units: "" +# depth_profile_var_name: "depth" +# data_min: 0 +# data_max: 0 +# data_error: 0 + + - class_name: "DateTime" + short_name: "dateTime" + name: "DateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + year: '*/YEAR' + month: '*/MNTH' + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + + - class_name: "DateTime" + short_name: "rcptDateTime" + name: "RcptDateTime" + descriptor: "MetaData" + units: "seconds since 1970-01-01T00:00:00Z" + depth_profile_var_name: "depth" + data_min: 0 + data_max: 0 + data_error: 0 + bufr_mnemonics: + ryear: '*/RCYR' + rmonth: '*/RCMO' + rday: '*/RCDY' + rhour: '*/RCHR' + rminute: '*/RCMI' diff --git a/test/marine/testref/bufr2ioda_insitu_profile_argo_subpfl_2019010700.ref b/test/marine/testref/bufr2ioda_insitu_profile_argo_subpfl_2019010700.ref new file mode 100644 index 000000000..8d9229480 --- /dev/null +++ b/test/marine/testref/bufr2ioda_insitu_profile_argo_subpfl_2019010700.ref @@ -0,0 +1,28 @@ +ObsValue/waterTemperature: 51731, float32 min, max = -1.682989478111267, 30.1660099029541 +ObsValue/waterTemperature hash = 119c9234acefad1277715f7dd8ac039d7bcdb16b8720aa6c9cc1b6a11dbe4213 +ObsValue/salinity: 51731, float32 min, max = 0.0, 38.48899841308594 +ObsValue/salinity hash = 6fb94cea16e0f580f286ea75f5364e3e17e4870ad757db9591f632d6072ebf46 +MetaData/Water depth: 51731, float32 min, max = 0.0, 3051.300048828125 +MetaData/Water depth hash = 1e1107b2d4f325e03cfaf37dcf67764653193ed91e1104204151121d0b47da0c +MetaData/Longitude: 51731, float32 min, max = -179.53500366210938, 175.97000122070312 +MetaData/Longitude hash = 075372768691f0d8b0d0e9e8fd392b7d8f046dc2da8882f1c303ecb60e281661 +MetaData/Latitude: 51731, float32 min, max = -66.93299865722656, 66.80599975585938 +MetaData/Latitude hash = 961938028557122f6a614ef887b549027ee3782b49df96cb48b882ab886419f4 +MetaData/Station Identification: 51731, self._data_min) & (self._data <= self._data_max) + + def get_short_name(self): + return self._short_name + + def get_name(self): + return self._name + + def set_name(self, name): + self._name = name + + def get_error(self): + return self._data_error + + def set_error(self, e): + self._data_error = e + + def get_units(self): + return self._units + + def get_descriptor(self): + return self._descriptor + + def get_depth_profile_var_name(self): + return self._depth_profile_var_name + + def read_from_bufr(self, bufr_file_path): + q = bufr.QuerySet() + q = self.add_query(q) + with bufr.File(bufr_file_path) as f: + r = f.execute(q) + self.set_from_query_result(r) + + def read_from_bufr_by_depth(self, bufr_file_path, depth): + q = bufr.QuerySet() + q = depth.add_query(q) + q = self.add_query(q) + with bufr.File(bufr_file_path) as f: + r = f.execute(q) + self.set_from_query_result(r) + + def print_data(self): + for index, value in np.ndenumerate(self._data): + print(f"{index[0]}: {value}") diff --git a/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/data_variable_dictionary.py b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/data_variable_dictionary.py new file mode 100644 index 000000000..0e3467bb0 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/data_variable_dictionary.py @@ -0,0 +1,115 @@ +import numpy as np +import yaml +from pyioda import ioda_obs_space as ioda_ospace +from .data_set import DataSet +from .data_variable import DataVariable +from .variables import * + + +# adding a dictionary to the data set +class DataVariableDictionary(DataSet): + def __init__(self, logger): + self.logger = logger + super().__init__(logger) + self._variable_dict = {} + + def add(self, variable: DataVariable): + # Use the short_name as the key in the dictionary + if variable._short_name in self._variable_dict: + self.logger.warning(f"'{variable._short_name}' already exists, overwriting.") + self._variable_dict[variable._short_name] = variable + super().add(variable) + + # remove a variable both from the data set and the dictionary + def remove(self, short_name): + v = self._variable_dict.pop(short_name, None) # None is returned if the key is not found + super().remove(v) + + def get(self, short_name: str): + # Return the variable associated with the short_name + return self._variable_dict.get(short_name, None) + + def print_var_list(self): + for name, value in self._variable_dict.items(): + print(name) + + def __repr__(self): + return f"DataVariableDictionary({len(self._variable_dict)} variables)" + + def read_from_yaml(self, yaml_file): + with open(yaml_file, 'r') as file: + data = yaml.safe_load(file) + # print("Loaded data from YAML:", data) + + for var_data in data: + class_name = var_data['class_name'] + + # Remove the 'class_name' key from the dictionary to get only the parameters + parameters = {k: v for k, v in var_data.items() if k != 'class_name'} + # add this dictionary to the parameter list + parameters["data_dictionary"] = self + + # Dynamically get the class by name + variable_class = globals()[class_name] + + if variable_class: + # Instantiate the class using the unpacked parameters + instance = variable_class(**parameters) # Unpack the dictionary into keyword arguments + self.add(instance) + + def add_preqc_vars(self): + for v in self.data_variables: + if v.get_descriptor() == "ObsValue": + self.add(PreQCVariable(v)) + self.logger.debug(f"added preqc variable for {v.get_name()}") + + def add_error_vars(self): + for v in self.data_variables: + if v.get_descriptor() == "ObsValue": + self.add(ErrorVariable(v)) + self.logger.debug(f"added error variable for {v.get_name()}") + + # needs to have PreQC for correct dtype, fill_value + def add_seq_num(self): + # find a preqc variable based on a descriptor + for v in self.data_variables: + if v.get_descriptor() == "PreQC": + preqc = v.get_data() + dtype = preqc.dtype + fill_value = preqc.fill_value + lat = self.get("latitude") + lon = self.get("longitude") + seq_num = SequenceNumber(lon.get_data(), lat.get_data(), dtype, fill_value) + self.add(seq_num) + self.logger.debug("added sequence number variable") + + def add_ocean_basin(self, nc_file_path): + # find a preqc variable based on a descriptor + for v in self.data_variables: + if v.get_descriptor() == "PreQC": + preqc = v.get_data() + dtype = preqc.dtype + fill_value = preqc.fill_value + lat = self.get("latitude") + lon = self.get("longitude") + ocean_basin = OceanBasinVariable(nc_file_path, lon.get_data(), lat.get_data(), dtype, fill_value) + self.add(ocean_basin) + self.logger.debug("added ocean basin variable") + + def write_to_ioda_file(self, b2i_config): + iodafile_path = b2i_config.ioda_filepath() + path, fname = os.path.split(iodafile_path) + os.makedirs(path, exist_ok=True) + + n = self.get_data_size() + dims = {'Location': np.arange(0, n)} + obsspace = ioda_ospace.ObsSpace(iodafile_path, mode='w', dim_dict=dims) + + date_time = self.get("dateTime") + min_date = date_time.get_data().min() + max_date = date_time.get_data().max() + date_range = [str(min_date), str(max_date)] + b2i_config.create_ioda_attributes(obsspace, date_range) + + self.create_ioda_objects(obsspace) + self.logger.debug(f"written ioda variables to {iodafile_path}") diff --git a/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/log.py b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/log.py new file mode 100644 index 000000000..9efa65c06 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/log.py @@ -0,0 +1,79 @@ +import logging + + +class B2ILogger: + def __init__(self, name, log_to_console=True, log_file=None): + self.log_to_console = log_to_console + self.log_file = log_file + + self.logger = logging.getLogger(name) + self.logger.setLevel(logging.DEBUG) # Set default logging level to DEBUG + + console_format = '%(message)s' + self.console_formatter = logging.Formatter(console_format) + self.console_handler = None + + file_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + self.file_formatter = logging.Formatter(file_format) + self.file_handler = None + + test_file_format = '%(message)s' + self.test_file_formatter = logging.Formatter(test_file_format) + self.test_file_handler = None + + self.enable_logging() + + def debug(self, message): + self.logger.debug(message) + + def info(self, message): + self.logger.info(message) + + def warning(self, message): + self.logger.warning(message) + + def error(self, message): + self.logger.error(message) + + def critical(self, message): + self.logger.critical(message) + + def disable_console_logging(self): + if self.console_handler: + self.logger.removeHandler(self.console_handler) + + def enable_console_logging(self): + if not self.console_handler: + self.console_handler = logging.StreamHandler() + self.console_handler.setFormatter(self.console_formatter) + self.logger.addHandler(self.console_handler) + + def disable_file_logging(self): + if self.file_handler: + self.logger.removeHandler(self.file_handler) + + def enable_file_logging(self): + if not self.file_handler and self.log_file: + self.file_handler = logging.FileHandler(self.log_file) + self.file_handler.setFormatter(self.file_formatter) + self.logger.addHandler(self.file_handler) + + def disable_test_file_logging(self): + if self.test_file_handler: + self.logger.removeHandler(self.test_file_handler) + + def enable_test_file_logging(self, test_log_file): + if not self.test_file_handler and test_log_file: + self.test_file_handler = logging.FileHandler(test_log_file) + self.test_file_handler.setFormatter(self.test_file_formatter) + self.logger.addHandler(self.test_file_handler) + + def disable_logging(self): + self.disable_file_logging() + self.disable_console_logging() + + def enable_logging(self): + if self.log_file: + self.enable_file_logging() + if self.log_to_console: + self.enable_console_logging() diff --git a/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/ocean.py b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/ocean.py new file mode 100644 index 000000000..60c0fbf16 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/ocean.py @@ -0,0 +1,64 @@ +import os +import sys +import numpy as np +import numpy.ma as ma +import math +import netCDF4 as nc +import xarray as xr + +# OceanBasin class provides a facility to add an OceanBasin +# metadata variable using lon and lat +# basic definition of ocean basins is read from an nc file, +# We search for the filename, depending on the system +# The path to the ocean basin nc file can be supplied +# in the implementation of the converter + +# the main method is get_station_basin which returns the ocean basin +# for a list of station coordinates + + +class OceanBasin: + def __init__(self, ocean_basin_nc_file_path): + self.read_nc_file(ocean_basin_nc_file_path) + + def read_nc_file(self, ocean_basin_nc_file_path): + try: + with nc.Dataset(ocean_basin_nc_file_path, 'r') as nc_file: + variable_name = 'open_ocean' + if variable_name in nc_file.variables: + lat_dim = nc_file.dimensions['lat'].size + lon_dim = nc_file.dimensions['lon'].size + self.__latitudes = nc_file.variables['lat'][:] + self.__longitudes = nc_file.variables['lon'][:] + + variable = nc_file.variables[variable_name] + # Read the variable data into a numpy array + variable_data = variable[:] + # Convert to 2D numpy array + self.__basin_array = np.reshape(variable_data, (lat_dim, lon_dim)) + except FileNotFoundError: + print(f"The file {file_path} does not exist.") + sys.exit(1) + except IOError as e: + # Handle other I/O errors, such as permission errors + print(f"An IOError occurred: {e}") + sys.exit(1) + + # input: 2 vectors of station coordinates + # output: a vector of station ocean basin values + def get_station_basin(self, lat, lon): + n = len(lon) + + lat0 = self.__latitudes[0] + dlat = self.__latitudes[1] - self.__latitudes[0] + lon0 = self.__longitudes[0] + dlon = self.__longitudes[1] - self.__longitudes[0] + + # the data may be a masked array + ocean_basin = [] + for i in range(n): + if not ma.is_masked(lat[i]): + i1 = round((lat[i] - lat0) / dlat) + i2 = round((lon[i] - lon0) / dlon) + ocean_basin.append(self.__basin_array[i1][i2]) + return np.array(ocean_basin, dtype=np.int32) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/util.py b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/util.py new file mode 100644 index 000000000..d650134ef --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/util.py @@ -0,0 +1,78 @@ +import os +import sys +import argparse +import subprocess +import numpy as np +import tempfile +import hashlib + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument( + '-c', '--config', + type=str, + help='Input JSON or YAML configuration', required=True + ) + parser.add_argument( + '-l', '--log_file', + type=str, + help='Output file for testing ioda variables' + ) + parser.add_argument( + '-t', '--test', + type=str, + help='Input test reference file' + ) + args = parser.parse_args() + config_file = args.config + log_file = args.log_file + test_file = args.test + script_name = sys.argv[0] + return script_name, config_file, log_file, test_file + + +def run_diff(file1, file2, logger): + try: + # Run the diff command + result = subprocess.run( + ['diff', file1, file2], + capture_output=True, text=True, check=False + ) + + # Check if diff command succeeded (return code 0) + if result.returncode == 0: + pass + elif result.returncode == 1: + logger.error("diff on files:") + logger.error(f"{file1}") + logger.error(f"{file2}") + logger.error("Files are different:") + logger.error(f"{result.stdout}") + else: + logger.error("Error occurred while running diff command.") + logger.error(f"{result.stdout}") + + except subprocess.CalledProcessError as e: + logger.error(f"Error occurred: {e}") + + return result.returncode + + +# use hash for testing; +def compute_hash(sequence, algorithm='sha256'): + """ + Compute a hash of the given sequence using the specified algorithm. + + :param sequence: A sequence of numbers (e.g., list of integers). + :param algorithm: The hash algorithm to use (e.g., 'sha256'). + :return: The hexadecimal digest of the hash. + """ + # Convert the sequence to a byte string + sequence_bytes = bytes(sequence) + # Create a hash object + hash_obj = hashlib.new(algorithm) + # Update the hash object with the byte string + hash_obj.update(sequence_bytes) + # Return the hexadecimal digest of the hash + return hash_obj.hexdigest() diff --git a/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/variables.py b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/variables.py new file mode 100644 index 000000000..3a6da5ea9 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/b2ibase/variables.py @@ -0,0 +1,224 @@ +import numpy as np +import sys +from .data_variable import DataVariable +from .ocean import OceanBasin +from .util import * + + +class Depth(DataVariable): + def create_ioda_objects(self, obsspace): + obsspace.create_var( + self._descriptor + "/" + self._short_name, + dtype=self._data.dtype, fillval=self._data.fill_value + ) \ + .write_attr('units', self._units) \ + .write_attr('long_name', self._name) \ + .write_data(self._data) + + +class DepthFromPressure(Depth): + def set_from_query_result(self, r): + super().set_from_query_result(r) + # convert depth in pressure units to meters (rho * g * h) + self._data = np.float32(self._data.astype(float) * 0.0001) + + +class Longitude(DataVariable): + pass + + +class Latitude(DataVariable): + pass + + +class DateTime(DataVariable): + def __init__(self, short_name, name, descriptor, units, depth_profile_var_name, data_min, data_max, data_error, bufr_mnemonics, data_dictionary): + # Pass empty string for bufr_mnemonic + super().__init__(short_name, name, None, descriptor, units, depth_profile_var_name, data_min, data_max, data_error, data_dictionary) + self._bufr_mnemonics = bufr_mnemonics +# WARNING: _bufr_mnemonics is a dictionary, but it should be a list +# because position is important in get_datetime + + def print__bufr_mnemonics(self): + for date_component in self._bufr_mnemonics: + print(f"\t{date_component}: {self._bufr_mnemonics[date_component]}") + + def describe(self): + super().describe() + print("_bufr_mnemonics:") + self.print__bufr_mnemonics() + # print(self._bufr_mnemonics) + + def add_query(self, q): + for date_component in self._bufr_mnemonics: + q.add(date_component, self._bufr_mnemonics[date_component]) + return q + + def set_from_query_result(self, r): + date_keys = list(self._bufr_mnemonics.keys()) + if self._depth_profile_var_name: + self._data = r.get_datetime(*date_keys, group_by=self._depth_profile_var_name) + else: + self._data = r.get_datetime(*date_keys) + # convert to seconds since 1970 + self._data = self._data.astype(np.int64) + + def create_ioda_objects(self, obsspace): + obsspace.create_var( + self._descriptor + "/" + self._short_name, + dtype=self._data.dtype, fillval=self._data.fill_value + ) \ + .write_attr('units', self._units) \ + .write_attr('long_name', self._name) \ + .write_data(self._data) + + +# when rcpt date time is 2d in bufr +# extract without grouping by depth +# discard the irrelevant dimension +# tile repeats the variable, +# so we are doing "by_depth" by hand +# we use the dict to get the depth variable +class RcptDateTime2D(DateTime): + def set_from_query_result(self, r): + date_keys = list(self._bufr_mnemonics.keys()) + self._data = r.get_datetime(*date_keys) + self._data = self._data[:, 0] + self._data = self._data.astype(np.int64) + n = self._data_dictionary.get_data_size() + k = int(n / self._data.size) + self._data = np.tile(self._data, k) + + +class StationID(DataVariable): + def create_ioda_objects(self, obsspace): + self.short_create_ioda_objects(obsspace) + + def log(self, logger): + logger.debug(f"{self._descriptor}/{self._name}: {len(self._data)}, {self._data.astype(str).dtype}") + if isinstance(self._data[0], str): + concatenated_string = ''.join(self._data) + # Compute the hash using SHA-256 + hash_object = hashlib.sha256(concatenated_string.encode()) + hash_hex = hash_object.hexdigest() + logger.debug(f"{self._descriptor}/{self._name} hash = {hash_hex}") + # # elif isinstance(self.stationID[0], int32): + else: + logger.debug(f"{self._descriptor}/{self._name} hash = {compute_hash(self._data)}") + + +class Temperature(DataVariable): + def set_from_query_result(self, r): + super().set_from_query_result(r) + self._data -= 273.15 + + +class Salinity(DataVariable): + def get_filter(self): + return (self._data >= self._data_min) & (self._data <= self._data_max) + + +class BuoyType(DataVariable): + def create_ioda_objects(self, obsspace): + self.short_create_ioda_objects(obsspace) + + +# Additional variables +class PreQCVariable(DataVariable): + def __init__(self, v): + super().__init__("PreQC_" + v.get_short_name(), + v.get_name(), + bufr_mnemonic=None, + descriptor="PreQC", + units=v.get_units(), + depth_profile_var_name=v.get_depth_profile_var_name(), + data_min=0, + data_max=0, + data_error=0, + data_dictionary=None) + n = v.get_data_size() + self._data = (np.ma.masked_array(np.full(n, 0))).astype(np.int32) + + def create_ioda_objects(self, obsspace): + # self.short_create_ioda_objects(obsspace) + obsspace.create_var(self._descriptor + "/" + self._name, + dtype=self._data.dtype, + fillval=self._data.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(self._data) + + +class ErrorVariable(DataVariable): + def __init__(self, v): + e = v.get_error() + super().__init__("ObsError_" + v.get_short_name(), + v.get_name(), + bufr_mnemonic=None, + descriptor="ObsError", + units=v.get_units(), + depth_profile_var_name=v.get_depth_profile_var_name(), + data_min=e, + data_max=e, + data_error=e, + data_dictionary=None) + n = v.get_data_size() + self._data = np.float32(np.ma.masked_array(np.full(n, e))) + + def create_ioda_objects(self, obsspace): + obsspace.create_var(self._descriptor + "/" + self._name, + dtype=self._data.dtype, + fillval=self._data.fill_value) \ + .write_attr('units', self._units) \ + .write_attr('long_name', 'ObsError') \ + .write_data(self._data) + + +class SequenceNumber(DataVariable): + def __init__(self, lon, lat, dtype, fill_value): + self.dtype = dtype + self.fill_value = fill_value + super().__init__("sequenceNumber", + 'Sequence Number', + bufr_mnemonic=None, + descriptor="MetaData", + units=None, + depth_profile_var_name=None, + data_min=None, + data_max=None, + data_error=None, + data_dictionary=None) + combined = np.stack((lon, lat), axis=-1) + unique_combined, seq_num = np.unique(combined, axis=0, return_inverse=True) + self._data = np.ma.masked_equal(seq_num.astype(np.int32), 1) + + def create_ioda_objects(self, obsspace): + obsspace.create_var(self._descriptor + "/" + self._short_name, + dtype=self.dtype, + fillval=self.fill_value) \ + .write_attr('long_name', self._name) \ + .write_data(self._data) + + +class OceanBasinVariable(DataVariable): + def __init__(self, nc_file_path, lon, lat, dtype, fill_value): + self.dtype = dtype + self.fill_value = fill_value + self.ocean = OceanBasin(nc_file_path) + super().__init__("oceanBasin", + 'Ocean basin', + bufr_mnemonic=None, + descriptor="MetaData", + units=None, + depth_profile_var_name=None, + data_min=0, + data_max=5, + data_error=0, + data_dictionary=None) + self._data = self.ocean.get_station_basin(lat, lon) + + def create_ioda_objects(self, obsspace): + obsspace.create_var(self._descriptor + "/" + self._short_name, + dtype=self.dtype, + fillval=self.fill_value) \ + .write_attr('long_name', self._name) \ + .write_data(self._data) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_argo.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_argo.py new file mode 100755 index 000000000..c5b5dac72 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_argo.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class ArgoConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_argo.{self.cycle_datetime}.nc4" + + +class ArgoData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("waterTemperature") + saln = self.get("salinity") + station_id = self.get("stationID").get_data() + id_mask = [True if str(x)[1] == '9' else False for x in station_id] + mask = id_mask & temp.get_filter() & saln.get_filter() + self.filter(mask) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = ArgoConfig(config_file, logger) + data = ArgoData(logger) + b2i = B2I(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_bathy.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_bathy.py new file mode 100755 index 000000000..059a38528 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_bathy.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class BathyData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("waterTemperature") + mask = temp.get_filter() + self.filter(mask) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = Config(config_file, logger) + data = BathyData(logger) + b2i = B2I(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_glider.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_glider.py new file mode 100755 index 000000000..117bf840b --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_glider.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class GliderConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_glider.{self.cycle_datetime}.nc4" + + +class GliderData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("waterTemperature") + saln = self.get("salinity") + id = self.get("stationID").get_data() + id_mask = (id >= 68900) & (id <= 68999) | \ + (id >= 1800000) & (id <= 1809999) | \ + (id >= 2800000) & (id <= 2809999) | \ + (id >= 3800000) & (id <= 3809999) | \ + (id >= 4800000) & (id <= 4809999) | \ + (id >= 5800000) & (id <= 5809999) | \ + (id >= 6800000) & (id <= 6809999) | \ + (id >= 7800000) & (id <= 7809999) + mask = id_mask & temp.get_filter() & saln.get_filter() + self.filter(mask) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = GliderConfig(config_file, logger) + data = GliderData(logger) + b2i = B2I(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_tesac.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_tesac.py new file mode 100755 index 000000000..709b125ee --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_tesac.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class TesacData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("waterTemperature") + saln = self.get("salinity") + station_id = self.get("stationID").get_data() + id_mask = [True if id.isdigit() and id != 0 else False for id in station_id] + mask = id_mask & temp.get_filter() & saln.get_filter() + self.filter(mask) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = Config(config_file, logger) + data = TesacData(logger) + b2i = B2I(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_tropical.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_tropical.py new file mode 100755 index 000000000..b8bca611f --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_tropical.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class TropicalConfig(Config): + + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_tropical.{self.cycle_datetime}.nc4" + + +class TropicalData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("waterTemperature") + saln = self.get("salinity") + # Separate tropical mooring profiles from dbuoy tank + # buoy_type: ATLAS is 21, TRITON is 22 + buoy_type = self.get("buoyType").get_data() + buoy_mask = [True if x == 21 or x == 22 else False for x in buoy_type] + mask = buoy_mask & temp.get_filter() & saln.get_filter() + self.filter(mask) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = TropicalConfig(config_file, logger) + data = TropicalData(logger) + b2i = B2I(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + print("result = ", result) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_xbtctd.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_xbtctd.py new file mode 100755 index 000000000..24f928dcc --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_profile_xbtctd.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class XbtctdData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("waterTemperature") + saln = self.get("salinity") + mask = temp.get_filter() & saln.get_filter() + self.filter(mask) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = Config(config_file, logger) + data = XbtctdData(logger) + b2i = B2I(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_altkob.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_altkob.py new file mode 100755 index 000000000..c4c563d63 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_altkob.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class AltkobConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_surface_{self.data_format}.{self.cycle_datetime}.nc4" + + +class AltkobData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("seaSurfaceTemperature") + saln = self.get("salinity") + mask = temp.get_filter() & saln.get_filter() + self.filter(mask) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = AltkobConfig(config_file, logger) + data = AltkobData(logger) + b2i = B2I(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_cstgd.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_cstgd.py new file mode 100755 index 000000000..5a7209df1 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_cstgd.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class CstgdConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_surface_{self.data_format}.{self.cycle_datetime}.nc4" + + +class CstgdData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("seaSurfaceTemperature") + mask = temp.get_filter() + self.filter(mask) + + +class CstgdConverter(B2I): + def process_data(self): + self.data.add_preqc_vars() + self.data.add_error_vars() + ocean_file_path = self.config.ocean_basin_nc_file_path() + self.data.add_ocean_basin(ocean_file_path) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = CstgdConfig(config_file, logger) + data = CstgdData(logger) + b2i = CstgdConverter(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_drifter.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_drifter.py new file mode 100755 index 000000000..f728ef49e --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_drifter.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import sys +import numpy as np +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class DrifterConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_surface_drifter..{self.cycle_datetime}.nc4" + + +class DrifterData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("waterTemperature") + buoy_type = self.get("buoyType").get_data() + # Separate Drifter profiles from dbuoy tank + # buoy_type: + # 1 - Standard Lagrangian drifter (Global Drifter Programme) + # 4 - Ice drifter + # 5 - SVPG Standard Lagrangian drifter with GPS + values_to_select = [1, 4, 5] + buoy_mask = np.isin(buoy_type, values_to_select) + mask = buoy_mask & temp.get_filter() + self.filter(mask) + + +class DrifterConverter(B2I): + def process_data(self): + self.data.remove("depth") + self.data.remove("salinity") + self.data.add_preqc_vars() + self.data.add_error_vars() + ocean_file_path = self.config.ocean_basin_nc_file_path() + self.data.add_ocean_basin(ocean_file_path) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = DrifterConfig(config_file, logger) + data = DrifterData(logger) + b2i = DrifterConverter(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_lcman.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_lcman.py new file mode 100755 index 000000000..7bbba748b --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_lcman.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class LcmanConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_surface_{self.data_format}.{self.cycle_datetime}.nc4" + + +class LcmanData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("seaSurfaceTemperature") + mask = temp.get_filter() + self.filter(mask) + + +class LcmanConverter(B2I): + def process_data(self): + self.data.add_preqc_vars() + self.data.add_error_vars() + ocean_file_path = self.config.ocean_basin_nc_file_path() + self.data.add_ocean_basin(ocean_file_path) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = LcmanConfig(config_file, logger) + data = LcmanData(logger) + b2i = LcmanConverter(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_shipsu.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_shipsu.py new file mode 100755 index 000000000..f06ea39a5 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_shipsu.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class ShipsuConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_surface_{self.data_format}.{self.cycle_datetime}.nc4" + + +class ShipsuData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("seaSurfaceTemperature") + mask = temp.get_filter() + self.filter(mask) + + +class ShipsuConverter(B2I): + def process_data(self): + self.data.add_preqc_vars() + self.data.add_error_vars() + ocean_file_path = self.config.ocean_basin_nc_file_path() + self.data.add_ocean_basin(ocean_file_path) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = ShipsuConfig(config_file, logger) + data = ShipsuData(logger) + b2i = ShipsuConverter(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result) diff --git a/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_trkob.py b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_trkob.py new file mode 100755 index 000000000..65b7c5032 --- /dev/null +++ b/ush/ioda/bufr2ioda/marine/newb2i/bufr2ioda_insitu_surface_trkob.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import sys +from b2ibase.util import parse_arguments +from b2ibase.config import Config +from b2ibase.data_variable_dictionary import DataVariableDictionary +from b2ibase.b2i import B2I +from b2ibase.log import B2ILogger + + +class TrkobConfig(Config): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_surface_{self.data_format}.{self.cycle_datetime}.nc4" + + +# same as altkob +class TrkobData(DataVariableDictionary): + def read_from_bufr(self, bufr_file_path): + super().read_from_bufr(bufr_file_path) + temp = self.get("seaSurfaceTemperature") + saln = self.get("seaSurfaceSalinity") + mask = temp.get_filter() & saln.get_filter() + self.filter(mask) + + +class TrkobConverter(B2I): + def process_data(self): + self.data.remove("depth") + self.data.add_preqc_vars() + self.data.add_error_vars() + ocean_file_path = self.config.ocean_basin_nc_file_path() + self.data.add_ocean_basin(ocean_file_path) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + log_to_console = True + logger = B2ILogger(script_name, log_to_console, log_file) + + config = TrkobConfig(config_file, logger) + data = TrkobData(logger) + b2i = TrkobConverter(config, data, logger) + b2i.run() + if test_file: + result = b2i.test(test_file) + sys.exit(result)