From 399e60972dabd005992632aced7e0e4284587927 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 15 Oct 2024 17:51:21 +0000 Subject: [PATCH 01/11] Per #2893, add obtype_as_obs_message_type_flag option for Point-Stat and Ensemble-Stat that applies when writing MPR, SEEPS_MPR, and ORANK line types. If true, write the input observation message type to the OBTYPE output column rather than the name of the group. --- src/basic/vx_config/config_constants.h | 1 + src/libcode/vx_stat_out/stat_columns.cc | 14 +++++++++- src/libcode/vx_stat_out/stat_columns.h | 9 ++++--- src/libcode/vx_statistics/pair_base.cc | 10 +++++-- src/libcode/vx_statistics/pair_base.h | 8 ++++-- .../vx_statistics/pair_data_ensemble.cc | 5 ++-- src/libcode/vx_statistics/pair_data_point.cc | 27 ++++++++++--------- src/libcode/vx_statistics/pair_data_point.h | 6 +++-- src/tools/core/ensemble_stat/ensemble_stat.cc | 9 ++++++- .../ensemble_stat/ensemble_stat_conf_info.cc | 5 ++++ .../ensemble_stat/ensemble_stat_conf_info.h | 1 + src/tools/core/point_stat/point_stat.cc | 26 +++++++++++++----- .../core/point_stat/point_stat_conf_info.cc | 5 ++++ .../core/point_stat/point_stat_conf_info.h | 1 + src/tools/other/gsi_tools/gsidens2orank.cc | 4 +-- 15 files changed, 97 insertions(+), 34 deletions(-) diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 7bba9e759e..802e37f28a 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -538,6 +538,7 @@ static const char conf_key_model[] = "model"; static const char conf_key_desc[] = "desc"; static const char conf_key_obtype[] = "obtype"; static const char conf_key_output_flag[] = "output_flag"; +static const char conf_key_obtype_as_obs_message_type_flag[] = "obtype_as_obs_message_type_flag"; static const char conf_key_obs_window[] = "obs_window"; static const char conf_key_beg[] = "beg"; static const char conf_key_end[] = "end"; diff --git a/src/libcode/vx_stat_out/stat_columns.cc b/src/libcode/vx_stat_out/stat_columns.cc index ac8530f6a5..c805d447fd 100644 --- a/src/libcode/vx_stat_out/stat_columns.cc +++ b/src/libcode/vx_stat_out/stat_columns.cc @@ -1548,6 +1548,7 @@ void write_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, STATOutputType out_type, AsciiTable &stat_at, int &stat_row, AsciiTable &txt_at, int &txt_row, + bool update_obtype, bool update_thresh) { // MPR line type @@ -1567,6 +1568,9 @@ void write_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, // Write a line for each matched pair for(int i=0; in_obs; i++) { + // MET #2893 write individual obs message type + if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str()); + // Set the observation valid time shc.set_obs_valid_beg(pd_ptr->vld_ta[i]); shc.set_obs_valid_end(pd_ptr->vld_ta[i]); @@ -1645,6 +1649,7 @@ void write_seeps_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, STATOutputType out_type, AsciiTable &stat_at, int &stat_row, AsciiTable &txt_at, int &txt_row, + bool update_obtype, bool update_thresh) { // SEEPS line type @@ -1668,6 +1673,9 @@ void write_seeps_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr, if(!pd_ptr->seeps_mpr[i] || is_bad_data(pd_ptr->seeps_mpr[i]->score)) continue; + // MET #2893 write individual obs message type + if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str()); + // Set the observation valid time shc.set_obs_valid_beg(pd_ptr->vld_ta[i]); shc.set_obs_valid_end(pd_ptr->vld_ta[i]); @@ -1899,7 +1907,8 @@ void write_phist_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, void write_orank_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, STATOutputType out_type, AsciiTable &stat_at, int &stat_row, - AsciiTable &txt_at, int &txt_row) { + AsciiTable &txt_at, int &txt_row, + bool update_obtype) { // Observation Rank line type shc.set_line_type(stat_orank_str); @@ -1913,6 +1922,9 @@ void write_orank_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, // Write a line for each ensemble pair for(int i=0; in_obs; i++) { + // MET #2893 write individual obs message type + if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str()); + // Set the observation valid time shc.set_obs_valid_beg(pd_ptr->vld_ta[i]); shc.set_obs_valid_end(pd_ptr->vld_ta[i]); diff --git a/src/libcode/vx_stat_out/stat_columns.h b/src/libcode/vx_stat_out/stat_columns.h index 7eb21c6fa0..6d2e0a15aa 100644 --- a/src/libcode/vx_stat_out/stat_columns.h +++ b/src/libcode/vx_stat_out/stat_columns.h @@ -106,13 +106,13 @@ extern void write_dmap_row (StatHdrColumns &, const DMAPInfo &, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_mpr_row (StatHdrColumns &, const PairDataPoint *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, - bool update_thresh = true); + bool update_obtype, bool update_thresh = true); extern void write_seeps_row (StatHdrColumns &, const SeepsAggScore *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, - bool update_thresh = true); + bool update_obtype, bool update_thresh = true); extern void write_seeps_mpr_row (StatHdrColumns &, const PairDataPoint *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, - bool update_thresh = true); + bool update_obtype, bool update_thresh = true); extern void write_isc_row (StatHdrColumns &, const ISCInfo &, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_ecnt_row (StatHdrColumns &, const ECNTInfo &, STATOutputType, @@ -124,7 +124,8 @@ extern void write_rhist_row (StatHdrColumns &, const PairDataEnsemble *, STATOut extern void write_phist_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_orank_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, - AsciiTable &, int &, AsciiTable &, int &); + AsciiTable &, int &, AsciiTable &, int &, + bool update_obtype = false); extern void write_ssvar_row (StatHdrColumns &, const PairDataEnsemble *, double, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); extern void write_relp_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, diff --git a/src/libcode/vx_statistics/pair_base.cc b/src/libcode/vx_statistics/pair_base.cc index c26d2d3459..9a18ee6b55 100644 --- a/src/libcode/vx_statistics/pair_base.cc +++ b/src/libcode/vx_statistics/pair_base.cc @@ -104,6 +104,7 @@ void PairBase::clear() { ocsd_na.clear(); ocdf_na.clear(); + typ_sa.clear(); sid_sa.clear(); lat_na.clear(); lon_na.clear(); @@ -155,6 +156,7 @@ void PairBase::erase() { ocsd_na.erase(); ocdf_na.erase(); + typ_sa.clear(); // no erase option sid_sa.clear(); // no erase option lat_na.erase(); lon_na.erase(); @@ -427,7 +429,7 @@ void PairBase::compute_climo_cdf() { //////////////////////////////////////////////////////////////////////// -bool PairBase::add_point_obs(const char *sid, +bool PairBase::add_point_obs(const char *typ, const char *sid, double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, double o, const char *qc, @@ -476,6 +478,7 @@ bool PairBase::add_point_obs(const char *sid, } else { station_values_t val; + val.typ = string(typ); val.sid = string(sid); val.lat = lat; val.lon = lon; @@ -497,6 +500,7 @@ bool PairBase::add_point_obs(const char *sid, } if(obs_summary == ObsSummary::None) { + typ_sa.add(typ); sid_sa.add(sid); lat_na.add(lat); lon_na.add(lon); @@ -520,7 +524,7 @@ bool PairBase::add_point_obs(const char *sid, //////////////////////////////////////////////////////////////////////// -void PairBase::set_point_obs(int i_obs, const char *sid, +void PairBase::set_point_obs(int i_obs, const char *typ, const char *sid, double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, double o, const char *qc, @@ -534,6 +538,7 @@ void PairBase::set_point_obs(int i_obs, const char *sid, exit(1); } + typ_sa.set(i_obs, typ); sid_sa.set(i_obs, sid); lat_na.set(i_obs, lat); lon_na.set(i_obs, lon); @@ -751,6 +756,7 @@ void PairBase::calc_obs_summary(){ // Store summarized value in the map svt.summary_val = ob.val; + typ_sa.add (svt.typ.c_str()); sid_sa.add (svt.sid.c_str()); lat_na.add (svt.lat); lon_na.add (svt.lon); diff --git a/src/libcode/vx_statistics/pair_base.h b/src/libcode/vx_statistics/pair_base.h index af92385f5c..9ad70e0266 100644 --- a/src/libcode/vx_statistics/pair_base.h +++ b/src/libcode/vx_statistics/pair_base.h @@ -36,6 +36,7 @@ struct station_values_t { void clear(); + std::string typ; std::string sid; double lat; double lon; @@ -110,6 +111,7 @@ class PairBase { NumArray ocdf_na; // Observation climatology cumulative distribution function [n_obs] // Point Observation Information + StringArray typ_sa; // Message type [n_obs] StringArray sid_sa; // Station ID [n_obs] NumArray lat_na; // Latitude [n_obs] NumArray lon_na; // Longitude [n_obs] @@ -168,11 +170,13 @@ class PairBase { ob_val_t compute_median(std::string sng_key); ob_val_t compute_percentile(std::string sng_key, int perc); - bool add_point_obs(const char *, double, double, double, double, + bool add_point_obs(const char *, const char *, + double, double, double, double, unixtime, double, double, double, const char *, const ClimoPntInfo &, double); - void set_point_obs(int, const char *, double, double, double, double, + void set_point_obs(int, const char *, const char *, + double, double, double, double, unixtime, double, double, double, const char *, const ClimoPntInfo &, double); diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index af9f80de15..9a2b0a7baf 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -1311,8 +1311,9 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr, // Add the observation value // Weight is from the nearest grid point int n = three_to_one(i_msg_typ, i_mask, i_interp); - if(!pd[n].add_point_obs(hdr_sid_str, hdr_lat, hdr_lon, - obs_x, obs_y, hdr_ut, obs_lvl, obs_hgt, + if(!pd[n].add_point_obs(hdr_typ_str, hdr_sid_str, + hdr_lat, hdr_lon, obs_x, obs_y, + hdr_ut, obs_lvl, obs_hgt, obs_v, obs_qty, cpi, wgt_v)) { if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { diff --git a/src/libcode/vx_statistics/pair_data_point.cc b/src/libcode/vx_statistics/pair_data_point.cc index a22194a297..17a86703bc 100644 --- a/src/libcode/vx_statistics/pair_data_point.cc +++ b/src/libcode/vx_statistics/pair_data_point.cc @@ -156,7 +156,8 @@ void PairDataPoint::assign(const PairDataPoint &pd) { ClimoPntInfo cpi(pd.fcmn_na[i], pd.fcsd_na[i], pd.ocmn_na[i], pd.ocsd_na[i]); - if(add_point_pair(pd.sid_sa[i].c_str(), pd.lat_na[i], pd.lon_na[i], + if(add_point_pair(pd.typ_sa[i].c_str(), pd.sid_sa[i].c_str(), + pd.lat_na[i], pd.lon_na[i], pd.x_na[i], pd.y_na[i], pd.vld_ta[i], pd.lvl_na[i], pd.elv_na[i], pd.f_na[i], pd.o_na[i], pd.o_qc_sa[i].c_str(), @@ -176,13 +177,14 @@ void PairDataPoint::assign(const PairDataPoint &pd) { //////////////////////////////////////////////////////////////////////// -bool PairDataPoint::add_point_pair(const char *sid, double lat, double lon, +bool PairDataPoint::add_point_pair(const char *typ, const char *sid, + double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, double f, double o, const char *qc, const ClimoPntInfo &cpi, double wgt) { - if(!add_point_obs(sid, lat, lon, x, y, ut, lvl, elv, o, qc, + if(!add_point_obs(typ, sid, lat, lon, x, y, ut, lvl, elv, o, qc, cpi, wgt)) return false; f_na.add(f); @@ -233,7 +235,8 @@ void PairDataPoint::set_seeps_score(SeepsScore *seeps, int index) { //////////////////////////////////////////////////////////////////////// -void PairDataPoint::set_point_pair(int i_obs, const char *sid, +void PairDataPoint::set_point_pair(int i_obs, + const char *typ, const char *sid, double lat, double lon, double x, double y, unixtime ut, double lvl, double elv, @@ -248,7 +251,7 @@ void PairDataPoint::set_point_pair(int i_obs, const char *sid, exit(1); } - set_point_obs(i_obs, sid, lat, lon, x, y, ut, lvl, elv, + set_point_obs(i_obs, typ, sid, lat, lon, x, y, ut, lvl, elv, o, qc, cpi, wgt); f_na.set(i_obs, f); @@ -380,8 +383,8 @@ PairDataPoint PairDataPoint::subset_pairs_cnt_thresh( // Handle point data if(is_point_vx()) { - if(out_pd.add_point_pair(sid_sa[i].c_str(), lat_na[i], - lon_na[i], x_na[i], y_na[i], + if(out_pd.add_point_pair(typ_sa[i].c_str(), sid_sa[i].c_str(), + lat_na[i], lon_na[i], x_na[i], y_na[i], vld_ta[i], lvl_na[i], elv_na[i], f_na[i], o_na[i], o_qc_sa[i].c_str(), cpi, wgt_na[i])) { @@ -631,7 +634,7 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Add the forecast, climatological, and observation data // Weight is from the nearest grid point int n = three_to_one(i_msg_typ, i_mask, i_interp); - if(!pd[n].add_point_pair(hdr_sid_str, + if(!pd[n].add_point_pair(hdr_typ_str, hdr_sid_str, hdr_lat, hdr_lon, obs_x, obs_y, hdr_ut, obs_lvl, obs_hgt, fcst_v, obs_v, obs_qty, cpi, wgt_v)) { @@ -996,14 +999,14 @@ void subset_wind_pairs(const PairDataPoint &pd_u, const PairDataPoint &pd_v, // Handle point data if(pd_u.is_point_vx()) { - out_pd_u.add_point_pair(pd_u.sid_sa[i].c_str(), + out_pd_u.add_point_pair(pd_u.typ_sa[i].c_str(), pd_u.sid_sa[i].c_str(), pd_u.lat_na[i], pd_u.lon_na[i], pd_u.x_na[i], pd_u.y_na[i], pd_u.vld_ta[i], pd_u.lvl_na[i], pd_u.elv_na[i], pd_u.f_na[i], pd_u.o_na[i], pd_u.o_qc_sa[i].c_str(), u_cpi, pd_u.wgt_na[i]); - out_pd_v.add_point_pair(pd_v.sid_sa[i].c_str(), + out_pd_v.add_point_pair(pd_v.typ_sa[i].c_str(), pd_v.sid_sa[i].c_str(), pd_v.lat_na[i], pd_v.lon_na[i], pd_v.x_na[i], pd_v.y_na[i], pd_v.vld_ta[i], pd_v.lvl_na[i], pd_v.elv_na[i], @@ -1074,8 +1077,8 @@ PairDataPoint subset_climo_cdf_bin(const PairDataPoint &pd, // Handle point data if(pd.is_point_vx()) { - out_pd.add_point_pair(pd.sid_sa[i].c_str(), pd.lat_na[i], - pd.lon_na[i], pd.x_na[i], pd.y_na[i], + out_pd.add_point_pair(pd.typ_sa[i].c_str(), pd.sid_sa[i].c_str(), + pd.lat_na[i], pd.lon_na[i], pd.x_na[i], pd.y_na[i], pd.vld_ta[i], pd.lvl_na[i], pd.elv_na[i], pd.f_na[i], pd.o_na[i], pd.o_qc_sa[i].c_str(), cpi, pd.wgt_na[i]); diff --git a/src/libcode/vx_statistics/pair_data_point.h b/src/libcode/vx_statistics/pair_data_point.h index 7669e122b2..b6a84579d9 100644 --- a/src/libcode/vx_statistics/pair_data_point.h +++ b/src/libcode/vx_statistics/pair_data_point.h @@ -56,14 +56,16 @@ class PairDataPoint : public PairBase { void extend(int); - bool add_point_pair(const char *, double, double, double, double, + bool add_point_pair(const char *, const char *, + double, double, double, double, unixtime, double, double, double, double, const char *, const ClimoPntInfo &, double); void load_seeps_climo(const ConcatString &seeps_climo_name); void set_seeps_thresh(const SingleThresh &p1_thresh); void set_seeps_score(SeepsScore *, int index=-1); - void set_point_pair(int, const char *, double, double, double, double, + void set_point_pair(int, const char *, const char *, + double, double, double, double, unixtime, double, double, double, double, const char *, const ClimoPntInfo &, double, const SeepsScore *); diff --git a/src/tools/core/ensemble_stat/ensemble_stat.cc b/src/tools/core/ensemble_stat/ensemble_stat.cc index 38abd51a3b..30ea808ee9 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -2236,10 +2236,17 @@ void write_txt_files(const EnsembleStatVxOpt &vx_opt, // Set the header column shc.set_obs_thresh(na_str); + // Store current obtype value + string cur_obtype = shc.get_obtype(); + write_orank_row(shc, &pd_all, vx_opt.output_flag[i_orank], stat_at, i_stat_row, - txt_at[i_orank], i_txt_row[i_orank]); + txt_at[i_orank], i_txt_row[i_orank], + conf_info.obtype_as_obs_message_type_flag); + + // Reset the obtype column + shc.set_obtype(cur_obtype.c_str()); // Reset the observation valid time shc.set_obs_valid_beg(vx_opt.vx_pd.beg_ut); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index de3af1256b..23460be1a6 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -70,6 +70,7 @@ void EnsembleStatConfInfo::clear() { vld_ens_thresh = bad_data_double; vld_data_thresh = bad_data_double; msg_typ_group_map.clear(); + obtype_as_obs_message_type = false; msg_typ_sfc.clear(); mask_area_map.clear(); mask_sid_map.clear(); @@ -173,6 +174,10 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype, msg_typ_sfc.parse_css(default_msg_typ_group_surface); } + // Conf: obtype_as_obs_message_type_flag + obtype_as_obs_message_type_flag = + conf.lookup_bool(conf_key_obtype_as_obs_message_type_flag); + // Conf: ens_member_ids ens_member_ids = parse_conf_ens_member_ids(&conf); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index 9d9177cdab..9b5cbf3bc3 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -221,6 +221,7 @@ class EnsembleStatConfInfo { // Message type groups that should be processed together std::map msg_typ_group_map; StringArray msg_typ_sfc; + bool obtype_as_obs_message_type; // Mapping of mask names to MaskPlanes std::map mask_area_map; diff --git a/src/tools/core/point_stat/point_stat.cc b/src/tools/core/point_stat/point_stat.cc index fcd67962b0..3d651274ba 100644 --- a/src/tools/core/point_stat/point_stat.cc +++ b/src/tools/core/point_stat/point_stat.cc @@ -1108,7 +1108,11 @@ void process_scores() { write_mpr_row(shc, pd_ptr, conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, - txt_at[i_mpr], i_txt_row[i_mpr]); + txt_at[i_mpr], i_txt_row[i_mpr], + conf_info.obtype_as_obs_message_type_flag); + + // Reset the obtype column + shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -1120,7 +1124,11 @@ void process_scores() { write_seeps_mpr_row(shc, pd_ptr, conf_info.vx_opt[i_vx].output_flag[i_seeps_mpr], stat_at, i_stat_row, - txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr]); + txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr], + conf_info.obtype_as_obs_message_type_flag); + + // Reset the obtype column + shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -1910,7 +1918,8 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { pd_ptr->ocmn_na[j], pd_ptr->ocsd_na[j]); // Store the observation value - hira_pd.add_point_obs(pd_ptr->sid_sa[j].c_str(), + hira_pd.add_point_obs( + pd_ptr->typ_sa[j].c_str(), pd_ptr->sid_sa[j].c_str(), pd_ptr->lat_na[j], pd_ptr->lon_na[j], pd_ptr->x_na[j], pd_ptr->y_na[j], pd_ptr->vld_ta[j], pd_ptr->lvl_na[j], pd_ptr->elv_na[j], @@ -1965,7 +1974,8 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { write_orank_row(shc, &hira_pd, conf_info.vx_opt[i_vx].output_flag[i_orank], stat_at, i_stat_row, - txt_at[i_orank], i_txt_row[i_orank]); + txt_at[i_orank], i_txt_row[i_orank], + conf_info.obtype_as_obs_message_type_flag); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -2107,7 +2117,9 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { } // Store the fractional coverage pair - hira_pd.add_point_pair(pd_ptr->sid_sa[k].c_str(), + hira_pd.add_point_pair( + pd_ptr->typ_sa[k].c_str(), + pd_ptr->sid_sa[k].c_str(), pd_ptr->lat_na[k], pd_ptr->lon_na[k], pd_ptr->x_na[k], pd_ptr->y_na[k], pd_ptr->vld_ta[k], pd_ptr->lvl_na[k], pd_ptr->elv_na[k], @@ -2155,7 +2167,9 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { write_mpr_row(shc, &hira_pd, conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, - txt_at[i_mpr], i_txt_row[i_mpr], false); + txt_at[i_mpr], i_txt_row[i_mpr], + conf_info.obtype_as_obs_message_type_flag, + false); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); diff --git a/src/tools/core/point_stat/point_stat_conf_info.cc b/src/tools/core/point_stat/point_stat_conf_info.cc index c8ea7c4f70..6cfa7b3914 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.cc +++ b/src/tools/core/point_stat/point_stat_conf_info.cc @@ -69,6 +69,7 @@ void PointStatConfInfo::clear() { topo_use_obs_thresh.clear(); topo_interp_fcst_thresh.clear(); msg_typ_group_map.clear(); + obtype_as_obs_message_type_flag = false; mask_area_map.clear(); mask_sid_map.clear(); tmp_dir.clear(); @@ -174,6 +175,10 @@ void PointStatConfInfo::process_config(GrdFileType ftype) { // Conf: message_type_group_map msg_typ_group_map = parse_conf_message_type_group_map(&conf); + // Conf: obtype_as_obs_message_type_flag + obtype_as_obs_message_type_flag = + conf.lookup_bool(conf_key_obtype_as_obs_message_type_flag); + // Conf: fcst.field and obs.field fdict = conf.lookup_array(conf_key_fcst_field); odict = conf.lookup_array(conf_key_obs_field); diff --git a/src/tools/core/point_stat/point_stat_conf_info.h b/src/tools/core/point_stat/point_stat_conf_info.h index 95bf59a7a7..0b123b950b 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.h +++ b/src/tools/core/point_stat/point_stat_conf_info.h @@ -235,6 +235,7 @@ class PointStatConfInfo { // Message type groups that should be processed together std::map msg_typ_group_map; + bool obtype_as_obs_message_type_flag; // Mapping of mask names to DataPlanes std::map mask_area_map; diff --git a/src/tools/other/gsi_tools/gsidens2orank.cc b/src/tools/other/gsi_tools/gsidens2orank.cc index 39edb87cc5..5bf48aeff3 100644 --- a/src/tools/other/gsi_tools/gsidens2orank.cc +++ b/src/tools/other/gsi_tools/gsidens2orank.cc @@ -269,7 +269,7 @@ void process_conv_data(ConvData &d, int i_mem) { // Store the current observation info ClimoPntInfo cpi(bad_data_double, bad_data_double, bad_data_double, bad_data_double); - ens_pd.add_point_obs(d.sid.c_str(), d.lat, d.lon, + ens_pd.add_point_obs(d.obtype.c_str(), d.sid.c_str(), d.lat, d.lon, bad_data_double, bad_data_double, d.obs_ut, d.prs, d.elv, d.obs, na_str, cpi, default_grid_weight); @@ -429,7 +429,7 @@ void process_rad_data(RadData &d, int i_mem) { // Store the current observation info ClimoPntInfo cpi(bad_data_double, bad_data_double, bad_data_double, bad_data_double); - ens_pd.add_point_obs(na_str, d.lat, d.lon, + ens_pd.add_point_obs(na_str, na_str, d.lat, d.lon, bad_data_double, bad_data_double, d.obs_ut, bad_data_double, d.elv, d.obs, na_str, cpi, default_grid_weight); From 168eef4346aa24320fd971265b7c856c331120c2 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 15 Oct 2024 18:04:14 +0000 Subject: [PATCH 02/11] Per #2893, add option to default config files and fix compilation errors. --- data/config/EnsembleStatConfig_default | 2 ++ data/config/PointStatConfig_default | 2 ++ src/libcode/vx_stat_out/stat_columns.h | 2 +- src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc | 4 ++-- src/tools/core/ensemble_stat/ensemble_stat_conf_info.h | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/data/config/EnsembleStatConfig_default b/data/config/EnsembleStatConfig_default index c496dd3507..18d4846aac 100644 --- a/data/config/EnsembleStatConfig_default +++ b/data/config/EnsembleStatConfig_default @@ -120,6 +120,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_obs_message_type_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/data/config/PointStatConfig_default b/data/config/PointStatConfig_default index 266c251e59..1785548267 100644 --- a/data/config/PointStatConfig_default +++ b/data/config/PointStatConfig_default @@ -115,6 +115,8 @@ message_type_group_map = [ { key = "WATERSF"; val = "SFCSHP"; } ]; +obtype_as_obs_message_type_flag = FALSE; + //////////////////////////////////////////////////////////////////////////////// // diff --git a/src/libcode/vx_stat_out/stat_columns.h b/src/libcode/vx_stat_out/stat_columns.h index 6d2e0a15aa..e8998e1fa4 100644 --- a/src/libcode/vx_stat_out/stat_columns.h +++ b/src/libcode/vx_stat_out/stat_columns.h @@ -109,7 +109,7 @@ extern void write_mpr_row (StatHdrColumns &, const PairDataPoint *, STATOutput bool update_obtype, bool update_thresh = true); extern void write_seeps_row (StatHdrColumns &, const SeepsAggScore *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, - bool update_obtype, bool update_thresh = true); + bool update_thresh = true); extern void write_seeps_mpr_row (StatHdrColumns &, const PairDataPoint *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &, bool update_obtype, bool update_thresh = true); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index 23460be1a6..07fa557065 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -70,7 +70,7 @@ void EnsembleStatConfInfo::clear() { vld_ens_thresh = bad_data_double; vld_data_thresh = bad_data_double; msg_typ_group_map.clear(); - obtype_as_obs_message_type = false; + obtype_as_obs_message_type_flag = false; msg_typ_sfc.clear(); mask_area_map.clear(); mask_sid_map.clear(); @@ -176,7 +176,7 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype, // Conf: obtype_as_obs_message_type_flag obtype_as_obs_message_type_flag = - conf.lookup_bool(conf_key_obtype_as_obs_message_type_flag); + conf.lookup_bool(conf_key_obtype_as_obs_message_type_flag); // Conf: ens_member_ids ens_member_ids = parse_conf_ens_member_ids(&conf); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index 9b5cbf3bc3..0c07582131 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -221,7 +221,7 @@ class EnsembleStatConfInfo { // Message type groups that should be processed together std::map msg_typ_group_map; StringArray msg_typ_sfc; - bool obtype_as_obs_message_type; + bool obtype_as_obs_message_type_flag; // Mapping of mask names to MaskPlanes std::map mask_area_map; From 3069a170bd1e160ad5eabe561fc36ab03a758d6f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 15 Oct 2024 18:07:16 +0000 Subject: [PATCH 03/11] Per #2893, change the name from obtype_as_obs_message_type_flag to obtype_as_group_val_flag. --- data/config/EnsembleStatConfig_default | 2 +- data/config/PointStatConfig_default | 2 +- src/basic/vx_config/config_constants.h | 2 +- src/tools/core/ensemble_stat/ensemble_stat.cc | 2 +- src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc | 8 ++++---- src/tools/core/ensemble_stat/ensemble_stat_conf_info.h | 2 +- src/tools/core/point_stat/point_stat.cc | 8 ++++---- src/tools/core/point_stat/point_stat_conf_info.cc | 8 ++++---- src/tools/core/point_stat/point_stat_conf_info.h | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/data/config/EnsembleStatConfig_default b/data/config/EnsembleStatConfig_default index 18d4846aac..daa914021e 100644 --- a/data/config/EnsembleStatConfig_default +++ b/data/config/EnsembleStatConfig_default @@ -120,7 +120,7 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; -obtype_as_obs_message_type_flag = FALSE; +obtype_as_group_val_flag = FALSE; // // Ensemble bin sizes diff --git a/data/config/PointStatConfig_default b/data/config/PointStatConfig_default index 1785548267..bb4dc9d464 100644 --- a/data/config/PointStatConfig_default +++ b/data/config/PointStatConfig_default @@ -115,7 +115,7 @@ message_type_group_map = [ { key = "WATERSF"; val = "SFCSHP"; } ]; -obtype_as_obs_message_type_flag = FALSE; +obtype_as_group_val_flag = FALSE; //////////////////////////////////////////////////////////////////////////////// diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 802e37f28a..58e20ed82c 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -538,7 +538,7 @@ static const char conf_key_model[] = "model"; static const char conf_key_desc[] = "desc"; static const char conf_key_obtype[] = "obtype"; static const char conf_key_output_flag[] = "output_flag"; -static const char conf_key_obtype_as_obs_message_type_flag[] = "obtype_as_obs_message_type_flag"; +static const char conf_key_obtype_as_group_val_flag[] = "obtype_as_group_val_flag"; static const char conf_key_obs_window[] = "obs_window"; static const char conf_key_beg[] = "beg"; static const char conf_key_end[] = "end"; diff --git a/src/tools/core/ensemble_stat/ensemble_stat.cc b/src/tools/core/ensemble_stat/ensemble_stat.cc index 30ea808ee9..1163716893 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -2243,7 +2243,7 @@ void write_txt_files(const EnsembleStatVxOpt &vx_opt, vx_opt.output_flag[i_orank], stat_at, i_stat_row, txt_at[i_orank], i_txt_row[i_orank], - conf_info.obtype_as_obs_message_type_flag); + conf_info.obtype_as_group_val_flag); // Reset the obtype column shc.set_obtype(cur_obtype.c_str()); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc index 07fa557065..e0a39685cf 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc @@ -70,7 +70,7 @@ void EnsembleStatConfInfo::clear() { vld_ens_thresh = bad_data_double; vld_data_thresh = bad_data_double; msg_typ_group_map.clear(); - obtype_as_obs_message_type_flag = false; + obtype_as_group_val_flag = false; msg_typ_sfc.clear(); mask_area_map.clear(); mask_sid_map.clear(); @@ -174,9 +174,9 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype, msg_typ_sfc.parse_css(default_msg_typ_group_surface); } - // Conf: obtype_as_obs_message_type_flag - obtype_as_obs_message_type_flag = - conf.lookup_bool(conf_key_obtype_as_obs_message_type_flag); + // Conf: obtype_as_group_val_flag + obtype_as_group_val_flag = + conf.lookup_bool(conf_key_obtype_as_group_val_flag); // Conf: ens_member_ids ens_member_ids = parse_conf_ens_member_ids(&conf); diff --git a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h index 0c07582131..e3738d3f88 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h +++ b/src/tools/core/ensemble_stat/ensemble_stat_conf_info.h @@ -221,7 +221,7 @@ class EnsembleStatConfInfo { // Message type groups that should be processed together std::map msg_typ_group_map; StringArray msg_typ_sfc; - bool obtype_as_obs_message_type_flag; + bool obtype_as_group_val_flag; // Mapping of mask names to MaskPlanes std::map mask_area_map; diff --git a/src/tools/core/point_stat/point_stat.cc b/src/tools/core/point_stat/point_stat.cc index 3d651274ba..76db60a522 100644 --- a/src/tools/core/point_stat/point_stat.cc +++ b/src/tools/core/point_stat/point_stat.cc @@ -1109,7 +1109,7 @@ void process_scores() { conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, txt_at[i_mpr], i_txt_row[i_mpr], - conf_info.obtype_as_obs_message_type_flag); + conf_info.obtype_as_group_val_flag); // Reset the obtype column shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); @@ -1125,7 +1125,7 @@ void process_scores() { conf_info.vx_opt[i_vx].output_flag[i_seeps_mpr], stat_at, i_stat_row, txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr], - conf_info.obtype_as_obs_message_type_flag); + conf_info.obtype_as_group_val_flag); // Reset the obtype column shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); @@ -1975,7 +1975,7 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { conf_info.vx_opt[i_vx].output_flag[i_orank], stat_at, i_stat_row, txt_at[i_orank], i_txt_row[i_orank], - conf_info.obtype_as_obs_message_type_flag); + conf_info.obtype_as_group_val_flag); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -2168,7 +2168,7 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, txt_at[i_mpr], i_txt_row[i_mpr], - conf_info.obtype_as_obs_message_type_flag, + conf_info.obtype_as_group_val_flag, false); // Reset the observation valid time diff --git a/src/tools/core/point_stat/point_stat_conf_info.cc b/src/tools/core/point_stat/point_stat_conf_info.cc index 6cfa7b3914..125c847f58 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.cc +++ b/src/tools/core/point_stat/point_stat_conf_info.cc @@ -69,7 +69,7 @@ void PointStatConfInfo::clear() { topo_use_obs_thresh.clear(); topo_interp_fcst_thresh.clear(); msg_typ_group_map.clear(); - obtype_as_obs_message_type_flag = false; + obtype_as_group_val_flag = false; mask_area_map.clear(); mask_sid_map.clear(); tmp_dir.clear(); @@ -175,9 +175,9 @@ void PointStatConfInfo::process_config(GrdFileType ftype) { // Conf: message_type_group_map msg_typ_group_map = parse_conf_message_type_group_map(&conf); - // Conf: obtype_as_obs_message_type_flag - obtype_as_obs_message_type_flag = - conf.lookup_bool(conf_key_obtype_as_obs_message_type_flag); + // Conf: obtype_as_group_val_flag + obtype_as_group_val_flag = + conf.lookup_bool(conf_key_obtype_as_group_val_flag); // Conf: fcst.field and obs.field fdict = conf.lookup_array(conf_key_fcst_field); diff --git a/src/tools/core/point_stat/point_stat_conf_info.h b/src/tools/core/point_stat/point_stat_conf_info.h index 0b123b950b..e07b0dffc7 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.h +++ b/src/tools/core/point_stat/point_stat_conf_info.h @@ -235,7 +235,7 @@ class PointStatConfInfo { // Message type groups that should be processed together std::map msg_typ_group_map; - bool obtype_as_obs_message_type_flag; + bool obtype_as_group_val_flag; // Mapping of mask names to DataPlanes std::map mask_area_map; From 7776212d09b9bc93a462c304b4aa5a4346f4db01 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 15 Oct 2024 18:21:30 +0000 Subject: [PATCH 04/11] Per #2893, add docs ci-run-unit --- docs/Users_Guide/config_options.rst | 19 +++++++++++++++++++ docs/Users_Guide/ensemble-stat.rst | 1 + docs/Users_Guide/point-stat.rst | 1 + 3 files changed, 21 insertions(+) diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst index 25515fc524..6128930c14 100644 --- a/docs/Users_Guide/config_options.rst +++ b/docs/Users_Guide/config_options.rst @@ -667,6 +667,25 @@ used. { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag +------------------------ + +The "obtype_as_group_val_flag" entry is a boolean that controls how the +OBTYPE header column is populated for message type groups defined in +"message_type_group_map". If set to TRUE and when writing matched pair +line types (MPR, SEEPS_MPR, and ORANK), write OBTYPE as the group map +*value*, i.e. the input message type for each individual observation. +If set to FALSE (default) and for all other line types, write OBTYPE +as the group map key, i.e. the name of the message type group. + +For example, if FALSE, write the OBTYPE column in the MPR line type +as the "ANYAIR" message type group name. If TRUE, write OBTYPE as "AIRCAR" +or "AIRCFT", based on the input message type of each point observation. + +.. code-block:: none + + obtyp_as_group_val_flag = FALSE; + message_type_map ---------------- diff --git a/docs/Users_Guide/ensemble-stat.rst b/docs/Users_Guide/ensemble-stat.rst index 8a2502c525..27e0e5ea34 100644 --- a/docs/Users_Guide/ensemble-stat.rst +++ b/docs/Users_Guide/ensemble-stat.rst @@ -181,6 +181,7 @@ ____________________ obs_summary = NONE; obs_perc_value = 50; message_type_group_map = [...]; + obtype_as_group_val_flag = FALSE; grid_weight_flag = NONE; output_prefix = ""; version = "VN.N"; diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst index 73b7ce9a33..349fca2cc8 100644 --- a/docs/Users_Guide/point-stat.rst +++ b/docs/Users_Guide/point-stat.rst @@ -362,6 +362,7 @@ ________________________ obs_summary = NONE; obs_perc_value = 50; message_type_group_map = [...]; + obtype_as_group_val_flag = FALSE; tmp_dir = "/tmp"; output_prefix = ""; version = "VN.N"; From 8a08ca719d1907c7335b1051d659327970d8e6b6 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 15 Oct 2024 18:29:01 +0000 Subject: [PATCH 05/11] Per #2893, add obtype_as_group_val_flag entry for all config files that already include the message_type_group_map entry. --- internal/test_unit/config/EnsembleStatConfig_python | 2 ++ internal/test_unit/config/EnsembleStatConfig_single_file_grib | 2 ++ internal/test_unit/config/EnsembleStatConfig_single_file_nc | 2 ++ internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK | 2 ++ internal/test_unit/config/PointStatConfig_WINDS | 2 ++ internal/test_unit/config/PointStatConfig_airnow | 2 ++ scripts/config/EnsembleStatConfig | 2 ++ 7 files changed, 14 insertions(+) diff --git a/internal/test_unit/config/EnsembleStatConfig_python b/internal/test_unit/config/EnsembleStatConfig_python index 4de2045664..666303582f 100644 --- a/internal/test_unit/config/EnsembleStatConfig_python +++ b/internal/test_unit/config/EnsembleStatConfig_python @@ -119,6 +119,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/internal/test_unit/config/EnsembleStatConfig_single_file_grib b/internal/test_unit/config/EnsembleStatConfig_single_file_grib index 20220f82b5..6d26a90531 100644 --- a/internal/test_unit/config/EnsembleStatConfig_single_file_grib +++ b/internal/test_unit/config/EnsembleStatConfig_single_file_grib @@ -131,6 +131,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/internal/test_unit/config/EnsembleStatConfig_single_file_nc b/internal/test_unit/config/EnsembleStatConfig_single_file_nc index b3faf78d95..62d3e8ef6c 100644 --- a/internal/test_unit/config/EnsembleStatConfig_single_file_nc +++ b/internal/test_unit/config/EnsembleStatConfig_single_file_nc @@ -137,6 +137,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry diff --git a/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK b/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK index 48a04ed227..c23e5518ba 100644 --- a/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK +++ b/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK @@ -81,6 +81,8 @@ message_type_group_map = [ { key = "WATERSF"; val = "SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + //////////////////////////////////////////////////////////////////////////////// climo_mean = fcst; diff --git a/internal/test_unit/config/PointStatConfig_WINDS b/internal/test_unit/config/PointStatConfig_WINDS index 1c87273e89..60879eb302 100644 --- a/internal/test_unit/config/PointStatConfig_WINDS +++ b/internal/test_unit/config/PointStatConfig_WINDS @@ -52,6 +52,8 @@ message_type_group_map = [ { key = "USERSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + fcst = { sid_inc = []; sid_exc = []; diff --git a/internal/test_unit/config/PointStatConfig_airnow b/internal/test_unit/config/PointStatConfig_airnow index 4223455902..0cc2791210 100644 --- a/internal/test_unit/config/PointStatConfig_airnow +++ b/internal/test_unit/config/PointStatConfig_airnow @@ -92,6 +92,8 @@ message_type_group_map = [ { key = "WATERSF"; val = "SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + //////////////////////////////////////////////////////////////////////////////// // diff --git a/scripts/config/EnsembleStatConfig b/scripts/config/EnsembleStatConfig index bc84b81d8e..7e79bbc00e 100644 --- a/scripts/config/EnsembleStatConfig +++ b/scripts/config/EnsembleStatConfig @@ -124,6 +124,8 @@ message_type_group_map = [ { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } ]; +obtype_as_group_val_flag = FALSE; + // // Ensemble bin sizes // May be set separately in each "obs.field" entry From 24b88aaa1a6fa05f8d2cd1ec835e12fb02d8d919 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 15 Oct 2024 19:10:46 +0000 Subject: [PATCH 06/11] Per #2893, modify existing PointStatConfig_WINDS config file to have it write MPR output with obtype_as_group_val_flag = TRUE. Note that I switched the message types from ADPSFC, SFCSHP, USERSF to just USERSF. The original reason for the 3 was to prove that the counts for ADPSFC + SFCSHP = USERSF. --- internal/test_unit/config/PointStatConfig_WINDS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/test_unit/config/PointStatConfig_WINDS b/internal/test_unit/config/PointStatConfig_WINDS index 60879eb302..2141bbb7cb 100644 --- a/internal/test_unit/config/PointStatConfig_WINDS +++ b/internal/test_unit/config/PointStatConfig_WINDS @@ -52,7 +52,7 @@ message_type_group_map = [ { key = "USERSF"; val = "ADPSFC,SFCSHP"; } ]; -obtype_as_group_val_flag = FALSE; +obtype_as_group_val_flag = TRUE; fcst = { sid_inc = []; @@ -60,7 +60,7 @@ fcst = { obs_quality_inc = []; obs_quality_exc = []; - message_type = [ "ADPSFC", "SFCSHP", "USERSF" ]; + message_type = [ "USERSF" ]; wind_thresh = [ NA, >=3, >=5 ]; field = [ @@ -151,7 +151,7 @@ output_flag = { rps = NONE; orank = NONE; eclv = NONE; - mpr = NONE; + mpr = STAT; seeps = NONE; seeps_mpr = NONE; } From d26ef0fdf133c965c47a9d79e7a8f3713ee2b3e4 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 16 Oct 2024 03:40:32 +0000 Subject: [PATCH 07/11] Per #2893, update commit history for ensemble_stat and point_stat --- src/tools/core/ensemble_stat/ensemble_stat.cc | 2 ++ src/tools/core/point_stat/point_stat.cc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/tools/core/ensemble_stat/ensemble_stat.cc b/src/tools/core/ensemble_stat/ensemble_stat.cc index 76b3267466..68cfe50cc1 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -77,6 +77,8 @@ // 044 06/17/24 Halley Gotway MET #2856 Reinitialize climo_cdf pointer // 045 07/05/24 Halley Gotway MET #2924 Support forecast climatology. // 046 10/08/24 Halley Gotway MET #2887 Compute weighted contingency tables. +// 047 10/14/24 Halley Gotway MET #2279 Add point_weight_flag option. +// 048 10/15/24 Halley Gotway MET #2893 Write individual pair OBTYPE. // //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/point_stat/point_stat.cc b/src/tools/core/point_stat/point_stat.cc index d646776e94..57d82725c1 100644 --- a/src/tools/core/point_stat/point_stat.cc +++ b/src/tools/core/point_stat/point_stat.cc @@ -105,6 +105,9 @@ // 053 10/03/22 Prestopnik MET #2227 Remove using namespace netCDF from header files. // 054 04/29/24 Halley Gotway MET #2795 Move level mismatch warning. // 055 07/05/24 Halley Gotway MET #2924 Support forecast climatology. +// 056 10/08/24 Halley Gotway MET #2887 Compute weighted contingency tables. +// 057 10/14/24 Halley Gotway MET #2279 Add point_weight_flag option. +// 058 10/15/24 Halley Gotway MET #2893 Write individual pair OBTYPE. // //////////////////////////////////////////////////////////////////////// From e3784ffd28119828c27c7149b45626b71804f3f8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 16 Oct 2024 03:54:31 +0000 Subject: [PATCH 08/11] Per #2893, revert back PointStatConfig_WINDS to minimize diffs --- internal/test_unit/config/PointStatConfig_WINDS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/test_unit/config/PointStatConfig_WINDS b/internal/test_unit/config/PointStatConfig_WINDS index 3856a3eb52..4dac7fe51e 100644 --- a/internal/test_unit/config/PointStatConfig_WINDS +++ b/internal/test_unit/config/PointStatConfig_WINDS @@ -52,7 +52,7 @@ message_type_group_map = [ { key = "USERSF"; val = "ADPSFC,SFCSHP"; } ]; -obtype_as_group_val_flag = TRUE; +obtype_as_group_val_flag = FALSE; fcst = { sid_inc = []; @@ -60,7 +60,7 @@ fcst = { obs_quality_inc = []; obs_quality_exc = []; - message_type = [ "USERSF" ]; + message_type = [ "ADPSFC", "SFCSHP", "USERSF" ]; wind_thresh = [ NA, >=3, >=5 ]; field = [ @@ -151,7 +151,7 @@ output_flag = { rps = NONE; orank = NONE; eclv = NONE; - mpr = STAT; + mpr = NONE; seeps = NONE; seeps_mpr = NONE; } From 3e7182b31063929c36f9367df856d60613c2210c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 16 Oct 2024 04:09:07 +0000 Subject: [PATCH 09/11] Per #2893, add a Point-Stat unit test to demonstrate setting the obtype_as_group_val_flag option. --- .../config/PointStatConfig_MPR_OBTYPE | 166 ++++++++++++++++++ internal/test_unit/xml/unit_point_stat.xml | 20 +++ 2 files changed, 186 insertions(+) create mode 100644 internal/test_unit/config/PointStatConfig_MPR_OBTYPE diff --git a/internal/test_unit/config/PointStatConfig_MPR_OBTYPE b/internal/test_unit/config/PointStatConfig_MPR_OBTYPE new file mode 100644 index 0000000000..8a346c4eed --- /dev/null +++ b/internal/test_unit/config/PointStatConfig_MPR_OBTYPE @@ -0,0 +1,166 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Point-Stat configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +model = "FCST"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +//////////////////////////////////////////////////////////////////////////////// + +regrid = { + to_grid = NONE; + method = NEAREST; + width = 1; +} + +//////////////////////////////////////////////////////////////////////////////// + +obs_window = { + beg = ${BEG_DS}; + end = ${END_DS}; +} + +//////////////////////////////////////////////////////////////////////////////// + +mpr_column = []; +mpr_thresh = []; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; +eclv_points = 0.05; + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, + { key = "LANDSF"; val = "ADPSFC,MSONET"; }, + { key = "WATERSF"; val = "SFCSHP"; } +]; + +obtype_as_group_val_flag = TRUE; + +fcst = { + sid_inc = []; + sid_exc = []; + obs_quality_inc = []; + obs_quality_exc = []; + + message_type = [ "SURFACE" ]; + + field = [ + { name = "TMP"; level = "Z2"; } + ]; + +} +obs = fcst; + +//////////////////////////////////////////////////////////////////////////////// + +climo_mean = fcst; +climo_mean = { + file_name = [ ${CLIMO_FILE} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +mask = { + grid = [ "FULL" ]; + poly = []; + sid = []; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 200; + rng = "mt19937"; + seed = "1"; +} + +//////////////////////////////////////////////////////////////////////////////// + +interp = { + vld_thresh = 1.0; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +hira = { + flag = FALSE; + width = [ 2, 3, 4, 5 ]; + vld_thresh = 1.0; + cov_thresh = [ ==0.25 ]; + shape = SQUARE; + prob_cat_thresh = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = STAT; + sal1l2 = STAT; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + ecnt = NONE; + rps = NONE; + orank = NONE; + eclv = NONE; + mpr = STAT; + seeps = NONE; + seeps_mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// +// Threshold for SEEPS p1 (Probability of being dry) + +seeps_p1_thresh = NA; + +//////////////////////////////////////////////////////////////////////////////// + +duplicate_flag = NONE; +rank_corr_flag = TRUE; + +point_weight_flag = NONE; + +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V12.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/internal/test_unit/xml/unit_point_stat.xml b/internal/test_unit/xml/unit_point_stat.xml index 2c90567c80..97f36ccc2b 100644 --- a/internal/test_unit/xml/unit_point_stat.xml +++ b/internal/test_unit/xml/unit_point_stat.xml @@ -72,6 +72,26 @@ + + &MET_BIN;/point_stat + + BEG_DS -1800 + END_DS 1800 + OUTPUT_PREFIX GRIB1_NAM_GDAS_MPR_OBTYPE + CONFIG_DIR &CONFIG_DIR; + CLIMO_FILE "&DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012_gNam.grib" + + \ + &DATA_DIR_MODEL;/grib1/nam/nam_2012040900_F012.grib \ + &OUTPUT_DIR;/pb2nc/gdas1.20120409.t12z.prepbufr.nc \ + &CONFIG_DIR;/PointStatConfig_MPR_OBTYPE \ + -outdir &OUTPUT_DIR;/point_stat -v 1 + + + &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_MPR_OBTYPE_120000L_20120409_120000V.stat + + + &MET_BIN;/point_stat From 82565f7c42043216e98c6f9afc7f14d77097bc03 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 16 Oct 2024 04:34:29 +0000 Subject: [PATCH 10/11] Per #2893, need to reset the obtype after writing all MPR, SEEPS_MPR, and ORANK line types in Point-Stat --- src/tools/core/point_stat/point_stat.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tools/core/point_stat/point_stat.cc b/src/tools/core/point_stat/point_stat.cc index 57d82725c1..63f333d734 100644 --- a/src/tools/core/point_stat/point_stat.cc +++ b/src/tools/core/point_stat/point_stat.cc @@ -1980,6 +1980,9 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { stat_at, i_stat_row, txt_at[i_orank], i_txt_row[i_orank], conf_info.obtype_as_group_val_flag); + + // Reset the obtype column + shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -2175,6 +2178,9 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { conf_info.obtype_as_group_val_flag, false); + // Reset the obtype column + shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); + // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); shc.set_obs_valid_end(conf_info.vx_opt[i_vx].vx_pd.end_ut); From 1c1fe7fff214932b22cb5fc398f2158903b52182 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 16 Oct 2024 05:00:28 +0000 Subject: [PATCH 11/11] Per #2893, tweak Point-Stat configuration to also write HIRA ORANK and PCT outputs. --- internal/test_unit/config/PointStatConfig_MPR_OBTYPE | 12 ++++++------ internal/test_unit/xml/unit_point_stat.xml | 4 ++-- src/libcode/vx_statistics/pair_data_ensemble.cc | 3 +-- src/tools/core/point_stat/point_stat.cc | 4 ++-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/internal/test_unit/config/PointStatConfig_MPR_OBTYPE b/internal/test_unit/config/PointStatConfig_MPR_OBTYPE index 8a346c4eed..6aa68e9842 100644 --- a/internal/test_unit/config/PointStatConfig_MPR_OBTYPE +++ b/internal/test_unit/config/PointStatConfig_MPR_OBTYPE @@ -112,12 +112,12 @@ interp = { //////////////////////////////////////////////////////////////////////////////// hira = { - flag = FALSE; - width = [ 2, 3, 4, 5 ]; + flag = TRUE; + width = [ 3 ]; vld_thresh = 1.0; cov_thresh = [ ==0.25 ]; shape = SQUARE; - prob_cat_thresh = []; + prob_cat_thresh = [ >273 ]; } //////////////////////////////////////////////////////////////////////////////// @@ -134,13 +134,13 @@ output_flag = { vl1l2 = NONE; val1l2 = NONE; vcnt = NONE; - pct = NONE; + pct = STAT; pstd = NONE; pjc = NONE; prc = NONE; - ecnt = NONE; + ecnt = STAT; rps = NONE; - orank = NONE; + orank = STAT; eclv = NONE; mpr = STAT; seeps = NONE; diff --git a/internal/test_unit/xml/unit_point_stat.xml b/internal/test_unit/xml/unit_point_stat.xml index 97f36ccc2b..8e798e03ec 100644 --- a/internal/test_unit/xml/unit_point_stat.xml +++ b/internal/test_unit/xml/unit_point_stat.xml @@ -75,8 +75,8 @@ &MET_BIN;/point_stat - BEG_DS -1800 - END_DS 1800 + BEG_DS -300 + END_DS 300 OUTPUT_PREFIX GRIB1_NAM_GDAS_MPR_OBTYPE CONFIG_DIR &CONFIG_DIR; CLIMO_FILE "&DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012_gNam.grib" diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index 16f68a78b9..0d80abce1a 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -1305,11 +1305,10 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr, // Add the observation value // Weight is from the nearest grid point int n = three_to_one(i_msg_typ, i_mask, i_interp); - if(!pd[n].add_point_obs(hdr_typ_str, hdr_sid_str, hdr_lat, hdr_lon, obs_x, obs_y, hdr_ut, obs_lvl, obs_hgt, - obs_v, obs_qty, cpi, wgt_v)) { + obs_v, obs_qty, cpi, default_weight)) { if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { mlog << Debug(REJECT_DEBUG_LEVEL) diff --git a/src/tools/core/point_stat/point_stat.cc b/src/tools/core/point_stat/point_stat.cc index 63f333d734..735f85107e 100644 --- a/src/tools/core/point_stat/point_stat.cc +++ b/src/tools/core/point_stat/point_stat.cc @@ -1982,7 +1982,7 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { conf_info.obtype_as_group_val_flag); // Reset the obtype column - shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); + shc.set_obtype(pd_ptr->msg_typ.c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut); @@ -2179,7 +2179,7 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { false); // Reset the obtype column - shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); + shc.set_obtype(pd_ptr->msg_typ.c_str()); // Reset the observation valid time shc.set_obs_valid_beg(conf_info.vx_opt[i_vx].vx_pd.beg_ut);