From 06b3fdbb17bbaa25fdbd27b025fb3d795281218f Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Wed, 20 Nov 2024 08:57:46 -0800 Subject: [PATCH 01/14] Version of pycbc_pygrb_plot_chisq_veto with vetoes (albeit with a mute apply_vetoes_to_found_injs for the time being) --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 205 ++++++++++++-------------- 1 file changed, 96 insertions(+), 109 deletions(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index 966798fc0f2..ec0c4df2784 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -48,79 +48,6 @@ __program__ = "pycbc_pygrb_plot_chisq_veto" # ============================================================================= # Functions # ============================================================================= -# Function to load trigger data: includes applying cut in reweighted SNR -def load_data(input_file, ifos, vetoes, opts, injections=False, slide_id=None): - """Load data from a trigger/injection file""" - - snr_type = opts.snr_type - veto_type = opts.y_variable - - # Initialize the dictionary - data = {} - data[snr_type] = None - data[veto_type] = None - data['dof'] = None - - # Ensure that newtwork power chi-square plots show all the data to see - # the impact of the reweighted SNR cut - rw_snr_threshold = 0. if veto_type=='network' else opts.newsnr_threshold - - if input_file: - if injections: - logging.info("Loading injections...") - # This will eventually become load_injections - trigs_or_injs = \ - ppu.load_triggers(input_file, ifos, vetoes, - rw_snr_threshold=rw_snr_threshold, - slide_id=slide_id) - else: - logging.info("Loading triggers...") - trigs_or_injs = \ - ppu.load_triggers(input_file, ifos, vetoes, - rw_snr_threshold=rw_snr_threshold, - slide_id=slide_id) - - # Count surviving points - num_trigs_or_injs = len(trigs_or_injs['network/reweighted_snr']) - - if snr_type in ['coherent', 'null', 'reweighted']: - data[snr_type] = trigs_or_injs['network/%s_snr' % snr_type][:] - elif snr_type == 'single': - key = opts.ifo + '/snr' - data[snr_type] = trigs_or_injs[key][:] - - # Calculate coincident SNR - elif snr_type == 'coincident': - data[snr_type] = ppu.get_coinc_snr(trigs_or_injs) - - # Tags to find vetoes in HDF files - veto_tags = {'power': 'chisq', - 'bank': 'bank_chisq', - 'auto': 'auto_chisq', - 'network': 'my_network_chisq'} - - # This chi-square is already normalized - if veto_type == 'network': - chisq_key = 'network/my_network_chisq' - data['dof'] = 1. - else: - chisq_key = opts.ifo + '/' + veto_tags[veto_type] - dof_key = '%s/%s_dof' % (opts.ifo, veto_tags[veto_type]) - data['dof'] = trigs_or_injs[dof_key][:] - - # Normalize - data[veto_type] = trigs_or_injs[chisq_key][:]/data['dof'] - - # Floor single IFO chi-square at 0.005 - numpy.putmask(data[veto_type], data[veto_type] == 0, 0.005) - - label = "injections" if injections else "triggers" - - logging.info("{0} {1} found.".format(num_trigs_or_injs, label)) - - return data - - # Function to calculate chi-square weight for the reweighted SNR def new_snr_chisq(snr, new_snr, chisq_index=4.0, chisq_nhigh=3.0): """Returns the chi-square value needed to weight SNR into new SNR""" @@ -133,7 +60,7 @@ def new_snr_chisq(snr, new_snr, chisq_index=4.0, chisq_nhigh=3.0): # Function that produces the contours to be plotted -def calculate_contours(trig_data, opts, new_snrs=None): +def calculate_contours(opts, new_snrs=None): """Generate the contours for the veto plots""" # Add the new SNR threshold contour to the list if necessary @@ -219,13 +146,13 @@ veto_labels = {'network': "Network Power", 'auto': "Auto", 'power': "Power"} if opts.plot_title is None: - opts.plot_title = " %s Chi Square" % veto_labels[veto_type] + opts.plot_title = veto_labels[veto_type] + " Chi Square" if veto_type != 'network': opts.plot_title = ifo + opts.plot_title if snr_type == 'single': - opts.plot_title += " vs %s SNR" % (ifo) + opts.plot_title += " vs " + ifo + " SNR" else: - opts.plot_title += " vs %s SNR" % snr_type.capitalize() + opts.plot_title += " vs " + snr_type.capitalize() + " SNR" if opts.plot_caption is None: opts.plot_caption = ("Blue crosses: background triggers. ") if found_missed_file: @@ -243,30 +170,91 @@ outdir = os.path.split(os.path.abspath(opts.output_file))[0] if not os.path.isdir(outdir): os.makedirs(outdir) -# Extract IFOs and vetoes -ifos, vetoes = ppu.extract_ifos_and_vetoes(trig_file, opts.veto_files, - opts.veto_category) - -# Exit gracefully if the requested IFO is not available -if ifo and ifo not in ifos: - err_msg = "The IFO selected with --ifo is unavailable in the data." - raise RuntimeError(err_msg) +# Extract IFOs +ifos = ppu.extract_ifos(trig_file) + +# Generate time-slides dictionary +slide_dict = ppu.load_time_slides(trig_file) + +# Generate segments dictionary +segment_dict = ppu.load_segment_dict(trig_file) + +# Construct trials removing vetoed times +trial_dict, total_trials = ppu.construct_trials( + opts.seg_files, + segment_dict, + ifos, + slide_dict, + opts.veto_file +) + +# Load trigger and injections data: ensure that newtwork power chi-square plots +# show all the data to see the impact of the reweighted SNR cut, otherwise remove +# points with reweighted SNR below threshold +rw_snr_threshold = None if veto_type == 'network' else opts.newsnr_threshold +trig_data = ppu.load_data(trig_file, ifos, data_tag='trigs', + rw_snr_threshold=rw_snr_threshold, + slide_id=opts.slide_id) +inj_data = ppu.load_data(found_missed_file, ifos, data_tag='injs', + rw_snr_threshold=rw_snr_threshold, + slide_id=0) + +# Dataset name for the horizontal direction +if snr_type == 'single': + x_key = ifo + '/snr' +else: + x_key = 'network/' + snr_type + '_snr' +# Dataset name for the vertical direction and for normalization +if veto_type == 'power': + y_key = opts.ifo + '/chisq' +elif veto_type in ['bank', 'auto']: + y_key = opts.ifo + '/' + veto_type +'_chisq' +else: + y_key = 'network/my_network_chisq' + +keys = [x_key, y_key] +# The network chi-square is already normalized so dof_key is not needed +if veto_type != 'network': + dof_key = y_key + '_dof' + keys += [dof_key] + +# Extract needed trigger properties and store them as dictionaries +# Based on trial_dict: if vetoes were applied, trig_* are the veto survivors +found_trigs_slides = ppu.extract_trig_properties( + trial_dict, + trig_data, + slide_dict, + segment_dict, + keys +) +found_trigs = {} +for key in keys: + found_trigs[key] = numpy.concatenate( + [found_trigs_slides[key][slide_id][:] for slide_id in slide_dict] + ) + +# Gather injections found surviving vetoes +found_injs, *_ = ppu.apply_vetoes_to_found_injs( + opts.found_missed_file, + inj_data, + ifos, + veto_file=opts.veto_file, + keys=keys +) -# Extract trigger data -trig_data = load_data(trig_file, ifos, vetoes, opts, - slide_id=opts.slide_id) +# Sanity checks +for test in zip(keys[0:2], ['x', 'y']): + if found_trigs[test[0]] is None and found_injs[test[0]] is None: + err_msg = "No data to be plotted on the " + test[1] + "-axis was found" + raise RuntimeError(err_msg) -# Extract (or initialize) injection data -inj_data = load_data(found_missed_file, ifos, vetoes, opts, - injections=True, slide_id=0) +# Normalize chi-squares with the number of degrees of freedom +if len(keys) == 3: + found_trigs[keys[1]] /= found_trigs[keys[2]] + found_injs[keys[1]] /= found_injs[keys[2]] -# Sanity checks -if trig_data[snr_type] is None and inj_data[snr_type] is None: - err_msg = "No data to be plotted on the x-axis was found" - raise RuntimeError(err_msg) -if trig_data[veto_type] is None and inj_data[veto_type] is None: - err_msg = "No data to be plotted on the y-axis was found" - raise RuntimeError(err_msg) +# Floor single IFO chi-square at 0.005 +numpy.putmask(found_trigs[y_key], found_trigs[y_key] == 0, 0.005) # Generate plots logging.info("Plotting...") @@ -274,23 +262,23 @@ logging.info("Plotting...") # Determine x-axis values of triggers and injections # Default is coherent SNR x_label = ifo if snr_type == 'single' else snr_type.capitalize() -x_label = "%s SNR" % x_label +x_label = x_label + " SNR" # Determine the minumum and maximum SNR value we are dealing with -x_min = 0.9*plu.axis_min_value(trig_data[snr_type], inj_data[snr_type], +x_min = 0.9*plu.axis_min_value(found_trigs[x_key], found_injs[x_key], found_missed_file) -x_max = 1.1*plu.axis_max_value(trig_data[snr_type], inj_data[snr_type], +x_max = 1.1*plu.axis_max_value(found_trigs[x_key], found_injs[x_key], found_missed_file) # Determine the minimum and maximum chi-square value we are dealing with -y_min = 0.9*plu.axis_min_value(trig_data[veto_type], inj_data[veto_type], +y_min = 0.9*plu.axis_min_value(found_trigs[y_key], found_injs[y_key], found_missed_file) -y_max = 1.1*plu.axis_max_value(trig_data[veto_type], inj_data[veto_type], +y_max = 1.1*plu.axis_max_value(found_trigs[y_key], found_injs[y_key], found_missed_file) # Determine y-axis minimum value and label y_label = "Network power chi-square" if veto_type == 'network' \ - else "%s Single %s chi-square" % (ifo, veto_labels[veto_type].lower()) + else ifo + " Single " + veto_labels[veto_type].lower() + " chi-square" # Determine contours for plots conts = None @@ -299,8 +287,7 @@ cont_value = None colors = None # Enable countours of constant reweighted SNR as a function of coherent SNR if snr_type == 'coherent': - conts, snr_vals, cont_value, colors = calculate_contours(trig_data, - opts, + conts, snr_vals, cont_value, colors = calculate_contours(opts, new_snrs=None) # The cut in reweighted SNR involves only the network power chi-square if veto_type != 'network': @@ -314,9 +301,9 @@ if not opts.x_lims: else: opts.x_lims = str(x_min)+','+str(x_max) opts.y_lims = str(y_min)+','+str(10*y_max) -trigs = [trig_data[snr_type], trig_data[veto_type]] -injs = [inj_data[snr_type], inj_data[veto_type]] -plu.pygrb_plotter(trigs, injs, x_label, y_label, opts, +plu.pygrb_plotter([found_trigs[x_key], found_trigs[y_key]], + [found_injs[x_key], found_injs[y_key]], + x_label, y_label, opts, snr_vals=snr_vals, conts=conts, colors=colors, shade_cont_value=cont_value, vert_spike=True, cmd=' '.join(sys.argv)) From 5bb4beacc6cc3ac121b28e539434fc191ac837ba Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Wed, 20 Nov 2024 09:00:11 -0800 Subject: [PATCH 02/14] Version of pycbc_pygrb_plot_coh_ifosnr with vetoes (albeit with a mute apply_vetoes_to_found_injs for the time being) --- bin/pygrb/pycbc_pygrb_plot_coh_ifosnr | 236 +++++++++++++------------- 1 file changed, 120 insertions(+), 116 deletions(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr index 5a2b88321e2..b84d11a538d 100644 --- a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr +++ b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr @@ -53,94 +53,6 @@ __program__ = "pycbc_pygrb_plot_coh_ifosnr" # ============================================================================= # Functions # ============================================================================= -# Function to load trigger data -def load_data(input_file, ifos, vetoes, opts, injections=False, slide_id=None): - """Load data from a trigger/injection file""" - - # Initialize the dictionary - data = {} - data['coherent'] = None - data['single'] = dict((ifo, None) for ifo in ifos) - data['f_resp_mean'] = dict((ifo, None) for ifo in ifos) - data['sigma_mean'] = dict((ifo, None) for ifo in ifos) - data['sigma_max'] = None - data['sigma_min'] = None - - if input_file: - if injections: - logging.info("Loading injections...") - # TODO: This will eventually become load_injections - trigs = ppu.load_triggers( - input_file, - ifos, - vetoes, - rw_snr_threshold=opts.newsnr_threshold, - slide_id=slide_id - ) - else: - logging.info("Loading triggers...") - trigs = ppu.load_triggers( - input_file, - ifos, - vetoes, - rw_snr_threshold=opts.newsnr_threshold, - slide_id=slide_id - ) - - # Load SNR data - data['coherent'] = trigs['network/coherent_snr'] - - # Get single ifo SNR data - ifo_trig_index = {} - for ifo in ifos: - sorted_ifo_ids = numpy.array( - [ - numpy.where(trigs['%s/event_id' % ifo] == idx)[0][0] - for idx in trigs['network/event_id'] - ] - ) - ifo_trig_index[ifo] = sorted_ifo_ids - data['single'][ifo] = trigs['%s/snr' % ifo][:][sorted_ifo_ids] - - # Get sigma for each ifo - sigma = {} - for ifo in ifos: - sigma[ifo] = trigs['%s/sigmasq' % ifo][:][ifo_trig_index[ifo]] - - # Create array for sigma_tot - sigma_tot = numpy.zeros(len(data['coherent'])) - - # Get parameters necessary for antenna responses - ra = trigs['network/ra'][:] - dec = trigs['network/dec'][:] - time = trigs['network/end_time_gc'][:] - - # Get antenna response based parameters - for ifo in ifos: - antenna = Detector(ifo) - ifo_f_resp = ppu.get_antenna_responses(antenna, ra, dec, time) - # Get the average for f_resp_mean and calculate sigma_tot - data['f_resp_mean'][ifo] = ifo_f_resp.mean() - sigma_tot += sigma[ifo] * ifo_f_resp - - for ifo in ifos: - try: - sigma_norm = sigma[ifo] / sigma_tot - data['sigma_mean'][ifo] = sigma_norm.mean() - if ifo == opts.ifo: - data['sigma_max'] = sigma_norm.max() - data['sigma_min'] = sigma_norm.min() - except ValueError: - data['sigma_mean'][ifo] = 0 - if ifo == opts.ifo: - data['sigma_max'] = 0 - data['sigma_min'] = 0 - - logging.info("%d triggers found.", len(data['coherent'])) - - return data - - # Plot lines representing deviations based on non-central chi-square def plot_deviation(percentile, snr_grid, y, ax, style): """Plot deviations based on non-central chi-square""" @@ -196,20 +108,19 @@ init_logging(opts.verbose, format="%(asctime)s: %(levelname)s: %(message)s") # Check options trig_file = os.path.abspath(opts.trig_file) -found_file = ( +found_missed_file = ( os.path.abspath(opts.found_missed_file) if opts.found_missed_file else None ) zoom_in = opts.zoom_in -ifo = opts.ifo -if ifo is None: +if opts.ifo is None: err_msg = "Please specify an interferometer" parser.error(err_msg) if opts.plot_title is None: - opts.plot_title = '%s SNR vs Coherent SNR' % ifo + opts.plot_title = opts.ifo + " SNR vs Coherent SNR" if opts.plot_caption is None: opts.plot_caption = "Blue crosses: background triggers. " - if found_file: + if found_missed_file: opts.plot_caption += "Red crosses: injections triggers. " opts.plot_caption = ( opts.plot_caption @@ -230,25 +141,116 @@ outdir = os.path.split(os.path.abspath(opts.output_file))[0] if not os.path.isdir(outdir): os.makedirs(outdir) -# Extract IFOs and vetoes -ifos, vetoes = ppu.extract_ifos_and_vetoes( - trig_file, opts.veto_files, opts.veto_category +# Extract IFOs +ifos = ppu.extract_ifos(trig_file) + +# Generate time-slides dictionary +slide_dict = ppu.load_time_slides(trig_file) + +# Generate segments dictionary +segment_dict = ppu.load_segment_dict(trig_file) + +# Construct trials removing vetoed times +trial_dict, total_trials = ppu.construct_trials( + opts.seg_files, + segment_dict, + ifos, + slide_dict, + opts.veto_file ) -# Extract trigger data -trig_data = load_data(trig_file, ifos, vetoes, opts, slide_id=opts.slide_id) +# Load triggers/injections (apply reweighted SNR cut, not vetoes) +trig_data = ppu.load_data(trig_file, ifos, data_tag='trigs', + rw_snr_threshold=opts.newsnr_threshold, + slide_id=opts.slide_id) +inj_data = ppu.load_data(found_missed_file, ifos, data_tag='injs', + rw_snr_threshold=opts.newsnr_threshold, + slide_id=0) + +# Extract needed trigger properties and store them as dictionaries +# Based on trial_dict: if vetoes were applied, trig_* are the veto survivors +# Coherent SNR is always used +x_key = 'network/coherent_snr' +keys = [x_key] +# Get parameters necessary for antenna responses +keys += ['network/ra', 'network/dec', 'network/end_time_gc'] +# Get event_ids +keys += [ifo+'/event_id' for ifo in ifos] +keys += ['network/event_id'] +# Get single ifo SNR data +keys += [ifo+'/snr' for ifo in ifos] +# Get sigma for each ifo +keys += [ifo+'/sigmasq' for ifo in ifos] +found_trigs_slides = ppu.extract_trig_properties( + trial_dict, + trig_data, + slide_dict, + segment_dict, + keys +) +found_trigs = {} +for key in keys: + found_trigs[key] = numpy.concatenate( + [found_trigs_slides[key][slide_id][:] for slide_id in slide_dict] + ) -# Extract (or initialize) injection data -inj_data = load_data(found_file, ifos, vetoes, opts, injections=True, slide_id=0) +# Complete the dictionary found_trigs +# 1) Do not assume individual IFO and network event_ids are sorted the same way +for ifo in ifos: + sorted_ifo_ids = numpy.array( + [ + numpy.where(found_trigs[ifo+'/event_id'] == idx)[0][0] + for idx in found_trigs['network/event_id'] + ] + ) + for key in [ifo+'/snr', ifo+'/sigmasq']: + found_trigs[key] = found_trigs[key][sorted_ifo_ids] + +# 2) Get antenna response based parameters +found_trigs['sigma_tot'] = numpy.zeros(len(found_trigs[x_key])) +for ifo in ifos: + antenna = Detector(ifo) + ifo_f_resp = ppu.get_antenna_responses( + antenna, + found_trigs['network/ra'], + found_trigs['network/dec'], + found_trigs['network/end_time_gc'] + ) + # Get the average for f_resp_mean and calculate sigma_tot + found_trigs[ifo+'/f_resp_mean'] = ifo_f_resp.mean() + found_trigs['sigma_tot'] += found_trigs[ifo+'/sigmasq'] * ifo_f_resp + +# 3) Calculate the mean, max, and min sigmas +for ifo in ifos: + try: + sigma_norm = found_trigs[ifo+'/sigmasq'] / found_trigs['sigma_tot'] + found_trigs[ifo+'/sigma_mean'] = sigma_norm.mean() + if ifo == opts.ifo: + found_trigs['sigma_max'] = sigma_norm.max() + found_trigs['sigma_min'] = sigma_norm.min() + except ValueError: + found_trigs[ifo+'/sigma_mean'] = 0 + if ifo == opts.ifo: + found_trigs['sigma_max'] = 0 + found_trigs['sigma_min'] = 0 + +# Gather injections found surviving vetoes +found_injs, *_ = ppu.apply_vetoes_to_found_injs( + found_missed_file, + inj_data, + ifos, + veto_file=opts.veto_file, + keys=keys +) # Generate plots logging.info("Plotting...") # Order the IFOs by sensitivity ifo_senstvty = {} -for i_ifo in ifos: - senstvty = trig_data['f_resp_mean'][i_ifo] * trig_data['sigma_mean'][i_ifo] - ifo_senstvty.update({i_ifo: senstvty}) +for ifo in ifos: + senstvty = found_trigs[ifo+'/f_resp_mean'] * found_trigs[ifo+'/sigma_mean'] + ifo_senstvty.update({ifo: senstvty}) ifo_senstvty = collections.OrderedDict( sorted(ifo_senstvty.items(), key=operator.itemgetter(1), reverse=True) ) @@ -256,39 +258,41 @@ loudness_labels = ['first', 'second', 'third'] # Determine the maximum coherent SNR value we are dealing with x_max = plu.axis_max_value( - trig_data['coherent'], inj_data['coherent'], found_file + found_trigs[x_key], found_injs[x_key], found_missed_file ) max_snr = x_max if x_max < 50.0: max_snr = 50.0 # Determine the maximum auto veto value we are dealing with +y_key = opts.ifo+'/snr' y_max = plu.axis_max_value( - trig_data['single'][ifo], inj_data['single'][ifo], found_file + found_trigs[y_key], found_injs[y_key], found_missed_file ) # Setup the plots x_label = "Coherent SNR" -y_label = "%s sngl SNR" % ifo +y_label = opts.ifo + " SNR" fig = plt.figure() ax = fig.gca() # Plot trigger data -ax.plot(trig_data['coherent'], trig_data['single'][ifo], 'bx') +ax.plot(found_trigs[x_key], found_trigs[y_key], 'bx') ax.grid() # Plot injection data -if found_file: - ax.plot(inj_data['coherent'], inj_data['single'][ifo], 'r+') +if found_missed_file: + ax.plot(found_injs[x_key], found_injs[y_key], 'r+') # Sigma-mean, min, max y_data = { - 'mean': trig_data['sigma_mean'][ifo], - 'min': trig_data['sigma_min'], - 'max': trig_data['sigma_max'], + 'mean': found_trigs[opts.ifo+'/sigma_mean'], + 'min': found_trigs['sigma_min'], + 'max': found_trigs['sigma_max'], } # Calculate: zoom-snr * sqrt(response * sigma-mean, min, max) snr_grid = numpy.arange(0.01, max_snr, 1) y_data = dict( - (key, snr_grid * (trig_data['f_resp_mean'][ifo] * y_data[key]) ** 0.5) - for key in y_data + (key, + snr_grid * (found_trigs[opts.ifo+'/f_resp_mean'] * val) ** 0.5) + for key, val in y_data.items() ) for key in y_data: ax.plot(snr_grid, y_data[key], 'g-') @@ -305,7 +309,7 @@ ax.set_ylabel(y_label) ax.set_xlim([0, 1.01 * x_max]) ax.set_ylim([0, 1.20 * y_max]) # Veto applies to the two most sensitive IFOs, so shade them -loudness_index = list(ifo_senstvty.keys()).index(ifo) +loudness_index = list(ifo_senstvty.keys()).index(opts.ifo) if loudness_index < 2: limy = ax.get_ylim()[0] polyx = [0, max_snr] @@ -314,7 +318,7 @@ if loudness_index < 2: polyy.extend([limy, limy]) ax.fill(polyx, polyy, color='#dddddd') opts.plot_title += ( - " (%s average sensitivity)" % loudness_labels[loudness_index] + " (" + loudness_labels[loudness_index] + " average sensitivity)" ) # Zoom in if asked to do so if opts.zoom_in: From a881da9b41aff35b7ede29761b20a851a6d09cdb Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 09:50:24 -0700 Subject: [PATCH 03/14] Update f-string in pycbc_pygrb_plot_coh_ifosnr Co-authored-by: Tito Dal Canton --- bin/pygrb/pycbc_pygrb_plot_coh_ifosnr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr index b84d11a538d..3013e6863f0 100644 --- a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr +++ b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr @@ -318,7 +318,7 @@ if loudness_index < 2: polyy.extend([limy, limy]) ax.fill(polyx, polyy, color='#dddddd') opts.plot_title += ( - " (" + loudness_labels[loudness_index] + " average sensitivity)" + f" ({loudness_labels[loudness_index]} average sensitivity)" ) # Zoom in if asked to do so if opts.zoom_in: From 63e41996b9b404776b6e52d129adbc3cc2b27472 Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 09:50:52 -0700 Subject: [PATCH 04/14] Update f-string in pycbc_pygrb_plot_chisq_veto Co-authored-by: Tito Dal Canton --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index ec0c4df2784..2771f668a03 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -150,7 +150,7 @@ if opts.plot_title is None: if veto_type != 'network': opts.plot_title = ifo + opts.plot_title if snr_type == 'single': - opts.plot_title += " vs " + ifo + " SNR" + opts.plot_title += f" vs {ifo} SNR" else: opts.plot_title += " vs " + snr_type.capitalize() + " SNR" if opts.plot_caption is None: From 67ce5444da0f5f8e714e0f8cedf2c67587253c6c Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 09:51:14 -0700 Subject: [PATCH 05/14] Update f-string in pycbc_pygrb_plot_chisq_veto Co-authored-by: Tito Dal Canton --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index 2771f668a03..3b764c709c9 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -152,7 +152,7 @@ if opts.plot_title is None: if snr_type == 'single': opts.plot_title += f" vs {ifo} SNR" else: - opts.plot_title += " vs " + snr_type.capitalize() + " SNR" + opts.plot_title += f" vs {snr_type.capitalize()} SNR" if opts.plot_caption is None: opts.plot_caption = ("Blue crosses: background triggers. ") if found_missed_file: From 2b3a380897de86a4177af47cc3ad798da630c878 Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 09:52:34 -0700 Subject: [PATCH 06/14] Update f-string in pycbc_pygrb_plot_chisq_veto Co-authored-by: Tito Dal Canton --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index 3b764c709c9..798da11125e 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -262,7 +262,7 @@ logging.info("Plotting...") # Determine x-axis values of triggers and injections # Default is coherent SNR x_label = ifo if snr_type == 'single' else snr_type.capitalize() -x_label = x_label + " SNR" +x_label += " SNR" # Determine the minumum and maximum SNR value we are dealing with x_min = 0.9*plu.axis_min_value(found_trigs[x_key], found_injs[x_key], From bad8389f3a67a245ed520654a0c5a1fff92d846c Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 10:12:38 -0800 Subject: [PATCH 07/14] Comment corrected --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index 798da11125e..6cbc716c804 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -276,7 +276,7 @@ y_min = 0.9*plu.axis_min_value(found_trigs[y_key], found_injs[y_key], y_max = 1.1*plu.axis_max_value(found_trigs[y_key], found_injs[y_key], found_missed_file) -# Determine y-axis minimum value and label +# Determine y-axis label y_label = "Network power chi-square" if veto_type == 'network' \ else ifo + " Single " + veto_labels[veto_type].lower() + " chi-square" From 1f500a003d7e871c31dad9879f1dd7b7d655b7f4 Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 12:13:58 -0800 Subject: [PATCH 08/14] Removed dof_key variable --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index 6cbc716c804..112db52c4a7 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -213,10 +213,10 @@ else: y_key = 'network/my_network_chisq' keys = [x_key, y_key] -# The network chi-square is already normalized so dof_key is not needed +# The network chi-square is already normalized so it does not require a key +# for the number of degrees of freedom if veto_type != 'network': - dof_key = y_key + '_dof' - keys += [dof_key] + keys += [y_key + '_dof'] # Extract needed trigger properties and store them as dictionaries # Based on trial_dict: if vetoes were applied, trig_* are the veto survivors From 92fcde42807cc3b9eed0090cde7570d6668a06b5 Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 13:29:31 -0700 Subject: [PATCH 09/14] Update bin/pygrb/pycbc_pygrb_plot_chisq_veto Co-authored-by: Tito Dal Canton --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index 112db52c4a7..48a8410f107 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -278,7 +278,7 @@ y_max = 1.1*plu.axis_max_value(found_trigs[y_key], found_injs[y_key], # Determine y-axis label y_label = "Network power chi-square" if veto_type == 'network' \ - else ifo + " Single " + veto_labels[veto_type].lower() + " chi-square" + else f"{ifo} Single {veto_labels[veto_type].lower()} chi-square" # Determine contours for plots conts = None From 7d426fddfe4697740b8995d19abb66b1be1cf471 Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 13:35:17 -0700 Subject: [PATCH 10/14] numpy.where --> numpy.nonzero in pycbc_pygrb_plot_coh_ifosnr Co-authored-by: Tito Dal Canton --- bin/pygrb/pycbc_pygrb_plot_coh_ifosnr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr index 3013e6863f0..4578ad06baf 100644 --- a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr +++ b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr @@ -199,7 +199,7 @@ for key in keys: for ifo in ifos: sorted_ifo_ids = numpy.array( [ - numpy.where(found_trigs[ifo+'/event_id'] == idx)[0][0] + numpy.nonzero(found_trigs[ifo + '/event_id'] == idx)[0][0] for idx in found_trigs['network/event_id'] ] ) From ef7c8e156d354fedd91024baf7ea32a2a0c93c8f Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 13:07:12 -0800 Subject: [PATCH 11/14] senstvty --> sensitivity --- bin/pygrb/pycbc_pygrb_plot_coh_ifosnr | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr index 4578ad06baf..b915d5b897b 100644 --- a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr +++ b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr @@ -247,12 +247,13 @@ found_injs, *_ = ppu.apply_vetoes_to_found_injs( logging.info("Plotting...") # Order the IFOs by sensitivity -ifo_senstvty = {} +ifo_sensitivity = {} for ifo in ifos: - senstvty = found_trigs[ifo+'/f_resp_mean'] * found_trigs[ifo+'/sigma_mean'] - ifo_senstvty.update({ifo: senstvty}) -ifo_senstvty = collections.OrderedDict( - sorted(ifo_senstvty.items(), key=operator.itemgetter(1), reverse=True) + sensitivity = \ + found_trigs[ifo+'/f_resp_mean'] * found_trigs[ifo+'/sigma_mean'] + ifo_sensitivity.update({ifo: sensitivity}) +ifo_sensitivity = collections.OrderedDict( + sorted(ifo_sensitivity.items(), key=operator.itemgetter(1), reverse=True) ) loudness_labels = ['first', 'second', 'third'] @@ -309,7 +310,7 @@ ax.set_ylabel(y_label) ax.set_xlim([0, 1.01 * x_max]) ax.set_ylim([0, 1.20 * y_max]) # Veto applies to the two most sensitive IFOs, so shade them -loudness_index = list(ifo_senstvty.keys()).index(opts.ifo) +loudness_index = list(ifo_sensitivity.keys()).index(opts.ifo) if loudness_index < 2: limy = ax.get_ylim()[0] polyx = [0, max_snr] From 380b32f72158cd9486fca7c4539352e35626d6de Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Thu, 21 Nov 2024 13:11:26 -0800 Subject: [PATCH 12/14] Comprehension for dictionary generation --- bin/pygrb/pycbc_pygrb_plot_coh_ifosnr | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr index b915d5b897b..3dc848a4503 100644 --- a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr +++ b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr @@ -247,11 +247,9 @@ found_injs, *_ = ppu.apply_vetoes_to_found_injs( logging.info("Plotting...") # Order the IFOs by sensitivity -ifo_sensitivity = {} -for ifo in ifos: - sensitivity = \ - found_trigs[ifo+'/f_resp_mean'] * found_trigs[ifo+'/sigma_mean'] - ifo_sensitivity.update({ifo: sensitivity}) +ifo_sensitivity = { + ifo: found_trigs[ifo+'/f_resp_mean'] * found_trigs[ifo+'/sigma_mean'] for ifo in ifos +} ifo_sensitivity = collections.OrderedDict( sorted(ifo_sensitivity.items(), key=operator.itemgetter(1), reverse=True) ) From a6f84b30f8885bb36a81143676da9a07303b3b78 Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Fri, 22 Nov 2024 08:36:37 -0800 Subject: [PATCH 13/14] Removed try-except --- bin/pygrb/pycbc_pygrb_plot_coh_ifosnr | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr index 3dc848a4503..b3fdb6ccb26 100644 --- a/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr +++ b/bin/pygrb/pycbc_pygrb_plot_coh_ifosnr @@ -222,17 +222,11 @@ for ifo in ifos: # 3) Calculate the mean, max, and min sigmas for ifo in ifos: - try: - sigma_norm = found_trigs[ifo+'/sigmasq'] / found_trigs['sigma_tot'] - found_trigs[ifo+'/sigma_mean'] = sigma_norm.mean() - if ifo == opts.ifo: - found_trigs['sigma_max'] = sigma_norm.max() - found_trigs['sigma_min'] = sigma_norm.min() - except ValueError: - found_trigs[ifo+'/sigma_mean'] = 0 - if ifo == opts.ifo: - found_trigs['sigma_max'] = 0 - found_trigs['sigma_min'] = 0 + sigma_norm = found_trigs[ifo+'/sigmasq'] / found_trigs['sigma_tot'] + found_trigs[ifo+'/sigma_mean'] = sigma_norm.mean() if len(sigma_norm) else 0 + if ifo == opts.ifo: + found_trigs['sigma_max'] = sigma_norm.max() if len(sigma_norm) else 0 + found_trigs['sigma_min'] = sigma_norm.min() if len(sigma_norm) else 0 # Gather injections found surviving vetoes found_injs, *_ = ppu.apply_vetoes_to_found_injs( From 90b138027bd00bca1dde2b717459ac861635bcbb Mon Sep 17 00:00:00 2001 From: Francesco Pannarale Date: Fri, 22 Nov 2024 08:52:14 -0800 Subject: [PATCH 14/14] Improved comments --- bin/pygrb/pycbc_pygrb_plot_chisq_veto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/pygrb/pycbc_pygrb_plot_chisq_veto b/bin/pygrb/pycbc_pygrb_plot_chisq_veto index 48a8410f107..e247311d23c 100644 --- a/bin/pygrb/pycbc_pygrb_plot_chisq_veto +++ b/bin/pygrb/pycbc_pygrb_plot_chisq_veto @@ -253,7 +253,8 @@ if len(keys) == 3: found_trigs[keys[1]] /= found_trigs[keys[2]] found_injs[keys[1]] /= found_injs[keys[2]] -# Floor single IFO chi-square at 0.005 +# Single detector chi-squares are initialized to 0: we floor possible +# remaining 0s to 0.005 to avoid asking for logarithms of 0 in the plot numpy.putmask(found_trigs[y_key], found_trigs[y_key] == 0, 0.005) # Generate plots