diff --git a/bin/workflows/pycbc_make_offline_search_workflow b/bin/workflows/pycbc_make_offline_search_workflow index 86371191fae..f3205438408 100755 --- a/bin/workflows/pycbc_make_offline_search_workflow +++ b/bin/workflows/pycbc_make_offline_search_workflow @@ -127,7 +127,8 @@ save_fig_with_metadata(time_str, time_file.storage_path, **kwds) # Get segments and find the data locations sci_seg_name = 'science' science_seg_file = wf.get_segments_file(workflow, sci_seg_name, 'segments-science', - rdir['analysis_time/segment_data']) + rdir['analysis_time/segment_data'], + tags=['science']) ssegs = {} for ifo in workflow.ifos: @@ -145,13 +146,15 @@ datafind_files, analyzable_file, analyzable_segs, analyzable_name = \ final_veto_name = 'vetoes' final_veto_file = wf.get_segments_file(workflow, final_veto_name, 'segments-vetoes', - rdir['analysis_time/segment_data']) + rdir['analysis_time/segment_data'], + tags=['veto']) # Get dq segments from veto definer and calculate data quality timeseries dq_flag_name = 'dq_flag' dq_segment_file = wf.get_flag_segments_file(workflow, dq_flag_name, 'segments-dq', - rdir['analysis_time/segment_data']) + rdir['analysis_time/segment_data'], + tags=['dq']) # Template bank stuff hdfbank = wf.setup_tmpltbank_workflow(workflow, analyzable_segs, diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 0667b10ff68..e00fb75ba3d 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -61,7 +61,7 @@ def save_veto_definer(cp, out_dir, tags=None): return veto_def_new_path -def get_segments_file(workflow, name, option_name, out_dir): +def get_segments_file(workflow, name, option_name, out_dir, tags=None): """Get cumulative segments from option name syntax for each ifo. Use syntax of configparser string to define the resulting segment_file @@ -76,6 +76,9 @@ def get_segments_file(workflow, name, option_name, out_dir): Name of the segment list being created option_name: str Name of option in the associated config parser to get the flag list + tags : list of strings + Used to retrieve subsections of the ini file for + configuration options. returns -------- @@ -88,25 +91,29 @@ def get_segments_file(workflow, name, option_name, out_dir): start = workflow.analysis_time[0] end = workflow.analysis_time[1] + if tags is None: + tags = [] + # Check for veto definer file veto_definer = None if cp.has_option("workflow-segments", "segments-veto-definer-url"): - veto_definer = save_veto_definer(workflow.cp, out_dir, []) + veto_definer = save_veto_definer(workflow.cp, out_dir) # Check for provided server server = "https://segments.ligo.org" - if cp.has_option("workflow-segments", "segments-database-url"): - server = cp.get("workflow-segments", - "segments-database-url") + if cp.has_option_tags("workflow-segments", "segments-database-url", tags): + server = cp.get_opt_tags("workflow-segments", + "segments-database-url", tags) - if cp.has_option("workflow-segments", "segments-source"): - source = cp.get("workflow-segments", "segments-source") + if cp.has_option_tags("workflow-segments", "segments-source", tags): + source = cp.get_opt_tags("workflow-segments", "segments-source", tags) else: source = "any" if source == "file": local_file_path = \ - resolve_url(cp.get("workflow-segments", option_name+"-file")) + resolve_url(cp.get_opt_tag("workflow-segments", + option_name+"-file", tags)) pfn = os.path.join(out_dir, os.path.basename(local_file_path)) shutil.move(local_file_path, pfn) return SegFile.from_segment_xml(pfn) @@ -380,72 +387,80 @@ def generate_triggered_segment(workflow, out_dir, sciencesegs): except UnboundLocalError: return None, min_seg -def get_flag_segments_file(workflow, name, option_name, out_dir): - """Get segments from option name syntax for each ifo for indivudal flags. - - Use syntax of configparser string to define the resulting segment_file - e.x. option_name = +up_flag1,+up_flag2,+up_flag3,-down_flag1,-down_flag2 - Each ifo may have a different string and is stored separately in the file. - Each flag is stored separately in the file. - Flags which add time must precede flags which subtract time. - - Parameters - ---------- - workflow: pycbc.workflow.Workflow - name: string - Name of the segment list being created - option_name: str - Name of option in the associated config parser to get the flag list - - returns - -------- - seg_file: pycbc.workflow.SegFile - SegFile intance that points to the segment xml file on disk. - """ - from pycbc.dq import query_str - make_analysis_dir(out_dir) - cp = workflow.cp - start = workflow.analysis_time[0] - end = workflow.analysis_time[1] - - # Check for veto definer file - veto_definer = None - if cp.has_option("workflow-segments", "segments-veto-definer-url"): - veto_definer = save_veto_definer(workflow.cp, out_dir, []) - - # Check for provided server - server = "https://segments.ligo.org" - if cp.has_option("workflow-segments", "segments-database-url"): - server = cp.get("workflow-segments", "segments-database-url") - - source = "any" - if cp.has_option("workflow-segments", "segments-source"): - source = cp.get("workflow-segments", "segments-source") - if source == "file": - local_file_path = \ - resolve_url(cp.get("workflow-segments", option_name+"-file")) - pfn = os.path.join(out_dir, os.path.basename(local_file_path)) - shutil.move(local_file_path, pfn) - return SegFile.from_segment_xml(pfn) - - segs = {} - for ifo in workflow.ifos: - if cp.has_option_tags("workflow-segments", option_name, [ifo]): - flag_str = cp.get_opt_tags("workflow-segments", option_name, [ifo]) - flag_list = flag_str.split(',') - for flag in flag_list: - flag_name = flag[1:] - key = flag_name - if len(key.split(':')) > 2: - key = ':'.join(key.split(':')[:2]) - segs[key] = query_str(ifo, flag, start, end, - source=source, server=server, - veto_definer=veto_definer) - logging.info("%s: got %s segments", ifo, flag_name) - else: - logging.info("%s: no segments requested", ifo) - - return SegFile.from_segment_list_dict(name, segs, - extension='.xml', - valid_segment=workflow.analysis_time, - directory=out_dir) +def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): + """Get segments from option name syntax for each ifo for indivudal flags. + + Use syntax of configparser string to define the resulting segment_file + e.x. option_name = +up_flag1,+up_flag2,+up_flag3,-down_flag1,-down_flag2 + Each ifo may have a different string and is stored separately in the file. + Each flag is stored separately in the file. + Flags which add time must precede flags which subtract time. + + Parameters + ---------- + workflow: pycbc.workflow.Workflow + name: string + Name of the segment list being created + option_name: str + Name of option in the associated config parser to get the flag list + tags : list of strings + Used to retrieve subsections of the ini file for + configuration options. + + returns + -------- + seg_file: pycbc.workflow.SegFile + SegFile intance that points to the segment xml file on disk. + """ + from pycbc.dq import query_str + make_analysis_dir(out_dir) + cp = workflow.cp + start = workflow.analysis_time[0] + end = workflow.analysis_time[1] + + if tags is None: + tags = [] + + # Check for veto definer file + veto_definer = None + if cp.has_option("workflow-segments", "segments-veto-definer-url"): + veto_definer = save_veto_definer(workflow.cp, out_dir) + + # Check for provided server + server = "https://segments.ligo.org" + if cp.has_option_tags("workflow-segments", "segments-database-url", tags): + server = cp.get_opt_tags("workflow-segments", + "segments-database-url", tags) + + source = "any" + if cp.has_option_tags("workflow-segments", "segments-source", tags): + source = cp.get_opt_tags("workflow-segments", "segments-source", tags) + if source == "file": + local_file_path = \ + resolve_url(cp.get_opt_tags("workflow-segments", + option_name+"-file", tags)) + pfn = os.path.join(out_dir, os.path.basename(local_file_path)) + shutil.move(local_file_path, pfn) + return SegFile.from_segment_xml(pfn) + + segs = {} + for ifo in workflow.ifos: + if cp.has_option_tags("workflow-segments", option_name, [ifo]): + flag_str = cp.get_opt_tags("workflow-segments", option_name, [ifo]) + flag_list = flag_str.split(',') + for flag in flag_list: + flag_name = flag[1:] + if len(flag_name.split(':')) > 1: + flag_name = name.split(':')[1] + key = ifo + ':' + flag_name + segs[key] = query_str(ifo, flag, start, end, + source=source, server=server, + veto_definer=veto_definer) + logging.info("%s: got %s segments", ifo, flag_name) + else: + logging.info("%s: no segments requested", ifo) + + return SegFile.from_segment_list_dict(name, segs, + extension='.xml', + valid_segment=workflow.analysis_time, + directory=out_dir)