From d1f9a77206ead6292d34f3e1fd7079f57a7bf9f8 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Mon, 28 Aug 2023 12:21:44 -0700 Subject: [PATCH 1/9] Added support for dq tags --- .../pycbc_make_offline_search_workflow | 18 +++++++++-- pycbc/workflow/segment.py | 32 +++++++++++-------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/bin/workflows/pycbc_make_offline_search_workflow b/bin/workflows/pycbc_make_offline_search_workflow index 86371191fae..9b82ed1ae9b 100755 --- a/bin/workflows/pycbc_make_offline_search_workflow +++ b/bin/workflows/pycbc_make_offline_search_workflow @@ -126,8 +126,12 @@ save_fig_with_metadata(time_str, time_file.storage_path, **kwds) # Get segments and find the data locations sci_seg_name = 'science' +science_tags = [] +if 'science' in workflow.cp.get_subsections('workflow-segments'): + science_tags = ['science'] science_seg_file = wf.get_segments_file(workflow, sci_seg_name, 'segments-science', - rdir['analysis_time/segment_data']) + rdir['analysis_time/segment_data'], + tags=science_tags) ssegs = {} for ifo in workflow.ifos: @@ -143,15 +147,23 @@ datafind_files, analyzable_file, analyzable_segs, analyzable_name = \ seg_file=science_seg_file, tags=hoft_tags) final_veto_name = 'vetoes' +veto_tags = [] +if 'veto' in workflow.cp.get_subsections('workflow-segments'): + veto_tags = ['veto'] final_veto_file = wf.get_segments_file(workflow, final_veto_name, 'segments-vetoes', - rdir['analysis_time/segment_data']) + rdir['analysis_time/segment_data'], + tags=veto_tags) # Get dq segments from veto definer and calculate data quality timeseries dq_flag_name = 'dq_flag' +dq_tags = [] +if 'dq' in workflow.cp.get_subsections('workflow-segments'): + dq_tags = ['dq'] dq_segment_file = wf.get_flag_segments_file(workflow, dq_flag_name, 'segments-dq', - rdir['analysis_time/segment_data']) + rdir['analysis_time/segment_data'], + tags=dq_tags) # Template bank stuff hdfbank = wf.setup_tmpltbank_workflow(workflow, analyzable_segs, diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 0667b10ff68..62c7dbe26f0 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -61,7 +61,7 @@ def save_veto_definer(cp, out_dir, tags=None): return veto_def_new_path -def get_segments_file(workflow, name, option_name, out_dir): +def get_segments_file(workflow, name, option_name, out_dir, tags=None): """Get cumulative segments from option name syntax for each ifo. Use syntax of configparser string to define the resulting segment_file @@ -76,6 +76,9 @@ def get_segments_file(workflow, name, option_name, out_dir): Name of the segment list being created option_name: str Name of option in the associated config parser to get the flag list + tags : list of strings + Used to retrieve subsections of the ini file for + configuration options. returns -------- @@ -95,18 +98,19 @@ def get_segments_file(workflow, name, option_name, out_dir): # Check for provided server server = "https://segments.ligo.org" - if cp.has_option("workflow-segments", "segments-database-url"): - server = cp.get("workflow-segments", - "segments-database-url") + if cp.has_option_tags("workflow-segments", "segments-database-url", tags): + server = cp.get_opt_tags("workflow-segments", + "segments-database-url", tags) - if cp.has_option("workflow-segments", "segments-source"): - source = cp.get("workflow-segments", "segments-source") + if cp.has_option_tags("workflow-segments", "segments-source", tags): + source = cp.get_opt_tags("workflow-segments", "segments-source", tags) else: source = "any" if source == "file": local_file_path = \ - resolve_url(cp.get("workflow-segments", option_name+"-file")) + resolve_url(cp.get_opt_tag("workflow-segments", + option_name+"-file", tags)) pfn = os.path.join(out_dir, os.path.basename(local_file_path)) shutil.move(local_file_path, pfn) return SegFile.from_segment_xml(pfn) @@ -380,7 +384,7 @@ def generate_triggered_segment(workflow, out_dir, sciencesegs): except UnboundLocalError: return None, min_seg -def get_flag_segments_file(workflow, name, option_name, out_dir): +def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): """Get segments from option name syntax for each ifo for indivudal flags. Use syntax of configparser string to define the resulting segment_file @@ -415,15 +419,17 @@ def get_flag_segments_file(workflow, name, option_name, out_dir): # Check for provided server server = "https://segments.ligo.org" - if cp.has_option("workflow-segments", "segments-database-url"): - server = cp.get("workflow-segments", "segments-database-url") + if cp.has_option_tags("workflow-segments", "segments-database-url", tags): + server = cp.get_opt_tags("workflow-segments", + "segments-database-url", tags) source = "any" - if cp.has_option("workflow-segments", "segments-source"): - source = cp.get("workflow-segments", "segments-source") + if cp.has_option_tags("workflow-segments", "segments-source", tags): + source = cp.get_opt_tags("workflow-segments", "segments-source", tags) if source == "file": local_file_path = \ - resolve_url(cp.get("workflow-segments", option_name+"-file")) + resolve_url(cp.get_opt_tags("workflow-segments", + option_name+"-file", tags)) pfn = os.path.join(out_dir, os.path.basename(local_file_path)) shutil.move(local_file_path, pfn) return SegFile.from_segment_xml(pfn) From ee0d8607307e02792c3431ee2c286b0e7ecff17d Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Mon, 28 Aug 2023 17:36:08 -0700 Subject: [PATCH 2/9] Small fix --- pycbc/workflow/segment.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 62c7dbe26f0..03a964dddbb 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -93,8 +93,11 @@ def get_segments_file(workflow, name, option_name, out_dir, tags=None): # Check for veto definer file veto_definer = None - if cp.has_option("workflow-segments", "segments-veto-definer-url"): - veto_definer = save_veto_definer(workflow.cp, out_dir, []) + veto_tags = [] + if 'veto' in workflow.cp.get_subsections('workflow-segments'): + veto_tags = ['veto'] + if cp.has_option_tags("workflow-segments", "segments-veto-definer-url", veto_tags): + veto_definer = save_veto_definer(workflow.cp, out_dir, veto_tags) # Check for provided server server = "https://segments.ligo.org" From 4c5ed694dfa2f8c5a54a1deed8f9a879a7e64b16 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Tue, 29 Aug 2023 09:37:53 -0700 Subject: [PATCH 3/9] Addressed some of Ian's comments --- .../pycbc_make_offline_search_workflow | 15 +++------------ pycbc/workflow/segment.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/bin/workflows/pycbc_make_offline_search_workflow b/bin/workflows/pycbc_make_offline_search_workflow index 9b82ed1ae9b..f3205438408 100755 --- a/bin/workflows/pycbc_make_offline_search_workflow +++ b/bin/workflows/pycbc_make_offline_search_workflow @@ -126,12 +126,9 @@ save_fig_with_metadata(time_str, time_file.storage_path, **kwds) # Get segments and find the data locations sci_seg_name = 'science' -science_tags = [] -if 'science' in workflow.cp.get_subsections('workflow-segments'): - science_tags = ['science'] science_seg_file = wf.get_segments_file(workflow, sci_seg_name, 'segments-science', rdir['analysis_time/segment_data'], - tags=science_tags) + tags=['science']) ssegs = {} for ifo in workflow.ifos: @@ -147,23 +144,17 @@ datafind_files, analyzable_file, analyzable_segs, analyzable_name = \ seg_file=science_seg_file, tags=hoft_tags) final_veto_name = 'vetoes' -veto_tags = [] -if 'veto' in workflow.cp.get_subsections('workflow-segments'): - veto_tags = ['veto'] final_veto_file = wf.get_segments_file(workflow, final_veto_name, 'segments-vetoes', rdir['analysis_time/segment_data'], - tags=veto_tags) + tags=['veto']) # Get dq segments from veto definer and calculate data quality timeseries dq_flag_name = 'dq_flag' -dq_tags = [] -if 'dq' in workflow.cp.get_subsections('workflow-segments'): - dq_tags = ['dq'] dq_segment_file = wf.get_flag_segments_file(workflow, dq_flag_name, 'segments-dq', rdir['analysis_time/segment_data'], - tags=dq_tags) + tags=['dq']) # Template bank stuff hdfbank = wf.setup_tmpltbank_workflow(workflow, analyzable_segs, diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 03a964dddbb..868a1c1dab7 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -91,13 +91,14 @@ def get_segments_file(workflow, name, option_name, out_dir, tags=None): start = workflow.analysis_time[0] end = workflow.analysis_time[1] + if tags is None: + tags = [] + # Check for veto definer file veto_definer = None - veto_tags = [] - if 'veto' in workflow.cp.get_subsections('workflow-segments'): - veto_tags = ['veto'] - if cp.has_option_tags("workflow-segments", "segments-veto-definer-url", veto_tags): - veto_definer = save_veto_definer(workflow.cp, out_dir, veto_tags) + if cp.has_option_tags("workflow-segments", + "segments-veto-definer-url", veto_tags): + veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) # Check for provided server server = "https://segments.ligo.org" @@ -415,10 +416,14 @@ def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): start = workflow.analysis_time[0] end = workflow.analysis_time[1] + if tags is None: + tags = [] + # Check for veto definer file veto_definer = None - if cp.has_option("workflow-segments", "segments-veto-definer-url"): - veto_definer = save_veto_definer(workflow.cp, out_dir, []) + if cp.has_option_tags("workflow-segments", + "segments-veto-definer-url", veto_tags): + veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) # Check for provided server server = "https://segments.ligo.org" From 9143a91a1378f99b14fab61bee1c3314103305e6 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Tue, 29 Aug 2023 09:45:59 -0700 Subject: [PATCH 4/9] Fixed indent mistake --- pycbc/workflow/segment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 868a1c1dab7..b9f03364f3c 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -416,12 +416,12 @@ def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): start = workflow.analysis_time[0] end = workflow.analysis_time[1] - if tags is None: + if tags is None: tags = [] # Check for veto definer file veto_definer = None - if cp.has_option_tags("workflow-segments", + if cp.has_option_tags("workflow-segments", "segments-veto-definer-url", veto_tags): veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) From 94bfa47e4c1135953d734b2309facb0a021eb881 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Tue, 29 Aug 2023 10:02:35 -0700 Subject: [PATCH 5/9] Added description of tags argument to docstring --- pycbc/workflow/segment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index b9f03364f3c..95eda19664d 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -404,6 +404,9 @@ def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): Name of the segment list being created option_name: str Name of option in the associated config parser to get the flag list + tags : list of strings + Used to retrieve subsections of the ini file for + configuration options. returns -------- From f567e8ac10fea61e02e823e084d093beb4ea28d2 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Tue, 29 Aug 2023 10:20:54 -0700 Subject: [PATCH 6/9] Small fix to tags --- pycbc/workflow/segment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 95eda19664d..2a6faf5e372 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -97,7 +97,7 @@ def get_segments_file(workflow, name, option_name, out_dir, tags=None): # Check for veto definer file veto_definer = None if cp.has_option_tags("workflow-segments", - "segments-veto-definer-url", veto_tags): + "segments-veto-definer-url", ['veto']): veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) # Check for provided server @@ -425,8 +425,8 @@ def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): # Check for veto definer file veto_definer = None if cp.has_option_tags("workflow-segments", - "segments-veto-definer-url", veto_tags): - veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) + "segments-veto-definer-url", ['veto']): + veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) # Check for provided server server = "https://segments.ligo.org" From 46bcb19b630979c9489a8d7fa3218eaf7314cce4 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Tue, 29 Aug 2023 13:11:53 -0700 Subject: [PATCH 7/9] Don't use tags with veto-definer --- pycbc/workflow/segment.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 2a6faf5e372..2ca5297b532 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -96,9 +96,8 @@ def get_segments_file(workflow, name, option_name, out_dir, tags=None): # Check for veto definer file veto_definer = None - if cp.has_option_tags("workflow-segments", - "segments-veto-definer-url", ['veto']): - veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) + if cp.has_option("workflow-segments", "segments-veto-definer-url"): + veto_definer = save_veto_definer(workflow.cp, out_dir) # Check for provided server server = "https://segments.ligo.org" @@ -424,9 +423,8 @@ def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): # Check for veto definer file veto_definer = None - if cp.has_option_tags("workflow-segments", - "segments-veto-definer-url", ['veto']): - veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto']) + if cp.has_option("workflow-segments", "segments-veto-definer-url"): + veto_definer = save_veto_definer(workflow.cp, out_dir) # Check for provided server server = "https://segments.ligo.org" From d4e5167d0aefad36ac1f663510330a1308afa472 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Tue, 29 Aug 2023 15:03:22 -0700 Subject: [PATCH 8/9] Don't require dq segments to be set for each ifo separately --- pycbc/workflow/segment.py | 140 +++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 2ca5297b532..35ed01e2aa7 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -388,79 +388,79 @@ def generate_triggered_segment(workflow, out_dir, sciencesegs): return None, min_seg def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): - """Get segments from option name syntax for each ifo for indivudal flags. - - Use syntax of configparser string to define the resulting segment_file - e.x. option_name = +up_flag1,+up_flag2,+up_flag3,-down_flag1,-down_flag2 - Each ifo may have a different string and is stored separately in the file. - Each flag is stored separately in the file. - Flags which add time must precede flags which subtract time. - - Parameters - ---------- - workflow: pycbc.workflow.Workflow - name: string - Name of the segment list being created - option_name: str - Name of option in the associated config parser to get the flag list - tags : list of strings - Used to retrieve subsections of the ini file for - configuration options. + """Get segments from option name syntax for each ifo for indivudal flags. + + Use syntax of configparser string to define the resulting segment_file + e.x. option_name = +up_flag1,+up_flag2,+up_flag3,-down_flag1,-down_flag2 + Each ifo may have a different string and is stored separately in the file. + Each flag is stored separately in the file. + Flags which add time must precede flags which subtract time. + + Parameters + ---------- + workflow: pycbc.workflow.Workflow + name: string + Name of the segment list being created + option_name: str + Name of option in the associated config parser to get the flag list + tags : list of strings + Used to retrieve subsections of the ini file for + configuration options. + + returns + -------- + seg_file: pycbc.workflow.SegFile + SegFile intance that points to the segment xml file on disk. + """ + from pycbc.dq import query_str + make_analysis_dir(out_dir) + cp = workflow.cp + start = workflow.analysis_time[0] + end = workflow.analysis_time[1] - returns - -------- - seg_file: pycbc.workflow.SegFile - SegFile intance that points to the segment xml file on disk. - """ - from pycbc.dq import query_str - make_analysis_dir(out_dir) - cp = workflow.cp - start = workflow.analysis_time[0] - end = workflow.analysis_time[1] - - if tags is None: + if tags is None: tags = [] - # Check for veto definer file - veto_definer = None - if cp.has_option("workflow-segments", "segments-veto-definer-url"): - veto_definer = save_veto_definer(workflow.cp, out_dir) + # Check for veto definer file + veto_definer = None + if cp.has_option("workflow-segments", "segments-veto-definer-url"): + veto_definer = save_veto_definer(workflow.cp, out_dir) - # Check for provided server - server = "https://segments.ligo.org" - if cp.has_option_tags("workflow-segments", "segments-database-url", tags): - server = cp.get_opt_tags("workflow-segments", - "segments-database-url", tags) + # Check for provided server + server = "https://segments.ligo.org" + if cp.has_option_tags("workflow-segments", "segments-database-url", tags): + server = cp.get_opt_tags("workflow-segments", + "segments-database-url", tags) - source = "any" - if cp.has_option_tags("workflow-segments", "segments-source", tags): + source = "any" + if cp.has_option_tags("workflow-segments", "segments-source", tags): source = cp.get_opt_tags("workflow-segments", "segments-source", tags) - if source == "file": - local_file_path = \ - resolve_url(cp.get_opt_tags("workflow-segments", - option_name+"-file", tags)) - pfn = os.path.join(out_dir, os.path.basename(local_file_path)) - shutil.move(local_file_path, pfn) - return SegFile.from_segment_xml(pfn) - - segs = {} - for ifo in workflow.ifos: - if cp.has_option_tags("workflow-segments", option_name, [ifo]): - flag_str = cp.get_opt_tags("workflow-segments", option_name, [ifo]) - flag_list = flag_str.split(',') - for flag in flag_list: - flag_name = flag[1:] - key = flag_name - if len(key.split(':')) > 2: - key = ':'.join(key.split(':')[:2]) - segs[key] = query_str(ifo, flag, start, end, - source=source, server=server, - veto_definer=veto_definer) - logging.info("%s: got %s segments", ifo, flag_name) - else: - logging.info("%s: no segments requested", ifo) - - return SegFile.from_segment_list_dict(name, segs, - extension='.xml', - valid_segment=workflow.analysis_time, - directory=out_dir) + if source == "file": + local_file_path = \ + resolve_url(cp.get_opt_tags("workflow-segments", + option_name+"-file", tags)) + pfn = os.path.join(out_dir, os.path.basename(local_file_path)) + shutil.move(local_file_path, pfn) + return SegFile.from_segment_xml(pfn) + + segs = {} + for ifo in workflow.ifos: + if cp.has_option_tags("workflow-segments", option_name, [ifo]): + flag_str = cp.get_opt_tags("workflow-segments", option_name, [ifo]) + flag_list = flag_str.split(',') + for flag in flag_list: + flag_name = flag[1:] + if len(flag_name.split(':')) > 1: + flag_name = name.split(':')[1] + key = ifo + ':' + flag_name + segs[key] = query_str(ifo, flag, start, end, + source=source, server=server, + veto_definer=veto_definer) + logging.info("%s: got %s segments", ifo, flag_name) + else: + logging.info("%s: no segments requested", ifo) + + return SegFile.from_segment_list_dict(name, segs, + extension='.xml', + valid_segment=workflow.analysis_time, + directory=out_dir) From 3cccb1d739b2fcdf194c9aec223ac3735e9cb2d8 Mon Sep 17 00:00:00 2001 From: maxtrevor Date: Tue, 29 Aug 2023 15:19:47 -0700 Subject: [PATCH 9/9] Fixed over-indent --- pycbc/workflow/segment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycbc/workflow/segment.py b/pycbc/workflow/segment.py index 35ed01e2aa7..e00fb75ba3d 100644 --- a/pycbc/workflow/segment.py +++ b/pycbc/workflow/segment.py @@ -434,7 +434,7 @@ def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None): source = "any" if cp.has_option_tags("workflow-segments", "segments-source", tags): - source = cp.get_opt_tags("workflow-segments", "segments-source", tags) + source = cp.get_opt_tags("workflow-segments", "segments-source", tags) if source == "file": local_file_path = \ resolve_url(cp.get_opt_tags("workflow-segments",