Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Segment tags #4468

Merged
merged 9 commits into from
Aug 30, 2023
Merged

Segment tags #4468

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d1f9a77
Added support for dq tags
maxtrevor Aug 28, 2023
ee0d860
Small fix
maxtrevor Aug 29, 2023
4c5ed69
Addressed some of Ian's comments
maxtrevor Aug 29, 2023
9143a91
Fixed indent mistake
maxtrevor Aug 29, 2023
94bfa47
Added description of tags argument to docstring
maxtrevor Aug 29, 2023
f567e8a
Small fix to tags
maxtrevor Aug 29, 2023
46bcb19
Don't use tags with veto-definer
maxtrevor Aug 29, 2023
d4e5167
Don't require dq segments to be set for each ifo separately
maxtrevor Aug 29, 2023
3cccb1d
Fixed over-indent
maxtrevor Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions bin/workflows/pycbc_make_offline_search_workflow
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,8 @@ save_fig_with_metadata(time_str, time_file.storage_path, **kwds)
# Get segments and find the data locations
sci_seg_name = 'science'
science_seg_file = wf.get_segments_file(workflow, sci_seg_name, 'segments-science',
rdir['analysis_time/segment_data'])
rdir['analysis_time/segment_data'],
tags=['science'])

ssegs = {}
for ifo in workflow.ifos:
Expand All @@ -145,13 +146,15 @@ datafind_files, analyzable_file, analyzable_segs, analyzable_name = \
final_veto_name = 'vetoes'
final_veto_file = wf.get_segments_file(workflow, final_veto_name,
'segments-vetoes',
rdir['analysis_time/segment_data'])
rdir['analysis_time/segment_data'],
tags=['veto'])

# Get dq segments from veto definer and calculate data quality timeseries
dq_flag_name = 'dq_flag'
dq_segment_file = wf.get_flag_segments_file(workflow, dq_flag_name,
'segments-dq',
rdir['analysis_time/segment_data'])
rdir['analysis_time/segment_data'],
tags=['dq'])

# Template bank stuff
hdfbank = wf.setup_tmpltbank_workflow(workflow, analyzable_segs,
Expand Down
51 changes: 34 additions & 17 deletions pycbc/workflow/segment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def save_veto_definer(cp, out_dir, tags=None):
return veto_def_new_path


def get_segments_file(workflow, name, option_name, out_dir):
def get_segments_file(workflow, name, option_name, out_dir, tags=None):
maxtrevor marked this conversation as resolved.
Show resolved Hide resolved
"""Get cumulative segments from option name syntax for each ifo.

Use syntax of configparser string to define the resulting segment_file
Expand All @@ -76,6 +76,9 @@ def get_segments_file(workflow, name, option_name, out_dir):
Name of the segment list being created
option_name: str
Name of option in the associated config parser to get the flag list
tags : list of strings
Used to retrieve subsections of the ini file for
configuration options.

returns
--------
Expand All @@ -88,25 +91,30 @@ def get_segments_file(workflow, name, option_name, out_dir):
start = workflow.analysis_time[0]
end = workflow.analysis_time[1]

if tags is None:
tags = []

# Check for veto definer file
veto_definer = None
if cp.has_option("workflow-segments", "segments-veto-definer-url"):
veto_definer = save_veto_definer(workflow.cp, out_dir, [])
if cp.has_option_tags("workflow-segments",
"segments-veto-definer-url", ['veto']):
veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto'])

# Check for provided server
server = "https://segments.ligo.org"
if cp.has_option("workflow-segments", "segments-database-url"):
server = cp.get("workflow-segments",
"segments-database-url")
if cp.has_option_tags("workflow-segments", "segments-database-url", tags):
server = cp.get_opt_tags("workflow-segments",
"segments-database-url", tags)

if cp.has_option("workflow-segments", "segments-source"):
source = cp.get("workflow-segments", "segments-source")
if cp.has_option_tags("workflow-segments", "segments-source", tags):
source = cp.get_opt_tags("workflow-segments", "segments-source", tags)
else:
source = "any"

if source == "file":
local_file_path = \
resolve_url(cp.get("workflow-segments", option_name+"-file"))
resolve_url(cp.get_opt_tag("workflow-segments",
option_name+"-file", tags))
pfn = os.path.join(out_dir, os.path.basename(local_file_path))
shutil.move(local_file_path, pfn)
return SegFile.from_segment_xml(pfn)
Expand Down Expand Up @@ -380,7 +388,7 @@ def generate_triggered_segment(workflow, out_dir, sciencesegs):
except UnboundLocalError:
return None, min_seg

def get_flag_segments_file(workflow, name, option_name, out_dir):
maxtrevor marked this conversation as resolved.
Show resolved Hide resolved
def get_flag_segments_file(workflow, name, option_name, out_dir, tags=None):
"""Get segments from option name syntax for each ifo for indivudal flags.

Use syntax of configparser string to define the resulting segment_file
Expand All @@ -396,6 +404,9 @@ def get_flag_segments_file(workflow, name, option_name, out_dir):
Name of the segment list being created
option_name: str
Name of option in the associated config parser to get the flag list
tags : list of strings
Used to retrieve subsections of the ini file for
configuration options.

returns
--------
Expand All @@ -408,22 +419,28 @@ def get_flag_segments_file(workflow, name, option_name, out_dir):
start = workflow.analysis_time[0]
end = workflow.analysis_time[1]

if tags is None:
tags = []

# Check for veto definer file
veto_definer = None
if cp.has_option("workflow-segments", "segments-veto-definer-url"):
veto_definer = save_veto_definer(workflow.cp, out_dir, [])
if cp.has_option_tags("workflow-segments",
"segments-veto-definer-url", ['veto']):
maxtrevor marked this conversation as resolved.
Show resolved Hide resolved
veto_definer = save_veto_definer(workflow.cp, out_dir, tags=['veto'])

# Check for provided server
server = "https://segments.ligo.org"
if cp.has_option("workflow-segments", "segments-database-url"):
server = cp.get("workflow-segments", "segments-database-url")
if cp.has_option_tags("workflow-segments", "segments-database-url", tags):
server = cp.get_opt_tags("workflow-segments",
"segments-database-url", tags)

source = "any"
if cp.has_option("workflow-segments", "segments-source"):
source = cp.get("workflow-segments", "segments-source")
if cp.has_option_tags("workflow-segments", "segments-source", tags):
source = cp.get_opt_tags("workflow-segments", "segments-source", tags)
if source == "file":
local_file_path = \
resolve_url(cp.get("workflow-segments", option_name+"-file"))
resolve_url(cp.get_opt_tags("workflow-segments",
option_name+"-file", tags))
pfn = os.path.join(out_dir, os.path.basename(local_file_path))
shutil.move(local_file_path, pfn)
return SegFile.from_segment_xml(pfn)
Expand Down
Loading