Skip to content

Commit

Permalink
Merge branch 'master' into feature.skip_mode
Browse files Browse the repository at this point in the history
  • Loading branch information
wxtim authored Nov 28, 2024
2 parents 9690389 + 354c7d5 commit 1426d3f
Show file tree
Hide file tree
Showing 21 changed files with 337 additions and 59 deletions.
1 change: 1 addition & 0 deletions .mailmap
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,4 @@ Utheri Wagura <[email protected]>
github-actions[bot] <[email protected]> <41898282+github-actions[bot]@users.noreply.github.com>
github-actions[bot] <[email protected]> GitHub Action
Diquan Jabbour <[email protected]>
Maxime Rio <[email protected]>
1 change: 1 addition & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ requests_).
- Diquan Jabbour
- Shixian Sheng
- Utheri Wagura
- Maxime Rio
- Paul Armstrong
- Paul Earnshaw
<!-- end-shortlog -->
Expand Down
2 changes: 2 additions & 0 deletions changes.d/6081.fix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix job submission when a batch of jobs is submitted to a runner that does
not return a newline with the job ID (did not affect built-in job runners).
1 change: 1 addition & 0 deletions changes.d/6456.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`cylc lint` now checks for unnecessary continuation characters in the graph section.
1 change: 1 addition & 0 deletions changes.d/6475.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Allow easy definition of multiple install targets in `global.cylc[install][symlink dirs]` using comma separated lists.
1 change: 1 addition & 0 deletions changes.d/6491.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The "cylc show" command now says if the target task is held, queued, or runahead limited.
73 changes: 64 additions & 9 deletions cylc/flow/cfgspec/globalcfg.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import os
from pathlib import Path
from sys import stderr
from textwrap import dedent
from textwrap import dedent, indent
from typing import List, Optional, Tuple, Any, Union

from contextlib import suppress
Expand Down Expand Up @@ -588,6 +588,47 @@
'''
}


def comma_sep_section_note(version_changed: str = '') -> str:
note_text = "This section can be a comma separated list."
if version_changed:
note_text = (
f".. versionchanged:: {version_changed}\n\n" +
indent(note_text, 3 * ' ')
)

example = dedent('''
.. spoiler:: Example
For example:
.. code-block:: cylc
[a, b]
setting = x
[a]
another_setting = y
Will become:
.. code-block:: cylc
[a]
setting = x
[b]
setting = x
[a]
another_setting = y
Which will then be combined according to
:ref:`the rules for Cylc config syntax<syntax>`.
''')

return "\n\n.. note::\n\n" + indent(note_text + example, 3 * ' ')


# ----------------------------------------------------------------------------


Expand Down Expand Up @@ -1132,9 +1173,12 @@ def default_for(
.. versionadded:: 8.0.0
"""):
with Conf('<install target>', desc="""
with Conf('<install target>', desc=dedent("""
:ref:`Host <Install targets>` on which to create the symlinks.
"""):
.. versionadded:: 8.0.0
""") + comma_sep_section_note(version_changed='8.4.0')):
Conf('run', VDR.V_STRING, None, desc="""
Alternative location for the run dir.
Expand Down Expand Up @@ -1176,7 +1220,9 @@ def default_for(
.. versionadded:: 8.0.0
'''):
with Conf('<platform name>', desc='''
with Conf(
'<platform name>',
desc=dedent('''
Configuration defining a platform.
Many of these settings have replaced those of the same name from
Expand Down Expand Up @@ -1220,8 +1266,10 @@ def default_for(
- :ref:`AdminGuide.PlatformConfigs`, an administrator's guide to
platform configurations.
.. versionadded:: 8.0.0
''') as Platform:
''')
+ comma_sep_section_note()
+ ".. versionadded:: 8.0.0",
) as Platform:
with Conf('meta', desc=PLATFORM_META_DESCR):
Conf('<custom metadata>', VDR.V_STRING, '', desc='''
Any user-defined metadata item.
Expand Down Expand Up @@ -1987,6 +2035,7 @@ def load(self) -> None:
self.dense.clear()
LOG.debug("Loading site/user config files")
conf_path_str = os.getenv("CYLC_CONF_PATH")

if conf_path_str:
# Explicit config file override.
fname = os.path.join(conf_path_str, self.CONF_BASENAME)
Expand All @@ -1999,7 +2048,7 @@ def load(self) -> None:

# Expand platforms needs to be performed first because it
# manipulates the sparse config.
self._expand_platforms()
self._expand_commas()

# Flesh out with defaults
self.expand()
Expand Down Expand Up @@ -2043,8 +2092,8 @@ def _no_platform_group_name_overlap(self):
msg += f'\n * {name}'
raise GlobalConfigError(msg)

def _expand_platforms(self):
"""Expand comma separated platform names.
def _expand_commas(self):
"""Expand comma separated section headers.
E.G. turn [platforms][foo, bar] into [platforms][foo] and
platforms[bar].
Expand All @@ -2053,6 +2102,12 @@ def _expand_platforms(self):
self.sparse['platforms'] = expand_many_section(
self.sparse['platforms']
)
if (
self.sparse.get("install", {}).get("symlink dirs")
):
self.sparse["install"]["symlink dirs"] = expand_many_section(
self.sparse["install"]["symlink dirs"]
)

def platform_dump(
self,
Expand Down
2 changes: 1 addition & 1 deletion cylc/flow/job_runner_handlers/background.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def submit(cls, job_file_path, submit_opts):
exc.filename = "nohup"
return (1, None, str(exc))
else:
return (0, "%d\n" % (proc.pid), None)
return (0, str(proc.pid), None)


JOB_RUNNER_HANDLER = BgCommandHandler()
3 changes: 1 addition & 2 deletions cylc/flow/job_runner_handlers/documentation.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,8 +424,7 @@ def submit(
ret_code:
Subprocess return code.
out:
Subprocess standard output, note this should be newline
terminated.
Subprocess standard output.
err:
Subprocess standard error.
Expand Down
22 changes: 10 additions & 12 deletions cylc/flow/job_runner_mgr.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,12 +210,10 @@ def jobs_kill(self, job_log_root, job_log_dirs):
self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code))
# Note: Print STDERR to STDOUT may look a bit strange, but it
# requires less logic for the workflow to parse the output.
if err.strip():
for line in err.splitlines(True):
if not line.endswith("\n"):
line += "\n"
sys.stdout.write("%s%s|%s|%s" % (
self.OUT_PREFIX_CMD_ERR, now, job_log_dir, line))
for line in err.strip().splitlines():
sys.stdout.write(
f"{self.OUT_PREFIX_CMD_ERR}{now}|{job_log_dir}|{line}\n"
)

def jobs_poll(self, job_log_root, job_log_dirs):
"""Poll multiple jobs.
Expand Down Expand Up @@ -303,13 +301,13 @@ def jobs_submit(self, job_log_root, job_log_dirs, remote_mode=False,
sys.stdout.write("%s%s|%s|%d|%s\n" % (
self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code, job_id))
for key, value in [("STDERR", err), ("STDOUT", out)]:
if value is None or not value.strip():
if value is None:
continue
for line in value.splitlines(True):
if not value.endswith("\n"):
value += "\n"
sys.stdout.write("%s%s|%s|[%s] %s" % (
self.OUT_PREFIX_COMMAND, now, job_log_dir, key, line))
for line in value.strip().splitlines():
sys.stdout.write(
f"{self.OUT_PREFIX_COMMAND}{now}"
f"|{job_log_dir}|[{key}] {line}\n"
)

def job_kill(self, st_file_path):
"""Ask job runner to terminate the job specified in "st_file_path".
Expand Down
3 changes: 2 additions & 1 deletion cylc/flow/scripts/cylc.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,8 @@ def get_version(long=False):
'gscan':
'cylc gscan has been removed, use the web UI',
'insert':
'inserting tasks is now done automatically',
'Insertion is no longer required, `cylc set` and `cylc trigger`'
' will insert tasks automatically.',
'jobscript':
'cylc jobscript has been removed',
'nudge':
Expand Down
14 changes: 11 additions & 3 deletions cylc/flow/scripts/lint.py
Original file line number Diff line number Diff line change
Expand Up @@ -653,12 +653,14 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
FUNCTION: check_wallclock_directives,
},
'S015': {
'short': 'Task outputs {outputs}: {description}.',
FUNCTION: check_skip_mode_outputs
'short': (
'``=>`` implies line continuation without ``\\``.'
),
FUNCTION: re.compile(r'=>\s*\\').findall
},
'S016': {
'short': 'Run mode is not live: This task will only appear to run.',
FUNCTION: re.compile(r'run mode\s*=\s*[^l][^i][^v][^e]$').findall
FUNCTION: check_skip_mode_outputs
},
}
# Subset of deprecations which are tricky (impossible?) to scrape from the
Expand Down Expand Up @@ -827,6 +829,12 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
FUNCTION: functools.partial(
list_wrapper, check=CHECK_FOR_OLD_VARS.findall),
},
'U017': {
'short': (
'``&`` and ``|`` imply line continuation without ``\\``'
),
FUNCTION: re.compile(r'[&|]\s*\\').findall
},
}
ALL_RULESETS = ['728', 'style', 'all']
EXTRA_TOML_VALIDATION = {
Expand Down
24 changes: 23 additions & 1 deletion cylc/flow/scripts/show.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@
name
cyclePoint
state
isHeld
isQueued
isRunahead
flowNums
task {
meta {
title
Expand Down Expand Up @@ -330,7 +334,25 @@ async def prereqs_and_outputs_query(
f'<bold>{key}:</bold>'
f' {value or "<m>(not given)</m>"}')

ansiprint(f'<bold>state:</bold> {state}')
# state and state attributes
attrs = []
if t_proxy['isHeld']:
attrs.append("held")
if t_proxy['isQueued']:
attrs.append("queued")
if t_proxy['isRunahead']:
attrs.append("runahead")
state_msg = state
if attrs:
state_msg += f" ({','.join(attrs)})"
ansiprint(f'<bold>state:</bold> {state_msg}')

# flow numbers, if not just 1
if t_proxy["flowNums"] != "[1]":
ansiprint(
f"<bold>flows:</bold> "
f"{t_proxy['flowNums'].replace(' ','')}"
)

# prerequisites
pre_txt = "<bold>prerequisites:</bold>"
Expand Down
29 changes: 14 additions & 15 deletions cylc/flow/scripts/workflow_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,13 @@
Check or poll a workflow database for task statuses or completed outputs.
The ID argument can target a workflow, or a cycle point, or a specific
task, with an optional selector on cycle or task to match task status,
output trigger (if not a status, or with --trigger) or output message
(with --message). All matching results will be printed.
The ID argument can target a workflow, or a cycle point, or a specific task,
with an optional selector on cycle or task to match final task statuses,
output trigger (with --triggers), or output message (with --messages).
You cannot poll for transient states such as "submitted" and "running";
poll for the corresponding output triggers instead ("submitted", "started").
All matching results will be printed.
If no results match, the command will repeatedly check (poll) until a match
is found or polling is exhausted (see --max-polls and --interval). For a
Expand All @@ -35,14 +38,12 @@
Legacy (pre-8.3.0) options are supported, but deprecated, for existing scripts:
cylc workflow-state --task=NAME --point=CYCLE --status=STATUS
--output=MESSAGE --message=MESSAGE --task-point WORKFLOW
(Note from 8.0 until 8.3.0 --output and --message both match task messages).
(Note from 8.0 until 8.3.0 --output and --message both matched task messages).
In "cycle/task:selector" the selector will match task statuses, unless:
- if it is not a known status, it will match task output triggers
(Cylc 8 DB) or task ouput messages (Cylc 7 DB)
- with --triggers, it will only match task output triggers
- with --messages (deprecated), it will only match task output messages.
Triggers are more robust - they match manually and naturally set outputs.
- with --triggers, it will only match task output triggers.
- with --messages, it will only match task output messages. It is recommended
to use triggers instead - they match both naturally & manually set outputs.
Selector does not default to "succeeded". If omitted, any status will match.
Expand All @@ -64,8 +65,6 @@
Warnings:
- Typos in the workflow or task ID will result in fruitless polling.
- To avoid missing transient states ("submitted", "running") poll for the
corresponding output trigger instead ("submitted", "started").
- Cycle points are auto-converted to the DB point format (and UTC mode).
- Task outputs manually completed by "cylc set" have "(force-completed)"
recorded as the task message in the DB, so it is best to query trigger
Expand Down Expand Up @@ -296,8 +295,8 @@ def get_option_parser() -> COP:

parser.add_option(
"--triggers",
help="Task selector should match output triggers rather than status."
" (Note this is not needed for custom outputs).",
help="Task selector should match output trigger names rather than "
"status.",
action="store_true", dest="is_trigger", default=False)

parser.add_option(
Expand Down Expand Up @@ -370,7 +369,7 @@ def main(parser: COP, options: 'Values', *ids: str) -> None:
[
options.depr_task,
options.depr_status,
options.depr_msg, # --message and --trigger
options.depr_msg,
options.depr_point,
options.depr_env_point
]
Expand Down
10 changes: 5 additions & 5 deletions cylc/flow/xtriggers/workflow_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ def workflow_state(
e.g. PT1H (1 hour) or P1 (1 integer cycle)
flow_num:
Flow number of the target task.
is_message:
Interpret the task:selector as a task output message
(the default is a task status or trigger)
is_trigger:
Interpret the task:selector as a task trigger name
(only needed if it is also a valid status name)
Interpret the task:selector as a task trigger name rather than a
task status.
is_message:
Interpret the task:selector as a task output message rather than a
task status.
alt_cylc_run_dir:
Alternate cylc-run directory, e.g. for another user.
Expand Down
4 changes: 4 additions & 0 deletions tests/flakyfunctional/cylc-show/00-simple.t
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,10 @@ cmp_json "${TEST_NAME}-taskinstance" "${TEST_NAME}-taskinstance" \
"id": "~${USER}/${WORKFLOW_NAME}//20141106T0900Z/foo",
"cyclePoint": "20141106T0900Z",
"state": "running",
"isHeld": false,
"isQueued": false,
"isRunahead": false,
"flowNums": "[1]",
"task": {
"meta": {
"title": "a task",
Expand Down
Loading

0 comments on commit 1426d3f

Please sign in to comment.