From d824324e60ab7092185eadb4ba42427babd647ba Mon Sep 17 00:00:00 2001 From: WXTIM <26465611+wxtim@users.noreply.github.com> Date: Thu, 25 Apr 2024 13:56:33 +0100 Subject: [PATCH] Implement Skip Mode * Add `[runtime][]run mode` and `[runtime][][skip]`. * Spin run mode functionality into separate modules. * Run sim mode check with every main loop - we don't know if any tasks are in sim mode from the scheduler, but it doesn't cost much to check if none are. * Implemented separate job "submission" pathway switching. * Implemented skip mode, including output control logic. * Add a linter and a validation check for tasks in nonlive modes, and for combinations of outputs * Enabled setting outputs as if task ran in skip mode using `cylc set --out skip`. * Testing for the above. --- changes.d/6039.feat.md | 1 + cylc/flow/cfgspec/workflow.py | 53 ++++- cylc/flow/commands.py | 6 +- cylc/flow/config.py | 10 +- cylc/flow/data_messages.proto | 1 + cylc/flow/data_messages_pb2.py | 100 +++++----- cylc/flow/data_messages_pb2.pyi | 68 +++---- cylc/flow/data_store_mgr.py | 2 + cylc/flow/network/schema.py | 2 + cylc/flow/prerequisite.py | 25 ++- cylc/flow/run_modes/dummy.py | 121 ++++++++++++ cylc/flow/run_modes/nonlive.py | 61 ++++++ cylc/flow/{ => run_modes}/simulation.py | 185 +++++++++++------- cylc/flow/run_modes/skip.py | 160 +++++++++++++++ cylc/flow/scheduler.py | 29 ++- cylc/flow/scheduler_cli.py | 12 +- cylc/flow/scripts/lint.py | 49 ++++- cylc/flow/scripts/validate.py | 2 +- cylc/flow/task_events_mgr.py | 16 +- cylc/flow/task_job_mgr.py | 110 +++++++---- cylc/flow/task_pool.py | 13 +- cylc/flow/task_proxy.py | 7 +- cylc/flow/task_state.py | 62 +++++- cylc/flow/unicode_rules.py | 4 +- cylc/flow/workflow_status.py | 4 + .../cylc-config/00-simple/section2.stdout | 52 +++++ tests/functional/cylc-set/09-set-skip.t | 28 +++ .../functional/cylc-set/09-set-skip/flow.cylc | 50 +++++ .../cylc-set/09-set-skip/reference.log | 8 + .../{modes => run_modes}/01-dummy.t | 0 .../{modes => run_modes}/01-dummy/flow.cylc | 0 .../01-dummy/reference.log | 0 .../02-dummy-message-outputs.t | 0 .../02-dummy-message-outputs/flow.cylc | 0 .../02-dummy-message-outputs/reference.log | 0 .../{modes => run_modes}/03-simulation.t | 0 .../03-simulation/flow.cylc | 0 .../03-simulation/reference.log | 0 .../04-simulation-runtime.t | 0 .../04-simulation-runtime/flow.cylc | 0 .../04-simulation-runtime/reference.log | 0 .../{modes => run_modes}/05-sim-trigger.t | 0 .../05-sim-trigger/flow.cylc | 0 .../05-sim-trigger/reference.log | 0 .../run_modes/06-run-mode-overrides.t | 72 +++++++ .../run_modes/06-run-mode-overrides/flow.cylc | 29 +++ .../{modes => run_modes}/test_header | 0 .../run_modes/test_mode_overrides.py | 109 +++++++++++ .../{ => run_modes}/test_simulation.py | 55 +++--- tests/integration/test_config.py | 45 +++++ tests/integration/test_task_pool.py | 16 +- tests/unit/run_modes/test_dummy.py | 40 ++++ tests/unit/run_modes/test_nonlive.py | 46 +++++ tests/unit/{ => run_modes}/test_simulation.py | 26 +-- tests/unit/run_modes/test_skip.py | 105 ++++++++++ tests/unit/scripts/test_lint.py | 3 + tests/unit/test_config.py | 36 ++-- tests/unit/test_task_state.py | 28 +++ 58 files changed, 1530 insertions(+), 321 deletions(-) create mode 100644 changes.d/6039.feat.md create mode 100644 cylc/flow/run_modes/dummy.py create mode 100644 cylc/flow/run_modes/nonlive.py rename cylc/flow/{ => run_modes}/simulation.py (66%) create mode 100644 cylc/flow/run_modes/skip.py create mode 100644 tests/functional/cylc-set/09-set-skip.t create mode 100644 tests/functional/cylc-set/09-set-skip/flow.cylc create mode 100644 tests/functional/cylc-set/09-set-skip/reference.log rename tests/functional/{modes => run_modes}/01-dummy.t (100%) rename tests/functional/{modes => run_modes}/01-dummy/flow.cylc (100%) rename tests/functional/{modes => run_modes}/01-dummy/reference.log (100%) rename tests/functional/{modes => run_modes}/02-dummy-message-outputs.t (100%) rename tests/functional/{modes => run_modes}/02-dummy-message-outputs/flow.cylc (100%) rename tests/functional/{modes => run_modes}/02-dummy-message-outputs/reference.log (100%) rename tests/functional/{modes => run_modes}/03-simulation.t (100%) rename tests/functional/{modes => run_modes}/03-simulation/flow.cylc (100%) rename tests/functional/{modes => run_modes}/03-simulation/reference.log (100%) rename tests/functional/{modes => run_modes}/04-simulation-runtime.t (100%) rename tests/functional/{modes => run_modes}/04-simulation-runtime/flow.cylc (100%) rename tests/functional/{modes => run_modes}/04-simulation-runtime/reference.log (100%) rename tests/functional/{modes => run_modes}/05-sim-trigger.t (100%) rename tests/functional/{modes => run_modes}/05-sim-trigger/flow.cylc (100%) rename tests/functional/{modes => run_modes}/05-sim-trigger/reference.log (100%) create mode 100644 tests/functional/run_modes/06-run-mode-overrides.t create mode 100644 tests/functional/run_modes/06-run-mode-overrides/flow.cylc rename tests/functional/{modes => run_modes}/test_header (100%) create mode 100644 tests/integration/run_modes/test_mode_overrides.py rename tests/integration/{ => run_modes}/test_simulation.py (90%) create mode 100644 tests/unit/run_modes/test_dummy.py create mode 100644 tests/unit/run_modes/test_nonlive.py rename tests/unit/{ => run_modes}/test_simulation.py (86%) create mode 100644 tests/unit/run_modes/test_skip.py diff --git a/changes.d/6039.feat.md b/changes.d/6039.feat.md new file mode 100644 index 00000000000..6b951fd7076 --- /dev/null +++ b/changes.d/6039.feat.md @@ -0,0 +1 @@ +Allow setting of run mode on a task by task basis. Add a new mode "skip". \ No newline at end of file diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index 934897bdbb4..74446620941 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -1334,6 +1334,22 @@ def get_script_common_text(this: str, example: Optional[str] = None): "[platforms][]submission retry delays" ) ) + Conf( + 'run mode', VDR.V_STRING, + options=['workflow', 'simulation', 'dummy', 'live', 'skip'], + default='workflow', + desc=''' + Override the workflow's run mode. + + By default workflows run in "live mode" - tasks run + in the way defined by the runtime config. + This setting allows individual tasks to be run using + a different run mode. + + .. TODO: Reference updated documention. + + .. versionadded:: 8.4.0 + ''') with Conf('meta', desc=r''' Metadata for the task or task family. @@ -1406,9 +1422,44 @@ def get_script_common_text(this: str, example: Optional[str] = None): determine how an event handler responds to task failure events. ''') + with Conf('skip', desc=''' + Task configuration for task :ref:`SkipMode`. + + For a full description of skip run mode see + :ref:`SkipMode`. + + .. versionadded:: 8.4.0 + '''): + Conf( + 'outputs', + VDR.V_STRING_LIST, + desc=''' + Outputs to be emitted by a task in skip mode. + + By default started, submitted, succeeded and all + required outputs will be emitted. + + If outputs are specified, but neither succeeded or + failed are specified, succeeded will automatically be + emitted. + .. versionadded:: 8.4.0 + ''' + ) + Conf( + 'disable task event handlers', + VDR.V_BOOLEAN, + default=True, + desc=''' + Task event handlers are turned off by default for + skip mode tasks. Changing this setting to ``False`` + will re-enable task event handlers. + + .. versionadded:: 8.4.0 + ''' + ) with Conf('simulation', desc=''' - Task configuration for workflow *simulation* and *dummy* run + Task configuration for *simulation* and *dummy* run modes. For a full description of simulation and dummy run modes see diff --git a/cylc/flow/commands.py b/cylc/flow/commands.py index 173984f17e0..e565737636d 100644 --- a/cylc/flow/commands.py +++ b/cylc/flow/commands.py @@ -77,14 +77,14 @@ from cylc.flow.network.schema import WorkflowStopMode from cylc.flow.parsec.exceptions import ParsecError from cylc.flow.task_id import TaskID -from cylc.flow.task_state import TASK_STATUSES_ACTIVE, TASK_STATUS_FAILED -from cylc.flow.workflow_status import RunMode, StopMode +from cylc.flow.task_state import ( + TASK_STATUSES_ACTIVE, TASK_STATUS_FAILED, RunMode) +from cylc.flow.workflow_status import StopMode from metomi.isodatetime.parsers import TimePointParser if TYPE_CHECKING: from cylc.flow.scheduler import Scheduler - # define a type for command implementations Command = Callable[ ..., diff --git a/cylc/flow/config.py b/cylc/flow/config.py index cb987ac6d4c..b7d199ed9c1 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -82,7 +82,7 @@ ) from cylc.flow.print_tree import print_tree from cylc.flow.task_qualifiers import ALT_QUALIFIERS -from cylc.flow.simulation import configure_sim_modes +from cylc.flow.run_modes.nonlive import mode_validate_checks from cylc.flow.subprocctx import SubFuncContext from cylc.flow.task_events_mgr import ( EventData, @@ -114,8 +114,8 @@ WorkflowFiles, check_deprecation, ) -from cylc.flow.workflow_status import RunMode from cylc.flow.xtrigger_mgr import XtriggerCollator +from cylc.flow.task_state import RunMode if TYPE_CHECKING: from optparse import Values @@ -513,10 +513,6 @@ def __init__( self.process_runahead_limit() - run_mode = self.run_mode() - if run_mode in {RunMode.SIMULATION, RunMode.DUMMY}: - configure_sim_modes(self.taskdefs.values(), run_mode) - self.configure_workflow_state_polling_tasks() self._check_task_event_handlers() @@ -567,6 +563,8 @@ def __init__( self.mem_log("config.py: end init config") + mode_validate_checks(self.taskdefs) + @staticmethod def _warn_if_queues_have_implicit_tasks( config, taskdefs, max_warning_lines diff --git a/cylc/flow/data_messages.proto b/cylc/flow/data_messages.proto index c0af5094c0d..f259a735f0a 100644 --- a/cylc/flow/data_messages.proto +++ b/cylc/flow/data_messages.proto @@ -128,6 +128,7 @@ message PbRuntime { optional string environment = 16; optional string outputs = 17; optional string completion = 18; + optional string run_mode = 19; } diff --git a/cylc/flow/data_messages_pb2.py b/cylc/flow/data_messages_pb2.py index 7fb5ae84d24..0f16888d6bd 100644 --- a/cylc/flow/data_messages_pb2.py +++ b/cylc/flow/data_messages_pb2.py @@ -14,7 +14,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18 \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\xe1\x06\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x12\x17\n\ncompletion\x18\x12 \x01(\tH\x11\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputsB\r\n\x0b_completion\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18 \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x64\x61ta_messages.proto\"\x96\x01\n\x06PbMeta\x12\x12\n\x05title\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03URL\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x19\n\x0cuser_defined\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_titleB\x0e\n\x0c_descriptionB\x06\n\x04_URLB\x0f\n\r_user_defined\"\xaa\x01\n\nPbTimeZone\x12\x12\n\x05hours\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07minutes\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x19\n\x0cstring_basic\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1c\n\x0fstring_extended\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x08\n\x06_hoursB\n\n\x08_minutesB\x0f\n\r_string_basicB\x12\n\x10_string_extended\"\'\n\x0fPbTaskProxyRefs\x12\x14\n\x0ctask_proxies\x18\x01 \x03(\t\"\xd4\x0c\n\nPbWorkflow\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06status\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x11\n\x04host\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\x04port\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x12\n\x05owner\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\r\n\x05tasks\x18\x08 \x03(\t\x12\x10\n\x08\x66\x61milies\x18\t \x03(\t\x12\x1c\n\x05\x65\x64ges\x18\n \x01(\x0b\x32\x08.PbEdgesH\x07\x88\x01\x01\x12\x18\n\x0b\x61pi_version\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x19\n\x0c\x63ylc_version\x18\x0c \x01(\tH\t\x88\x01\x01\x12\x19\n\x0clast_updated\x18\r \x01(\x01H\n\x88\x01\x01\x12\x1a\n\x04meta\x18\x0e \x01(\x0b\x32\x07.PbMetaH\x0b\x88\x01\x01\x12&\n\x19newest_active_cycle_point\x18\x10 \x01(\tH\x0c\x88\x01\x01\x12&\n\x19oldest_active_cycle_point\x18\x11 \x01(\tH\r\x88\x01\x01\x12\x15\n\x08reloaded\x18\x12 \x01(\x08H\x0e\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x0f\x88\x01\x01\x12\x19\n\x0c\x63ycling_mode\x18\x14 \x01(\tH\x10\x88\x01\x01\x12\x32\n\x0cstate_totals\x18\x15 \x03(\x0b\x32\x1c.PbWorkflow.StateTotalsEntry\x12\x1d\n\x10workflow_log_dir\x18\x16 \x01(\tH\x11\x88\x01\x01\x12(\n\x0etime_zone_info\x18\x17 \x01(\x0b\x32\x0b.PbTimeZoneH\x12\x88\x01\x01\x12\x17\n\ntree_depth\x18\x18 \x01(\x05H\x13\x88\x01\x01\x12\x15\n\rjob_log_names\x18\x19 \x03(\t\x12\x14\n\x0cns_def_order\x18\x1a \x03(\t\x12\x0e\n\x06states\x18\x1b \x03(\t\x12\x14\n\x0ctask_proxies\x18\x1c \x03(\t\x12\x16\n\x0e\x66\x61mily_proxies\x18\x1d \x03(\t\x12\x17\n\nstatus_msg\x18\x1e \x01(\tH\x14\x88\x01\x01\x12\x1a\n\ris_held_total\x18\x1f \x01(\x05H\x15\x88\x01\x01\x12\x0c\n\x04jobs\x18 \x03(\t\x12\x15\n\x08pub_port\x18! \x01(\x05H\x16\x88\x01\x01\x12\x17\n\nbroadcasts\x18\" \x01(\tH\x17\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18# \x01(\x05H\x18\x88\x01\x01\x12=\n\x12latest_state_tasks\x18$ \x03(\x0b\x32!.PbWorkflow.LatestStateTasksEntry\x12\x13\n\x06pruned\x18% \x01(\x08H\x19\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18& \x01(\x05H\x1a\x88\x01\x01\x12\x1b\n\x0estates_updated\x18\' \x01(\x08H\x1b\x88\x01\x01\x12\x1c\n\x0fn_edge_distance\x18( \x01(\x05H\x1c\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1aI\n\x15LatestStateTasksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1f\n\x05value\x18\x02 \x01(\x0b\x32\x10.PbTaskProxyRefs:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\t\n\x07_statusB\x07\n\x05_hostB\x07\n\x05_portB\x08\n\x06_ownerB\x08\n\x06_edgesB\x0e\n\x0c_api_versionB\x0f\n\r_cylc_versionB\x0f\n\r_last_updatedB\x07\n\x05_metaB\x1c\n\x1a_newest_active_cycle_pointB\x1c\n\x1a_oldest_active_cycle_pointB\x0b\n\t_reloadedB\x0b\n\t_run_modeB\x0f\n\r_cycling_modeB\x13\n\x11_workflow_log_dirB\x11\n\x0f_time_zone_infoB\r\n\x0b_tree_depthB\r\n\x0b_status_msgB\x10\n\x0e_is_held_totalB\x0b\n\t_pub_portB\r\n\x0b_broadcastsB\x12\n\x10_is_queued_totalB\t\n\x07_prunedB\x14\n\x12_is_runahead_totalB\x11\n\x0f_states_updatedB\x12\n\x10_n_edge_distance\"\x85\x07\n\tPbRuntime\x12\x15\n\x08platform\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06script\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0binit_script\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\nenv_script\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\nerr_script\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x18\n\x0b\x65xit_script\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\npre_script\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0bpost_script\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cwork_sub_dir\x18\t \x01(\tH\x08\x88\x01\x01\x12(\n\x1b\x65xecution_polling_intervals\x18\n \x01(\tH\t\x88\x01\x01\x12#\n\x16\x65xecution_retry_delays\x18\x0b \x01(\tH\n\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0c \x01(\tH\x0b\x88\x01\x01\x12)\n\x1csubmission_polling_intervals\x18\r \x01(\tH\x0c\x88\x01\x01\x12$\n\x17submission_retry_delays\x18\x0e \x01(\tH\r\x88\x01\x01\x12\x17\n\ndirectives\x18\x0f \x01(\tH\x0e\x88\x01\x01\x12\x18\n\x0b\x65nvironment\x18\x10 \x01(\tH\x0f\x88\x01\x01\x12\x14\n\x07outputs\x18\x11 \x01(\tH\x10\x88\x01\x01\x12\x17\n\ncompletion\x18\x12 \x01(\tH\x11\x88\x01\x01\x12\x15\n\x08run_mode\x18\x13 \x01(\tH\x12\x88\x01\x01\x42\x0b\n\t_platformB\t\n\x07_scriptB\x0e\n\x0c_init_scriptB\r\n\x0b_env_scriptB\r\n\x0b_err_scriptB\x0e\n\x0c_exit_scriptB\r\n\x0b_pre_scriptB\x0e\n\x0c_post_scriptB\x0f\n\r_work_sub_dirB\x1e\n\x1c_execution_polling_intervalsB\x19\n\x17_execution_retry_delaysB\x17\n\x15_execution_time_limitB\x1f\n\x1d_submission_polling_intervalsB\x1a\n\x18_submission_retry_delaysB\r\n\x0b_directivesB\x0e\n\x0c_environmentB\n\n\x08_outputsB\r\n\x0b_completionB\x0b\n\t_run_mode\"\x9d\x05\n\x05PbJob\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nsubmit_num\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x17\n\ntask_proxy\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1b\n\x0esubmitted_time\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x19\n\x0cstarted_time\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x1a\n\rfinished_time\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x06job_id\x18\t \x01(\tH\x08\x88\x01\x01\x12\x1c\n\x0fjob_runner_name\x18\n \x01(\tH\t\x88\x01\x01\x12!\n\x14\x65xecution_time_limit\x18\x0e \x01(\x02H\n\x88\x01\x01\x12\x15\n\x08platform\x18\x0f \x01(\tH\x0b\x88\x01\x01\x12\x18\n\x0bjob_log_dir\x18\x11 \x01(\tH\x0c\x88\x01\x01\x12\x11\n\x04name\x18\x1e \x01(\tH\r\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x1f \x01(\tH\x0e\x88\x01\x01\x12\x10\n\x08messages\x18 \x03(\t\x12 \n\x07runtime\x18! \x01(\x0b\x32\n.PbRuntimeH\x0f\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\r\n\x0b_submit_numB\x08\n\x06_stateB\r\n\x0b_task_proxyB\x11\n\x0f_submitted_timeB\x0f\n\r_started_timeB\x10\n\x0e_finished_timeB\t\n\x07_job_idB\x12\n\x10_job_runner_nameB\x17\n\x15_execution_time_limitB\x0b\n\t_platformB\x0e\n\x0c_job_log_dirB\x07\n\x05_nameB\x0e\n\x0c_cycle_pointB\n\n\x08_runtimeJ\x04\x08\x1d\x10\x1e\"\xe2\x02\n\x06PbTask\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x1e\n\x11mean_elapsed_time\x18\x05 \x01(\x02H\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x0f\n\x07proxies\x18\x07 \x03(\t\x12\x11\n\tnamespace\x18\x08 \x03(\t\x12\x0f\n\x07parents\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x06\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x07\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x14\n\x12_mean_elapsed_timeB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xd8\x01\n\nPbPollTask\x12\x18\n\x0blocal_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08workflow\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0cremote_proxy\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\treq_state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x19\n\x0cgraph_string\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0c_local_proxyB\x0b\n\t_workflowB\x0f\n\r_remote_proxyB\x0c\n\n_req_stateB\x0f\n\r_graph_string\"\xcb\x01\n\x0bPbCondition\x12\x17\n\ntask_proxy\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nexpr_alias\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\treq_state\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x14\n\x07message\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\r\n\x0b_task_proxyB\r\n\x0b_expr_aliasB\x0c\n\n_req_stateB\x0c\n\n_satisfiedB\n\n\x08_message\"\x96\x01\n\x0ePbPrerequisite\x12\x17\n\nexpression\x18\x01 \x01(\tH\x00\x88\x01\x01\x12 \n\nconditions\x18\x02 \x03(\x0b\x32\x0c.PbCondition\x12\x14\n\x0c\x63ycle_points\x18\x03 \x03(\t\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\r\n\x0b_expressionB\x0c\n\n_satisfied\"\x8c\x01\n\x08PbOutput\x12\x12\n\x05label\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07message\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tsatisfied\x18\x03 \x01(\x08H\x02\x88\x01\x01\x12\x11\n\x04time\x18\x04 \x01(\x01H\x03\x88\x01\x01\x42\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\xa5\x01\n\tPbTrigger\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07message\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tsatisfied\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04time\x18\x05 \x01(\x01H\x04\x88\x01\x01\x42\x05\n\x03_idB\x08\n\x06_labelB\n\n\x08_messageB\x0c\n\n_satisfiedB\x07\n\x05_time\"\x91\x08\n\x0bPbTaskProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04task\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x12\n\x05state\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bjob_submits\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12*\n\x07outputs\x18\t \x03(\x0b\x32\x19.PbTaskProxy.OutputsEntry\x12\x11\n\tnamespace\x18\x0b \x03(\t\x12&\n\rprerequisites\x18\x0c \x03(\x0b\x32\x0f.PbPrerequisite\x12\x0c\n\x04jobs\x18\r \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\x0f \x01(\tH\x07\x88\x01\x01\x12\x11\n\x04name\x18\x10 \x01(\tH\x08\x88\x01\x01\x12\x14\n\x07is_held\x18\x11 \x01(\x08H\t\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x12 \x03(\t\x12\x11\n\tancestors\x18\x13 \x03(\t\x12\x16\n\tflow_nums\x18\x14 \x01(\tH\n\x88\x01\x01\x12=\n\x11\x65xternal_triggers\x18\x17 \x03(\x0b\x32\".PbTaskProxy.ExternalTriggersEntry\x12.\n\txtriggers\x18\x18 \x03(\x0b\x32\x1b.PbTaskProxy.XtriggersEntry\x12\x16\n\tis_queued\x18\x19 \x01(\x08H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x1a \x01(\x08H\x0c\x88\x01\x01\x12\x16\n\tflow_wait\x18\x1b \x01(\x08H\r\x88\x01\x01\x12 \n\x07runtime\x18\x1c \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x1d \x01(\x05H\x0f\x88\x01\x01\x1a\x39\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x18\n\x05value\x18\x02 \x01(\x0b\x32\t.PbOutput:\x02\x38\x01\x1a\x43\n\x15\x45xternalTriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x1a<\n\x0eXtriggersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x19\n\x05value\x18\x02 \x01(\x0b\x32\n.PbTrigger:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_taskB\x08\n\x06_stateB\x0e\n\x0c_cycle_pointB\x08\n\x06_depthB\x0e\n\x0c_job_submitsB\x0f\n\r_first_parentB\x07\n\x05_nameB\n\n\x08_is_heldB\x0c\n\n_flow_numsB\x0c\n\n_is_queuedB\x0e\n\x0c_is_runaheadB\x0c\n\n_flow_waitB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xc8\x02\n\x08PbFamily\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\x04meta\x18\x04 \x01(\x0b\x32\x07.PbMetaH\x03\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0f\n\x07proxies\x18\x06 \x03(\t\x12\x0f\n\x07parents\x18\x07 \x03(\t\x12\x13\n\x0b\x63hild_tasks\x18\x08 \x03(\t\x12\x16\n\x0e\x63hild_families\x18\t \x03(\t\x12\x19\n\x0c\x66irst_parent\x18\n \x01(\tH\x05\x88\x01\x01\x12 \n\x07runtime\x18\x0b \x01(\x0b\x32\n.PbRuntimeH\x06\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x07\n\x05_nameB\x07\n\x05_metaB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_runtime\"\xae\x06\n\rPbFamilyProxy\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0b\x63ycle_point\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x11\n\x04name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06\x66\x61mily\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x12\n\x05state\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05\x64\x65pth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x19\n\x0c\x66irst_parent\x18\x08 \x01(\tH\x07\x88\x01\x01\x12\x13\n\x0b\x63hild_tasks\x18\n \x03(\t\x12\x16\n\x0e\x63hild_families\x18\x0b \x03(\t\x12\x14\n\x07is_held\x18\x0c \x01(\x08H\x08\x88\x01\x01\x12\x11\n\tancestors\x18\r \x03(\t\x12\x0e\n\x06states\x18\x0e \x03(\t\x12\x35\n\x0cstate_totals\x18\x0f \x03(\x0b\x32\x1f.PbFamilyProxy.StateTotalsEntry\x12\x1a\n\ris_held_total\x18\x10 \x01(\x05H\t\x88\x01\x01\x12\x16\n\tis_queued\x18\x11 \x01(\x08H\n\x88\x01\x01\x12\x1c\n\x0fis_queued_total\x18\x12 \x01(\x05H\x0b\x88\x01\x01\x12\x18\n\x0bis_runahead\x18\x13 \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11is_runahead_total\x18\x14 \x01(\x05H\r\x88\x01\x01\x12 \n\x07runtime\x18\x15 \x01(\x0b\x32\n.PbRuntimeH\x0e\x88\x01\x01\x12\x18\n\x0bgraph_depth\x18\x16 \x01(\x05H\x0f\x88\x01\x01\x1a\x32\n\x10StateTotalsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\x0e\n\x0c_cycle_pointB\x07\n\x05_nameB\t\n\x07_familyB\x08\n\x06_stateB\x08\n\x06_depthB\x0f\n\r_first_parentB\n\n\x08_is_heldB\x10\n\x0e_is_held_totalB\x0c\n\n_is_queuedB\x12\n\x10_is_queued_totalB\x0e\n\x0c_is_runaheadB\x14\n\x12_is_runahead_totalB\n\n\x08_runtimeB\x0e\n\x0c_graph_depth\"\xbc\x01\n\x06PbEdge\x12\x12\n\x05stamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x02id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x13\n\x06source\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x13\n\x06target\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x14\n\x07suicide\x18\x05 \x01(\x08H\x04\x88\x01\x01\x12\x11\n\x04\x63ond\x18\x06 \x01(\x08H\x05\x88\x01\x01\x42\x08\n\x06_stampB\x05\n\x03_idB\t\n\x07_sourceB\t\n\x07_targetB\n\n\x08_suicideB\x07\n\x05_cond\"{\n\x07PbEdges\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\r\n\x05\x65\x64ges\x18\x02 \x03(\t\x12+\n\x16workflow_polling_tasks\x18\x03 \x03(\x0b\x32\x0b.PbPollTask\x12\x0e\n\x06leaves\x18\x04 \x03(\t\x12\x0c\n\x04\x66\x65\x65t\x18\x05 \x03(\tB\x05\n\x03_id\"\xf2\x01\n\x10PbEntireWorkflow\x12\"\n\x08workflow\x18\x01 \x01(\x0b\x32\x0b.PbWorkflowH\x00\x88\x01\x01\x12\x16\n\x05tasks\x18\x02 \x03(\x0b\x32\x07.PbTask\x12\"\n\x0ctask_proxies\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x14\n\x04jobs\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x1b\n\x08\x66\x61milies\x18\x05 \x03(\x0b\x32\t.PbFamily\x12&\n\x0e\x66\x61mily_proxies\x18\x06 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x16\n\x05\x65\x64ges\x18\x07 \x03(\x0b\x32\x07.PbEdgeB\x0b\n\t_workflow\"\xaf\x01\n\x07\x45\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbEdge\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbEdge\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xb3\x01\n\x07\x46\x44\x65ltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x18\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\t.PbFamily\x12\x1a\n\x07updated\x18\x04 \x03(\x0b\x32\t.PbFamily\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xbe\x01\n\x08\x46PDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1d\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x1f\n\x07updated\x18\x04 \x03(\x0b\x32\x0e.PbFamilyProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xad\x01\n\x07JDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x15\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x06.PbJob\x12\x17\n\x07updated\x18\x04 \x03(\x0b\x32\x06.PbJob\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xaf\x01\n\x07TDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x16\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x07.PbTask\x12\x18\n\x07updated\x18\x04 \x03(\x0b\x32\x07.PbTask\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xba\x01\n\x08TPDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\x08\x63hecksum\x18\x02 \x01(\x03H\x01\x88\x01\x01\x12\x1b\n\x05\x61\x64\x64\x65\x64\x18\x03 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x1d\n\x07updated\x18\x04 \x03(\x0b\x32\x0c.PbTaskProxy\x12\x0e\n\x06pruned\x18\x05 \x03(\t\x12\x15\n\x08reloaded\x18\x06 \x01(\x08H\x02\x88\x01\x01\x42\x07\n\x05_timeB\x0b\n\t_checksumB\x0b\n\t_reloaded\"\xc3\x01\n\x07WDeltas\x12\x11\n\x04time\x18\x01 \x01(\x01H\x00\x88\x01\x01\x12\x1f\n\x05\x61\x64\x64\x65\x64\x18\x02 \x01(\x0b\x32\x0b.PbWorkflowH\x01\x88\x01\x01\x12!\n\x07updated\x18\x03 \x01(\x0b\x32\x0b.PbWorkflowH\x02\x88\x01\x01\x12\x15\n\x08reloaded\x18\x04 \x01(\x08H\x03\x88\x01\x01\x12\x13\n\x06pruned\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x07\n\x05_timeB\x08\n\x06_addedB\n\n\x08_updatedB\x0b\n\t_reloadedB\t\n\x07_pruned\"\xd1\x01\n\tAllDeltas\x12\x1a\n\x08\x66\x61milies\x18\x01 \x01(\x0b\x32\x08.FDeltas\x12!\n\x0e\x66\x61mily_proxies\x18\x02 \x01(\x0b\x32\t.FPDeltas\x12\x16\n\x04jobs\x18\x03 \x01(\x0b\x32\x08.JDeltas\x12\x17\n\x05tasks\x18\x04 \x01(\x0b\x32\x08.TDeltas\x12\x1f\n\x0ctask_proxies\x18\x05 \x01(\x0b\x32\t.TPDeltas\x12\x17\n\x05\x65\x64ges\x18\x06 \x01(\x0b\x32\x08.EDeltas\x12\x1a\n\x08workflow\x18\x07 \x01(\x0b\x32\x08.WDeltasb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -46,55 +46,55 @@ _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_start=1493 _globals['_PBWORKFLOW_LATESTSTATETASKSENTRY']._serialized_end=1566 _globals['_PBRUNTIME']._serialized_start=2014 - _globals['_PBRUNTIME']._serialized_end=2879 - _globals['_PBJOB']._serialized_start=2882 - _globals['_PBJOB']._serialized_end=3551 - _globals['_PBTASK']._serialized_start=3554 - _globals['_PBTASK']._serialized_end=3908 - _globals['_PBPOLLTASK']._serialized_start=3911 - _globals['_PBPOLLTASK']._serialized_end=4127 - _globals['_PBCONDITION']._serialized_start=4130 - _globals['_PBCONDITION']._serialized_end=4333 - _globals['_PBPREREQUISITE']._serialized_start=4336 - _globals['_PBPREREQUISITE']._serialized_end=4486 - _globals['_PBOUTPUT']._serialized_start=4489 - _globals['_PBOUTPUT']._serialized_end=4629 - _globals['_PBTRIGGER']._serialized_start=4632 - _globals['_PBTRIGGER']._serialized_end=4797 - _globals['_PBTASKPROXY']._serialized_start=4800 - _globals['_PBTASKPROXY']._serialized_end=5841 - _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5451 - _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5508 - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5510 - _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5577 - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5579 - _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5639 - _globals['_PBFAMILY']._serialized_start=5844 - _globals['_PBFAMILY']._serialized_end=6172 - _globals['_PBFAMILYPROXY']._serialized_start=6175 - _globals['_PBFAMILYPROXY']._serialized_end=6989 + _globals['_PBRUNTIME']._serialized_end=2915 + _globals['_PBJOB']._serialized_start=2918 + _globals['_PBJOB']._serialized_end=3587 + _globals['_PBTASK']._serialized_start=3590 + _globals['_PBTASK']._serialized_end=3944 + _globals['_PBPOLLTASK']._serialized_start=3947 + _globals['_PBPOLLTASK']._serialized_end=4163 + _globals['_PBCONDITION']._serialized_start=4166 + _globals['_PBCONDITION']._serialized_end=4369 + _globals['_PBPREREQUISITE']._serialized_start=4372 + _globals['_PBPREREQUISITE']._serialized_end=4522 + _globals['_PBOUTPUT']._serialized_start=4525 + _globals['_PBOUTPUT']._serialized_end=4665 + _globals['_PBTRIGGER']._serialized_start=4668 + _globals['_PBTRIGGER']._serialized_end=4833 + _globals['_PBTASKPROXY']._serialized_start=4836 + _globals['_PBTASKPROXY']._serialized_end=5877 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_start=5487 + _globals['_PBTASKPROXY_OUTPUTSENTRY']._serialized_end=5544 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_start=5546 + _globals['_PBTASKPROXY_EXTERNALTRIGGERSENTRY']._serialized_end=5613 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_start=5615 + _globals['_PBTASKPROXY_XTRIGGERSENTRY']._serialized_end=5675 + _globals['_PBFAMILY']._serialized_start=5880 + _globals['_PBFAMILY']._serialized_end=6208 + _globals['_PBFAMILYPROXY']._serialized_start=6211 + _globals['_PBFAMILYPROXY']._serialized_end=7025 _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_start=1441 _globals['_PBFAMILYPROXY_STATETOTALSENTRY']._serialized_end=1491 - _globals['_PBEDGE']._serialized_start=6992 - _globals['_PBEDGE']._serialized_end=7180 - _globals['_PBEDGES']._serialized_start=7182 - _globals['_PBEDGES']._serialized_end=7305 - _globals['_PBENTIREWORKFLOW']._serialized_start=7308 - _globals['_PBENTIREWORKFLOW']._serialized_end=7550 - _globals['_EDELTAS']._serialized_start=7553 - _globals['_EDELTAS']._serialized_end=7728 - _globals['_FDELTAS']._serialized_start=7731 - _globals['_FDELTAS']._serialized_end=7910 - _globals['_FPDELTAS']._serialized_start=7913 - _globals['_FPDELTAS']._serialized_end=8103 - _globals['_JDELTAS']._serialized_start=8106 - _globals['_JDELTAS']._serialized_end=8279 - _globals['_TDELTAS']._serialized_start=8282 - _globals['_TDELTAS']._serialized_end=8457 - _globals['_TPDELTAS']._serialized_start=8460 - _globals['_TPDELTAS']._serialized_end=8646 - _globals['_WDELTAS']._serialized_start=8649 - _globals['_WDELTAS']._serialized_end=8844 - _globals['_ALLDELTAS']._serialized_start=8847 - _globals['_ALLDELTAS']._serialized_end=9056 + _globals['_PBEDGE']._serialized_start=7028 + _globals['_PBEDGE']._serialized_end=7216 + _globals['_PBEDGES']._serialized_start=7218 + _globals['_PBEDGES']._serialized_end=7341 + _globals['_PBENTIREWORKFLOW']._serialized_start=7344 + _globals['_PBENTIREWORKFLOW']._serialized_end=7586 + _globals['_EDELTAS']._serialized_start=7589 + _globals['_EDELTAS']._serialized_end=7764 + _globals['_FDELTAS']._serialized_start=7767 + _globals['_FDELTAS']._serialized_end=7946 + _globals['_FPDELTAS']._serialized_start=7949 + _globals['_FPDELTAS']._serialized_end=8139 + _globals['_JDELTAS']._serialized_start=8142 + _globals['_JDELTAS']._serialized_end=8315 + _globals['_TDELTAS']._serialized_start=8318 + _globals['_TDELTAS']._serialized_end=8493 + _globals['_TPDELTAS']._serialized_start=8496 + _globals['_TPDELTAS']._serialized_end=8682 + _globals['_WDELTAS']._serialized_start=8685 + _globals['_WDELTAS']._serialized_end=8880 + _globals['_ALLDELTAS']._serialized_start=8883 + _globals['_ALLDELTAS']._serialized_end=9092 # @@protoc_insertion_point(module_scope) diff --git a/cylc/flow/data_messages_pb2.pyi b/cylc/flow/data_messages_pb2.pyi index 4e96c6ed2da..8c80f7f8f10 100644 --- a/cylc/flow/data_messages_pb2.pyi +++ b/cylc/flow/data_messages_pb2.pyi @@ -6,7 +6,7 @@ from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Map DESCRIPTOR: _descriptor.FileDescriptor class PbMeta(_message.Message): - __slots__ = ["title", "description", "URL", "user_defined"] + __slots__ = ("title", "description", "URL", "user_defined") TITLE_FIELD_NUMBER: _ClassVar[int] DESCRIPTION_FIELD_NUMBER: _ClassVar[int] URL_FIELD_NUMBER: _ClassVar[int] @@ -18,7 +18,7 @@ class PbMeta(_message.Message): def __init__(self, title: _Optional[str] = ..., description: _Optional[str] = ..., URL: _Optional[str] = ..., user_defined: _Optional[str] = ...) -> None: ... class PbTimeZone(_message.Message): - __slots__ = ["hours", "minutes", "string_basic", "string_extended"] + __slots__ = ("hours", "minutes", "string_basic", "string_extended") HOURS_FIELD_NUMBER: _ClassVar[int] MINUTES_FIELD_NUMBER: _ClassVar[int] STRING_BASIC_FIELD_NUMBER: _ClassVar[int] @@ -30,22 +30,22 @@ class PbTimeZone(_message.Message): def __init__(self, hours: _Optional[int] = ..., minutes: _Optional[int] = ..., string_basic: _Optional[str] = ..., string_extended: _Optional[str] = ...) -> None: ... class PbTaskProxyRefs(_message.Message): - __slots__ = ["task_proxies"] + __slots__ = ("task_proxies",) TASK_PROXIES_FIELD_NUMBER: _ClassVar[int] task_proxies: _containers.RepeatedScalarFieldContainer[str] def __init__(self, task_proxies: _Optional[_Iterable[str]] = ...) -> None: ... class PbWorkflow(_message.Message): - __slots__ = ["stamp", "id", "name", "status", "host", "port", "owner", "tasks", "families", "edges", "api_version", "cylc_version", "last_updated", "meta", "newest_active_cycle_point", "oldest_active_cycle_point", "reloaded", "run_mode", "cycling_mode", "state_totals", "workflow_log_dir", "time_zone_info", "tree_depth", "job_log_names", "ns_def_order", "states", "task_proxies", "family_proxies", "status_msg", "is_held_total", "jobs", "pub_port", "broadcasts", "is_queued_total", "latest_state_tasks", "pruned", "is_runahead_total", "states_updated", "n_edge_distance"] + __slots__ = ("stamp", "id", "name", "status", "host", "port", "owner", "tasks", "families", "edges", "api_version", "cylc_version", "last_updated", "meta", "newest_active_cycle_point", "oldest_active_cycle_point", "reloaded", "run_mode", "cycling_mode", "state_totals", "workflow_log_dir", "time_zone_info", "tree_depth", "job_log_names", "ns_def_order", "states", "task_proxies", "family_proxies", "status_msg", "is_held_total", "jobs", "pub_port", "broadcasts", "is_queued_total", "latest_state_tasks", "pruned", "is_runahead_total", "states_updated", "n_edge_distance") class StateTotalsEntry(_message.Message): - __slots__ = ["key", "value"] + __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str value: int def __init__(self, key: _Optional[str] = ..., value: _Optional[int] = ...) -> None: ... class LatestStateTasksEntry(_message.Message): - __slots__ = ["key", "value"] + __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str @@ -132,7 +132,7 @@ class PbWorkflow(_message.Message): def __init__(self, stamp: _Optional[str] = ..., id: _Optional[str] = ..., name: _Optional[str] = ..., status: _Optional[str] = ..., host: _Optional[str] = ..., port: _Optional[int] = ..., owner: _Optional[str] = ..., tasks: _Optional[_Iterable[str]] = ..., families: _Optional[_Iterable[str]] = ..., edges: _Optional[_Union[PbEdges, _Mapping]] = ..., api_version: _Optional[int] = ..., cylc_version: _Optional[str] = ..., last_updated: _Optional[float] = ..., meta: _Optional[_Union[PbMeta, _Mapping]] = ..., newest_active_cycle_point: _Optional[str] = ..., oldest_active_cycle_point: _Optional[str] = ..., reloaded: bool = ..., run_mode: _Optional[str] = ..., cycling_mode: _Optional[str] = ..., state_totals: _Optional[_Mapping[str, int]] = ..., workflow_log_dir: _Optional[str] = ..., time_zone_info: _Optional[_Union[PbTimeZone, _Mapping]] = ..., tree_depth: _Optional[int] = ..., job_log_names: _Optional[_Iterable[str]] = ..., ns_def_order: _Optional[_Iterable[str]] = ..., states: _Optional[_Iterable[str]] = ..., task_proxies: _Optional[_Iterable[str]] = ..., family_proxies: _Optional[_Iterable[str]] = ..., status_msg: _Optional[str] = ..., is_held_total: _Optional[int] = ..., jobs: _Optional[_Iterable[str]] = ..., pub_port: _Optional[int] = ..., broadcasts: _Optional[str] = ..., is_queued_total: _Optional[int] = ..., latest_state_tasks: _Optional[_Mapping[str, PbTaskProxyRefs]] = ..., pruned: bool = ..., is_runahead_total: _Optional[int] = ..., states_updated: bool = ..., n_edge_distance: _Optional[int] = ...) -> None: ... class PbRuntime(_message.Message): - __slots__ = ["platform", "script", "init_script", "env_script", "err_script", "exit_script", "pre_script", "post_script", "work_sub_dir", "execution_polling_intervals", "execution_retry_delays", "execution_time_limit", "submission_polling_intervals", "submission_retry_delays", "directives", "environment", "outputs", "completion"] + __slots__ = ("platform", "script", "init_script", "env_script", "err_script", "exit_script", "pre_script", "post_script", "work_sub_dir", "execution_polling_intervals", "execution_retry_delays", "execution_time_limit", "submission_polling_intervals", "submission_retry_delays", "directives", "environment", "outputs", "completion", "run_mode") PLATFORM_FIELD_NUMBER: _ClassVar[int] SCRIPT_FIELD_NUMBER: _ClassVar[int] INIT_SCRIPT_FIELD_NUMBER: _ClassVar[int] @@ -151,6 +151,7 @@ class PbRuntime(_message.Message): ENVIRONMENT_FIELD_NUMBER: _ClassVar[int] OUTPUTS_FIELD_NUMBER: _ClassVar[int] COMPLETION_FIELD_NUMBER: _ClassVar[int] + RUN_MODE_FIELD_NUMBER: _ClassVar[int] platform: str script: str init_script: str @@ -169,10 +170,11 @@ class PbRuntime(_message.Message): environment: str outputs: str completion: str - def __init__(self, platform: _Optional[str] = ..., script: _Optional[str] = ..., init_script: _Optional[str] = ..., env_script: _Optional[str] = ..., err_script: _Optional[str] = ..., exit_script: _Optional[str] = ..., pre_script: _Optional[str] = ..., post_script: _Optional[str] = ..., work_sub_dir: _Optional[str] = ..., execution_polling_intervals: _Optional[str] = ..., execution_retry_delays: _Optional[str] = ..., execution_time_limit: _Optional[str] = ..., submission_polling_intervals: _Optional[str] = ..., submission_retry_delays: _Optional[str] = ..., directives: _Optional[str] = ..., environment: _Optional[str] = ..., outputs: _Optional[str] = ..., completion: _Optional[str] = ...) -> None: ... + run_mode: str + def __init__(self, platform: _Optional[str] = ..., script: _Optional[str] = ..., init_script: _Optional[str] = ..., env_script: _Optional[str] = ..., err_script: _Optional[str] = ..., exit_script: _Optional[str] = ..., pre_script: _Optional[str] = ..., post_script: _Optional[str] = ..., work_sub_dir: _Optional[str] = ..., execution_polling_intervals: _Optional[str] = ..., execution_retry_delays: _Optional[str] = ..., execution_time_limit: _Optional[str] = ..., submission_polling_intervals: _Optional[str] = ..., submission_retry_delays: _Optional[str] = ..., directives: _Optional[str] = ..., environment: _Optional[str] = ..., outputs: _Optional[str] = ..., completion: _Optional[str] = ..., run_mode: _Optional[str] = ...) -> None: ... class PbJob(_message.Message): - __slots__ = ["stamp", "id", "submit_num", "state", "task_proxy", "submitted_time", "started_time", "finished_time", "job_id", "job_runner_name", "execution_time_limit", "platform", "job_log_dir", "name", "cycle_point", "messages", "runtime"] + __slots__ = ("stamp", "id", "submit_num", "state", "task_proxy", "submitted_time", "started_time", "finished_time", "job_id", "job_runner_name", "execution_time_limit", "platform", "job_log_dir", "name", "cycle_point", "messages", "runtime") STAMP_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] SUBMIT_NUM_FIELD_NUMBER: _ClassVar[int] @@ -210,7 +212,7 @@ class PbJob(_message.Message): def __init__(self, stamp: _Optional[str] = ..., id: _Optional[str] = ..., submit_num: _Optional[int] = ..., state: _Optional[str] = ..., task_proxy: _Optional[str] = ..., submitted_time: _Optional[str] = ..., started_time: _Optional[str] = ..., finished_time: _Optional[str] = ..., job_id: _Optional[str] = ..., job_runner_name: _Optional[str] = ..., execution_time_limit: _Optional[float] = ..., platform: _Optional[str] = ..., job_log_dir: _Optional[str] = ..., name: _Optional[str] = ..., cycle_point: _Optional[str] = ..., messages: _Optional[_Iterable[str]] = ..., runtime: _Optional[_Union[PbRuntime, _Mapping]] = ...) -> None: ... class PbTask(_message.Message): - __slots__ = ["stamp", "id", "name", "meta", "mean_elapsed_time", "depth", "proxies", "namespace", "parents", "first_parent", "runtime"] + __slots__ = ("stamp", "id", "name", "meta", "mean_elapsed_time", "depth", "proxies", "namespace", "parents", "first_parent", "runtime") STAMP_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] NAME_FIELD_NUMBER: _ClassVar[int] @@ -236,7 +238,7 @@ class PbTask(_message.Message): def __init__(self, stamp: _Optional[str] = ..., id: _Optional[str] = ..., name: _Optional[str] = ..., meta: _Optional[_Union[PbMeta, _Mapping]] = ..., mean_elapsed_time: _Optional[float] = ..., depth: _Optional[int] = ..., proxies: _Optional[_Iterable[str]] = ..., namespace: _Optional[_Iterable[str]] = ..., parents: _Optional[_Iterable[str]] = ..., first_parent: _Optional[str] = ..., runtime: _Optional[_Union[PbRuntime, _Mapping]] = ...) -> None: ... class PbPollTask(_message.Message): - __slots__ = ["local_proxy", "workflow", "remote_proxy", "req_state", "graph_string"] + __slots__ = ("local_proxy", "workflow", "remote_proxy", "req_state", "graph_string") LOCAL_PROXY_FIELD_NUMBER: _ClassVar[int] WORKFLOW_FIELD_NUMBER: _ClassVar[int] REMOTE_PROXY_FIELD_NUMBER: _ClassVar[int] @@ -250,7 +252,7 @@ class PbPollTask(_message.Message): def __init__(self, local_proxy: _Optional[str] = ..., workflow: _Optional[str] = ..., remote_proxy: _Optional[str] = ..., req_state: _Optional[str] = ..., graph_string: _Optional[str] = ...) -> None: ... class PbCondition(_message.Message): - __slots__ = ["task_proxy", "expr_alias", "req_state", "satisfied", "message"] + __slots__ = ("task_proxy", "expr_alias", "req_state", "satisfied", "message") TASK_PROXY_FIELD_NUMBER: _ClassVar[int] EXPR_ALIAS_FIELD_NUMBER: _ClassVar[int] REQ_STATE_FIELD_NUMBER: _ClassVar[int] @@ -264,7 +266,7 @@ class PbCondition(_message.Message): def __init__(self, task_proxy: _Optional[str] = ..., expr_alias: _Optional[str] = ..., req_state: _Optional[str] = ..., satisfied: bool = ..., message: _Optional[str] = ...) -> None: ... class PbPrerequisite(_message.Message): - __slots__ = ["expression", "conditions", "cycle_points", "satisfied"] + __slots__ = ("expression", "conditions", "cycle_points", "satisfied") EXPRESSION_FIELD_NUMBER: _ClassVar[int] CONDITIONS_FIELD_NUMBER: _ClassVar[int] CYCLE_POINTS_FIELD_NUMBER: _ClassVar[int] @@ -276,7 +278,7 @@ class PbPrerequisite(_message.Message): def __init__(self, expression: _Optional[str] = ..., conditions: _Optional[_Iterable[_Union[PbCondition, _Mapping]]] = ..., cycle_points: _Optional[_Iterable[str]] = ..., satisfied: bool = ...) -> None: ... class PbOutput(_message.Message): - __slots__ = ["label", "message", "satisfied", "time"] + __slots__ = ("label", "message", "satisfied", "time") LABEL_FIELD_NUMBER: _ClassVar[int] MESSAGE_FIELD_NUMBER: _ClassVar[int] SATISFIED_FIELD_NUMBER: _ClassVar[int] @@ -288,7 +290,7 @@ class PbOutput(_message.Message): def __init__(self, label: _Optional[str] = ..., message: _Optional[str] = ..., satisfied: bool = ..., time: _Optional[float] = ...) -> None: ... class PbTrigger(_message.Message): - __slots__ = ["id", "label", "message", "satisfied", "time"] + __slots__ = ("id", "label", "message", "satisfied", "time") ID_FIELD_NUMBER: _ClassVar[int] LABEL_FIELD_NUMBER: _ClassVar[int] MESSAGE_FIELD_NUMBER: _ClassVar[int] @@ -302,23 +304,23 @@ class PbTrigger(_message.Message): def __init__(self, id: _Optional[str] = ..., label: _Optional[str] = ..., message: _Optional[str] = ..., satisfied: bool = ..., time: _Optional[float] = ...) -> None: ... class PbTaskProxy(_message.Message): - __slots__ = ["stamp", "id", "task", "state", "cycle_point", "depth", "job_submits", "outputs", "namespace", "prerequisites", "jobs", "first_parent", "name", "is_held", "edges", "ancestors", "flow_nums", "external_triggers", "xtriggers", "is_queued", "is_runahead", "flow_wait", "runtime", "graph_depth"] + __slots__ = ("stamp", "id", "task", "state", "cycle_point", "depth", "job_submits", "outputs", "namespace", "prerequisites", "jobs", "first_parent", "name", "is_held", "edges", "ancestors", "flow_nums", "external_triggers", "xtriggers", "is_queued", "is_runahead", "flow_wait", "runtime", "graph_depth") class OutputsEntry(_message.Message): - __slots__ = ["key", "value"] + __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str value: PbOutput def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[PbOutput, _Mapping]] = ...) -> None: ... class ExternalTriggersEntry(_message.Message): - __slots__ = ["key", "value"] + __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str value: PbTrigger def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[PbTrigger, _Mapping]] = ...) -> None: ... class XtriggersEntry(_message.Message): - __slots__ = ["key", "value"] + __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str @@ -375,7 +377,7 @@ class PbTaskProxy(_message.Message): def __init__(self, stamp: _Optional[str] = ..., id: _Optional[str] = ..., task: _Optional[str] = ..., state: _Optional[str] = ..., cycle_point: _Optional[str] = ..., depth: _Optional[int] = ..., job_submits: _Optional[int] = ..., outputs: _Optional[_Mapping[str, PbOutput]] = ..., namespace: _Optional[_Iterable[str]] = ..., prerequisites: _Optional[_Iterable[_Union[PbPrerequisite, _Mapping]]] = ..., jobs: _Optional[_Iterable[str]] = ..., first_parent: _Optional[str] = ..., name: _Optional[str] = ..., is_held: bool = ..., edges: _Optional[_Iterable[str]] = ..., ancestors: _Optional[_Iterable[str]] = ..., flow_nums: _Optional[str] = ..., external_triggers: _Optional[_Mapping[str, PbTrigger]] = ..., xtriggers: _Optional[_Mapping[str, PbTrigger]] = ..., is_queued: bool = ..., is_runahead: bool = ..., flow_wait: bool = ..., runtime: _Optional[_Union[PbRuntime, _Mapping]] = ..., graph_depth: _Optional[int] = ...) -> None: ... class PbFamily(_message.Message): - __slots__ = ["stamp", "id", "name", "meta", "depth", "proxies", "parents", "child_tasks", "child_families", "first_parent", "runtime"] + __slots__ = ("stamp", "id", "name", "meta", "depth", "proxies", "parents", "child_tasks", "child_families", "first_parent", "runtime") STAMP_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] NAME_FIELD_NUMBER: _ClassVar[int] @@ -401,9 +403,9 @@ class PbFamily(_message.Message): def __init__(self, stamp: _Optional[str] = ..., id: _Optional[str] = ..., name: _Optional[str] = ..., meta: _Optional[_Union[PbMeta, _Mapping]] = ..., depth: _Optional[int] = ..., proxies: _Optional[_Iterable[str]] = ..., parents: _Optional[_Iterable[str]] = ..., child_tasks: _Optional[_Iterable[str]] = ..., child_families: _Optional[_Iterable[str]] = ..., first_parent: _Optional[str] = ..., runtime: _Optional[_Union[PbRuntime, _Mapping]] = ...) -> None: ... class PbFamilyProxy(_message.Message): - __slots__ = ["stamp", "id", "cycle_point", "name", "family", "state", "depth", "first_parent", "child_tasks", "child_families", "is_held", "ancestors", "states", "state_totals", "is_held_total", "is_queued", "is_queued_total", "is_runahead", "is_runahead_total", "runtime", "graph_depth"] + __slots__ = ("stamp", "id", "cycle_point", "name", "family", "state", "depth", "first_parent", "child_tasks", "child_families", "is_held", "ancestors", "states", "state_totals", "is_held_total", "is_queued", "is_queued_total", "is_runahead", "is_runahead_total", "runtime", "graph_depth") class StateTotalsEntry(_message.Message): - __slots__ = ["key", "value"] + __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str @@ -454,7 +456,7 @@ class PbFamilyProxy(_message.Message): def __init__(self, stamp: _Optional[str] = ..., id: _Optional[str] = ..., cycle_point: _Optional[str] = ..., name: _Optional[str] = ..., family: _Optional[str] = ..., state: _Optional[str] = ..., depth: _Optional[int] = ..., first_parent: _Optional[str] = ..., child_tasks: _Optional[_Iterable[str]] = ..., child_families: _Optional[_Iterable[str]] = ..., is_held: bool = ..., ancestors: _Optional[_Iterable[str]] = ..., states: _Optional[_Iterable[str]] = ..., state_totals: _Optional[_Mapping[str, int]] = ..., is_held_total: _Optional[int] = ..., is_queued: bool = ..., is_queued_total: _Optional[int] = ..., is_runahead: bool = ..., is_runahead_total: _Optional[int] = ..., runtime: _Optional[_Union[PbRuntime, _Mapping]] = ..., graph_depth: _Optional[int] = ...) -> None: ... class PbEdge(_message.Message): - __slots__ = ["stamp", "id", "source", "target", "suicide", "cond"] + __slots__ = ("stamp", "id", "source", "target", "suicide", "cond") STAMP_FIELD_NUMBER: _ClassVar[int] ID_FIELD_NUMBER: _ClassVar[int] SOURCE_FIELD_NUMBER: _ClassVar[int] @@ -470,7 +472,7 @@ class PbEdge(_message.Message): def __init__(self, stamp: _Optional[str] = ..., id: _Optional[str] = ..., source: _Optional[str] = ..., target: _Optional[str] = ..., suicide: bool = ..., cond: bool = ...) -> None: ... class PbEdges(_message.Message): - __slots__ = ["id", "edges", "workflow_polling_tasks", "leaves", "feet"] + __slots__ = ("id", "edges", "workflow_polling_tasks", "leaves", "feet") ID_FIELD_NUMBER: _ClassVar[int] EDGES_FIELD_NUMBER: _ClassVar[int] WORKFLOW_POLLING_TASKS_FIELD_NUMBER: _ClassVar[int] @@ -484,7 +486,7 @@ class PbEdges(_message.Message): def __init__(self, id: _Optional[str] = ..., edges: _Optional[_Iterable[str]] = ..., workflow_polling_tasks: _Optional[_Iterable[_Union[PbPollTask, _Mapping]]] = ..., leaves: _Optional[_Iterable[str]] = ..., feet: _Optional[_Iterable[str]] = ...) -> None: ... class PbEntireWorkflow(_message.Message): - __slots__ = ["workflow", "tasks", "task_proxies", "jobs", "families", "family_proxies", "edges"] + __slots__ = ("workflow", "tasks", "task_proxies", "jobs", "families", "family_proxies", "edges") WORKFLOW_FIELD_NUMBER: _ClassVar[int] TASKS_FIELD_NUMBER: _ClassVar[int] TASK_PROXIES_FIELD_NUMBER: _ClassVar[int] @@ -502,7 +504,7 @@ class PbEntireWorkflow(_message.Message): def __init__(self, workflow: _Optional[_Union[PbWorkflow, _Mapping]] = ..., tasks: _Optional[_Iterable[_Union[PbTask, _Mapping]]] = ..., task_proxies: _Optional[_Iterable[_Union[PbTaskProxy, _Mapping]]] = ..., jobs: _Optional[_Iterable[_Union[PbJob, _Mapping]]] = ..., families: _Optional[_Iterable[_Union[PbFamily, _Mapping]]] = ..., family_proxies: _Optional[_Iterable[_Union[PbFamilyProxy, _Mapping]]] = ..., edges: _Optional[_Iterable[_Union[PbEdge, _Mapping]]] = ...) -> None: ... class EDeltas(_message.Message): - __slots__ = ["time", "checksum", "added", "updated", "pruned", "reloaded"] + __slots__ = ("time", "checksum", "added", "updated", "pruned", "reloaded") TIME_FIELD_NUMBER: _ClassVar[int] CHECKSUM_FIELD_NUMBER: _ClassVar[int] ADDED_FIELD_NUMBER: _ClassVar[int] @@ -518,7 +520,7 @@ class EDeltas(_message.Message): def __init__(self, time: _Optional[float] = ..., checksum: _Optional[int] = ..., added: _Optional[_Iterable[_Union[PbEdge, _Mapping]]] = ..., updated: _Optional[_Iterable[_Union[PbEdge, _Mapping]]] = ..., pruned: _Optional[_Iterable[str]] = ..., reloaded: bool = ...) -> None: ... class FDeltas(_message.Message): - __slots__ = ["time", "checksum", "added", "updated", "pruned", "reloaded"] + __slots__ = ("time", "checksum", "added", "updated", "pruned", "reloaded") TIME_FIELD_NUMBER: _ClassVar[int] CHECKSUM_FIELD_NUMBER: _ClassVar[int] ADDED_FIELD_NUMBER: _ClassVar[int] @@ -534,7 +536,7 @@ class FDeltas(_message.Message): def __init__(self, time: _Optional[float] = ..., checksum: _Optional[int] = ..., added: _Optional[_Iterable[_Union[PbFamily, _Mapping]]] = ..., updated: _Optional[_Iterable[_Union[PbFamily, _Mapping]]] = ..., pruned: _Optional[_Iterable[str]] = ..., reloaded: bool = ...) -> None: ... class FPDeltas(_message.Message): - __slots__ = ["time", "checksum", "added", "updated", "pruned", "reloaded"] + __slots__ = ("time", "checksum", "added", "updated", "pruned", "reloaded") TIME_FIELD_NUMBER: _ClassVar[int] CHECKSUM_FIELD_NUMBER: _ClassVar[int] ADDED_FIELD_NUMBER: _ClassVar[int] @@ -550,7 +552,7 @@ class FPDeltas(_message.Message): def __init__(self, time: _Optional[float] = ..., checksum: _Optional[int] = ..., added: _Optional[_Iterable[_Union[PbFamilyProxy, _Mapping]]] = ..., updated: _Optional[_Iterable[_Union[PbFamilyProxy, _Mapping]]] = ..., pruned: _Optional[_Iterable[str]] = ..., reloaded: bool = ...) -> None: ... class JDeltas(_message.Message): - __slots__ = ["time", "checksum", "added", "updated", "pruned", "reloaded"] + __slots__ = ("time", "checksum", "added", "updated", "pruned", "reloaded") TIME_FIELD_NUMBER: _ClassVar[int] CHECKSUM_FIELD_NUMBER: _ClassVar[int] ADDED_FIELD_NUMBER: _ClassVar[int] @@ -566,7 +568,7 @@ class JDeltas(_message.Message): def __init__(self, time: _Optional[float] = ..., checksum: _Optional[int] = ..., added: _Optional[_Iterable[_Union[PbJob, _Mapping]]] = ..., updated: _Optional[_Iterable[_Union[PbJob, _Mapping]]] = ..., pruned: _Optional[_Iterable[str]] = ..., reloaded: bool = ...) -> None: ... class TDeltas(_message.Message): - __slots__ = ["time", "checksum", "added", "updated", "pruned", "reloaded"] + __slots__ = ("time", "checksum", "added", "updated", "pruned", "reloaded") TIME_FIELD_NUMBER: _ClassVar[int] CHECKSUM_FIELD_NUMBER: _ClassVar[int] ADDED_FIELD_NUMBER: _ClassVar[int] @@ -582,7 +584,7 @@ class TDeltas(_message.Message): def __init__(self, time: _Optional[float] = ..., checksum: _Optional[int] = ..., added: _Optional[_Iterable[_Union[PbTask, _Mapping]]] = ..., updated: _Optional[_Iterable[_Union[PbTask, _Mapping]]] = ..., pruned: _Optional[_Iterable[str]] = ..., reloaded: bool = ...) -> None: ... class TPDeltas(_message.Message): - __slots__ = ["time", "checksum", "added", "updated", "pruned", "reloaded"] + __slots__ = ("time", "checksum", "added", "updated", "pruned", "reloaded") TIME_FIELD_NUMBER: _ClassVar[int] CHECKSUM_FIELD_NUMBER: _ClassVar[int] ADDED_FIELD_NUMBER: _ClassVar[int] @@ -598,7 +600,7 @@ class TPDeltas(_message.Message): def __init__(self, time: _Optional[float] = ..., checksum: _Optional[int] = ..., added: _Optional[_Iterable[_Union[PbTaskProxy, _Mapping]]] = ..., updated: _Optional[_Iterable[_Union[PbTaskProxy, _Mapping]]] = ..., pruned: _Optional[_Iterable[str]] = ..., reloaded: bool = ...) -> None: ... class WDeltas(_message.Message): - __slots__ = ["time", "added", "updated", "reloaded", "pruned"] + __slots__ = ("time", "added", "updated", "reloaded", "pruned") TIME_FIELD_NUMBER: _ClassVar[int] ADDED_FIELD_NUMBER: _ClassVar[int] UPDATED_FIELD_NUMBER: _ClassVar[int] @@ -612,7 +614,7 @@ class WDeltas(_message.Message): def __init__(self, time: _Optional[float] = ..., added: _Optional[_Union[PbWorkflow, _Mapping]] = ..., updated: _Optional[_Union[PbWorkflow, _Mapping]] = ..., reloaded: bool = ..., pruned: _Optional[str] = ...) -> None: ... class AllDeltas(_message.Message): - __slots__ = ["families", "family_proxies", "jobs", "tasks", "task_proxies", "edges", "workflow"] + __slots__ = ("families", "family_proxies", "jobs", "tasks", "task_proxies", "edges", "workflow") FAMILIES_FIELD_NUMBER: _ClassVar[int] FAMILY_PROXIES_FIELD_NUMBER: _ClassVar[int] JOBS_FIELD_NUMBER: _ClassVar[int] diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py index b98a055f882..592022fc688 100644 --- a/cylc/flow/data_store_mgr.py +++ b/cylc/flow/data_store_mgr.py @@ -258,6 +258,7 @@ def runtime_from_config(rtconfig): pre_script=rtconfig['pre-script'], post_script=rtconfig['post-script'], work_sub_dir=rtconfig['work sub-directory'], + run_mode=rtconfig['run mode'], execution_time_limit=str(rtconfig['execution time limit'] or ''), execution_polling_intervals=listjoin( rtconfig['execution polling intervals'] @@ -1227,6 +1228,7 @@ def generate_ghost_task( graph_depth=n_depth, name=name, ) + self.all_n_window_nodes.add(tp_id) self.n_window_depths.setdefault(n_depth, set()).add(tp_id) diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index b962b582b70..ffc76146d2c 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -51,6 +51,7 @@ from cylc.flow.id import Tokens from cylc.flow.task_outputs import SORT_ORDERS from cylc.flow.task_state import ( + RunMode, TASK_STATUSES_ORDERED, TASK_STATUS_DESC, TASK_STATUS_WAITING, @@ -821,6 +822,7 @@ class Meta: directives = graphene.List(RuntimeSetting, resolver=resolve_json_dump) environment = graphene.List(RuntimeSetting, resolver=resolve_json_dump) outputs = graphene.List(RuntimeSetting, resolver=resolve_json_dump) + run_mode = graphene.String(default_value=RunMode.WORKFLOW) RUNTIME_FIELD_TO_CFG_MAP = { diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py index 449a5009656..b2952087502 100644 --- a/cylc/flow/prerequisite.py +++ b/cylc/flow/prerequisite.py @@ -57,6 +57,7 @@ class Prerequisite: MESSAGE_TEMPLATE = r'%s/%s %s' DEP_STATE_SATISFIED = 'satisfied naturally' + DEP_STATE_ARTIFICIAL = 'Artificially satisfied' DEP_STATE_OVERRIDDEN = 'force satisfied' DEP_STATE_UNSATISFIED = False @@ -199,20 +200,26 @@ def _conditional_is_satisfied(self): ) from None return res - def satisfy_me(self, outputs: Iterable['Tokens']) -> 'Set[Tokens]': + def satisfy_me( + self, outputs: Iterable['Tokens'], mode: str = 'live' + ) -> 'Set[Tokens]': """Attempt to satisfy me with given outputs. Updates cache with the result. Return outputs that match. """ + if mode != 'live': + satisfied_message = self.DEP_STATE_ARTIFICIAL + f' by {mode} mode' + else: + satisfied_message = self.DEP_STATE_SATISFIED valid = set() for output in outputs: prereq = (output['cycle'], output['task'], output['task_sel']) if prereq not in self.satisfied: continue valid.add(output) - self.satisfied[prereq] = self.DEP_STATE_SATISFIED + self.satisfied[prereq] = satisfied_message if self.conditional_expression is None: self._all_satisfied = all(self.satisfied.values()) else: @@ -293,6 +300,14 @@ def get_resolved_dependencies(self): E.G: ['1/foo', '2/bar'] """ - return [f'{point}/{name}' for - (point, name, _), satisfied in self.satisfied.items() if - satisfied == self.DEP_STATE_SATISFIED] + return [ + f'{point}/{name}' for + (point, name, _), satisfied in self.satisfied.items() + if ( + satisfied == self.DEP_STATE_SATISFIED + or ( + isinstance(satisfied, str) + and satisfied.startswith(self.DEP_STATE_ARTIFICIAL) + ) + ) + ] diff --git a/cylc/flow/run_modes/dummy.py b/cylc/flow/run_modes/dummy.py new file mode 100644 index 00000000000..56d99b2c626 --- /dev/null +++ b/cylc/flow/run_modes/dummy.py @@ -0,0 +1,121 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Utilities supporting dummy mode. +""" + +from logging import INFO +from typing import TYPE_CHECKING, Any, Dict, Tuple + +from cylc.flow.task_outputs import TASK_OUTPUT_SUBMITTED +from cylc.flow.run_modes.simulation import ( + ModeSettings, + disable_platforms, + get_simulated_run_len, + parse_fail_cycle_points +) +from cylc.flow.task_state import RunMode +from cylc.flow.platforms import get_platform + + +if TYPE_CHECKING: + from cylc.flow.task_job_mgr import TaskJobManager + from cylc.flow.task_proxy import TaskProxy + from typing_extensions import Literal + + +def submit_task_job( + task_job_mgr: 'TaskJobManager', + itask: 'TaskProxy', + rtconfig: Dict[str, Any], + workflow: str, + now: Tuple[float, str] +) -> 'Literal[False]': + """Submit a task in dummy mode. + + Returns: + False - indicating that TaskJobManager needs to continue running the + live mode path. + """ + configure_dummy_mode( + rtconfig, itask.tdef.rtconfig['simulation']['fail cycle points']) + + itask.summary['started_time'] = now[0] + task_job_mgr._set_retry_timers(itask, rtconfig) + itask.mode_settings = ModeSettings( + itask, + task_job_mgr.workflow_db_mgr, + rtconfig + ) + + itask.waiting_on_job_prep = False + itask.submit_num += 1 + + itask.platform = get_platform() + itask.platform['name'] = RunMode.DUMMY + itask.summary['job_runner_name'] = RunMode.DUMMY + itask.summary[task_job_mgr.KEY_EXECUTE_TIME_LIMIT] = ( + itask.mode_settings.simulated_run_length) + itask.jobs.append( + task_job_mgr.get_simulation_job_conf(itask, workflow)) + task_job_mgr.task_events_mgr.process_message( + itask, INFO, TASK_OUTPUT_SUBMITTED) + task_job_mgr.workflow_db_mgr.put_insert_task_jobs( + itask, { + 'time_submit': now[1], + 'try_num': itask.get_try_num(), + } + ) + return False + + +def configure_dummy_mode(rtc, fallback): + """Adjust task defs for simulation and dummy mode. + + """ + rtc['submission retry delays'] = [1] + # Generate dummy scripting. + rtc['init-script'] = "" + rtc['env-script'] = "" + rtc['pre-script'] = "" + rtc['post-script'] = "" + rtc['script'] = build_dummy_script( + rtc, get_simulated_run_len(rtc)) + disable_platforms(rtc) + # Disable environment, in case it depends on env-script. + rtc['environment'] = {} + rtc["simulation"][ + "fail cycle points" + ] = parse_fail_cycle_points( + rtc["simulation"]["fail cycle points"], fallback + ) + + +def build_dummy_script(rtc: Dict[str, Any], sleep_sec: int) -> str: + """Create fake scripting for dummy mode. + + This is for Dummy mode only. + """ + script = "sleep %d" % sleep_sec + # Dummy message outputs. + for msg in rtc['outputs'].values(): + script += "\ncylc message '%s'" % msg + if rtc['simulation']['fail try 1 only']: + arg1 = "true" + else: + arg1 = "false" + arg2 = " ".join(rtc['simulation']['fail cycle points']) + script += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2) + return script diff --git a/cylc/flow/run_modes/nonlive.py b/cylc/flow/run_modes/nonlive.py new file mode 100644 index 00000000000..69c35e634ff --- /dev/null +++ b/cylc/flow/run_modes/nonlive.py @@ -0,0 +1,61 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Utilities supporting all nonlive modes +""" +from typing import TYPE_CHECKING, Dict, List + +from cylc.flow import LOG +from cylc.flow.run_modes.skip import check_task_skip_config +from cylc.flow.task_state import RunMode + +if TYPE_CHECKING: + from cylc.flow.taskdef import TaskDef + + +def mode_validate_checks(taskdefs: 'Dict[str, TaskDef]'): + """Warn user if any tasks have "run mode" set to simulation or + dummy in flow.cylc. + + Setting run mode to simulation/dummy in the config should only + be done during development of a workflow. + + Additionally, run specific checks for each mode's config settings. + """ + warn_nonlive: Dict[str, List[str]] = { + RunMode.SIMULATION: [], + RunMode.DUMMY: [], + } + + # Run through taskdefs looking for those with nonlive modes + for _, taskdef in taskdefs.items(): + # Add to list of tasks to be run in non-live modes: + if ( + taskdef.rtconfig.get('run mode', None) + in {RunMode.SIMULATION, RunMode.DUMMY} + ): + warn_nonlive[taskdef.rtconfig['run mode']].append(taskdef.name) + + # Run any mode specific validation checks: + check_task_skip_config(taskdef) + + if any(warn_nonlive.values()): + message = 'The following tasks are set to run in non-live mode:' + for mode, tasknames in warn_nonlive.items(): + if tasknames: + message += f'\n{mode} mode:' + for taskname in tasknames: + message += f'\n * {taskname}' + LOG.warning(message) diff --git a/cylc/flow/simulation.py b/cylc/flow/run_modes/simulation.py similarity index 66% rename from cylc/flow/simulation.py rename to cylc/flow/run_modes/simulation.py index 8ec4d279cb9..659808f7451 100644 --- a/cylc/flow/simulation.py +++ b/cylc/flow/run_modes/simulation.py @@ -13,33 +13,85 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Utilities supporting simulation and skip modes +"""Utilities supporting simulation mode """ from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from logging import INFO +from typing import ( + TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union) from time import time from metomi.isodatetime.parsers import DurationParser from cylc.flow import LOG +from cylc.flow.cycling import PointBase from cylc.flow.cycling.loader import get_point from cylc.flow.exceptions import PointParsingError -from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM +from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM, get_platform +from cylc.flow.task_outputs import TASK_OUTPUT_SUBMITTED from cylc.flow.task_state import ( TASK_STATUS_RUNNING, TASK_STATUS_FAILED, TASK_STATUS_SUCCEEDED, ) from cylc.flow.wallclock import get_unix_time_from_time_string -from cylc.flow.workflow_status import RunMode +from cylc.flow.task_state import RunMode if TYPE_CHECKING: from cylc.flow.task_events_mgr import TaskEventsManager + from cylc.flow.task_job_mgr import TaskJobManager from cylc.flow.task_proxy import TaskProxy from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager - from cylc.flow.cycling import PointBase + from typing_extensions import Literal + + +def submit_task_job( + task_job_mgr: 'TaskJobManager', + itask: 'TaskProxy', + rtconfig: Dict[str, Any], + workflow: str, + now: Tuple[float, str] +) -> 'Literal[True]': + """Submit a task in simulation mode. + + Returns: + True - indicating that TaskJobManager need take no further action. + """ + configure_sim_mode( + rtconfig, + itask.tdef.rtconfig['simulation']['fail cycle points']) + itask.summary['started_time'] = now[0] + task_job_mgr._set_retry_timers(itask, rtconfig) + itask.mode_settings = ModeSettings( + itask, + task_job_mgr.workflow_db_mgr, + rtconfig + ) + itask.waiting_on_job_prep = False + itask.submit_num += 1 + + itask.platform = get_platform() + itask.platform['name'] = RunMode.SIMULATION + itask.summary['job_runner_name'] = RunMode.SIMULATION + itask.summary[task_job_mgr.KEY_EXECUTE_TIME_LIMIT] = ( + itask.mode_settings.simulated_run_length + ) + itask.jobs.append( + task_job_mgr.get_simulation_job_conf(itask, workflow) + ) + task_job_mgr.task_events_mgr.process_message( + itask, INFO, TASK_OUTPUT_SUBMITTED, + ) + task_job_mgr.workflow_db_mgr.put_insert_task_jobs( + itask, { + 'time_submit': now[1], + 'try_num': itask.get_try_num(), + } + ) + itask.state.status = TASK_STATUS_RUNNING + return True @dataclass @@ -79,7 +131,6 @@ def __init__( db_mgr: 'WorkflowDatabaseManager', rtconfig: Dict[str, Any] ): - # itask.summary['started_time'] and mode_settings.timeout need # repopulating from the DB on workflow restart: started_time = itask.summary['started_time'] @@ -104,22 +155,15 @@ def __init__( try_num = db_info["try_num"] # Parse fail cycle points: - if rtconfig != itask.tdef.rtconfig: - try: - rtconfig["simulation"][ - "fail cycle points" - ] = parse_fail_cycle_points( - rtconfig["simulation"]["fail cycle points"] - ) - except PointParsingError as exc: - # Broadcast Fail CP didn't parse - LOG.warning( - 'Broadcast fail cycle point was invalid:\n' - f' {exc.args[0]}' - ) - rtconfig['simulation'][ - 'fail cycle points' - ] = itask.tdef.rtconfig['simulation']['fail cycle points'] + if not rtconfig: + rtconfig = itask.tdef.rtconfig + if rtconfig and rtconfig != itask.tdef.rtconfig: + rtconfig["simulation"][ + "fail cycle points" + ] = parse_fail_cycle_points( + rtconfig["simulation"]["fail cycle points"], + itask.tdef.rtconfig['simulation']['fail cycle points'] + ) # Calculate simulation info: self.simulated_run_length = ( @@ -132,37 +176,39 @@ def __init__( self.timeout = started_time + self.simulated_run_length -def configure_sim_modes(taskdefs, sim_mode): +def configure_sim_mode(rtc, fallback): """Adjust task defs for simulation and dummy mode. + Example: + >>> this = configure_sim_mode + >>> rtc = { + ... 'submission retry delays': [42, 24, 23], + ... 'environment': {'DoNot': '"WantThis"'}, + ... 'simulation': {'fail cycle points': ['all']} + ... } + >>> this(rtc, [53]) + >>> rtc['submission retry delays'] + [1] + >>> rtc['environment'] + {} + >>> rtc['simulation'] + {'fail cycle points': None} + >>> rtc['platform'] + 'localhost' """ - dummy_mode = (sim_mode == RunMode.DUMMY) - - for tdef in taskdefs: - # Compute simulated run time by scaling the execution limit. - rtc = tdef.rtconfig - - rtc['submission retry delays'] = [1] + rtc['submission retry delays'] = [1] - if dummy_mode: - # Generate dummy scripting. - rtc['init-script'] = "" - rtc['env-script'] = "" - rtc['pre-script'] = "" - rtc['post-script'] = "" - rtc['script'] = build_dummy_script( - rtc, get_simulated_run_len(rtc)) + disable_platforms(rtc) - disable_platforms(rtc) + # Disable environment, in case it depends on env-script. + rtc['environment'] = {} - # Disable environment, in case it depends on env-script. - rtc['environment'] = {} - - rtc["simulation"][ - "fail cycle points" - ] = parse_fail_cycle_points( - rtc["simulation"]["fail cycle points"] - ) + rtc["simulation"][ + "fail cycle points" + ] = parse_fail_cycle_points( + rtc["simulation"]["fail cycle points"], + fallback + ) def get_simulated_run_len(rtc: Dict[str, Any]) -> int: @@ -184,24 +230,6 @@ def get_simulated_run_len(rtc: Dict[str, Any]) -> int: return sleep_sec -def build_dummy_script(rtc: Dict[str, Any], sleep_sec: int) -> str: - """Create fake scripting for dummy mode. - - This is for Dummy mode only. - """ - script = "sleep %d" % sleep_sec - # Dummy message outputs. - for msg in rtc['outputs'].values(): - script += "\ncylc message '%s'" % msg - if rtc['simulation']['fail try 1 only']: - arg1 = "true" - else: - arg1 = "false" - arg2 = " ".join(rtc['simulation']['fail cycle points']) - script += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2) - return script - - def disable_platforms( rtc: Dict[str, Any] ) -> None: @@ -222,7 +250,7 @@ def disable_platforms( def parse_fail_cycle_points( - f_pts_orig: List[str] + f_pts_orig: List[str], fallback ) -> 'Union[None, List[PointBase]]': """Parse `[simulation][fail cycle points]`. @@ -231,11 +259,11 @@ def parse_fail_cycle_points( Examples: >>> this = parse_fail_cycle_points - >>> this(['all']) is None + >>> this(['all'], ['42']) is None True - >>> this([]) + >>> this([], ['42']) [] - >>> this(None) is None + >>> this(None, ['42']) is None True """ f_pts: 'Optional[List[PointBase]]' = [] @@ -247,7 +275,16 @@ def parse_fail_cycle_points( elif f_pts_orig: f_pts = [] for point_str in f_pts_orig: - f_pts.append(get_point(point_str).standardise()) + if isinstance(point_str, PointBase): + f_pts.append(point_str) + else: + try: + f_pts.append(get_point(point_str).standardise()) + except PointParsingError: + LOG.warning( + f'Invalid ISO 8601 date representation: {point_str}' + ) + return fallback return f_pts @@ -266,13 +303,19 @@ def sim_time_check( now = time() sim_task_state_changed: bool = False for itask in itasks: - if itask.state.status != TASK_STATUS_RUNNING: + if ( + itask.state.status != TASK_STATUS_RUNNING + or itask.tdef.run_mode != RunMode.SIMULATION + ): continue # This occurs if the workflow has been restarted. if itask.mode_settings is None: rtconfig = task_events_manager.broadcast_mgr.get_updated_rtconfig( itask) + rtconfig = configure_sim_mode( + rtconfig, + itask.tdef.rtconfig['simulation']['fail cycle points']) itask.mode_settings = ModeSettings( itask, db_mgr, diff --git a/cylc/flow/run_modes/skip.py b/cylc/flow/run_modes/skip.py new file mode 100644 index 00000000000..cdcd23a6b6e --- /dev/null +++ b/cylc/flow/run_modes/skip.py @@ -0,0 +1,160 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Utilities supporting skip modes +""" +from logging import INFO +from typing import ( + TYPE_CHECKING, List, Set, Tuple) + +from cylc.flow import LOG +from cylc.flow.exceptions import WorkflowConfigError +from cylc.flow.platforms import get_platform +from cylc.flow.task_outputs import ( + TASK_OUTPUT_SUBMITTED, + TASK_OUTPUT_SUCCEEDED, + TASK_OUTPUT_FAILED, + TASK_OUTPUT_STARTED +) +from cylc.flow.task_state import RunMode + +if TYPE_CHECKING: + from cylc.flow.taskdef import TaskDef + from cylc.flow.task_job_mgr import TaskJobManager + from cylc.flow.task_proxy import TaskProxy + from typing_extensions import Literal + + +def submit_task_job( + task_job_mgr: 'TaskJobManager', + itask: 'TaskProxy', + now: Tuple[float, str] +) -> 'Literal[True]': + """Submit a task in skip mode. + + Returns: + True - indicating that TaskJobManager need take no further action. + """ + itask.summary['started_time'] = now[0] + itask.waiting_on_job_prep = False + itask.submit_num += 1 + + itask.platform = get_platform() + itask.platform['name'] = RunMode.SKIP + itask.summary['job_runner_name'] = RunMode.SKIP + itask.tdef.run_mode = RunMode.SKIP + task_job_mgr.task_events_mgr.process_message( + itask, INFO, TASK_OUTPUT_SUBMITTED, + ) + task_job_mgr.workflow_db_mgr.put_insert_task_jobs( + itask, { + 'time_submit': now[1], + 'try_num': itask.get_try_num(), + } + ) + for output in process_outputs(itask): + task_job_mgr.task_events_mgr.process_message(itask, INFO, output) + + return True + + +def process_outputs(itask: 'TaskProxy') -> List[str]: + """Process Skip Mode Outputs: + + * By default, all required outputs will be generated plus succeeded + if success is optional. + * The outputs submitted and started are always produced and do not + need to be defined in outputs. + * If outputs is specified and does not include either + succeeded or failed then succeeded will be produced. + + Return: + A list of outputs to emit. + """ + result: List[str] = [] + conf_outputs = itask.tdef.rtconfig['skip']['outputs'] + + # Remove started or submitted from our list of outputs: + for out in get_unecessary_outputs(conf_outputs): + conf_outputs.remove(out) + + # Always produce `submitted` output: + result.append(TASK_OUTPUT_SUBMITTED) + # (No need to produce `started` as this is automatically handled by + # task event manager for jobless modes) + + # Send the rest of our outputs, unless they are succeed or failed, + # which we hold back, to prevent warnings about pre-requisites being + # unmet being shown because a "finished" output happens to come first. + for message in itask.state.outputs.iter_required_messages(): + trigger = itask.state.outputs._message_to_trigger[message] + # Send message unless it be succeeded/failed. + if trigger in [TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED]: + continue + if not conf_outputs or trigger in conf_outputs: + result.append(message) + + # Send succeeded/failed last. + if TASK_OUTPUT_FAILED in conf_outputs: + result.append(TASK_OUTPUT_FAILED) + else: + result.append(TASK_OUTPUT_SUCCEEDED) + + return result + + +def check_task_skip_config(tdef: 'TaskDef') -> None: + """Ensure that skip mode configurations are sane at validation time: + + Args: + tdef: of task + + Logs: + * Warn that outputs need not include started and submitted as these + are always emitted. + + Raises: + * Error if outputs include succeeded and failed. + """ + skip_config = tdef.rtconfig.get('skip', {}) + if not skip_config: + return + skip_outputs = skip_config.get('outputs', {}) + if not skip_outputs: + return + + # Error if outputs include succeded and failed: + if ( + TASK_OUTPUT_SUCCEEDED in skip_outputs + and TASK_OUTPUT_FAILED in skip_outputs + ): + raise WorkflowConfigError( + f'Skip mode settings for task {tdef.name} has' + ' mutually exclusive outputs: succeeded AND failed.') + LOG.info(f'Task {tdef.name} will be run in skip mode.') + + +def get_unecessary_outputs(skip_outputs: List[str]) -> Set[str]: + """Get a list of outputs which we will always run, and don't need + setting config. + + Examples: + >>> this = get_unecessary_outputs + >>> this(['foo', 'started', 'succeeded']) + {'started'} + """ + return {TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED}.intersection( + skip_outputs + ) diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index 92702b0b55e..9f6b0b33dfc 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -107,8 +107,13 @@ ) from cylc.flow.profiler import Profiler from cylc.flow.resources import get_resources -from cylc.flow.simulation import sim_time_check +from cylc.flow.run_modes.simulation import sim_time_check from cylc.flow.subprocpool import SubProcPool +from cylc.flow.templatevars import eval_var +from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager +from cylc.flow.workflow_events import WorkflowEventHandler +from cylc.flow.workflow_status import StopMode, AutoRestartMode +from cylc.flow.taskdef import TaskDef from cylc.flow.task_events_mgr import TaskEventsManager from cylc.flow.task_job_mgr import TaskJobManager from cylc.flow.task_pool import TaskPool @@ -127,9 +132,7 @@ TASK_STATUS_RUNNING, TASK_STATUS_SUBMITTED, TASK_STATUS_WAITING, -) -from cylc.flow.taskdef import TaskDef -from cylc.flow.templatevars import eval_var + RunMode) from cylc.flow.templatevars import get_template_vars from cylc.flow.timer import Timer from cylc.flow.util import cli_format @@ -138,9 +141,6 @@ get_time_string_from_unix_time as time2str, get_utc_mode, ) -from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager -from cylc.flow.workflow_events import WorkflowEventHandler -from cylc.flow.workflow_status import AutoRestartMode, RunMode, StopMode from cylc.flow.xtrigger_mgr import XtriggerManager if TYPE_CHECKING: @@ -1269,7 +1269,7 @@ def release_queued_tasks(self) -> bool: pre_prep_tasks, self.server.curve_auth, self.server.client_pub_key_dir, - is_simulation=(self.get_run_mode() == RunMode.SIMULATION) + run_mode=self.get_run_mode() ): if itask.flow_nums: flow = ','.join(str(i) for i in itask.flow_nums) @@ -1516,17 +1516,12 @@ async def _main_loop(self) -> None: if self.xtrigger_mgr.do_housekeeping: self.xtrigger_mgr.housekeep(self.pool.get_tasks()) - self.pool.clock_expire_tasks() self.release_queued_tasks() - - if ( - self.get_run_mode() == RunMode.SIMULATION - and sim_time_check( - self.task_events_mgr, - self.pool.get_tasks(), - self.workflow_db_mgr, - ) + if sim_time_check( + self.task_events_mgr, + self.pool.get_tasks(), + self.workflow_db_mgr, ): # A simulated task state change occurred. self.reset_inactivity_timer() diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py index c014697160a..4a11f7fffa3 100644 --- a/cylc/flow/scheduler_cli.py +++ b/cylc/flow/scheduler_cli.py @@ -54,6 +54,7 @@ from cylc.flow.remote import cylc_server_cmd from cylc.flow.scheduler import Scheduler, SchedulerError from cylc.flow.scripts.common import cylc_header +from cylc.flow.task_state import RunMode from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager from cylc.flow.workflow_files import ( SUITERC_DEPR_MSG, @@ -65,7 +66,6 @@ is_terminal, prompt, ) -from cylc.flow.workflow_status import RunMode if TYPE_CHECKING: from optparse import Values @@ -129,9 +129,15 @@ RUN_MODE = OptionSettings( ["-m", "--mode"], - help="Run mode: live, dummy, simulation (default live).", + help=( + f"Run mode: {RunMode.WORKFLOW_MODES} (default live)." + " Live mode executes the tasks as defined in the runtime section." + " Simulation, skip and dummy modes ignore part of tasks'" + " runtime configurations. Simulation and dummy modes are" + " designed for testing, and skip mode is for flow control." + ), metavar="STRING", action='store', dest="run_mode", - choices=[RunMode.LIVE, RunMode.DUMMY, RunMode.SIMULATION], + choices=list(RunMode.WORKFLOW_MODES), ) PLAY_RUN_MODE = deepcopy(RUN_MODE) diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py index c8aa7279590..eb9fb0a2f96 100755 --- a/cylc/flow/scripts/lint.py +++ b/cylc/flow/scripts/lint.py @@ -89,9 +89,15 @@ from cylc.flow.cfgspec.workflow import upg, SPEC from cylc.flow.id_cli import parse_id from cylc.flow.parsec.config import ParsecConfig +from cylc.flow.run_modes.skip import get_unecessary_outputs from cylc.flow.scripts.cylc import DEAD_ENDS +from cylc.flow.task_outputs import ( + TASK_OUTPUT_SUCCEEDED, + TASK_OUTPUT_FAILED, +) from cylc.flow.terminal import cli_function + if TYPE_CHECKING: # BACK COMPAT: typing_extensions.Literal # FROM: Python 3.7 @@ -350,6 +356,39 @@ def check_for_deprecated_task_event_template_vars( return None +BAD_SKIP_OUTS = re.compile(r'outputs\s*=\s*(.*)') + + +def check_skip_mode_outputs(line: str) -> Dict: + """Ensure skip mode output setting doesn't include: + + * succeeded _and_ failed: Mutually exclusive. + * submitted and started: These are emitted by skip mode anyway. + + n.b. + + This should be separable from ``[[outputs]]`` because it's a key + value pair not a section heading. + """ + + outputs = BAD_SKIP_OUTS.findall(line) + if outputs: + outputs = [i.strip() for i in outputs[0].split(',')] + if TASK_OUTPUT_FAILED in outputs and TASK_OUTPUT_SUCCEEDED in outputs: + return { + 'description': + 'are mutually exclusive and cannot be used together', + 'outputs': f'{TASK_OUTPUT_FAILED} and {TASK_OUTPUT_SUCCEEDED}' + } + pointless_outputs = get_unecessary_outputs(outputs) + if pointless_outputs: + return { + 'description': 'are not required, they will be emitted anyway', + 'outputs': f'{pointless_outputs}' + } + return {} + + INDENTATION = re.compile(r'^(\s*)(.*)') @@ -588,7 +627,15 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]: ' directive can make your workflow more portable.' ), FUNCTION: check_wallclock_directives, - } + }, + 'S015': { + 'short': 'Task outputs {outputs}: {description}.', + FUNCTION: check_skip_mode_outputs + }, + 'S016': { + 'short': 'Run mode is not live: This task will only appear to run.', + FUNCTION: re.compile(r'run mode\s*=\s*[^l][^i][^v][^e]$').findall + }, } # Subset of deprecations which are tricky (impossible?) to scrape from the # upgrader. diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py index f3a77087069..e16aabbc162 100755 --- a/cylc/flow/scripts/validate.py +++ b/cylc/flow/scripts/validate.py @@ -55,7 +55,7 @@ from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function from cylc.flow.scheduler_cli import RUN_MODE -from cylc.flow.workflow_status import RunMode +from cylc.flow.task_state import RunMode if TYPE_CHECKING: from cylc.flow.option_parsers import Values diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py index bf9c2ba3a9b..3737ed88bad 100644 --- a/cylc/flow/task_events_mgr.py +++ b/cylc/flow/task_events_mgr.py @@ -78,7 +78,8 @@ TASK_STATUS_FAILED, TASK_STATUS_EXPIRED, TASK_STATUS_SUCCEEDED, - TASK_STATUS_WAITING + TASK_STATUS_WAITING, + RunMode, ) from cylc.flow.task_outputs import ( TASK_OUTPUT_EXPIRED, @@ -98,7 +99,6 @@ get_template_variables as get_workflow_template_variables, process_mail_footer, ) -from cylc.flow.workflow_status import RunMode if TYPE_CHECKING: @@ -770,7 +770,7 @@ def process_message( # ... but either way update the job ID in the job proxy (it only # comes in via the submission message). - if itask.tdef.run_mode != RunMode.SIMULATION: + if itask.tdef.run_mode not in RunMode.JOBLESS_MODES: job_tokens = itask.tokens.duplicate( job=str(itask.submit_num) ) @@ -889,7 +889,7 @@ def _process_message_check( if ( itask.state(TASK_STATUS_WAITING) - # Polling in live mode only: + # Polling in live mode only. and itask.tdef.run_mode == RunMode.LIVE and ( ( @@ -935,7 +935,7 @@ def _process_message_check( def setup_event_handlers(self, itask, event, message): """Set up handlers for a task event.""" - if itask.tdef.run_mode != RunMode.LIVE: + if RunMode.disable_task_event_handlers(itask): return msg = "" if message != f"job {event}": @@ -1460,7 +1460,7 @@ def _process_message_submitted( ) itask.set_summary_time('submitted', event_time) - if itask.tdef.run_mode == RunMode.SIMULATION: + if itask.tdef.run_mode in RunMode.JOBLESS_MODES: # Simulate job started as well. itask.set_summary_time('started', event_time) if itask.state_reset(TASK_STATUS_RUNNING, forced=forced): @@ -1497,7 +1497,7 @@ def _process_message_submitted( 'submitted', event_time, ) - if itask.tdef.run_mode == RunMode.SIMULATION: + if itask.tdef.run_mode in RunMode.JOBLESS_MODES: # Simulate job started as well. self.data_store_mgr.delta_job_time( job_tokens, @@ -1530,7 +1530,7 @@ def _insert_task_job( # not see previous submissions (so can't use itask.jobs[submit_num-1]). # And transient tasks, used for setting outputs and spawning children, # do not submit jobs. - if (itask.tdef.run_mode == RunMode.SIMULATION) or forced: + if itask.tdef.run_mode in RunMode.JOBLESS_MODES or forced: job_conf = {"submit_num": itask.submit_num} else: job_conf = itask.jobs[-1] diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py index 185966ff12d..89553ac1e13 100644 --- a/cylc/flow/task_job_mgr.py +++ b/cylc/flow/task_job_mgr.py @@ -35,7 +35,7 @@ ) from shutil import rmtree from time import time -from typing import TYPE_CHECKING, Any, Union, Optional +from typing import TYPE_CHECKING, Any, List, Tuple, Union, Optional from cylc.flow import LOG from cylc.flow.job_runner_mgr import JobPollContext @@ -63,7 +63,12 @@ get_platform, ) from cylc.flow.remote import construct_ssh_cmd -from cylc.flow.simulation import ModeSettings +from cylc.flow.run_modes.simulation import ( + submit_task_job as simulation_submit_task_job) +from cylc.flow.run_modes.skip import ( + submit_task_job as skip_submit_task_job) +from cylc.flow.run_modes.dummy import ( + submit_task_job as dummy_submit_task_job) from cylc.flow.subprocctx import SubProcContext from cylc.flow.subprocpool import SubProcPool from cylc.flow.task_action_timer import ( @@ -103,7 +108,8 @@ TASK_STATUS_SUBMITTED, TASK_STATUS_RUNNING, TASK_STATUS_WAITING, - TASK_STATUSES_ACTIVE + TASK_STATUSES_ACTIVE, + RunMode ) from cylc.flow.wallclock import ( get_current_time_string, @@ -247,7 +253,7 @@ def prep_submit_task_jobs(self, workflow, itasks, check_syntax=True): return [prepared_tasks, bad_tasks] def submit_task_jobs(self, workflow, itasks, curve_auth, - client_pub_key_dir, is_simulation=False): + client_pub_key_dir, run_mode='live'): """Prepare for job submission and submit task jobs. Preparation (host selection, remote host init, and remote install) @@ -262,8 +268,8 @@ def submit_task_jobs(self, workflow, itasks, curve_auth, Return (list): list of tasks that attempted submission. """ - if is_simulation: - return self._simulation_submit_task_jobs(itasks, workflow) + itasks, jobless_tasks = self._nonlive_submit_task_jobs( + itasks, workflow, run_mode) # Prepare tasks for job submission prepared_tasks, bad_tasks = self.prep_submit_task_jobs( @@ -272,9 +278,10 @@ def submit_task_jobs(self, workflow, itasks, curve_auth, # Reset consumed host selection results self.task_remote_mgr.subshell_eval_reset() - if not prepared_tasks: + if not prepared_tasks and not jobless_tasks: return bad_tasks - + elif not prepared_tasks: + return jobless_tasks auth_itasks = {} # {platform: [itask, ...], ...} for itask in prepared_tasks: @@ -282,8 +289,7 @@ def submit_task_jobs(self, workflow, itasks, curve_auth, auth_itasks.setdefault(platform_name, []) auth_itasks[platform_name].append(itask) # Submit task jobs for each platform - # Non-prepared tasks can be considered done for now: - done_tasks = bad_tasks + done_tasks = bad_tasks + jobless_tasks for _, itasks in sorted(auth_itasks.items()): # Find the first platform where >1 host has not been tried and @@ -997,44 +1003,64 @@ def _set_retry_timers( except KeyError: itask.try_timers[key] = TaskActionTimer(delays=delays) - def _simulation_submit_task_jobs(self, itasks, workflow): - """Simulation mode task jobs submission.""" + def _nonlive_submit_task_jobs( + self: 'TaskJobManager', + itasks: 'List[TaskProxy]', + workflow: str, + workflow_run_mode: str, + ) -> 'Tuple[List[TaskProxy], List[TaskProxy]]': + """Simulation mode task jobs submission. + + Returns: + lively_tasks: + A list of tasks which require subsequent + processing **as if** they were live mode tasks. + (This includes live and dummy mode tasks) + ghostly_tasks: + A list of tasks which require no further processing + because their apparent execution is done entirely inside + the scheduler. (This includes skip and simulation mode tasks). + """ + lively_tasks: 'List[TaskProxy]' = [] + jobless_tasks: 'List[TaskProxy]' = [] now = time() - now_str = get_time_string_from_unix_time(now) + now = (now, get_time_string_from_unix_time(now)) + for itask in itasks: - # Handle broadcasts + # Handle broadcasts: rtconfig = self.task_events_mgr.broadcast_mgr.get_updated_rtconfig( itask) - itask.summary['started_time'] = now - self._set_retry_timers(itask, rtconfig) - itask.mode_settings = ModeSettings( - itask, - self.workflow_db_mgr, - rtconfig - ) - - itask.waiting_on_job_prep = False - itask.submit_num += 1 + # Apply task run mode - itask.platform = {'name': 'SIMULATION'} - itask.summary['job_runner_name'] = 'SIMULATION' - itask.summary[self.KEY_EXECUTE_TIME_LIMIT] = ( - itask.mode_settings.simulated_run_length - ) - itask.jobs.append( - self.get_simulation_job_conf(itask, workflow) - ) - self.task_events_mgr.process_message( - itask, INFO, TASK_OUTPUT_SUBMITTED, - ) - self.workflow_db_mgr.put_insert_task_jobs( - itask, { - 'time_submit': now_str, - 'try_num': itask.get_try_num(), - } - ) - return itasks + if rtconfig['run mode'] == 'workflow': + run_mode = workflow_run_mode + else: + if rtconfig['run mode'] != workflow_run_mode: + LOG.info( + f'[{itask.identity}] run mode set by task settings' + f' to: {rtconfig["run mode"]} mode.') + run_mode = rtconfig['run mode'] + itask.tdef.run_mode = run_mode + + # Submit ghost tasks, or add live-like tasks to list + # of tasks to put through live submission pipeline: + is_done = False + if run_mode == RunMode.DUMMY: + is_done = dummy_submit_task_job( + self, itask, rtconfig, workflow, now) + elif run_mode == RunMode.SIMULATION: + is_done = simulation_submit_task_job( + self, itask, rtconfig, workflow, now) + elif run_mode == RunMode.SKIP: + is_done = skip_submit_task_job( + self, itask, now) + # Assign task to list: + if is_done: + jobless_tasks.append(itask) + else: + lively_tasks.append(itask) + return lively_tasks, jobless_tasks def _submit_task_jobs_callback(self, ctx, workflow, itasks): """Callback when submit task jobs command exits.""" diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py index 78a9d0c6f9f..d422c25fdd4 100644 --- a/cylc/flow/task_pool.py +++ b/cylc/flow/task_pool.py @@ -53,6 +53,7 @@ from cylc.flow.task_id import TaskID from cylc.flow.task_proxy import TaskProxy from cylc.flow.task_state import ( + RunMode, TASK_STATUSES_ACTIVE, TASK_STATUSES_FINAL, TASK_STATUS_WAITING, @@ -70,6 +71,8 @@ ) from cylc.flow.wallclock import get_current_time_string from cylc.flow.platforms import get_platform +from cylc.flow.run_modes.skip import ( + process_outputs as get_skip_mode_outputs) from cylc.flow.task_outputs import ( TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_EXPIRED, @@ -1414,7 +1417,10 @@ def spawn_on_output(self, itask, output, forced=False): tasks = [c_task] for t in tasks: - t.satisfy_me([itask.tokens.duplicate(task_sel=output)]) + t.satisfy_me( + [itask.tokens.duplicate(task_sel=output)], + getattr(itask.tdef, 'run_mode', RunMode.LIVE) + ) self.data_store_mgr.delta_task_prerequisite(t) if not in_pool: self.add_to_pool(t) @@ -1538,7 +1544,8 @@ def spawn_on_all_outputs( continue if completed_only: c_task.satisfy_me( - [itask.tokens.duplicate(task_sel=message)] + [itask.tokens.duplicate(task_sel=message)], + itask.tdef.run_mode ) self.data_store_mgr.delta_task_prerequisite(c_task) self.add_to_pool(c_task) @@ -1952,6 +1959,8 @@ def _set_outputs_itask( """Set requested outputs on a task proxy and spawn children.""" if not outputs: outputs = list(itask.state.outputs.iter_required_messages()) + elif outputs == [RunMode.SKIP]: + outputs = get_skip_mode_outputs(itask) else: outputs = self._standardise_outputs( itask.point, itask.tdef, outputs diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py index 4e7b60d6e0a..80fe210fd80 100644 --- a/cylc/flow/task_proxy.py +++ b/cylc/flow/task_proxy.py @@ -39,6 +39,7 @@ from cylc.flow.platforms import get_platform from cylc.flow.task_action_timer import TimerFlags from cylc.flow.task_state import ( + RunMode, TaskState, TASK_STATUS_WAITING, TASK_STATUS_EXPIRED, @@ -53,7 +54,7 @@ if TYPE_CHECKING: from cylc.flow.cycling import PointBase - from cylc.flow.simulation import ModeSettings + from cylc.flow.run_modes.simulation import ModeSettings from cylc.flow.task_action_timer import TaskActionTimer from cylc.flow.taskdef import TaskDef from cylc.flow.id import Tokens @@ -544,7 +545,7 @@ def state_reset( return False def satisfy_me( - self, task_messages: 'List[Tokens]' + self, task_messages: 'List[Tokens]', mode=RunMode.LIVE ) -> 'Set[Tokens]': """Try to satisfy my prerequisites with given output messages. @@ -554,7 +555,7 @@ def satisfy_me( Return a set of unmatched task messages. """ - used = self.state.satisfy_me(task_messages) + used = self.state.satisfy_me(task_messages, mode) return set(task_messages) - used def clock_expire(self) -> bool: diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py index ebb3dbc985b..6fe5a200c1d 100644 --- a/cylc/flow/task_state.py +++ b/cylc/flow/task_state.py @@ -23,6 +23,7 @@ from cylc.flow.wallclock import get_current_time_string if TYPE_CHECKING: + from cylc.flow.option_parsers import Values from cylc.flow.id import Tokens @@ -154,6 +155,62 @@ } +class RunMode: + """The possible run modes of a task/workflow.""" + + LIVE = 'live' + """Task will run normally.""" + + SIMULATION = 'simulation' + """Task will run in simulation mode.""" + + DUMMY = 'dummy' + """Task will run in dummy mode.""" + + SKIP = 'skip' + """Task will run in skip mode.""" + + WORKFLOW = 'workflow' + """Default to workflow run mode""" + + MODES = {LIVE, SIMULATION, DUMMY, SKIP, WORKFLOW} + + WORKFLOW_MODES = [LIVE, DUMMY, SIMULATION, SKIP] + """Workflow mode not sensible mode for workflow. + + n.b. converted to a list to ensure ordering doesn't change in + CLI + """ + + JOB_MODES = {LIVE, DUMMY} + """Modes which need to have real jobs submitted.""" + + JOBLESS_MODES = {SKIP, SIMULATION} + """Modes which completely ignore the standard submission path.""" + + @staticmethod + def get(options: 'Values') -> str: + """Return the run mode from the options.""" + return getattr(options, 'run_mode', None) or RunMode.LIVE + + @staticmethod + def disable_task_event_handlers(itask): + """Should we disable event handlers for this task? + + No event handlers in simulation mode, or in skip mode + if we don't deliberately enable them: + """ + mode = itask.tdef.run_mode + return ( + mode == RunMode.SIMULATION + or ( + mode == RunMode.SKIP + and itask.tdef.rtconfig['skip'][ + 'disable task event handlers'] is True + ) + ) + + def status_leq(status_a, status_b): """"Return True if status_a <= status_b""" return (TASK_STATUSES_ORDERED.index(status_a) <= @@ -312,7 +369,8 @@ def __call__( def satisfy_me( self, - outputs: Iterable['Tokens'] + outputs: Iterable['Tokens'], + mode, ) -> Set['Tokens']: """Try to satisfy my prerequisites with given outputs. @@ -320,7 +378,7 @@ def satisfy_me( """ valid: Set[Tokens] = set() for prereq in (*self.prerequisites, *self.suicide_prerequisites): - yep = prereq.satisfy_me(outputs) + yep = prereq.satisfy_me(outputs, mode) if yep: valid = valid.union(yep) continue diff --git a/cylc/flow/unicode_rules.py b/cylc/flow/unicode_rules.py index a6974888248..4e29e4eff07 100644 --- a/cylc/flow/unicode_rules.py +++ b/cylc/flow/unicode_rules.py @@ -23,7 +23,7 @@ _TASK_NAME_PREFIX, ) from cylc.flow.task_qualifiers import TASK_QUALIFIERS -from cylc.flow.task_state import TASK_STATUSES_ORDERED +from cylc.flow.task_state import TASK_STATUSES_ORDERED, RunMode ENGLISH_REGEX_MAP = { r'\w': 'alphanumeric', @@ -351,6 +351,8 @@ class TaskOutputValidator(UnicodeRuleChecker): not_starts_with('_cylc'), # blacklist keywords not_equals('required', 'optional', 'all', 'and', 'or'), + # blacklist Run Modes: + not_equals(*RunMode.MODES), # blacklist built-in task qualifiers and statuses (e.g. "waiting") not_equals(*sorted({*TASK_QUALIFIERS, *TASK_STATUSES_ORDERED})), ] diff --git a/cylc/flow/workflow_status.py b/cylc/flow/workflow_status.py index d6d6fb587dc..a8d8db38802 100644 --- a/cylc/flow/workflow_status.py +++ b/cylc/flow/workflow_status.py @@ -156,6 +156,7 @@ def get_workflow_status(schd: 'Scheduler') -> WorkflowStatus: return WorkflowStatus.PAUSED return WorkflowStatus.RUNNING +<<<<<<< HEAD def get_workflow_status_msg(schd: 'Scheduler') -> str: """Return a short, concise status message for the provided workflow.""" @@ -220,3 +221,6 @@ class RunMode: def get(options: 'Values') -> str: """Return the run mode from the options.""" return getattr(options, 'run_mode', None) or RunMode.LIVE +======= + return (status.value, status_msg) +>>>>>>> 8932dbfeb (Implement Skip Mode) diff --git a/tests/functional/cylc-config/00-simple/section2.stdout b/tests/functional/cylc-config/00-simple/section2.stdout index 3d83ac15278..a85ed449573 100644 --- a/tests/functional/cylc-config/00-simple/section2.stdout +++ b/tests/functional/cylc-config/00-simple/section2.stdout @@ -15,10 +15,14 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -90,10 +94,14 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -165,10 +173,14 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -240,12 +252,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = serial [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -316,12 +332,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -392,12 +412,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = serial [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -468,12 +492,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = serial [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -544,12 +572,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -620,12 +652,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -696,12 +732,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = serial [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -772,12 +812,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = serial [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -848,12 +892,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = @@ -924,12 +972,16 @@ execution time limit = submission polling intervals = submission retry delays = + run mode = workflow [[[directives]]] job_type = parallel [[[meta]]] title = description = URL = + [[[skip]]] + outputs = + disable task event handlers = True [[[simulation]]] default run length = PT10S speedup factor = diff --git a/tests/functional/cylc-set/09-set-skip.t b/tests/functional/cylc-set/09-set-skip.t new file mode 100644 index 00000000000..dd314283700 --- /dev/null +++ b/tests/functional/cylc-set/09-set-skip.t @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# +# Skip Mode proposal example: +# https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md +# The cylc set --out option should accept the skip value +# which should set the outputs defined in +# [runtime][][skip]outputs. + +. "$(dirname "$0")/test_header" +set_test_number 2 +reftest +exit diff --git a/tests/functional/cylc-set/09-set-skip/flow.cylc b/tests/functional/cylc-set/09-set-skip/flow.cylc new file mode 100644 index 00000000000..ef74c362773 --- /dev/null +++ b/tests/functional/cylc-set/09-set-skip/flow.cylc @@ -0,0 +1,50 @@ +[meta] + test_description = """ + Test that cylc set --out skip satisfies + all outputs which are required by the graph. + """ + proposal url = https://github.com/cylc/cylc-admin/blob/master/docs/proposal-skip-mode.md + +[scheduler] + allow implicit tasks = true + [[events]] + expected task failures = 1/bar + +[scheduling] + [[graph]] + R1 = """ + # Optional out not created by set --out skip + foo:no? => not_this_task? + + # set --out skip creates required, started, submitted + # and succeeded (unless failed is set): + foo:yes => require_this_task + foo:submitted => submitted_emitted + foo:succeeded => succeeded_emitted + foo:started => skip_foo + + # set --out skip creates failed if that is required + # by skip mode settings: + bar:started => skip_bar + bar:failed? => bar_failed + """ + +[runtime] + [[foo]] + script = sleep 100 + [[[skip]]] + outputs = yes + [[[outputs]]] + no = 'Don\'t require this task' + yes = 'Require this task' + + [[bar]] + script = sleep 100 + [[[skip]]] + outputs = failed + + [[skip_foo]] + script = cylc set ${CYLC_WORKFLOW_ID}//1/foo --out skip + + [[skip_bar]] + script = cylc set ${CYLC_WORKFLOW_ID}//1/bar --out skip diff --git a/tests/functional/cylc-set/09-set-skip/reference.log b/tests/functional/cylc-set/09-set-skip/reference.log new file mode 100644 index 00000000000..6e7b636f540 --- /dev/null +++ b/tests/functional/cylc-set/09-set-skip/reference.log @@ -0,0 +1,8 @@ +1/bar -triggered off [] in flow 1 +1/foo -triggered off [] in flow 1 +1/submitted_emitted -triggered off ['1/foo'] in flow 1 +1/skip_bar -triggered off ['1/bar'] in flow 1 +1/skip_foo -triggered off ['1/foo'] in flow 1 +1/succeeded_emitted -triggered off ['1/foo'] in flow 1 +1/bar_failed -triggered off ['1/bar'] in flow 1 +1/require_this_task -triggered off ['1/foo'] in flow 1 diff --git a/tests/functional/modes/01-dummy.t b/tests/functional/run_modes/01-dummy.t similarity index 100% rename from tests/functional/modes/01-dummy.t rename to tests/functional/run_modes/01-dummy.t diff --git a/tests/functional/modes/01-dummy/flow.cylc b/tests/functional/run_modes/01-dummy/flow.cylc similarity index 100% rename from tests/functional/modes/01-dummy/flow.cylc rename to tests/functional/run_modes/01-dummy/flow.cylc diff --git a/tests/functional/modes/01-dummy/reference.log b/tests/functional/run_modes/01-dummy/reference.log similarity index 100% rename from tests/functional/modes/01-dummy/reference.log rename to tests/functional/run_modes/01-dummy/reference.log diff --git a/tests/functional/modes/02-dummy-message-outputs.t b/tests/functional/run_modes/02-dummy-message-outputs.t similarity index 100% rename from tests/functional/modes/02-dummy-message-outputs.t rename to tests/functional/run_modes/02-dummy-message-outputs.t diff --git a/tests/functional/modes/02-dummy-message-outputs/flow.cylc b/tests/functional/run_modes/02-dummy-message-outputs/flow.cylc similarity index 100% rename from tests/functional/modes/02-dummy-message-outputs/flow.cylc rename to tests/functional/run_modes/02-dummy-message-outputs/flow.cylc diff --git a/tests/functional/modes/02-dummy-message-outputs/reference.log b/tests/functional/run_modes/02-dummy-message-outputs/reference.log similarity index 100% rename from tests/functional/modes/02-dummy-message-outputs/reference.log rename to tests/functional/run_modes/02-dummy-message-outputs/reference.log diff --git a/tests/functional/modes/03-simulation.t b/tests/functional/run_modes/03-simulation.t similarity index 100% rename from tests/functional/modes/03-simulation.t rename to tests/functional/run_modes/03-simulation.t diff --git a/tests/functional/modes/03-simulation/flow.cylc b/tests/functional/run_modes/03-simulation/flow.cylc similarity index 100% rename from tests/functional/modes/03-simulation/flow.cylc rename to tests/functional/run_modes/03-simulation/flow.cylc diff --git a/tests/functional/modes/03-simulation/reference.log b/tests/functional/run_modes/03-simulation/reference.log similarity index 100% rename from tests/functional/modes/03-simulation/reference.log rename to tests/functional/run_modes/03-simulation/reference.log diff --git a/tests/functional/modes/04-simulation-runtime.t b/tests/functional/run_modes/04-simulation-runtime.t similarity index 100% rename from tests/functional/modes/04-simulation-runtime.t rename to tests/functional/run_modes/04-simulation-runtime.t diff --git a/tests/functional/modes/04-simulation-runtime/flow.cylc b/tests/functional/run_modes/04-simulation-runtime/flow.cylc similarity index 100% rename from tests/functional/modes/04-simulation-runtime/flow.cylc rename to tests/functional/run_modes/04-simulation-runtime/flow.cylc diff --git a/tests/functional/modes/04-simulation-runtime/reference.log b/tests/functional/run_modes/04-simulation-runtime/reference.log similarity index 100% rename from tests/functional/modes/04-simulation-runtime/reference.log rename to tests/functional/run_modes/04-simulation-runtime/reference.log diff --git a/tests/functional/modes/05-sim-trigger.t b/tests/functional/run_modes/05-sim-trigger.t similarity index 100% rename from tests/functional/modes/05-sim-trigger.t rename to tests/functional/run_modes/05-sim-trigger.t diff --git a/tests/functional/modes/05-sim-trigger/flow.cylc b/tests/functional/run_modes/05-sim-trigger/flow.cylc similarity index 100% rename from tests/functional/modes/05-sim-trigger/flow.cylc rename to tests/functional/run_modes/05-sim-trigger/flow.cylc diff --git a/tests/functional/modes/05-sim-trigger/reference.log b/tests/functional/run_modes/05-sim-trigger/reference.log similarity index 100% rename from tests/functional/modes/05-sim-trigger/reference.log rename to tests/functional/run_modes/05-sim-trigger/reference.log diff --git a/tests/functional/run_modes/06-run-mode-overrides.t b/tests/functional/run_modes/06-run-mode-overrides.t new file mode 100644 index 00000000000..2e90deaa460 --- /dev/null +++ b/tests/functional/run_modes/06-run-mode-overrides.t @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Test that broadcasting a change in [runtime][]run mode +# Leads to the next submission from that task to be in the updated +# mode. + +. "$(dirname "$0")/test_header" +set_test_number 15 + +install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" +run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" +workflow_run_ok "${TEST_NAME_BASE}-run" \ + cylc play "${WORKFLOW_NAME}" \ + --no-detach + +JOB_LOGS="${WORKFLOW_RUN_DIR}/log/job/1000" + +# Ghost modes do not leave log folders: +for MODE in simulation skip; do + run_fail "${TEST_NAME_BASE}-no-${MODE}-task-folder" ls "${JOB_LOGS}/${MODE}_" +done + +# Live modes leave log folders: +for MODE in default live dummy; do + run_ok "${TEST_NAME_BASE}-${MODE}-task-folder" ls "${JOB_LOGS}/${MODE}_" +done + +# Default defaults to live, and live is live: +for MODE in default live; do + named_grep_ok "${TEST_NAME_BASE}-default-task-live" "===.*===" "${JOB_LOGS}/${MODE}_/NN/job.out" +done + +# Dummy produces a job.out, containing dummy message: +named_grep_ok "${TEST_NAME_BASE}-default-task-live" "dummy job succeed" "${JOB_LOGS}/dummy_/NN/job.out" + +purge + +# Do it again with a workflow in simulation. +install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" +run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" +workflow_run_ok "${TEST_NAME_BASE}-run" \ + cylc play "${WORKFLOW_NAME}" \ + --no-detach \ + --mode simulation + +JOB_LOGS="${WORKFLOW_RUN_DIR}/log/job/1000" + +# Live modes leave log folders: +for MODE in live dummy; do + run_ok "${TEST_NAME_BASE}-${MODE}-task-folder" ls "${JOB_LOGS}/${MODE}_" +done + +# Ghost modes do not leave log folders: +run_fail "${TEST_NAME_BASE}-no-default-task-folder" ls "${JOB_LOGS}/default_" + +purge +exit 0 diff --git a/tests/functional/run_modes/06-run-mode-overrides/flow.cylc b/tests/functional/run_modes/06-run-mode-overrides/flow.cylc new file mode 100644 index 00000000000..b7693232149 --- /dev/null +++ b/tests/functional/run_modes/06-run-mode-overrides/flow.cylc @@ -0,0 +1,29 @@ +[scheduler] + cycle point format = %Y + +[scheduling] + initial cycle point = 1000 + final cycle point = 1000 + [[graph]] + P1Y = """ + default_ + live_ + dummy_ + simulation_ + skip_ + """ + +[runtime] + [[root]] + script = echo "=== this task ran in live mode ===" + [[[simulation]]] + default run length = PT0S + [[default_]] + [[live_]] + run mode = live + [[dummy_]] + run mode = dummy + [[simulation_]] + run mode = simulation + [[skip_]] + run mode = skip diff --git a/tests/functional/modes/test_header b/tests/functional/run_modes/test_header similarity index 100% rename from tests/functional/modes/test_header rename to tests/functional/run_modes/test_header diff --git a/tests/integration/run_modes/test_mode_overrides.py b/tests/integration/run_modes/test_mode_overrides.py new file mode 100644 index 00000000000..209837ebb7e --- /dev/null +++ b/tests/integration/run_modes/test_mode_overrides.py @@ -0,0 +1,109 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Test that using [runtime][TASK]run mode works in each mode. + +TODO: This is pretty much a functional test and +probably ought to be labelled as such, but uses the +integration test framework. +""" + +import pytest + + +@pytest.mark.parametrize( + 'workflow_run_mode', [('live'), ('simulation'), ('dummy')]) #, ('skip')]) +async def test_run_mode_override( + workflow_run_mode, flow, scheduler, run, complete, log_filter +): + """Test that ``[runtime][TASK]run mode`` overrides workflow modes. + + Can only be run for tasks which run in ghost modes. + """ + default_ = ( + '\ndefault_' if workflow_run_mode in ['simulation', 'skip'] else '') + + cfg = { + "scheduler": {"cycle point format": "%Y"}, + "scheduling": { + "initial cycle point": "1000", + "final cycle point": "1000", + "graph": {"P1Y": f"sim_\nskip_{default_}"}}, + "runtime": { + "sim_": { + "run mode": "simulation", + 'simulation': {'default run length': 'PT0S'} + }, + "skip_": {"run mode": "skip"}, + } + } + id_ = flow(cfg) + schd = scheduler(id_, run_mode=workflow_run_mode, paused_start=False) + expect = ('[1000/sim_] run mode set by task settings to: simulation mode.') + + async with run(schd) as log: + await complete(schd) + if workflow_run_mode == 'simulation': + # No message in simulation mode. + assert not log_filter(log, contains=expect) + else: + assert log_filter(log, contains=expect) + + +@pytest.mark.parametrize('mode', (('skip'), ('simulation'), ('dummy'))) +async def test_force_trigger_does_not_override_run_mode( + flow, + scheduler, + start, + mode, +): + """Force-triggering a task will not override the run mode. + + Tasks with run mode = skip will continue to abide by + the is_held flag as normal. + + Taken from spec at + https://github.com/cylc/cylc-admin/blob/master/ + docs/proposal-skip-mode.md#proposal + """ + wid = flow({ + 'scheduling': {'graph': {'R1': 'foo'}}, + 'runtime': {'foo': {'run mode': mode}} + }) + schd = scheduler(wid) + async with start(schd): + # Check that task isn't held at first + foo = schd.pool.get_tasks()[0] + assert foo.state.is_held is False + + # Hold task, check that it's held: + schd.pool.hold_tasks('1/foo') + assert foo.state.is_held is True + + # Trigger task, check that it's _still_ held: + schd.pool.force_trigger_tasks('1/foo', [1]) + assert foo.state.is_held is True + + # run_mode will always be simulation from test + # workflow before submit routine... + assert foo.tdef.run_mode == 'simulation' + + # ... but job submission will always change this to the correct mode: + schd.task_job_mgr.submit_task_jobs( + schd.workflow, + [foo], + schd.server.curve_auth, + schd.server.client_pub_key_dir) + assert foo.tdef.run_mode == mode diff --git a/tests/integration/test_simulation.py b/tests/integration/run_modes/test_simulation.py similarity index 90% rename from tests/integration/test_simulation.py rename to tests/integration/run_modes/test_simulation.py index c7e1b42fe27..03006220f2c 100644 --- a/tests/integration/test_simulation.py +++ b/tests/integration/run_modes/test_simulation.py @@ -20,7 +20,7 @@ from cylc.flow import commands from cylc.flow.cycling.iso8601 import ISO8601Point -from cylc.flow.simulation import sim_time_check +from cylc.flow.run_modes.simulation import sim_time_check @pytest.fixture @@ -28,7 +28,8 @@ def monkeytime(monkeypatch): """Convenience function monkeypatching time.""" def _inner(time_: int): monkeypatch.setattr('cylc.flow.task_job_mgr.time', lambda: time_) - monkeypatch.setattr('cylc.flow.simulation.time', lambda: time_) + monkeypatch.setattr( + 'cylc.flow.run_modes.simulation.time', lambda: time_) return _inner @@ -42,8 +43,8 @@ def _run_simjob(schd, point, task): itask = schd.pool.get_task(point, task) itask.state.is_queued = False monkeytime(0) - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') monkeytime(itask.mode_settings.timeout + 1) # Run Time Check @@ -150,8 +151,8 @@ def test_fail_once(sim_time_check_setup, itask, point, results, monkeypatch): for i, result in enumerate(results): itask.try_timers['execution-retry'].num = i - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') assert itask.mode_settings.sim_task_fails is result @@ -170,11 +171,11 @@ def test_task_finishes(sim_time_check_setup, monkeytime, caplog): fail_all_1066 = schd.pool.get_task(ISO8601Point('1066'), 'fail_all') fail_all_1066.state.status = 'running' fail_all_1066.state.is_queued = False - schd.task_job_mgr._simulation_submit_task_jobs( - [fail_all_1066], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [fail_all_1066], schd.workflow, 'simulation') # For the purpose of the test delete the started time set by - # _simulation_submit_task_jobs. + # _nonlive_submit_task_jobs. fail_all_1066.summary['started_time'] = 0 # Before simulation time is up: @@ -200,8 +201,8 @@ def test_task_sped_up(sim_time_check_setup, monkeytime): # Run the job submission method: monkeytime(0) - schd.task_job_mgr._simulation_submit_task_jobs( - [fast_forward_1066], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [fast_forward_1066], schd.workflow, 'simulation') fast_forward_1066.state.is_queued = False result = sim_time_check(schd.task_events_mgr, [fast_forward_1066], '') @@ -254,8 +255,8 @@ async def test_settings_restart( async with start(schd): og_timeouts = {} for itask in schd.pool.get_tasks(): - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') og_timeouts[itask.identity] = itask.mode_settings.timeout @@ -379,8 +380,8 @@ async def test_settings_broadcast( itask.state.is_queued = False # Submit the first - the sim task will fail: - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') assert itask.mode_settings.sim_task_fails is True # Let task finish. @@ -398,14 +399,14 @@ async def test_settings_broadcast( 'simulation': {'fail cycle points': ''} }]) # Submit again - result is different: - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') assert itask.mode_settings.sim_task_fails is False # Assert Clearing the broadcast works schd.broadcast_mgr.clear_broadcast() - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') assert itask.mode_settings.sim_task_fails is True # Assert that list of broadcasts doesn't change if we submit @@ -415,18 +416,22 @@ async def test_settings_broadcast( ['1066'], ['one'], [{ 'simulation': {'fail cycle points': 'higadfuhasgiurguj'} }]) - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') assert ( 'Invalid ISO 8601 date representation: higadfuhasgiurguj' in log.messages[-1]) + # Check that the invalid broadcast hasn't + # changed the itask sim mode settings: + assert itask.mode_settings.sim_task_fails is True + schd.broadcast_mgr.put_broadcast( ['1066'], ['one'], [{ 'simulation': {'fail cycle points': '1'} }]) - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') assert ( 'Invalid ISO 8601 date representation: 1' in log.messages[-1]) @@ -437,8 +442,8 @@ async def test_settings_broadcast( 'simulation': {'fail cycle points': '1945, 1977, 1066'}, 'execution retry delays': '3*PT2S' }]) - schd.task_job_mgr._simulation_submit_task_jobs( - [itask], schd.workflow) + schd.task_job_mgr._nonlive_submit_task_jobs( + [itask], schd.workflow, 'simulation') assert itask.mode_settings.sim_task_fails is True assert itask.try_timers['execution-retry'].delays == [2.0, 2.0, 2.0] # n.b. rtconfig should remain unchanged, lest we cancel broadcasts: diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py index 6e7454f1293..e2022f61ea4 100644 --- a/tests/integration/test_config.py +++ b/tests/integration/test_config.py @@ -15,7 +15,9 @@ # along with this program. If not, see . from pathlib import Path +import re import sqlite3 +from textwrap import dedent from typing import Any import pytest @@ -503,3 +505,46 @@ def test_special_task_non_word_names(flow: Fixture, validate: Fixture): }, }) validate(wid) + + +def test_nonlive_mode_validation(flow, validate, caplog): + """Nonlive tasks return a warning at validation. + """ + msg1 = dedent( + ''' The following tasks are set to run in non-live mode: + simulation mode: + * simulation + dummy mode: + * dummy''') + + wid = flow({ + 'scheduling': { + 'graph': { + 'R1': 'live => skip => simulation => dummy => default' + } + }, + 'runtime': { + 'default': {}, + 'live': {'run mode': 'live'}, + 'simulation': {'run mode': 'simulation'}, + 'dummy': {'run mode': 'dummy'}, + 'skip': { + 'run mode': 'skip', + 'skip': {'outputs': 'started, submitted'} + }, + }, + }) + + validate(wid) + assert msg1 in caplog.messages + + +@pytest.mark.parametrize('mode', (('simulation'), ('skip'), ('dummy'))) +def test_nonlive_mode_forbidden_as_outputs(flow, validate, mode): + """Run mode names are forbidden as task output names.""" + wid = flow({ + 'scheduling': {'graph': {'R1': 'task'}}, + 'runtime': {'task': {'outputs': {mode: f'message for {mode}'}}} + }) + with pytest.raises(WorkflowConfigError, match=f'message for {mode}'): + validate(wid) diff --git a/tests/integration/test_task_pool.py b/tests/integration/test_task_pool.py index beba9075bd3..995e289ed3f 100644 --- a/tests/integration/test_task_pool.py +++ b/tests/integration/test_task_pool.py @@ -644,7 +644,8 @@ def list_tasks(schd): ('1', 'z', 'waiting'), ], [ - {('1', 'a', 'succeeded'): 'satisfied naturally'}, + {('1', 'a', 'succeeded'): + 'Artificially satisfied by simulation mode'}, {('1', 'b', 'succeeded'): False}, {('1', 'c', 'succeeded'): False}, ], @@ -672,7 +673,8 @@ def list_tasks(schd): ('1', 'z', 'waiting'), ], [ - {('1', 'a', 'succeeded'): 'satisfied naturally'}, + {('1', 'a', 'succeeded'): + 'Artificially satisfied by simulation mode'}, {('1', 'b', 'succeeded'): False}, ], id='removed' @@ -767,7 +769,8 @@ async def test_restart_prereqs( ('1', 'z', 'waiting'), ], [ - {('1', 'a', 'succeeded'): 'satisfied naturally'}, + {('1', 'a', 'succeeded'): + 'Artificially satisfied by simulation mode'}, {('1', 'b', 'succeeded'): False}, {('1', 'c', 'succeeded'): False}, ], @@ -795,7 +798,8 @@ async def test_restart_prereqs( ('1', 'z', 'waiting'), ], [ - {('1', 'a', 'succeeded'): 'satisfied naturally'}, + {('1', 'a', 'succeeded'): + 'Artificially satisfied by simulation mode'}, {('1', 'b', 'succeeded'): False}, ], id='removed' @@ -893,7 +897,7 @@ async def _test_restart_prereqs_sat(): for prereq in task_c.state.prerequisites for key, satisfied in prereq.satisfied.items() ) == [ - ('1', 'a', 'succeeded', 'satisfied naturally'), + ('1', 'a', 'succeeded', 'Artificially satisfied by simulation mode'), ('1', 'b', 'succeeded', 'satisfied from database') ] @@ -910,7 +914,7 @@ async def _test_restart_prereqs_sat(): for prereq in task_c_prereqs for condition in prereq.conditions ) == [ - ('1/a', True, 'satisfied naturally'), + ('1/a', True, 'Artificially satisfied by simulation mode'), ('1/b', True, 'satisfied from database'), ] diff --git a/tests/unit/run_modes/test_dummy.py b/tests/unit/run_modes/test_dummy.py new file mode 100644 index 00000000000..998c13767c9 --- /dev/null +++ b/tests/unit/run_modes/test_dummy.py @@ -0,0 +1,40 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Tests for utilities supporting dummy mode. +""" +import pytest +from cylc.flow.run_modes.dummy import build_dummy_script + + +@pytest.mark.parametrize( + 'fail_one_time_only', (True, False) +) +def test_build_dummy_script(fail_one_time_only): + rtc = { + 'outputs': {'foo': '1', 'bar': '2'}, + 'simulation': { + 'fail try 1 only': fail_one_time_only, + 'fail cycle points': '1', + } + } + result = build_dummy_script(rtc, 60) + assert result.split('\n') == [ + 'sleep 60', + "cylc message '1'", + "cylc message '2'", + f"cylc__job__dummy_result {str(fail_one_time_only).lower()}" + " 1 || exit 1" + ] diff --git a/tests/unit/run_modes/test_nonlive.py b/tests/unit/run_modes/test_nonlive.py new file mode 100644 index 00000000000..6377e315007 --- /dev/null +++ b/tests/unit/run_modes/test_nonlive.py @@ -0,0 +1,46 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Unit tests for utilities supporting all nonlive modes +""" + +from types import SimpleNamespace + +from cylc.flow.run_modes.nonlive import mode_validate_checks + + +def test_mode_validate_checks(monkeypatch, caplog): + """It warns us if we've set a task config to nonlive mode. + + (And not otherwise) + """ + taskdefs = { + f'{run_mode}_task': SimpleNamespace( + rtconfig={'run mode': run_mode}, + name=f'{run_mode}_task' + ) + for run_mode + in ['live', 'workflow', 'dummy', 'simulation', 'skip'] + } + + mode_validate_checks(taskdefs) + + message = caplog.messages[0] + + assert 'skip mode:\n * skip_task' not in message + assert 'simulation mode:\n * simulation_task' in message + assert 'dummy mode:\n * dummy_task' in message + assert ' live mode' not in message # Avoid matching "non-live mode" + assert 'workflow mode' not in message diff --git a/tests/unit/test_simulation.py b/tests/unit/run_modes/test_simulation.py similarity index 86% rename from tests/unit/test_simulation.py rename to tests/unit/run_modes/test_simulation.py index 920a872503a..109174c8b43 100644 --- a/tests/unit/test_simulation.py +++ b/tests/unit/run_modes/test_simulation.py @@ -20,9 +20,8 @@ from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point -from cylc.flow.simulation import ( +from cylc.flow.run_modes.simulation import ( parse_fail_cycle_points, - build_dummy_script, disable_platforms, get_simulated_run_len, sim_task_failed, @@ -56,27 +55,6 @@ def test_get_simulated_run_len( assert get_simulated_run_len(rtc) == 3600 -@pytest.mark.parametrize( - 'fail_one_time_only', (True, False) -) -def test_set_simulation_script(fail_one_time_only): - rtc = { - 'outputs': {'foo': '1', 'bar': '2'}, - 'simulation': { - 'fail try 1 only': fail_one_time_only, - 'fail cycle points': '1', - } - } - result = build_dummy_script(rtc, 60) - assert result.split('\n') == [ - 'sleep 60', - "cylc message '1'", - "cylc message '2'", - f"cylc__job__dummy_result {str(fail_one_time_only).lower()}" - " 1 || exit 1" - ] - - @pytest.mark.parametrize( 'rtc, expect', ( ({'platform': 'skarloey'}, 'localhost'), @@ -100,7 +78,7 @@ def test_disable_platforms(rtc, expect): def test_parse_fail_cycle_points(set_cycling_type): before = ['2', '4'] set_cycling_type() - assert parse_fail_cycle_points(before) == [ + assert parse_fail_cycle_points(before, ['']) == [ IntegerPoint(i) for i in before ] diff --git a/tests/unit/run_modes/test_skip.py b/tests/unit/run_modes/test_skip.py new file mode 100644 index 00000000000..1de6e79a493 --- /dev/null +++ b/tests/unit/run_modes/test_skip.py @@ -0,0 +1,105 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Unit tests for utilities supporting skip modes +""" +import pytest +from pytest import param, raises +from types import SimpleNamespace + +from cylc.flow.exceptions import WorkflowConfigError +from cylc.flow.run_modes.skip import check_task_skip_config, process_outputs + + +@pytest.mark.parametrize( + 'conf', + ( + param({}, id='no-skip-config'), + param({'skip': {'outputs': []}}, id='no-skip-outputs'), + param({'skip': {'outputs': ['foo1', 'failed']}}, id='ok-skip-outputs'), + ) +) +def test_good_check_task_skip_config(conf): + """It returns none if the problems this function checks are not present. + """ + tdef = SimpleNamespace(rtconfig=conf) + tdef.name = 'foo' + assert check_task_skip_config(tdef) is None + + +def test_raises_check_task_skip_config(): + """It raises an error if succeeded and failed are set. + """ + tdef = SimpleNamespace( + rtconfig={'skip': {'outputs': ['foo1', 'failed', 'succeeded']}} + ) + tdef.name = 'foo' + with raises(WorkflowConfigError, match='succeeded AND failed'): + check_task_skip_config(tdef) + + +@pytest.mark.parametrize( + 'outputs, required, expect', + ( + param([], [], ['succeeded'], id='implicit-succeded'), + param( + ['succeeded'], ['succeeded'], ['succeeded'], + id='explicit-succeded' + ), + param(['submitted'], [], ['succeeded'], id='only-1-submit'), + param( + ['foo', 'bar', 'baz', 'qux'], + ['bar', 'qux'], + ['bar', 'qux', 'succeeded'], + id='required-only' + ), + param( + ['foo', 'baz'], + ['bar', 'qux'], + ['succeeded'], + id='no-required' + ), + param( + ['failed'], + [], + ['failed'], + id='explicit-failed' + ), + ) +) +def test_process_outputs(outputs, required, expect): + """Check that skip outputs: + + 1. Doesn't send submitted twice. + 2. Sends every required output. + 3. If failed is set send failed + 4. If failed in not set send succeeded. + + n.b: The real process message function sends the TASK_OUTPUT_STARTED + message for free, so there is no reference to that here. + """ + + + # Create a mocked up task-proxy: + itask = SimpleNamespace( + tdef=SimpleNamespace( + rtconfig={'skip': {'outputs': outputs}}), + state=SimpleNamespace( + outputs=SimpleNamespace( + iter_required_messages=lambda: iter(required), + _message_to_trigger={v: v for v in required} + ))) + + assert process_outputs(itask) == ['submitted'] + expect diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py index 527a33bdfb5..dbb2e014071 100644 --- a/tests/unit/scripts/test_lint.py +++ b/tests/unit/scripts/test_lint.py @@ -181,7 +181,10 @@ [[[directives]]] -l walltime = 666 [[baz]] + run mode = skip platform = `no backticks` + [[[skip]]] + outputs = succeeded, failed """ + ( '\nscript = the quick brown fox jumps over the lazy dog until it becomes ' 'clear that this line is longer than the default 130 character limit.' diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 9cdcee89003..b830228103a 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -17,8 +17,8 @@ import os import sys from optparse import Values -from typing import Any, Callable, Dict, List, Optional, Tuple, Type -from pathlib import Path +from typing import ( + TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type) import pytest import logging from types import SimpleNamespace @@ -47,8 +47,9 @@ from cylc.flow.cycling.iso8601 import ISO8601Point - -Fixture = Any +if TYPE_CHECKING: + from pathlib import Path + Fixture = Any def _tmp_flow_config(tmp_run_dir: Callable): @@ -60,8 +61,8 @@ def _tmp_flow_config(tmp_run_dir: Callable): Returns the path to the flow file. """ - def __tmp_flow_config(id_: str, config: str) -> Path: - run_dir: Path = tmp_run_dir(id_) + def __tmp_flow_config(id_: str, config: str) -> 'Path': + run_dir: 'Path' = tmp_run_dir(id_) flow_file = run_dir / WorkflowFiles.FLOW_FILE flow_file.write_text(config) return flow_file @@ -82,7 +83,7 @@ class TestWorkflowConfig: """Test class for the Cylc WorkflowConfig object.""" def test_xfunction_imports( - self, mock_glbl_cfg: Fixture, tmp_path: Path): + self, mock_glbl_cfg: 'Fixture', tmp_path: 'Path'): """Test for a workflow configuration with valid xtriggers""" mock_glbl_cfg( 'cylc.flow.platforms.glbl_cfg', @@ -175,7 +176,8 @@ def test_xfunction_attribute_error(self, mock_glbl_cfg, tmp_path): with pytest.raises(XtriggerConfigError) as excinfo: WorkflowConfig(workflow="capybara_workflow", fpath=flow_file, options=SimpleNamespace()) - assert "module 'capybara' has no attribute 'capybara'" in str(excinfo.value) + assert "module 'capybara' has no attribute 'capybara'" in str( + excinfo.value) def test_xfunction_not_callable(self, mock_glbl_cfg, tmp_path): """Test for error when a xtrigger function is not callable.""" @@ -358,7 +360,7 @@ def test_process_icp( expected_icp: Optional[str], expected_opt_icp: Optional[str], expected_err: Optional[Tuple[Type[Exception], str]], - monkeypatch: pytest.MonkeyPatch, set_cycling_type: Fixture + monkeypatch: pytest.MonkeyPatch, set_cycling_type: 'Fixture' ) -> None: """Test WorkflowConfig.process_initial_cycle_point(). @@ -445,7 +447,7 @@ def test_process_startcp( starttask: Optional[str], expected: str, expected_err: Optional[Tuple[Type[Exception], str]], - monkeypatch: pytest.MonkeyPatch, set_cycling_type: Fixture + monkeypatch: pytest.MonkeyPatch, set_cycling_type: 'Fixture' ) -> None: """Test WorkflowConfig.process_start_cycle_point(). @@ -648,7 +650,7 @@ def test_process_fcp( options_fcp: Optional[str], expected_fcp: Optional[str], expected_err: Optional[Tuple[Type[Exception], str]], - set_cycling_type: Fixture + set_cycling_type: 'Fixture' ) -> None: """Test WorkflowConfig.process_final_cycle_point(). @@ -671,7 +673,7 @@ def test_process_fcp( initial_point=loader.get_point( scheduling_cfg['initial cycle point'] ).standardise(), - final_point = None, + final_point=None, options=SimpleNamespace(fcp=options_fcp), ) @@ -812,7 +814,7 @@ def test_stopcp_after_fcp( cycle point is handled correctly.""" caplog.set_level(logging.WARNING, CYLC_LOG) id_ = 'cassini' - flow_file: Path = tmp_flow_config(id_, f""" + flow_file: 'Path' = tmp_flow_config(id_, f""" [scheduler] allow implicit tasks = True [scheduling] @@ -1366,7 +1368,7 @@ def test_implicit_tasks( """ # Setup id_ = 'rincewind' - flow_file: Path = tmp_flow_config(id_, f""" + flow_file: 'Path' = tmp_flow_config(id_, f""" [scheduler] { f'allow implicit tasks = {allow_implicit_tasks}' @@ -1470,7 +1472,7 @@ def test_zero_interval( """Test that a zero-duration recurrence with >1 repetition gets an appropriate warning.""" id_ = 'ordinary' - flow_file: Path = tmp_flow_config(id_, f""" + flow_file: 'Path' = tmp_flow_config(id_, f""" [scheduler] UTC mode = True allow implicit tasks = True @@ -1514,7 +1516,7 @@ def test_chain_expr( Note the order matters when "nominal" units (years, months) are used. """ id_ = 'osgiliath' - flow_file: Path = tmp_flow_config(id_, f""" + flow_file: 'Path' = tmp_flow_config(id_, f""" [scheduler] UTC mode = True allow implicit tasks = True @@ -1693,7 +1695,7 @@ def test__warn_if_queues_have_implicit_tasks(caplog): ] ) def test_cylc_env_at_parsing( - tmp_path: Path, + tmp_path: 'Path', monkeypatch: pytest.MonkeyPatch, installed, run_dir, diff --git a/tests/unit/test_task_state.py b/tests/unit/test_task_state.py index e655c74b7bb..106f4f8e2de 100644 --- a/tests/unit/test_task_state.py +++ b/tests/unit/test_task_state.py @@ -15,11 +15,13 @@ # along with this program. If not, see . import pytest +from types import SimpleNamespace from cylc.flow.taskdef import TaskDef from cylc.flow.cycling.integer import IntegerSequence, IntegerPoint from cylc.flow.task_trigger import Dependency, TaskTrigger from cylc.flow.task_state import ( + RunMode, TaskState, TASK_STATUS_PREPARING, TASK_STATUS_SUBMIT_FAILED, @@ -119,3 +121,29 @@ def test_task_state_order(): assert not tstate.is_gt(TASK_STATUS_RUNNING) assert not tstate.is_gte(TASK_STATUS_RUNNING) + +@pytest.mark.parametrize( + 'itask_run_mode, disable_handlers, expect', + ( + ('live', True, False), + ('live', False, False), + ('dummy', True, False), + ('dummy', False, False), + ('simulation', True, True), + ('simulation', False, True), + ('skip', True, True), + ('skip', False, False), + ) +) +def test_disable_task_event_handlers(itask_run_mode, disable_handlers, expect): + """Conditions under which task event handlers should not be used. + """ + # Construct a fake itask object: + itask = SimpleNamespace( + tdef=SimpleNamespace( + run_mode=itask_run_mode, + rtconfig={ + 'skip': {'disable task event handlers': disable_handlers}}) + ) + # Check method: + assert RunMode.disable_task_event_handlers(itask) is expect