Add a new entry_point for xtriggers (#5831)

* Add a new entry_point for xtriggers

* This creates a clean way to add xtriggers from python packages.
  xtriggers no longer need to be directly in the PYTHONPATH,
  but instead can be added via a cylc.xtriggers entry point.
* Built-in cylc-flow xtriggers are defined via new entry_points.

---------

Co-authored-by: Ronnie Dutta <[email protected]>

changes.d/5831.feat.md

@@ -0,0 +1 @@
+Add capability to install xtriggers via a new cylc.xtriggers entry point

cylc/flow/subprocpool.py

@@ -28,7 +28,7 @@
 from subprocess import DEVNULL, run  # nosec
 from typing import Any, Callable, List, Optional
 
-from cylc.flow import LOG
+from cylc.flow import LOG, iter_entry_points
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.cylc_subproc import procopen
 from cylc.flow.exceptions import PlatformLookupError
@@ -69,29 +69,41 @@ def _killpg(proc, signal):
 def get_func(func_name, src_dir):
     """Find and return an xtrigger function from a module of the same name.
 
-    Can be in <src_dir>/lib/python, CYLC_MOD_LOC, or in Python path.
+    These locations are checked in this order:
+    - <src_dir>/lib/python/
+    - `$CYLC_PYTHONPATH`
+    - defined via a `cylc.xtriggers` entry point for an
+      installed Python package.
+
     Workflow source directory passed in as this is executed in an independent
     process in the command pool and therefore doesn't know about the workflow.
 
     """
     if func_name in _XTRIG_FUNCS:
         return _XTRIG_FUNCS[func_name]
+
     # First look in <src-dir>/lib/python.
     sys.path.insert(0, os.path.join(src_dir, 'lib', 'python'))
     mod_name = func_name
     try:
         mod_by_name = __import__(mod_name, fromlist=[mod_name])
     except ImportError:
-        # Then look in built-in xtriggers.
-        mod_name = "%s.%s" % ("cylc.flow.xtriggers", func_name)
-        try:
-            mod_by_name = __import__(mod_name, fromlist=[mod_name])
-        except ImportError:
-            raise
+        # Look for xtriggers via entry_points for external sources.
+        # Do this after the lib/python and PYTHONPATH approaches to allow
+        # users to override entry_point definitions with local/custom
+        # implementations.
+        for entry_point in iter_entry_points('cylc.xtriggers'):
+            if func_name == entry_point.name:
+                _XTRIG_FUNCS[func_name] = entry_point.load()
+                return _XTRIG_FUNCS[func_name]
+
+        # Still unable to find anything so abort
+        raise
+
     try:
         _XTRIG_FUNCS[func_name] = getattr(mod_by_name, func_name)
     except AttributeError:
-        # Module func_name has no function func_name.
+        # Module func_name has no function func_name, nor an entry_point entry.
         raise
     return _XTRIG_FUNCS[func_name]
 

setup.cfg

@@ -214,6 +214,12 @@ cylc.main_loop =
 cylc.pre_configure =
 cylc.post_install =
     log_vc_info = cylc.flow.install_plugins.log_vc_info:main
+# NOTE: Built-in xtriggers
+cylc.xtriggers =
+    echo = cylc.flow.xtriggers.echo:echo
+    wall_clock = cylc.flow.xtriggers.wall_clock:wall_clock
+    workflow_state = cylc.flow.xtriggers.workflow_state:workflow_state
+    xrandom = cylc.flow.xtriggers.xrandom:xrandom
 
 [bdist_rpm]
 requires =

tests/functional/xtriggers/04-respect-cylc-pythonpath.t

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#------------------------------------------------------------------------------
+
+# An xtrigger added to $CYLC_PYTHONPATH should take precedence
+# over a `cylc.xtriggers` entry point of the same name
+
+. "$(dirname "$0")/test_header"
+set_test_number 3
+
+install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+# Install the succeeding xtrigger function.
+export CYLC_PYTHONPATH=${WORKFLOW_RUN_DIR}/dir:${CYLC_PYTHONPATH:-}
+
+# Validate the test workflow.
+run_ok "${TEST_NAME_BASE}-val" cylc validate --debug "${WORKFLOW_NAME}"
+
+TEST_NAME="${TEST_NAME_BASE}-run"
+workflow_run_ok "${TEST_NAME}" cylc play --no-detach --debug "${WORKFLOW_NAME}"
+
+# Check the result of xtrigger.
+grep_workflow_log_ok "${TEST_NAME_BASE}-grep" "echo overridden, args=('the_args',)"
+
+purge

tests/functional/xtriggers/04-respect-cylc-pythonpath/dir/echo.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def echo(*args, **kwargs):
+    print(f"echo overridden, args={args}")
+    return (True, {})

tests/functional/xtriggers/04-respect-cylc-pythonpath/flow.cylc

@@ -0,0 +1,8 @@
+[scheduling]
+    [[xtriggers]]
+        x1 = echo("the_args")
+    [[graph]]
+        R1 = @x1 => foo
+[runtime]
+    [[foo]]
+        script = true