Add callback function for catching MAX_RUNTIME failure

Co-authored-by: Morten Bendiksen <[email protected]>

python/res/job_queue/job_queue_node.py

@@ -53,9 +53,11 @@ def __init__(
         exit_callback_function,
         callback_arguments,
         max_runtime=None,
+        callback_timeout=None,
     ):
         self.done_callback_function = done_callback_function
         self.exit_callback_function = exit_callback_function
+        self.callback_timeout = callback_timeout
         self.callback_arguments = callback_arguments
         argc = 1
         argv = StringList()
@@ -159,6 +161,12 @@ def _job_monitor(self, driver, pool_sema, max_submit):
             self.update_status(driver)
             if self._should_be_killed():
                 self._kill(driver)
+                if (
+                    self._max_runtime
+                    and self.runtime >= self._max_runtime
+                    and self.callback_timeout
+                ):
+                    self.callback_timeout(self.callback_arguments)
 
         self._end_time = time.time()
 

python/res/job_queue/queue.py

@@ -506,7 +506,7 @@ def add_job_from_run_arg(self, run_arg, res_config, max_runtime, ok_cb, exit_cb)
             return
         run_arg._set_queue_index(self.add_job(job, run_arg.iens))
 
-    def add_ee_stage(self, stage):
+    def add_ee_stage(self, stage, callback_timeout):
         job = JobQueueNode(
             job_script=stage.get_job_script(),
             job_name=stage.get_job_name(),
@@ -519,6 +519,7 @@ def add_ee_stage(self, stage):
             exit_callback_function=stage.get_exit_callback(),
             callback_arguments=stage.get_callback_arguments(),
             max_runtime=stage.get_max_runtime(),
+            callback_timeout=callback_timeout,
         )
         if job is None:
             raise ValueError("JobQueueNode constructor created None job")