From ed0b6390a8078db94be3ebef0932389692f7211b Mon Sep 17 00:00:00 2001 From: Madhur Tandon Date: Fri, 17 Jan 2025 01:22:22 +0400 Subject: [PATCH] write to file as well --- metaflow/plugins/argo/argo_workflows.py | 1 + metaflow/plugins/kubernetes/kubernetes_decorator.py | 3 +++ metaflow/plugins/kubernetes/spot_monitor_sidecar.py | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/metaflow/plugins/argo/argo_workflows.py b/metaflow/plugins/argo/argo_workflows.py index 2f1a1d57f6..a3ddc52669 100644 --- a/metaflow/plugins/argo/argo_workflows.py +++ b/metaflow/plugins/argo/argo_workflows.py @@ -1705,6 +1705,7 @@ def _container_templates(self): }, **{ # Some optional values for bookkeeping + "MF_FLOW_FILENAME": os.path.basename(sys.argv[0]), "METAFLOW_FLOW_NAME": self.flow.name, "METAFLOW_STEP_NAME": node.name, "METAFLOW_RUN_ID": run_id, diff --git a/metaflow/plugins/kubernetes/kubernetes_decorator.py b/metaflow/plugins/kubernetes/kubernetes_decorator.py index 5a98e67138..551feecd1e 100644 --- a/metaflow/plugins/kubernetes/kubernetes_decorator.py +++ b/metaflow/plugins/kubernetes/kubernetes_decorator.py @@ -548,6 +548,9 @@ def task_pre_step( self._save_logs_sidecar.start() # Start spot termination monitor sidecar. + current._update_env( + {"spot_termination_notice": "/tmp/spot_termination_notice"} + ) self._spot_monitor_sidecar = Sidecar("spot_termination_monitor") self._spot_monitor_sidecar.start() diff --git a/metaflow/plugins/kubernetes/spot_monitor_sidecar.py b/metaflow/plugins/kubernetes/spot_monitor_sidecar.py index 52dec87d5b..a1c96587f4 100644 --- a/metaflow/plugins/kubernetes/spot_monitor_sidecar.py +++ b/metaflow/plugins/kubernetes/spot_monitor_sidecar.py @@ -82,6 +82,10 @@ def _emit_termination_metadata(self, termination_time): pathspec = os.getenv("MF_PATHSPEC") _, run_id, step_name, task_id = pathspec.split("/") retry_count = os.getenv("MF_ATTEMPT") + + with open("/tmp/spot_termination_notice", "w") as fp: + fp.write(termination_time) + command = [ sys.executable, f"/metaflow/{flow_filename}",