From de1c9cd83fbc17d644e15e1a4d69901c7ba2d372 Mon Sep 17 00:00:00 2001 From: cam Date: Fri, 8 Nov 2024 17:12:54 +1300 Subject: [PATCH] Add retry for running commands --- salt/schedule-commands/schedule-commands.py | 25 +++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/salt/schedule-commands/schedule-commands.py b/salt/schedule-commands/schedule-commands.py index 4cfdd21..f49f1bf 100755 --- a/salt/schedule-commands/schedule-commands.py +++ b/salt/schedule-commands/schedule-commands.py @@ -7,12 +7,13 @@ import subprocess import sys import shlex +import time import salt.client from salt_listener import SaltListener -COMMAND_FILE = "/opt/ops-tools/salt/commands.txt" +COMMAND_FILE = "/opt/ops-tools/salt/schedule-commands/commands.txt" def main(): @@ -29,12 +30,22 @@ def main(): print("listening for minion ping events") for minion_id in minion_ids: command = getMinionCommand(minion_id) - while command != None: - print(f"'{minion_id}' connected, running '{command}'") - result = subprocess.run(command.split(), capture_output=True, text=True) - print(result.stdout) - print(result.stderr) - + if command != None: + command_success = False + for i in range(0, 5): # Try 5 times to run command. If it fails, just skip it. + print(f"'{minion_id}' connected, running '{command}'") + result = subprocess.run(command.split(), capture_output=True, text=True) + print(result.stdout) + print(result.stderr) + if result.returncode == 0: + print("Command ran successfully") + break + else: + if i == 4: + print("Command failed") + else: + print(f"Command failed, waiting 10 seconds and trying {4-i} more times") + time.sleep(10) def getMinionCommand(minion_id): with open(COMMAND_FILE, "r") as file: