Change log collection for diagnostics

unskript-ctl/diagnostics.py

@@ -106,7 +106,6 @@ def execute_diagnostics(self, diag_commands):
                         if function:
                             # Call the function with the commands
                             diag_outputs[entry_function] = function(commands)
-                            print(f"Function '{function_name}' is accessible.")
                         else:
                             raise ValueError(f"Function '{function_name}' not found in the global namespace.")
                     except Exception as e:
@@ -127,7 +126,7 @@ def main(self):
         if not diag_commands:
             print("Skipping Diagnostics: No diagnostic command found. You can define them in the YAML configuration file")
             return 
-
+        print("\nRunning Diagnostics...")
         diag_outputs = self.execute_diagnostics(diag_commands)
 
         if diag_outputs:
@@ -144,6 +143,10 @@ def main(args):
     parser.add_argument("--output-dir-path", '-o', help="Path to output directory", required=True)
     ap = parser.parse_args(args)
 
+    print("\nFetching logs...")
+    fetch_pod_logs_not_running(ap.output_dir_path)
+    fetch_pod_logs_high_restarts(ap.output_dir_path)
+
     diagnostics_script = DiagnosticsScript(ap)
     diagnostics_script.main()
 

unskript-ctl/diagnostics_worker.py

@@ -6,11 +6,11 @@
 import subprocess
 import json
 from unskript_ctl_factory import UctlLogger
+import yaml
 
 
 logger = UctlLogger('UnskriptDiagnostics')
 
-
 def mongodb_diagnostics(commands:list):
     """
     mongodb_diagnostics runs mongocli command with command as the parameter
@@ -42,36 +42,39 @@ def mongodb_diagnostics(commands:list):
         except Exception as e:
             command_outputs.append({command: f"Exception: {str(e)}"})
 
-    for result_dict in command_outputs:
-        for command, cmd_output in result_dict.items():
-            logger.debug("\nMongodb Diagnostics")
-            logger.debug(f"Mongosh Command: {command}\nOutput: {cmd_output}\n")
+    # for result_dict in command_outputs:
+    #     for command, cmd_output in result_dict.items():
+    #         logger.debug("\nMongodb Diagnostics")
+    #         logger.debug(f"Mongosh Command: {command}\nOutput: {cmd_output}\n")
     return command_outputs
 
-def fetch_logs(namespace, pod, container):
+def fetch_logs(namespace, pod, container, output_path):
     """
-    Fetches logs and previous logs for a specified container in a pod.
+    Fetches logs and previous logs for a specified container in a pod and writes directly to a file with headers and separators.
     """
-    outputs = []
-    cmd_logs = ["kubectl", "logs", "--namespace", namespace, pod, "-c", container]
-    result_logs = subprocess.run(cmd_logs, capture_output=True, text=True)
-    if result_logs.stderr:
-        outputs.append(f"Error: {result_logs.stderr.strip()}")
-    else:
-        outputs.append(result_logs.stdout.strip())
-
-    cmd_logs_previous = ["kubectl", "logs", "--namespace", namespace, pod, "-c", container, "--previous"]
-    result_logs_previous = subprocess.run(cmd_logs_previous, capture_output=True, text=True)
-    if result_logs_previous.stderr:
-        outputs.append(f"Error: {result_logs_previous.stderr.strip()}")
-    else:
-        outputs.append(result_logs_previous.stdout.strip())
-
-    return outputs
-
-def fetch_pod_logs_not_running():
+    logs_file_path = os.path.join(output_path, 'logs.txt')
+    separator = f"\n{'=' * 40}\n"
+    header = f"Logs for Namespace: {namespace}, Pod: {pod}, Container: {container}\n"
+    header_previous = f"Previous Logs for Namespace: {namespace}, Pod: {pod}, Container: {container}\n"
+
+    try:
+        # Write header and current logs to file
+        with open(logs_file_path, 'a') as f:
+            f.write(separator + header)
+            subprocess.run(["kubectl", "logs", "--namespace", namespace, pod, "-c", container],
+                           stdout=f, stderr=f, text=True, check=False)
+
+        # Write header for previous logs and the logs themselves to file
+        with open(logs_file_path, 'a') as f:
+            f.write(separator + header_previous)
+            subprocess.run(["kubectl", "logs", "--namespace", namespace, pod, "-c", container, "--previous"],
+                           stdout=f, stderr=f, text=True, check=False)
+
+    except Exception as e:
+        logger.error(f"Failed to fetch and write logs for {namespace}/{pod}/{container}: {e}")
+
+def fetch_pod_logs_not_running(output_path):
     logger.debug("\nK8s Diagnostics: Fetching logs for pods not running")
-    command_outputs = []
     cmd = ["kubectl", "get", "pods", "--all-namespaces", "-o", "json"]
     result = subprocess.run(cmd, capture_output=True, text=True)
     pods = json.loads(result.stdout)['items']
@@ -81,18 +84,13 @@ def fetch_pod_logs_not_running():
         name = pod['metadata']['name']
         status = pod['status']['phase']
         if status != "Running":
-            logger.debug(f"Fetching logs for Pod: {name} in Namespace: {namespace} (Not Running)")
+            # logger.debug(f"Fetching logs for Pod: {name} in Namespace: {namespace} (Not Running)")
             containers = [c['name'] for c in pod['spec'].get('initContainers', []) + pod['spec'].get('containers', [])]
             for container in containers:
-                logs_output = fetch_logs(namespace, name, container)
-                for output in logs_output:
-                    logger.debug({f"Pod Not Running: {name}, Container: {container}": output})
-                    command_outputs.append({f"Pod Not Running: {name}, Container: {container}": output})
-    return command_outputs
+                fetch_logs(namespace, name, container, output_path)
 
-def fetch_pod_logs_high_restarts():
+def fetch_pod_logs_high_restarts(output_path):
     logger.debug("\nK8s Diagnostics: Fetching logs for pods with high restarts")
-    command_outputs = []
     cmd = ["kubectl", "get", "pods", "--all-namespaces", "-o", "json"]
     result = subprocess.run(cmd, capture_output=True, text=True)
     pods = json.loads(result.stdout)['items']
@@ -101,33 +99,18 @@ def fetch_pod_logs_high_restarts():
         namespace = pod['metadata']['namespace']
         name = pod['metadata']['name']
         pod_status = pod['status'].get('containerStatuses', [])
-        restarts = sum(cs['restartCount'] for cs in pod_status)
-        if restarts > 25:
-            logger.debug(f"Fetching logs for Pod: {name} in Namespace: {namespace} with high restarts")
-            result_logs = subprocess.run(["kubectl", "logs", "--namespace", namespace, name], capture_output=True, text=True)
-            if result_logs.stderr:
-                logger.debug({f"Pod high restarts: {name}": f"Error: {result_logs.stderr.strip()}"})
-                command_outputs.append({f"Pod high restarts: {name}": f"Error: {result_logs.stderr.strip()}"})
-            else:
-                logger.debug({f"Pod high restarts: {name}": result_logs.stdout.strip()})
-                command_outputs.append({f"Pod high restarts: {name}": result_logs.stdout.strip()})
-    return command_outputs
+        for container_status in pod_status:
+            if container_status['restartCount'] > 25:
+                container_name = container_status['name']
+                # logger.debug(f"Fetching logs for Pod: {name}, Container: {container_name} in Namespace: {namespace} with high restarts")
+                fetch_logs(namespace, name, container_name, output_path)
 
 def k8s_diagnostics(commands:list):
     """
     k8s_diagnostics runs kubectl command
 
     """
     command_outputs = []
-    if not hasattr(k8s_diagnostics, "already_called"):
-        command_outputs.extend(fetch_pod_logs_high_restarts())
-        command_outputs.extend(fetch_pod_logs_not_running())
-
-        k8s_diagnostics.already_called = True
-        logger.debug("Logs have been fetched.")
-    else:
-        command_outputs = []
-        logger.debug("Subsequent execution: Skipping logs")
 
     for command in commands:
         cmd_list = command.split()
@@ -141,10 +124,10 @@ def k8s_diagnostics(commands:list):
         except Exception as e:
             command_outputs.append({command: f"Exception: {str(e)}"})
 
-    for result_dict in command_outputs:
-        for command, cmd_output in result_dict.items():
-            logger.debug("\n Kubernetes Diagnostics")
-            logger.debug(f"K8S Command: {command}\nOutput: {cmd_output}\n")
+    # for result_dict in command_outputs:
+    #     for command, cmd_output in result_dict.items():
+    #         logger.debug("\n Kubernetes Diagnostics")
+    #         logger.debug(f"K8S Command: {command}\nOutput: {cmd_output}\n")
     return command_outputs
 
 def redis_diagnostics(commands:list):
@@ -181,10 +164,10 @@ def redis_diagnostics(commands:list):
                 command_outputs.append({command: output})
         except Exception as e:
             command_outputs.append({command: f"Exception: {str(e)}"})
-    for result_dict in command_outputs:
-        for command, cmd_output in result_dict.items():
-            logger.debug("\nRedis Diagnostics")
-            logger.debug(f"Redis Command: {command}\nOutput: {cmd_output}\n")
+    # for result_dict in command_outputs:
+    #     for command, cmd_output in result_dict.items():
+    #         logger.debug("\nRedis Diagnostics")
+    #         logger.debug(f"Redis Command: {command}\nOutput: {cmd_output}\n")
     return command_outputs
 
 def postgresql_diagnostics(commands:list):
@@ -217,10 +200,10 @@ def postgresql_diagnostics(commands:list):
         except Exception as e:
             command_outputs.append({command: f"Exception: {str(e)}"})
 
-    for result_dict in command_outputs:
-        for command, cmd_output in result_dict.items():
-            logger.debug("\nPostgresql Diagnostics")
-            logger.debug(f"Postgres Command: {command}\nOutput: {cmd_output}\n")
+    # for result_dict in command_outputs:
+    #     for command, cmd_output in result_dict.items():
+    #         logger.debug("\nPostgresql Diagnostics")
+    #         logger.debug(f"Postgres Command: {command}\nOutput: {cmd_output}\n")
     return command_outputs
 
 def elasticsearch_diagnostics(commands: list) -> list:
@@ -247,10 +230,10 @@ def elasticsearch_diagnostics(commands: list) -> list:
         except Exception as e:
             command_outputs.append({command: f"Exception: {str(e)}"})
 
-    for result_dict in command_outputs:
-        for command, cmd_output in result_dict.items():
-            logger.debug("\nElasticsearch Diagnostics")
-            logger.debug(f"Elasticsearch curl command: {command}\nOutput: {cmd_output}\n")
+    # for result_dict in command_outputs:
+    #     for command, cmd_output in result_dict.items():
+    #         logger.debug("\nElasticsearch Diagnostics")
+    #         logger.debug(f"Elasticsearch curl command: {command}\nOutput: {cmd_output}\n")
     return command_outputs
 
 def keycloak_diagnostics(commands: list):
@@ -276,10 +259,10 @@ def keycloak_diagnostics(commands: list):
         except Exception as e:
             command_outputs.append({command: f"Exception: {str(e)}"})
 
-    for result_dict in command_outputs:
-        for command, cmd_output in result_dict.items():
-            logger.debug("\nKeycloak Diagnostics")
-            logger.debug(f"Keycloak curl command: {command}\nOutput: {cmd_output}\n")
+    # for result_dict in command_outputs:
+    #     for command, cmd_output in result_dict.items():
+    #         logger.debug("\nKeycloak Diagnostics")
+    #         logger.debug(f"Keycloak curl command: {command}\nOutput: {cmd_output}\n")
     return command_outputs
 
 def vault_diagnostics(commands: list):
@@ -313,8 +296,8 @@ def vault_diagnostics(commands: list):
         except Exception as e:
             command_outputs.append({command: f"Exception: {str(e)}"})
 
-    for result_dict in command_outputs:
-        for command, cmd_output in result_dict.items():
-            logger.debug("\nVault Diagnostics")
-            logger.debug(f"Vault Command: {command}\nOutput: {cmd_output}\n")
+    # for result_dict in command_outputs:
+    #     for command, cmd_output in result_dict.items():
+    #         logger.debug("\nVault Diagnostics")
+    #         logger.debug(f"Vault Command: {command}\nOutput: {cmd_output}\n")
     return command_outputs

unskript-ctl/unskript_ctl_main.py

@@ -766,5 +766,7 @@ def rearrange_argv(argv):
     if args.command == 'run' and  args.report:
         uc.notify(args)
 
+    os._exit(0)
+
 if __name__ == '__main__':
     main()