Update checks: Crashloopback and Pending pods for time based interval…

… check (#1068)

Co-authored-by: Jayasimha Raghavan <[email protected]>

Kubernetes/legos/k8s_get_pending_pods/k8s_get_pending_pods.py

@@ -6,10 +6,16 @@
 from pydantic import BaseModel, Field
 import json
 from tabulate import tabulate
+from datetime import datetime, timedelta, timezone
 
 
 class InputSchema(BaseModel):
     namespace: Optional[str] = Field('', description='k8s Namespace', title='Namespace')
+    time_interval_to_check: int = Field(
+        24,
+        description='Time interval in hours. This time window is used to check if POD was in Pending state. Default is 24 hours.',
+        title="Time Interval"
+    )
 
 
 
@@ -24,18 +30,25 @@ def k8s_get_pending_pods_printer(output):
         print(tabulate(data, headers=headers, tablefmt="grid"))
 
 
-def k8s_get_pending_pods(handle, namespace:str="") -> Tuple:
+def format_datetime(dt):
+    return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
+
+def k8s_get_pending_pods(handle, namespace: str = "", time_interval_to_check=24) -> Tuple:
     """
-    k8s_get_pending_pods checks if any pod in the Kubernetes cluster is in 'Pending' status.
+    k8s_get_pending_pods checks if any pod in the Kubernetes cluster is in 'Pending' status within the specified time interval.
 
     :type handle: object
     :param handle: Object returned from the Task validate method
 
     :type namespace: string
     :param namespace: Namespace in which to look for the resources. If not provided, all namespaces are considered
 
+    :type time_interval_to_check: int
+    :param time_interval_to_check: (Optional) Integer, in hours, the interval within which the
+            state of the POD should be checked.
+
     :rtype: tuple
-    :return: Status,list of pending pods with their namespace
+    :return: Status, list of pending pods with their namespace and the time they became pending
     """
     if handle.client_side_validation is not True:
         print(f"K8S Connector is invalid: {handle}")
@@ -48,21 +61,35 @@ def k8s_get_pending_pods(handle, namespace:str="") -> Tuple:
     result = handle.run_native_cmd(cmd)
 
     if result.stderr:
-        raise Exception(f"Error occurred while executing command {cmd} {result.stderr}")
+        raise Exception(f"Error occurred while executing command {cmd}: {result.stderr}")
 
     pods = json.loads(result.stdout)['items']
     pending_pods = []
 
+    current_time = datetime.now(timezone.utc)
+    interval_time_to_check = current_time - timedelta(hours=time_interval_to_check)
+    interval_time_to_check = interval_time_to_check.replace(tzinfo=timezone.utc)
+
     for pod in pods:
         name = pod['metadata']['name']
         status = pod['status']['phase']
         pod_namespace = pod['metadata']['namespace']
 
         if status == 'Pending':
-            pending_pods.append({"pod_name":name,"namespace":pod_namespace})
-
-    if len(pending_pods) != 0:
+            # Check if the pod has been in Pending state within the specified the last 24 hours
+            start_time = pod['status'].get('startTime')
+            if start_time:
+                start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+                if start_time >= interval_time_to_check:
+                    formatted_start_time = format_datetime(start_time)
+                    formatted_interval_time_to_check = format_datetime(interval_time_to_check)
+                    pending_pods.append({
+                        "pod": name,
+                        "namespace": pod_namespace,
+                        "start_time": formatted_start_time,
+                        "interval_time_to_check": formatted_interval_time_to_check
+                    })
+
+    if pending_pods:
         return (False, pending_pods)
     return (True, None)
-
-

Kubernetes/legos/k8s_get_pods_in_crashloopbackoff_state/k8s_get_pods_in_crashloopbackoff_state.py

@@ -8,13 +8,20 @@
 from kubernetes import client
 from kubernetes.client.rest import ApiException
 from tabulate import tabulate
+import datetime
+from datetime import timezone 
 
 
 class InputSchema(BaseModel):
     namespace: Optional[str] = Field(
         default='',
         title='Namespace',
         description='k8s Namespace')
+    time_interval_to_check: int = Field(
+        24,
+        description='Time interval in hours. This time window is used to check if POD was in Crashloopback. Default is 24 hours.',
+        title="Time Interval"
+    )
 
 
 def k8s_get_pods_in_crashloopbackoff_state_printer(output):
@@ -27,16 +34,24 @@ def k8s_get_pods_in_crashloopbackoff_state_printer(output):
         table_data = [(entry["pod"], entry["namespace"], entry["container"]) for entry in data]
         print(tabulate(table_data, headers=headers, tablefmt="grid"))
 
-def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '') -> Tuple:
+def format_datetime(dt):
+    # Format datetime to a string 'YYYY-MM-DD HH:MM:SS UTC'
+    return dt.strftime('%Y-%m-%d %H:%M:%S UTC')
+
+def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '', time_interval_to_check=24) -> Tuple:
     """
-    k8s_get_pods_in_crashloopbackoff_state returns the pods that have CrashLoopBackOff state in their container statuses.
+    k8s_get_pods_in_crashloopbackoff_state returns the pods that have CrashLoopBackOff state in their container statuses within the specified time interval.
 
     :type handle: Object
     :param handle: Object returned from the task.validate(...) function
 
     :type namespace: str
     :param namespace: (Optional) String, K8S Namespace as python string
 
+    :type time_interval_to_check: int
+    :param time_interval_to_check: (Optional) Integer, in hours, the interval within which the
+            state of the POD should be checked.
+
     :rtype: Status, List of objects of pods, namespaces, and containers that are in CrashLoopBackOff state
     """
     result = []
@@ -47,12 +62,25 @@ def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '') -> Tuple
 
     try:
         if namespace:
-            pods = v1.list_namespaced_pod(namespace).items
+            response = v1.list_namespaced_pod(namespace)
         else:
-            pods = v1.list_pod_for_all_namespaces().items
+            response = v1.list_pod_for_all_namespaces()
+
+        if response is None or not hasattr(response, 'items'):
+            raise ApiException("Unexpected response from the Kubernetes API. 'items' not found in the response.")
+
+        pods = response.items
+
     except ApiException as e:
         raise e
 
+    if pods is None:
+        raise ApiException("No pods returned from the Kubernetes API.")
+
+    current_time = datetime.datetime.now(timezone.utc)
+    interval_time_to_check = current_time - datetime.timedelta(hours=time_interval_to_check)
+    interval_time_to_check = interval_time_to_check.replace(tzinfo=timezone.utc)
+
     for pod in pods:
         pod_name = pod.metadata.name
         namespace = pod.metadata.namespace
@@ -62,6 +90,20 @@ def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '') -> Tuple
         for container_status in container_statuses:
             container_name = container_status.name
             if container_status.state and container_status.state.waiting and container_status.state.waiting.reason == "CrashLoopBackOff":
-                result.append({"pod": pod_name, "namespace": namespace, "container": container_name})
+                # Check if the last transition time to CrashLoopBackOff is within the specified interval
+                if container_status.last_state and container_status.last_state.terminated:
+                    last_transition_time = container_status.last_state.terminated.finished_at
+                    if last_transition_time:
+                        last_transition_time = last_transition_time.replace(tzinfo=timezone.utc)
+                        if last_transition_time >= interval_time_to_check:
+                            formatted_transition_time = format_datetime(last_transition_time)
+                            formatted_interval_time_to_check = format_datetime(interval_time_to_check)
+                            result.append({
+                                "pod": pod_name,
+                                "namespace": namespace,
+                                "container": container_name,
+                                "last_transition_time": formatted_transition_time,
+                                "interval_time_to_check": formatted_interval_time_to_check
+                            })
 
     return (False, result) if result else (True, None)