Merge branch 'master' into EN-5468-upload-unskript-ctl-reports-to-dat…

…asource-s-3

Kubernetes/legos/k8s_get_pending_pods/k8s_get_pending_pods.py

@@ -6,10 +6,16 @@
 from pydantic import BaseModel, Field
 import json
 from tabulate import tabulate
+from datetime import datetime, timedelta, timezone
 
 
 class InputSchema(BaseModel):
     namespace: Optional[str] = Field('', description='k8s Namespace', title='Namespace')
+    time_interval_to_check: int = Field(
+        24,
+        description='Time interval in hours. This time window is used to check if POD was in Pending state. Default is 24 hours.',
+        title="Time Interval"
+    )
 
 
 
@@ -24,18 +30,25 @@ def k8s_get_pending_pods_printer(output):
         print(tabulate(data, headers=headers, tablefmt="grid"))
 
 
-def k8s_get_pending_pods(handle, namespace:str="") -> Tuple:
+def format_datetime(dt):
+    return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
+
+def k8s_get_pending_pods(handle, namespace: str = "", time_interval_to_check=24) -> Tuple:
     """
-    k8s_get_pending_pods checks if any pod in the Kubernetes cluster is in 'Pending' status.
+    k8s_get_pending_pods checks if any pod in the Kubernetes cluster is in 'Pending' status within the specified time interval.
 
     :type handle: object
     :param handle: Object returned from the Task validate method
 
     :type namespace: string
     :param namespace: Namespace in which to look for the resources. If not provided, all namespaces are considered
 
+    :type time_interval_to_check: int
+    :param time_interval_to_check: (Optional) Integer, in hours, the interval within which the
+            state of the POD should be checked.
+
     :rtype: tuple
-    :return: Status,list of pending pods with their namespace
+    :return: Status, list of pending pods with their namespace and the time they became pending
     """
     if handle.client_side_validation is not True:
         print(f"K8S Connector is invalid: {handle}")
@@ -48,21 +61,35 @@ def k8s_get_pending_pods(handle, namespace:str="") -> Tuple:
     result = handle.run_native_cmd(cmd)
 
     if result.stderr:
-        raise Exception(f"Error occurred while executing command {cmd} {result.stderr}")
+        raise Exception(f"Error occurred while executing command {cmd}: {result.stderr}")
 
     pods = json.loads(result.stdout)['items']
     pending_pods = []
 
+    current_time = datetime.now(timezone.utc)
+    interval_time_to_check = current_time - timedelta(hours=time_interval_to_check)
+    interval_time_to_check = interval_time_to_check.replace(tzinfo=timezone.utc)
+
     for pod in pods:
         name = pod['metadata']['name']
         status = pod['status']['phase']
         pod_namespace = pod['metadata']['namespace']
 
         if status == 'Pending':
-            pending_pods.append({"pod_name":name,"namespace":pod_namespace})
-
-    if len(pending_pods) != 0:
+            # Check if the pod has been in Pending state within the specified the last 24 hours
+            start_time = pod['status'].get('startTime')
+            if start_time:
+                start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+                if start_time >= interval_time_to_check:
+                    formatted_start_time = format_datetime(start_time)
+                    formatted_interval_time_to_check = format_datetime(interval_time_to_check)
+                    pending_pods.append({
+                        "pod": name,
+                        "namespace": pod_namespace,
+                        "start_time": formatted_start_time,
+                        "interval_time_to_check": formatted_interval_time_to_check
+                    })
+
+    if pending_pods:
         return (False, pending_pods)
     return (True, None)
-
-

Kubernetes/legos/k8s_get_pods_in_crashloopbackoff_state/k8s_get_pods_in_crashloopbackoff_state.py

@@ -8,13 +8,20 @@
 from kubernetes import client
 from kubernetes.client.rest import ApiException
 from tabulate import tabulate
+import datetime
+from datetime import timezone 
 
 
 class InputSchema(BaseModel):
     namespace: Optional[str] = Field(
         default='',
         title='Namespace',
         description='k8s Namespace')
+    time_interval_to_check: int = Field(
+        24,
+        description='Time interval in hours. This time window is used to check if POD was in Crashloopback. Default is 24 hours.',
+        title="Time Interval"
+    )
 
 
 def k8s_get_pods_in_crashloopbackoff_state_printer(output):
@@ -27,16 +34,24 @@ def k8s_get_pods_in_crashloopbackoff_state_printer(output):
         table_data = [(entry["pod"], entry["namespace"], entry["container"]) for entry in data]
         print(tabulate(table_data, headers=headers, tablefmt="grid"))
 
-def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '') -> Tuple:
+def format_datetime(dt):
+    # Format datetime to a string 'YYYY-MM-DD HH:MM:SS UTC'
+    return dt.strftime('%Y-%m-%d %H:%M:%S UTC')
+
+def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '', time_interval_to_check=24) -> Tuple:
     """
-    k8s_get_pods_in_crashloopbackoff_state returns the pods that have CrashLoopBackOff state in their container statuses.
+    k8s_get_pods_in_crashloopbackoff_state returns the pods that have CrashLoopBackOff state in their container statuses within the specified time interval.
 
     :type handle: Object
     :param handle: Object returned from the task.validate(...) function
 
     :type namespace: str
     :param namespace: (Optional) String, K8S Namespace as python string
 
+    :type time_interval_to_check: int
+    :param time_interval_to_check: (Optional) Integer, in hours, the interval within which the
+            state of the POD should be checked.
+
     :rtype: Status, List of objects of pods, namespaces, and containers that are in CrashLoopBackOff state
     """
     result = []
@@ -47,12 +62,25 @@ def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '') -> Tuple
 
     try:
         if namespace:
-            pods = v1.list_namespaced_pod(namespace).items
+            response = v1.list_namespaced_pod(namespace)
         else:
-            pods = v1.list_pod_for_all_namespaces().items
+            response = v1.list_pod_for_all_namespaces()
+
+        if response is None or not hasattr(response, 'items'):
+            raise ApiException("Unexpected response from the Kubernetes API. 'items' not found in the response.")
+
+        pods = response.items
+
     except ApiException as e:
         raise e
 
+    if pods is None:
+        raise ApiException("No pods returned from the Kubernetes API.")
+
+    current_time = datetime.datetime.now(timezone.utc)
+    interval_time_to_check = current_time - datetime.timedelta(hours=time_interval_to_check)
+    interval_time_to_check = interval_time_to_check.replace(tzinfo=timezone.utc)
+
     for pod in pods:
         pod_name = pod.metadata.name
         namespace = pod.metadata.namespace
@@ -62,6 +90,20 @@ def k8s_get_pods_in_crashloopbackoff_state(handle, namespace: str = '') -> Tuple
         for container_status in container_statuses:
             container_name = container_status.name
             if container_status.state and container_status.state.waiting and container_status.state.waiting.reason == "CrashLoopBackOff":
-                result.append({"pod": pod_name, "namespace": namespace, "container": container_name})
+                # Check if the last transition time to CrashLoopBackOff is within the specified interval
+                if container_status.last_state and container_status.last_state.terminated:
+                    last_transition_time = container_status.last_state.terminated.finished_at
+                    if last_transition_time:
+                        last_transition_time = last_transition_time.replace(tzinfo=timezone.utc)
+                        if last_transition_time >= interval_time_to_check:
+                            formatted_transition_time = format_datetime(last_transition_time)
+                            formatted_interval_time_to_check = format_datetime(interval_time_to_check)
+                            result.append({
+                                "pod": pod_name,
+                                "namespace": namespace,
+                                "container": container_name,
+                                "last_transition_time": formatted_transition_time,
+                                "interval_time_to_check": formatted_interval_time_to_check
+                            })
 
     return (False, result) if result else (True, None)

unskript-ctl/unskript_ctl_notification.py

@@ -198,10 +198,10 @@ def create_temp_files_of_failed_check_results(self,
         return list_of_failed_files
 
     def create_script_summary_message(self, output_metadata_file: str):
-        # message = ''
+        message = ''
         if os.path.exists(output_metadata_file) is False:
             self.logger.error(f"ERROR: The metadata file is missing, please check if file exists? {output_metadata_file}")
-            return ''
+            return message
 
         metadata = ''
         with open(output_metadata_file, 'r', encoding='utf-8') as f:
@@ -215,24 +215,24 @@ def create_script_summary_message(self, output_metadata_file: str):
         self.logger.debug(f"\tStatus: {metadata.get('status')} \n\tTime (in seconds): {metadata.get('time_taken')} \n\tError: {metadata.get('error')} \n")
 
         # Remove from email 
-        # message += f'''
-        #         <br>
-        #         <h3> Custom Script Run Result </h3>
-        #         <table border="1">
-        #             <tr>
-        #                 <th> Status </th>
-        #                 <th> Time (in seconds) </th>
-        #                 <th> Error </th>
-        #             </tr>
-        #             <tr>
-        #                 <td>{metadata.get('status')}</td>
-        #                 <td>{metadata.get('time_taken')}</td>
-        #                 <td>{metadata.get('error')}</td>
-        #             </tr>
-        #         </table>
-        # '''
-
-        return ''
+        message += f'''
+                <br>
+                <h3> Custom Script Run Result </h3>
+                <table border="1">
+                    <tr>
+                        <th> Status </th>
+                        <th> Time (in seconds) </th>
+                        <th> Error </th>
+                    </tr>
+                    <tr>
+                        <td>{metadata.get('status')}</td>
+                        <td>{metadata.get('time_taken')}</td>
+                        <td>{metadata.get('error')}</td>
+                    </tr>
+                </table>
+        '''
+
+        return message
 
     def create_info_legos_output_file(self):
         """create_info_legos_output_file: This function creates a file that will
@@ -469,6 +469,10 @@ def prepare_combined_email(self,
         if info_result:
             message += info_result
             self.create_info_legos_output_file()
+        # print("Output Metadata File\n",output_metadata_file)
+        if output_metadata_file:
+            message += self.create_script_summary_message(output_metadata_file=output_metadata_file)
+            temp_attachment = self.create_email_attachment(output_metadata_file=output_metadata_file)
 
         if len(os.listdir(self.execution_dir)) == 0 or not self.create_tarball_archive(tar_file_name=tar_file_name, output_metadata_file=None, parent_folder=parent_folder):
             self.logger.error("Execution directory is empty , tarball creation unsuccessful!")
@@ -560,6 +564,9 @@ def send_sendgrid_notification(self,
             if len(os.listdir(self.execution_dir)) == 0 or not self.create_tarball_archive(tar_file_name=tar_file_name, output_metadata_file=None, parent_folder=parent_folder):
                 self.logger.error("Execution directory is empty , tarball creation unsuccessful!")
 
+            if output_metadata_file:
+                html_message += self.create_script_summary_message(output_metadata_file=output_metadata_file)
+
             info_result = self.create_info_gathering_action_result()
             if info_result:
                 html_message += info_result