diff --git a/unskript-ctl/config/unskript_ctl_config.yaml b/unskript-ctl/config/unskript_ctl_config.yaml index d7470d7b6..cf1e8739d 100644 --- a/unskript-ctl/config/unskript_ctl_config.yaml +++ b/unskript-ctl/config/unskript_ctl_config.yaml @@ -178,6 +178,15 @@ notification: verbose: true #Not yet supported enable: false email_subject_line: "" + # Skip Generating Summary pdf + skip_generating_summary_report: false + # Specify if SMTP credentials vault path + vault: + enable: false + smtp_credential_path: "v1/lb-secrets/smtp-server/credentials" + # Auth Type: "Basic Auth" or OAuth2 + auth_type: "Basic Auth" + # provider for the email. Possible values: # - SMTP - SMTP server # - SES - AWS SES @@ -199,6 +208,7 @@ notification: api_key: "" to-email: "" from-email: "" + # # Job section diff --git a/unskript-ctl/unskript_ctl_main.py b/unskript-ctl/unskript_ctl_main.py index cf9fa3fcb..ecf13f798 100644 --- a/unskript-ctl/unskript_ctl_main.py +++ b/unskript-ctl/unskript_ctl_main.py @@ -520,7 +520,7 @@ def start_debug(self, args): return openvpn_log_file = "/tmp/openvpn_client.log" - command = [f"openvpn --config {remote_config_file} > {openvpn_log_file}"] + command = [f"sudo openvpn --config {remote_config_file} > {openvpn_log_file}"] try: process = subprocess.Popen(command, stdout=subprocess.PIPE, @@ -577,9 +577,17 @@ def stop_debug(self): # Search for openvpn process. On Docker, we dont expect # Multiple process of openvpn to run. if proc.info['name'] == "openvpn": - process = psutil.Process(proc.info['pid']) - process.terminate() - process.wait() + p_id = proc.info['pid'] + command = [f"sudo kill -9 {p_id}"] + try: + process = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + except Exception as e: + self.logger.debug(f"Unable to stop debug session: {e}") + print(f"ERROR: Unable to stop the debug session {e}") + return self.logger.debug("Stopped Active Debug session successfully") print("Stopped Active Debug session successfully") diff --git a/unskript-ctl/unskript_ctl_notification.py b/unskript-ctl/unskript_ctl_notification.py index 70ef0a0da..42b5fb21a 100644 --- a/unskript-ctl/unskript_ctl_notification.py +++ b/unskript-ctl/unskript_ctl_notification.py @@ -15,6 +15,7 @@ import smtplib import os import base64 +import logging from pathlib import Path from datetime import datetime @@ -23,6 +24,8 @@ from email.mime.application import MIMEApplication from jsonschema import validate, ValidationError +from tenacity import retry, stop_after_attempt, wait_exponential, before_log, after_log + from unskript_utils import * @@ -131,6 +134,9 @@ def __init__(self, **kwargs): def notify(self, **kwargs): failed_result = kwargs.get('failed_result', {}) + if failed_result is None: + failed_result = {} + failed_object_character_count = sum((len(str(value)) for value in failed_result.values())) if failed_object_character_count >= MAX_CHARACTER_COUNT_FOR_FAILED_OBJECTS: @@ -157,17 +163,29 @@ def create_tarball_archive(self, output_metadata_file: str, parent_folder: str): + log_file_path = None + + if os.path.exists(os.path.expanduser("~/unskript_ctl.log")): + log_file_path = os.path.expanduser("~/unskript_ctl.log") + if not tar_file_name.startswith('/tmp'): tar_file_name = os.path.join('/tmp', tar_file_name) - if not output_metadata_file: - tar_cmd = ["tar", "jcvf", tar_file_name, f"--exclude={output_metadata_file}", "-C" , parent_folder, "."] + if output_metadata_file: + tar_cmd = ["tar", "Jcvf", tar_file_name, f"--exclude={output_metadata_file}", "-C" , parent_folder, "."] else: - tar_cmd = ["tar", "jcvf", tar_file_name, "-C" , parent_folder, "."] + tar_cmd = ["tar", "Jcvf", tar_file_name, "-C" , parent_folder, "."] + + if log_file_path: + tar_cmd.append(str(log_file_path)) + try: - subprocess.run(tar_cmd, + result = subprocess.run(tar_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode != 0: + self.logger.error(f"ERROR: Tar command returned Non Zero value {result.returncode}") + return False except Exception as e: self.logger.error(f"ERROR: {e}") return False @@ -873,48 +891,64 @@ def notify(self, **kwargs): return retval def _send_email(self, **kwargs): - retval = False - summary_results = kwargs.get('summary_results') - failed_objects = kwargs.get('failed_objects') - if not failed_objects: - failed_objects = {} - - if self.email_config.get('provider').lower() == 'smtp': - smtp = self.email_config.get('SMTP') - retval = SmtpNotification().notify( - summary_result = summary_results, - failed_result = failed_objects, - output_metadata_file = kwargs.get('output_metadata_file'), - smtp_host = kwargs.get('smtp_host', smtp.get('smtp-host')), - smtp_user = kwargs.get('smtp_user', smtp.get('smtp-user')), - smtp_password = kwargs.get('smtp_password', smtp.get('smtp-password')), - to_email = kwargs.get('to_email', smtp.get('to-email')), - from_email = kwargs.get('from_email', smtp.get('from-email')), - subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result')) - ) - elif self.email_config.get('provider').lower() == 'sendgrid': - sendgrid = self.email_config.get('Sendgrid') - retval = SendgridNotification().notify( - summary_result = summary_results, - failed_result = failed_objects, - output_metadata_file = kwargs.get('output_metadata_file'), - from_email = kwargs.get('from_email', sendgrid.get('from-email')), - to_email = kwargs.get('to_email', sendgrid.get('to-email')), - api_key = kwargs.get('api_key', sendgrid.get('api_key')), - subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result')) - ) - elif self.email_config.get('provider').lower() == 'ses': - aws = self.email_config.get('SES') - retval = AWSEmailNotification().notify( - summary_result = summary_results, - failed_result = failed_objects, - output_metadata_file = kwargs.get('output_metadata_file'), - access_key = kwargs.get('access_key', aws.get('access_key')), - secret_access = kwargs.get('secret_access', aws.get('secret_access')), - to_email = kwargs.get('to_email', aws.get('to-email')), - from_email = kwargs.get('from_email', aws.get('from-email')), - region = kwargs.get('region', aws.get('region')), - subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result')) - ) - - return retval \ No newline at end of file + @retry( + stop=stop_after_attempt(5), # Retry up to 5 times + wait=wait_exponential(multiplier=1, min=4, max=60), # Exponential backoff, min 4s, max 60s + before=before_log(self.logger, logging.INFO), + after=after_log(self.logger, logging.INFO) + ) + def _do_send_email(): + retval = False + summary_results = kwargs.get('summary_results') + failed_objects = kwargs.get('failed_objects') + if failed_objects is None: + failed_objects = {} + + + if self.email_config.get('provider').lower() == 'smtp': + smtp = self.email_config.get('SMTP') + retval = SmtpNotification().notify( + summary_result = summary_results, + failed_result = failed_objects, + output_metadata_file = kwargs.get('output_metadata_file'), + smtp_host = kwargs.get('smtp_host', smtp.get('smtp-host')), + smtp_user = kwargs.get('smtp_user', smtp.get('smtp-user')), + smtp_password = kwargs.get('smtp_password', smtp.get('smtp-password')), + to_email = kwargs.get('to_email', smtp.get('to-email')), + from_email = kwargs.get('from_email', smtp.get('from-email')), + subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result')) + ) + elif self.email_config.get('provider').lower() == 'sendgrid': + sendgrid = self.email_config.get('Sendgrid') + retval = SendgridNotification().notify( + summary_result = summary_results, + failed_result = failed_objects, + output_metadata_file = kwargs.get('output_metadata_file'), + from_email = kwargs.get('from_email', sendgrid.get('from-email')), + to_email = kwargs.get('to_email', sendgrid.get('to-email')), + api_key = kwargs.get('api_key', sendgrid.get('api_key')), + subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result')) + ) + elif self.email_config.get('provider').lower() == 'ses': + aws = self.email_config.get('SES') + retval = AWSEmailNotification().notify( + summary_result = summary_results, + failed_result = failed_objects, + output_metadata_file = kwargs.get('output_metadata_file'), + access_key = kwargs.get('access_key', aws.get('access_key')), + secret_access = kwargs.get('secret_access', aws.get('secret_access')), + to_email = kwargs.get('to_email', aws.get('to-email')), + from_email = kwargs.get('from_email', aws.get('from-email')), + region = kwargs.get('region', aws.get('region')), + subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result')) + ) + + return retval + + try: + # Call the retry-wrapped do send email + return _do_send_email() + + except Exception as e: + self.logger.error(f"Error sending email: {e}") + return False \ No newline at end of file