Skip to content

Commit

Permalink
Email Retries and Log file inclusion in the attachment (#1105)
Browse files Browse the repository at this point in the history 
Co-authored-by: Jayasimha Raghavan <[email protected]>
  • Loading branch information
@jayasimha-raghavan-unskript
jayasimha-raghavan-unskript and Jayasimha Raghavan authored Sep 12, 2024
1 parent 433e7d2 commit 0806f14
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 53 deletions.
10 changes: 10 additions & 0 deletions unskript-ctl/config/unskript_ctl_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,15 @@ notification:
verbose: true #Not yet supported
enable: false
email_subject_line: ""
# Skip Generating Summary pdf
skip_generating_summary_report: false
# Specify if SMTP credentials vault path
vault:
enable: false
smtp_credential_path: "v1/lb-secrets/smtp-server/credentials"
# Auth Type: "Basic Auth" or OAuth2
auth_type: "Basic Auth"

# provider for the email. Possible values:
# - SMTP - SMTP server
# - SES - AWS SES
Expand All @@ -199,6 +208,7 @@ notification:
api_key: ""
to-email: ""
from-email: ""


#
# Job section
Expand Down
16 changes: 12 additions & 4 deletions unskript-ctl/unskript_ctl_main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -520,7 +520,7 @@ def start_debug(self, args):
return

openvpn_log_file = "/tmp/openvpn_client.log"
command = [f"openvpn --config {remote_config_file} > {openvpn_log_file}"]
command = [f"sudo openvpn --config {remote_config_file} > {openvpn_log_file}"]
try:
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
Expand Down Expand Up @@ -577,9 +577,17 @@ def stop_debug(self):
# Search for openvpn process. On Docker, we dont expect
# Multiple process of openvpn to run.
if proc.info['name'] == "openvpn":
process = psutil.Process(proc.info['pid'])
process.terminate()
process.wait()
p_id = proc.info['pid']
command = [f"sudo kill -9 {p_id}"]
try:
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
except Exception as e:
self.logger.debug(f"Unable to stop debug session: {e}")
print(f"ERROR: Unable to stop the debug session {e}")
return

self.logger.debug("Stopped Active Debug session successfully")
print("Stopped Active Debug session successfully")
Expand Down
132 changes: 83 additions & 49 deletions unskript-ctl/unskript_ctl_notification.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import smtplib
import os
import base64
import logging

from pathlib import Path
from datetime import datetime
Expand All @@ -23,6 +24,8 @@
from email.mime.application import MIMEApplication

from jsonschema import validate, ValidationError
from tenacity import retry, stop_after_attempt, wait_exponential, before_log, after_log



from unskript_utils import *
Expand Down Expand Up @@ -131,6 +134,9 @@ def __init__(self, **kwargs):

def notify(self, **kwargs):
failed_result = kwargs.get('failed_result', {})
if failed_result is None:
failed_result = {}

failed_object_character_count = sum((len(str(value)) for value in failed_result.values()))

if failed_object_character_count >= MAX_CHARACTER_COUNT_FOR_FAILED_OBJECTS:
Expand All @@ -157,17 +163,29 @@ def create_tarball_archive(self,
output_metadata_file: str,
parent_folder: str):

log_file_path = None

if os.path.exists(os.path.expanduser("~/unskript_ctl.log")):
log_file_path = os.path.expanduser("~/unskript_ctl.log")

if not tar_file_name.startswith('/tmp'):
tar_file_name = os.path.join('/tmp', tar_file_name)

if not output_metadata_file:
tar_cmd = ["tar", "jcvf", tar_file_name, f"--exclude={output_metadata_file}", "-C" , parent_folder, "."]
if output_metadata_file:
tar_cmd = ["tar", "Jcvf", tar_file_name, f"--exclude={output_metadata_file}", "-C" , parent_folder, "."]
else:
tar_cmd = ["tar", "jcvf", tar_file_name, "-C" , parent_folder, "."]
tar_cmd = ["tar", "Jcvf", tar_file_name, "-C" , parent_folder, "."]

if log_file_path:
tar_cmd.append(str(log_file_path))

try:
subprocess.run(tar_cmd,
result = subprocess.run(tar_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if result.returncode != 0:
self.logger.error(f"ERROR: Tar command returned Non Zero value {result.returncode}")
return False
except Exception as e:
self.logger.error(f"ERROR: {e}")
return False
Expand Down Expand Up @@ -873,48 +891,64 @@ def notify(self, **kwargs):
return retval

def _send_email(self, **kwargs):
retval = False
summary_results = kwargs.get('summary_results')
failed_objects = kwargs.get('failed_objects')
if not failed_objects:
failed_objects = {}

if self.email_config.get('provider').lower() == 'smtp':
smtp = self.email_config.get('SMTP')
retval = SmtpNotification().notify(
summary_result = summary_results,
failed_result = failed_objects,
output_metadata_file = kwargs.get('output_metadata_file'),
smtp_host = kwargs.get('smtp_host', smtp.get('smtp-host')),
smtp_user = kwargs.get('smtp_user', smtp.get('smtp-user')),
smtp_password = kwargs.get('smtp_password', smtp.get('smtp-password')),
to_email = kwargs.get('to_email', smtp.get('to-email')),
from_email = kwargs.get('from_email', smtp.get('from-email')),
subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result'))
)
elif self.email_config.get('provider').lower() == 'sendgrid':
sendgrid = self.email_config.get('Sendgrid')
retval = SendgridNotification().notify(
summary_result = summary_results,
failed_result = failed_objects,
output_metadata_file = kwargs.get('output_metadata_file'),
from_email = kwargs.get('from_email', sendgrid.get('from-email')),
to_email = kwargs.get('to_email', sendgrid.get('to-email')),
api_key = kwargs.get('api_key', sendgrid.get('api_key')),
subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result'))
)
elif self.email_config.get('provider').lower() == 'ses':
aws = self.email_config.get('SES')
retval = AWSEmailNotification().notify(
summary_result = summary_results,
failed_result = failed_objects,
output_metadata_file = kwargs.get('output_metadata_file'),
access_key = kwargs.get('access_key', aws.get('access_key')),
secret_access = kwargs.get('secret_access', aws.get('secret_access')),
to_email = kwargs.get('to_email', aws.get('to-email')),
from_email = kwargs.get('from_email', aws.get('from-email')),
region = kwargs.get('region', aws.get('region')),
subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result'))
)

return retval
@retry(
stop=stop_after_attempt(5), # Retry up to 5 times
wait=wait_exponential(multiplier=1, min=4, max=60), # Exponential backoff, min 4s, max 60s
before=before_log(self.logger, logging.INFO),
after=after_log(self.logger, logging.INFO)
)
def _do_send_email():
retval = False
summary_results = kwargs.get('summary_results')
failed_objects = kwargs.get('failed_objects')
if failed_objects is None:
failed_objects = {}


if self.email_config.get('provider').lower() == 'smtp':
smtp = self.email_config.get('SMTP')
retval = SmtpNotification().notify(
summary_result = summary_results,
failed_result = failed_objects,
output_metadata_file = kwargs.get('output_metadata_file'),
smtp_host = kwargs.get('smtp_host', smtp.get('smtp-host')),
smtp_user = kwargs.get('smtp_user', smtp.get('smtp-user')),
smtp_password = kwargs.get('smtp_password', smtp.get('smtp-password')),
to_email = kwargs.get('to_email', smtp.get('to-email')),
from_email = kwargs.get('from_email', smtp.get('from-email')),
subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result'))
)
elif self.email_config.get('provider').lower() == 'sendgrid':
sendgrid = self.email_config.get('Sendgrid')
retval = SendgridNotification().notify(
summary_result = summary_results,
failed_result = failed_objects,
output_metadata_file = kwargs.get('output_metadata_file'),
from_email = kwargs.get('from_email', sendgrid.get('from-email')),
to_email = kwargs.get('to_email', sendgrid.get('to-email')),
api_key = kwargs.get('api_key', sendgrid.get('api_key')),
subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result'))
)
elif self.email_config.get('provider').lower() == 'ses':
aws = self.email_config.get('SES')
retval = AWSEmailNotification().notify(
summary_result = summary_results,
failed_result = failed_objects,
output_metadata_file = kwargs.get('output_metadata_file'),
access_key = kwargs.get('access_key', aws.get('access_key')),
secret_access = kwargs.get('secret_access', aws.get('secret_access')),
to_email = kwargs.get('to_email', aws.get('to-email')),
from_email = kwargs.get('from_email', aws.get('from-email')),
region = kwargs.get('region', aws.get('region')),
subject = kwargs.get('subject', self.email_config.get('email_subject_line', 'Run Result'))
)

return retval

try:
# Call the retry-wrapped do send email
return _do_send_email()

except Exception as e:
self.logger.error(f"Error sending email: {e}")
return False

0 comments on commit 0806f14

Please sign in to comment.