Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add bucket exists check before uploading to S3 #1072

Merged
merged 1 commit into from
May 31, 2024
Merged

Add bucket exists check before uploading to S3 #1072

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions unskript-ctl/unskript_upload_results_to_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ def __init__(self):
logger.debug("AWS credentials are not set in environment variables")
return

self.bucket_name = 'lightbeam-reports'
self.uglobals = UnskriptGlobals()

try:
Expand All @@ -48,6 +47,23 @@ def __init__(self):
def create_s3_folder_path(self):
# Initialize folder_exists
folder_exists = False

# Ensure the bucket exists
try:
self.s3_client.head_bucket(Bucket=self.bucket_name)
logger.debug(f"S3 bucket {self.bucket_name} exists")
except ClientError as e:
if e.response['Error']['Code'] == '404':
logger.debug(f"S3 bucket {self.bucket_name} does not exist, creating bucket")
try:
self.s3_client.create_bucket(Bucket=self.bucket_name)
logger.debug(f"S3 bucket {self.bucket_name} created")
except ClientError as e:
logger.debug(f"Failed to create bucket: {e}")
return False # Exit if the bucket cannot be created
else:
logger.debug(f"Error checking bucket existence: {e}")
return False # Exit if there is any other error

# Ensure the folder structure exists in the bucket
try:
Expand All @@ -69,6 +85,8 @@ def create_s3_folder_path(self):
except ClientError as e:
logger.debug(f"Failed to create folder: {e}")

return True

def rename_and_upload_failed_objects(self, checks_output):
try:
# Convert checks_output to JSON format
Expand All @@ -86,7 +104,9 @@ def rename_and_upload_failed_objects(self, checks_output):
logger.debug(f"Failed to write JSON data to local file: {e}")
return

self.create_s3_folder_path()
if not self.create_s3_folder_path():
logger.debug("Unable to create bucket")
return

# Upload the JSON file
try:
Expand All @@ -102,7 +122,9 @@ def rename_and_upload_failed_objects(self, checks_output):
os.remove(self.local_file_name)

def rename_and_upload_other_items(self):
self.create_s3_folder_path()
if not self.create_s3_folder_path():
logger.debug("Unable to create bucket")
return
# Upload the files in the CURRENT_EXECUTION_RUN_DIRECTORY
file_list_to_upload = [self.local_file_name]
if self.uglobals.get('CURRENT_EXECUTION_RUN_DIRECTORY') and \
Expand Down
Loading