Merge pull request #20 from cwrc/19_media_without_node

Add additional exception handling to gracefully handle AIP package generation failures
cwrc · Aug 23, 2024 · 6e1d50d · 6e1d50d
2 parents 569654a + 68bae48
commit 6e1d50d
Showing 1 changed file with 49 additions and 38 deletions.
diff --git a/rootfs/var/www/leaf-isle-bagger/swift/utilities.py b/rootfs/var/www/leaf-isle-bagger/swift/utilities.py
@@ -37,7 +37,7 @@ def upload_aip(node_list, aip_dir, swift_options, container_dst, database_csv):
             for key, item_values in node_list.items():
                 aip_path = generate_aip_path(aip_dir, key)
                 aip_id = generate_aip_id(key)
-                logging.info(f"  uploading: {aip_path}")
+                logging.info(f"  adding to upload: {aip_path}")
                 checksums = file_checksum(aip_path)
                 item_options = {
                     "header": {
@@ -133,12 +133,16 @@ def build_swift_upload_object(item, aip_path, swift_options, item_options):
 def file_checksum(path):
     hash_md5 = hashlib.md5()
     hash_sha256 = hashlib.sha256()
-    with open(path, "rb") as f:
-        # read and buffer to prevent high memory usage
-        for chunk in iter(lambda: f.read(4096), b""):
-            hash_md5.update(chunk)
-            hash_sha256.update(chunk)
-    return {"md5sum": hash_md5.hexdigest(), "sha256sum": hash_sha256.hexdigest()}
+    try:
+        with open(path, "rb") as f:
+            # read and buffer to prevent high memory usage
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_md5.update(chunk)
+                hash_sha256.update(chunk)
+    except FileNotFoundError as e:
+        logging.error(f"{e}")
+    finally:
+        return {"md5sum": hash_md5.hexdigest(), "sha256sum": hash_sha256.hexdigest()}
 
 
 #
@@ -155,40 +159,47 @@ def validate_checksum(path, etag, id):
 def upload(swift_conn_dst, dst_objs, container_dst, db_writer=None):
 
     for dst_item in swift_conn_dst.upload(container_dst, dst_objs):
-        # test if segmented large object: https://docs.openstack.org/swift/newton/overview_large_objects.html
-        if dst_item["action"] == "upload_object":
-            logging.debug(f"{dst_item}")
-        if not dst_item["success"]:
-            if "object" in dst_item:
-                logging.error(f"{dst_item}")
-                raise SwiftError(dst_item["error"], container_dst, dst_item["object"])
-            # Swift segmented object
-            elif "for_object" in dst_item:
-                logging.error(f"{dst_item}")
-                raise SwiftError(
-                    dst_item["error"],
-                    container_dst,
+        try:
+            # test if segmented large object: https://docs.openstack.org/swift/newton/overview_large_objects.html
+            if dst_item["action"] == "upload_object":
+                logging.info(f"  uploading: {dst_item['object']}")
+                logging.debug(f"{dst_item}")
+            if not dst_item["success"]:
+                if "object" in dst_item:
+                    logging.error(f"{dst_item}")
+                    raise SwiftError(dst_item["error"], container_dst, dst_item["object"])
+                # Swift segmented object
+                elif "for_object" in dst_item:
+                    logging.error(f"{dst_item}")
+                    raise SwiftError(
+                        dst_item["error"],
+                        container_dst,
+                        dst_item["object"],
+                        dst_item["segment_index"],
+                    )
+
+            if dst_item["action"] == "upload_object" and os.path.isfile(dst_item["path"]):
+                # test upload file against Swift header etag to verify
+                checksums = validate_checksum(
+                    dst_item["path"],
+                    dst_item["response_dict"]["headers"]["etag"],
                     dst_item["object"],
-                    dst_item["segment_index"],
                 )
+                # log upload
+                logging.debug(f"swift stat - [{dst_item}]")
+                if db_writer:
+                    log_upload(
+                        db_writer,
+                        dst_item,
+                        container_dst,
+                        checksums,
+                        os.getenv("OS_USERNAME"),
+                    )
+        except Exception as e:
+            logging.error(f"swift stat - [{dst_item}]")
+            logging.error(f"{e}")
+
 
-        if dst_item["action"] == "upload_object" and os.path.isfile(dst_item["path"]):
-            # test upload file against Swift header etag to verify
-            checksums = validate_checksum(
-                dst_item["path"],
-                dst_item["response_dict"]["headers"]["etag"],
-                dst_item["object"],
-            )
-            # log upload
-            logging.debug(f"swift stat - [{dst_item}]")
-            if db_writer:
-                log_upload(
-                    db_writer,
-                    dst_item,
-                    container_dst,
-                    checksums,
-                    os.getenv("OS_USERNAME"),
-                )
 
 
 #