Merge pull request #13 from mlibrary/DWI-37-expect-zips-in-s3

DWI-37: assume s3 bucket has zip files when listing barcodes
mlibrary · Oct 31, 2024 · f9c9fbb · f9c9fbb
2 parents 7665ce4 + b730821
commit f9c9fbb
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 9 deletions.
diff --git a/aim/digifeeds/list_barcodes_in_bucket.py b/aim/digifeeds/list_barcodes_in_bucket.py
@@ -1,5 +1,6 @@
 import boto3
 from aim.services import S
+from pathlib import Path
 
 
 def list_barcodes_in_bucket():
@@ -9,11 +10,6 @@ def list_barcodes_in_bucket():
         aws_secret_access_key=S.digifeeds_s3_secret_access_key,
     )
     prefix = S.digifeeds_s3_input_path + "/"
-    response = s3.list_objects_v2(
-        Bucket=S.digifeeds_s3_bucket,
-        Prefix=prefix,
-        Delimiter="/",
-    )
-    paths = [object["Prefix"] for object in response["CommonPrefixes"]]
-    barcodes = [path.split("/")[1] for path in paths]
+    response = s3.list_objects_v2(Bucket=S.digifeeds_s3_bucket, Prefix=prefix)
+    barcodes = [Path(object["Key"]).stem for object in response["Contents"]]
     return barcodes
diff --git a/tests/digifeeds/test_list_barcodes_in_bucket.py b/tests/digifeeds/test_list_barcodes_in_bucket.py
@@ -14,11 +14,11 @@ def test_list_barcodes_in_bucket():
     conn.create_bucket(Bucket=S.digifeeds_s3_bucket)
 
     barcode1 = conn.Object(
-        S.digifeeds_s3_bucket, f"{S.digifeeds_s3_input_path}/barcode1/some_file.txt"
+        S.digifeeds_s3_bucket, f"{S.digifeeds_s3_input_path}/barcode1.zip"
     )
     barcode1.put(Body="some text")
     barcode2 = conn.Object(
-        S.digifeeds_s3_bucket, f"{S.digifeeds_s3_input_path}/barcode2/some_file.txt"
+        S.digifeeds_s3_bucket, f"{S.digifeeds_s3_input_path}/barcode2.zip"
     )
     barcode2.put(Body="some text")