From e80719648c362d1a8e7d6df660242812772d7292 Mon Sep 17 00:00:00 2001
From: Phil Snyder <subfivemile@gmail.com>
Date: Wed, 21 Aug 2024 16:32:06 -0700
Subject: [PATCH] squash bug where we upload part file before close (#136)

---
 src/glue/jobs/s3_to_json.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/glue/jobs/s3_to_json.py b/src/glue/jobs/s3_to_json.py
index 0e3002e..0f146d7 100644
--- a/src/glue/jobs/s3_to_json.py
+++ b/src/glue/jobs/s3_to_json.py
@@ -547,7 +547,6 @@ def write_file_to_json_dataset(
     # fmt: off
     # <python3.10 requires this backslash syntax, we currently run 3.7
     with z.open(json_path, "r") as input_json, \
-         open(current_part_path, "a") as f_out, \
          gzip.open(compressed_output_path, "wt", encoding="utf-8") as f_compressed_out:
         for transformed_block in transform_block(
             input_json=input_json, metadata=metadata, logger_context=logger_context
@@ -574,7 +573,8 @@ def write_file_to_json_dataset(
             for transformed_record in transformed_block:
                 line_count += 1
                 record_with_newline = "{}\n".format(json.dumps(transformed_record))
-                f_out.write(record_with_newline)
+                with open(current_part_path, "a") as f_out:
+                    f_out.write(record_with_newline)
                 f_compressed_out.write(record_with_newline)
     # fmt: on
     # Upload final part
@@ -695,6 +695,16 @@ def _upload_file_to_json_dataset(
     )
     if delete_upon_successful_upload:
         os.remove(file_path)
+        logger.info(
+            "Delete file upon successful upload",
+            extra={
+                **basic_file_info,
+                "event.kind": "event",
+                "event.category": ["file"],
+                "event.type": ["deletion"],
+                "event.action": "delete-upon-successful-upload",
+            },
+        )
     return s3_output_key