Merge pull request #104 from spacetelescope/release/v0.2.23

Release/v0.2.23

caldp-setup

@@ -6,10 +6,10 @@ export PATH=`pwd`/scripts:${PATH}
 # Define the exact version of reference files you want to use.
 # Specify "hst-operational" if you want it defined automatically
 # by the CRDS server and/or cache.
-export CRDS_CONTEXT=hst_1064.pmap
+export CRDS_CONTEXT=hst_1089.pmap
 
 # Version of stsci/hst-pipeline base image to use
-export BASE_IMAGE_TAG=CALDP_20230208_CAL_final
+export BASE_IMAGE_TAG=CALDP_20230810_CAL_final
 
 # Docker repo
 #

caldp/process.py

@@ -76,16 +76,25 @@ def is_within_directory(directory, target):
     return prefix == abs_directory
 
 
-def safe_extractall(tar, path=".", members=None, *, numeric_owner=False):
+def checked_tar_members(tar):
     """uses is_within_directory to ensure the tarfile is safe to extract
-    (see CVE-2007-4559 for details on the vulnerability)"""
-
-    for member in tar.getmembers():
-        member_path = os.path.join(path, member.name)
-        if not is_within_directory(path, member_path):  # pragma: no cover
+    (see CVE-2007-4559 for details on the vulnerability)
+    """
+    members = tar.getmembers()
+    for member in members:
+        if not is_within_directory(".", member.name):  # pragma: no cover
             raise Exception("Attempted Path Traversal in Tar File")
+        if member.isdev():
+            raise ValueError(f"Tarfile member {member.name} is a device file.")
+    return members
 
-    tar.extractall(path, members, numeric_owner=numeric_owner)
+
+def safe_tar_extrct(tarpath, mode="r:gz"):
+    """Extract all members of tar file at `path` opened using mode 'r:gz'
+    after verifying member paths and types are safe.
+    """
+    with tarfile.open(tarpath, mode) as tar:
+        tar.extractall(members=checked_tar_members(tar))
 
 
 def get_instrument(ipppssoot):
@@ -471,9 +480,8 @@ def get_objects(self, input_path, key=None):
 
         with sysexit.exit_on_exception(exit_codes.INPUT_TAR_FILE_ERROR, "Failed extracting inputs from", key):
             self.divider(f"Extracting files from {key}")
-            with tarfile.open(key, "r:gz") as tar_ref:
-                safe_extractall(tar_ref)
-                # then delete tars
+            safe_tar_extrct(key)
+            # then delete tars
         os.remove(key)
         self.divider("Gathering fits files for calibration")
         files = []
@@ -516,8 +524,7 @@ def find_input_files(self):
             elif len(tar_files) == 1:
                 log.info("Extracting inputs from: ", tar_files)
                 os.chdir(base_path)
-                with tarfile.open(tar_files[0], "r:gz") as tar_ref:
-                    safe_extractall(tar_ref)
+                safe_tar_extrct(tar_files[0])
             else:
                 raise RuntimeError(f"Too many tar files for: {repr(search_tar)} = {tar_files}")
         os.chdir(cwd)

caldp/tests/test_all.py

@@ -19,7 +19,7 @@
 # Set default CRDS Context
 CRDS_CONTEXT = os.environ.get("CRDS_CONTEXT")
 if CRDS_CONTEXT == "":
-    os.environ["CRDS_CONTEXT"] = "hst_1064.pmap"
+    os.environ["CRDS_CONTEXT"] = "hst_1089.pmap"
 
 # For applicable tests,  the product files associated with each ipppssoot below
 # must be present in the CWD after processing and be within 10% of the listed sizes.

changelog.md

@@ -1,11 +1,3 @@
-- default base docker image set to CALDP_20230208_CAL_final
-- default crds update to hst_1064.pmap
-- significant refactoring to generalize processing terms and support
-  HAP processing workflows
-  - enabled S3 output tests and made related adjustments
-  - overhauled logging, dropping Python logging for print(),  to:
-  - reduce complexity
-  - obtain log output running under pytest
-  - avoid writing to same log file from two processes concurrently: the
-    .py’s and caldp-process
-  - re-enable console output of caldp .py logs
+- default base docker image set to CALDP_cosandpin_CAL_rc1
+- default crds update to hst_1089.pmap
+- update tarball extraction method to address security vulnerability