From 54c30f6d5cd4d5b89e4ea66140b97e01cee973db Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 13:46:42 -0400
Subject: [PATCH 01/14] refactor

---
 .../clp_package_utils/general.py              |  52 +++++++++
 .../clp_package_utils/scripts/compress.py     |  46 +++-----
 .../clp_package_utils/scripts/decompress.py   |  50 +++------
 .../scripts/native/decompress.py              |  44 ++++----
 .../scripts/native/search.py                  | 100 ++++--------------
 .../clp_package_utils/scripts/search.py       |  43 +++-----
 6 files changed, 135 insertions(+), 200 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index ce0f10309..9a5825161 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -6,6 +6,9 @@
 import socket
 import subprocess
 import typing
+import uuid
+from enum import auto
+from typing import List, Tuple
 
 import yaml
 from clp_py_utils.clp_config import (
@@ -24,6 +27,7 @@
     read_yaml_config_file,
     validate_path_could_be_dir,
 )
+from strenum import KebabCaseStrEnum
 
 # CONSTANTS
 # Paths
@@ -38,6 +42,13 @@ class DockerMountType(enum.IntEnum):
     BIND = 0
 
 
+class JobType(KebabCaseStrEnum):
+    COMPRESSION = auto()
+    DECOMPRESSION = auto()
+    IR_EXTRACTION = auto()
+    SEARCH = auto()
+
+
 class DockerMount:
     def __init__(
         self,
@@ -91,6 +102,10 @@ def get_clp_home():
     return clp_home.resolve()
 
 
+def generate_container_name(job_type: JobType) -> str:
+    return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}"
+
+
 def check_dependencies():
     try:
         subprocess.run(
@@ -241,6 +256,43 @@ def generate_container_config(clp_config: CLPConfig, clp_home: pathlib.Path):
     return container_clp_config, docker_mounts
 
 
+def dump_container_config(
+    clp_config: CLPConfig, container_clp_config, container_name: str
+) -> Tuple[pathlib.Path, pathlib.Path]:
+    container_config_filename = f".{container_name}-config.yml"
+    config_file_path_on_host = clp_config.logs_directory / container_config_filename
+    config_file_path_on_container = container_clp_config.logs_directory / container_config_filename
+    with open(config_file_path_on_host, "w") as f:
+        yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f)
+
+    return config_file_path_on_container, config_file_path_on_host
+
+
+def generate_container_start_cmd(
+    container_name: str, container_mounts: List[DockerMount], execution_container: str
+):
+    clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
+    # fmt: off
+    container_start_cmd = [
+        "docker", "run",
+        "-i",
+        "--rm",
+        "--network", "host",
+        "-w", str(CONTAINER_CLP_HOME),
+        "-e", f"PYTHONPATH={clp_site_packages_dir}",
+        "-u", f"{os.getuid()}:{os.getgid()}",
+        "--name", container_name,
+        "--log-driver", "local"
+    ]
+    for mount in container_mounts:
+        if mount:
+            container_start_cmd.append("--mount")
+            container_start_cmd.append(str(mount))
+    container_start_cmd.append(execution_container)
+
+    return container_start_cmd
+
+
 def validate_config_key_existence(config, key):
     try:
         value = get_config_value(config, key)
diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index 61495a4cd..4a0c98d03 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -1,19 +1,19 @@
 import argparse
 import logging
-import os
 import pathlib
 import subprocess
 import sys
 import uuid
 
-import yaml
-
 from clp_package_utils.general import (
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
-    CONTAINER_CLP_HOME,
     CONTAINER_INPUT_LOGS_ROOT_DIR,
+    dump_container_config,
     generate_container_config,
+    generate_container_name,
+    generate_container_start_cmd,
     get_clp_home,
+    JobType,
     validate_and_load_config_file,
     validate_and_load_db_credentials_file,
 )
@@ -67,41 +67,23 @@ def main(argv):
         logger.exception("Failed to load config.")
         return -1
 
-    container_name = f"clp-compressor-{str(uuid.uuid4())[-4:]}"
+    container_name = generate_container_name(JobType.COMPRESSION)
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
-    container_config_filename = f".{container_name}-config.yml"
-    container_config_file_path_on_host = clp_config.logs_directory / container_config_filename
-    with open(container_config_file_path_on_host, "w") as f:
-        yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f)
+    config_file_path_on_container, config_file_path_on_host = dump_container_config(
+        clp_config, container_clp_config, container_name
+    )
 
-    clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
-    # fmt: off
-    container_start_cmd = [
-        "docker", "run",
-        "-i",
-        "--rm",
-        "--network", "host",
-        "-w", str(CONTAINER_CLP_HOME),
-        "-e", f"PYTHONPATH={clp_site_packages_dir}",
-        "-u", f"{os.getuid()}:{os.getgid()}",
-        "--name", container_name,
-        "--log-driver", "local",
-        "--mount", str(mounts.clp_home),
-    ]
-    # fmt: on
-    necessary_mounts = [mounts.input_logs_dir, mounts.data_dir, mounts.logs_dir]
-    for mount in necessary_mounts:
-        if mount:
-            container_start_cmd.append("--mount")
-            container_start_cmd.append(str(mount))
-    container_start_cmd.append(clp_config.execution_container)
+    necessary_mounts = [mounts.clp_home, mounts.input_logs_dir, mounts.data_dir, mounts.logs_dir]
+    container_start_cmd = generate_container_start_cmd(
+        container_name, necessary_mounts, clp_config.execution_container
+    )
 
     # fmt: off
     compress_cmd = [
         "python3",
         "-m", "clp_package_utils.scripts.native.compress",
-        "--config", str(container_clp_config.logs_directory / container_config_filename),
+        "--config", str(config_file_path_on_container),
         "--remove-path-prefix", str(CONTAINER_INPUT_LOGS_ROOT_DIR),
     ]
     # fmt: on
@@ -140,7 +122,7 @@ def main(argv):
     subprocess.run(cmd, check=True)
 
     # Remove generated files
-    container_config_file_path_on_host.unlink()
+    config_file_path_on_host.unlink()
 
     return 0
 
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index f20291b50..44297e4c7 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -1,20 +1,21 @@
 import argparse
 import logging
-import os
 import pathlib
 import subprocess
 import sys
-import uuid
 
-import yaml
+from clp_py_utils.clp_config import CLPConfig
 
 from clp_package_utils.general import (
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
-    CONTAINER_CLP_HOME,
     DockerMount,
     DockerMountType,
+    dump_container_config,
     generate_container_config,
+    generate_container_name,
+    generate_container_start_cmd,
     get_clp_home,
+    JobType,
     validate_and_load_config_file,
     validate_and_load_db_credentials_file,
     validate_path_could_be_dir,
@@ -76,33 +77,15 @@ def main(argv):
         return -1
     extraction_dir.mkdir(exist_ok=True)
 
-    container_name = f"clp-decompressor-{str(uuid.uuid4())[-4:]}"
-
+    container_name = generate_container_name(JobType.DECOMPRESSION)
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
-    container_config_filename = f".{container_name}-config.yml"
-    container_config_file_path_on_host = clp_config.logs_directory / container_config_filename
-    with open(container_config_file_path_on_host, "w") as f:
-        yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f)
-
-    clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
-    # fmt: off
-    container_start_cmd = [
-        "docker", "run",
-        "-i",
-        "--rm",
-        "--network", "host",
-        "-w", str(CONTAINER_CLP_HOME),
-        "-e", f"PYTHONPATH={clp_site_packages_dir}",
-        "-u", f"{os.getuid()}:{os.getgid()}",
-        "--name", container_name,
-        "--log-driver", "local",
-        "--mount", str(mounts.clp_home),
-    ]
-    # fmt: on
-
+    config_file_path_on_container, config_file_path_on_host = dump_container_config(
+        clp_config, container_clp_config, container_name
+    )
     # Set up mounts
     container_extraction_dir = pathlib.Path("/") / "mnt" / "extraction-dir"
     necessary_mounts = [
+        mounts.clp_home,
         mounts.data_dir,
         mounts.logs_dir,
         mounts.archives_output_dir,
@@ -120,18 +103,15 @@ def main(argv):
                 container_paths_to_decompress_file_path,
             )
         )
-    for mount in necessary_mounts:
-        if mount:
-            container_start_cmd.append("--mount")
-            container_start_cmd.append(str(mount))
-
-    container_start_cmd.append(clp_config.execution_container)
+    container_start_cmd = generate_container_start_cmd(
+        container_name, necessary_mounts, clp_config.execution_container
+    )
 
     # fmt: off
     decompress_cmd = [
         "python3",
         "-m", "clp_package_utils.scripts.native.decompress",
-        "--config", str(container_clp_config.logs_directory / container_config_filename),
+        "--config", str(config_file_path_on_container),
         "-d", str(container_extraction_dir),
     ]
     # fmt: on
@@ -145,7 +125,7 @@ def main(argv):
     subprocess.run(cmd, check=True)
 
     # Remove generated files
-    container_config_file_path_on_host.unlink()
+    config_file_path_on_host.unlink()
 
     return 0
 
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index 9ca3ab7b6..2468f869b 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -25,15 +25,24 @@
 logger.addHandler(logging_console_handler)
 
 
-def decompress_paths(
+def handle_decompression(
+    parsed_args,
     clp_home: pathlib.Path,
-    paths,
-    list_path: pathlib.Path,
     clp_config: CLPConfig,
-    archives_dir: pathlib.Path,
-    logs_dir: pathlib.Path,
-    extraction_dir: pathlib.Path,
 ):
+    # Validate paths were specified using only one method
+    if len(parsed_args.paths) > 0 and parsed_args.files_from is not None:
+        logger.error("Paths cannot be specified both on the command line and through a file.")
+
+    # Validate extraction directory
+    extraction_dir = pathlib.Path(parsed_args.extraction_dir)
+    if not extraction_dir.is_dir():
+        logger.error(f"extraction-dir ({extraction_dir}) is not a valid directory.")
+        return -1
+
+    logs_dir = clp_config.logs_directory
+    archives_dir = clp_config.archive_output.directory
+
     # Generate database config file for clp
     db_config_file_path = logs_dir / f".decompress-db-config-{uuid.uuid4()}.yml"
     with open(db_config_file_path, "w") as f:
@@ -46,6 +55,8 @@ def decompress_paths(
         "--db-config-file", str(db_config_file_path),
     ]
     # fmt: on
+    paths = parsed_args.paths
+    list_path = parsed_args.files_from
     files_to_decompress_list_path = None
     if list_path is not None:
         decompression_cmd.append("-f")
@@ -93,16 +104,6 @@ def main(argv):
     )
     parsed_args = args_parser.parse_args(argv[1:])
 
-    # Validate paths were specified using only one method
-    if len(parsed_args.paths) > 0 and parsed_args.files_from is not None:
-        args_parser.error("Paths cannot be specified both on the command line and through a file.")
-
-    # Validate extraction directory
-    extraction_dir = pathlib.Path(parsed_args.extraction_dir)
-    if not extraction_dir.is_dir():
-        logger.error(f"extraction-dir ({extraction_dir}) is not a valid directory.")
-        return -1
-
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
@@ -115,15 +116,8 @@ def main(argv):
         logger.exception("Failed to load config.")
         return -1
 
-    return decompress_paths(
-        clp_home,
-        parsed_args.paths,
-        parsed_args.files_from,
-        clp_config,
-        clp_config.archive_output.directory,
-        clp_config.logs_directory,
-        extraction_dir,
-    )
+    return handle_decompression(parsed_args, clp_home, clp_config)
+
 
 
 if "__main__" == __name__:
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py
index 9041b0006..df015f25f 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py
@@ -4,16 +4,13 @@
 import asyncio
 import ipaddress
 import logging
-import multiprocessing
 import pathlib
 import socket
 import sys
-import time
-from contextlib import closing
 
 import msgpack
 import pymongo
-from clp_py_utils.clp_config import Database, QUERY_JOBS_TABLE_NAME, ResultsCache
+from clp_py_utils.clp_config import Database, ResultsCache
 from clp_py_utils.sql_adapter import SQL_Adapter
 from job_orchestration.scheduler.constants import QueryJobStatus, QueryJobType
 from job_orchestration.scheduler.job_config import AggregationConfig, SearchJobConfig
@@ -23,6 +20,11 @@
     get_clp_home,
     validate_and_load_config_file,
 )
+from clp_package_utils.scripts.native.utils import (
+    run_function_in_process,
+    submit_query_job,
+    wait_for_query_job,
+)
 
 # Setup logging
 # Create logger
@@ -35,41 +37,6 @@
 logger.addHandler(logging_console_handler)
 
 
-async def run_function_in_process(function, *args, initializer=None, init_args=None):
-    """
-    Runs the given function in a separate process wrapped in a *cancellable*
-    asyncio task. This is necessary because asyncio's multiprocessing process
-    cannot be cancelled once it's started.
-    :param function: Method to run
-    :param args: Arguments for the method
-    :param initializer: Initializer for each process in the pool
-    :param init_args: Arguments for the initializer
-    :return: Return value of the method
-    """
-    pool = multiprocessing.Pool(1, initializer, init_args)
-
-    loop = asyncio.get_event_loop()
-    fut = loop.create_future()
-
-    def process_done_callback(obj):
-        loop.call_soon_threadsafe(fut.set_result, obj)
-
-    def process_error_callback(err):
-        loop.call_soon_threadsafe(fut.set_exception, err)
-
-    pool.apply_async(
-        function, args, callback=process_done_callback, error_callback=process_error_callback
-    )
-
-    try:
-        return await fut
-    except asyncio.CancelledError:
-        pass
-    finally:
-        pool.terminate()
-        pool.close()
-
-
 def create_and_monitor_job_in_db(
     db_config: Database,
     results_cache: ResultsCache,
@@ -106,47 +73,22 @@ def create_and_monitor_job_in_db(
             search_config.tags = tag_list
 
     sql_adapter = SQL_Adapter(db_config)
-    with closing(sql_adapter.create_connection(True)) as db_conn, closing(
-        db_conn.cursor(dictionary=True)
-    ) as db_cursor:
-        # Create job
-        db_cursor.execute(
-            f"INSERT INTO `{QUERY_JOBS_TABLE_NAME}` (`job_config`, `type`) VALUES (%s, %s)",
-            (msgpack.packb(search_config.dict()), QueryJobType.SEARCH_OR_AGGREGATION),
-        )
-        db_conn.commit()
-        job_id = db_cursor.lastrowid
+    job_id = submit_query_job(sql_adapter, search_config, QueryJobType.SEARCH_OR_AGGREGATION)
+    job_status = wait_for_query_job(sql_adapter, job_id)
 
-        # Wait for the job to be marked complete
-        while True:
-            db_cursor.execute(
-                f"SELECT `status` FROM `{QUERY_JOBS_TABLE_NAME}` WHERE `id` = {job_id}"
-            )
-            # There will only ever be one row since it's impossible to have more than one job with
-            # the same ID
-            new_status = db_cursor.fetchall()[0]["status"]
-            db_conn.commit()
-            if new_status in (
-                QueryJobStatus.SUCCEEDED,
-                QueryJobStatus.FAILED,
-                QueryJobStatus.CANCELLED,
-            ):
-                break
-
-            time.sleep(0.5)
-
-        if do_count_aggregation is None and count_by_time_bucket_size is None:
-            return
-        with pymongo.MongoClient(results_cache.get_uri()) as client:
-            search_results_collection = client[results_cache.db_name][str(job_id)]
-            if do_count_aggregation is not None:
-                for document in search_results_collection.find():
-                    print(
-                        f"tags: {document['group_tags']} count: {document['records'][0]['count']}"
-                    )
-            elif count_by_time_bucket_size is not None:
-                for document in search_results_collection.find():
-                    print(f"timestamp: {document['timestamp']} count: {document['count']}")
+    if do_count_aggregation is None and count_by_time_bucket_size is None:
+        return
+    with pymongo.MongoClient(results_cache.get_uri()) as client:
+        search_results_collection = client[results_cache.db_name][str(job_id)]
+        if do_count_aggregation is not None:
+            for document in search_results_collection.find():
+                print(f"tags: {document['group_tags']} count: {document['records'][0]['count']}")
+        elif count_by_time_bucket_size is not None:
+            for document in search_results_collection.find():
+                print(f"timestamp: {document['timestamp']} count: {document['count']}")
+
+    if job_status != QueryJobStatus.SUCCEEDED:
+        logger.error(f"job {job_id} finished with unexpected status: {job_status}")
 
 
 async def worker_connection_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index 2f2450430..34acbda40 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -10,9 +10,12 @@
 
 from clp_package_utils.general import (
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
-    CONTAINER_CLP_HOME,
+    dump_container_config,
     generate_container_config,
+    generate_container_name,
+    generate_container_start_cmd,
     get_clp_home,
+    JobType,
     validate_and_load_config_file,
     validate_and_load_db_credentials_file,
 )
@@ -81,41 +84,23 @@ def main(argv):
         logger.exception("Failed to load config.")
         return -1
 
-    container_name = f"clp-search-{str(uuid.uuid4())[-4:]}"
+    container_name = generate_container_name(JobType.SEARCH)
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
-    container_config_filename = f".{container_name}-config.yml"
-    container_config_file_path_on_host = clp_config.logs_directory / container_config_filename
-    with open(container_config_file_path_on_host, "w") as f:
-        yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f)
+    config_file_path_on_container, config_file_path_on_host = dump_container_config(
+        clp_config, container_clp_config, container_name
+    )
 
-    clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
-    # fmt: off
-    container_start_cmd = [
-        "docker", "run",
-        "-i",
-        "--rm",
-        "--network", "host",
-        "-w", str(CONTAINER_CLP_HOME),
-        "-e", f"PYTHONPATH={clp_site_packages_dir}",
-        "-u", f"{os.getuid()}:{os.getgid()}",
-        "--name", container_name,
-        "--log-driver", "local",
-        "--mount", str(mounts.clp_home),
-    ]
-    # fmt: on
-    necessary_mounts = [mounts.logs_dir]
-    for mount in necessary_mounts:
-        if mount:
-            container_start_cmd.append("--mount")
-            container_start_cmd.append(str(mount))
-    container_start_cmd.append(clp_config.execution_container)
+    necessary_mounts = [mounts.clp_home, mounts.logs_dir]
+    container_start_cmd = generate_container_start_cmd(
+        container_name, necessary_mounts, clp_config.execution_container
+    )
 
     # fmt: off
     search_cmd = [
         "python3",
         "-m", "clp_package_utils.scripts.native.search",
-        "--config", str(container_clp_config.logs_directory / container_config_filename),
+        "--config", str(config_file_path_on_container),
         parsed_args.wildcard_query,
     ]
     # fmt: on
@@ -142,7 +127,7 @@ def main(argv):
     subprocess.run(cmd, check=True)
 
     # Remove generated files
-    container_config_file_path_on_host.unlink()
+    config_file_path_on_host.unlink()
 
     return 0
 

From 3dc95795e7b1f391304bdfa90550ca660631cd90 Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 13:50:43 -0400
Subject: [PATCH 02/14] remove IR extraction for now

---
 components/clp-package-utils/clp_package_utils/general.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 9a5825161..39126c335 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -45,7 +45,6 @@ class DockerMountType(enum.IntEnum):
 class JobType(KebabCaseStrEnum):
     COMPRESSION = auto()
     DECOMPRESSION = auto()
-    IR_EXTRACTION = auto()
     SEARCH = auto()
 
 

From 702c9c2fa7f321ffcf12d88294be1b2a8ff03b79 Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 14:10:50 -0400
Subject: [PATCH 03/14] Add missing types

---
 .../clp-package-utils/clp_package_utils/general.py     | 10 ++++++----
 .../clp_package_utils/scripts/native/decompress.py     |  1 -
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 39126c335..644eed22a 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -191,7 +191,9 @@ def is_path_already_mounted(
     return host_path_relative_to_mounted_root == container_path_relative_to_mounted_root
 
 
-def generate_container_config(clp_config: CLPConfig, clp_home: pathlib.Path):
+def generate_container_config(
+    clp_config: CLPConfig, clp_home: pathlib.Path
+) -> Tuple[CLPConfig, CLPDockerMounts]:
     """
     Copies the given config and sets up mounts mapping the relevant host paths into the container
 
@@ -256,7 +258,7 @@ def generate_container_config(clp_config: CLPConfig, clp_home: pathlib.Path):
 
 
 def dump_container_config(
-    clp_config: CLPConfig, container_clp_config, container_name: str
+    clp_config: CLPConfig, container_clp_config: CLPConfig, container_name: str
 ) -> Tuple[pathlib.Path, pathlib.Path]:
     container_config_filename = f".{container_name}-config.yml"
     config_file_path_on_host = clp_config.logs_directory / container_config_filename
@@ -268,8 +270,8 @@ def dump_container_config(
 
 
 def generate_container_start_cmd(
-    container_name: str, container_mounts: List[DockerMount], execution_container: str
-):
+    container_name: str, container_mounts: List[CLPDockerMounts], execution_container: str
+) -> List[str]:
     clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
     # fmt: off
     container_start_cmd = [
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index 2468f869b..024a3d761 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -119,6 +119,5 @@ def main(argv):
     return handle_decompression(parsed_args, clp_home, clp_config)
 
 
-
 if "__main__" == __name__:
     sys.exit(main(sys.argv))

From efdc165f16488c588c390e29b48a11b8d3806f00 Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 14:14:38 -0400
Subject: [PATCH 04/14] add missing file

---
 .../clp_package_utils/scripts/native/utils.py | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 components/clp-package-utils/clp_package_utils/scripts/native/utils.py

diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/utils.py b/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
new file mode 100644
index 000000000..3de57a330
--- /dev/null
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
@@ -0,0 +1,85 @@
+import asyncio
+import multiprocessing
+import time
+from contextlib import closing
+
+import msgpack
+from clp_py_utils.clp_config import (
+    QUERY_JOBS_TABLE_NAME,
+)
+from clp_py_utils.sql_adapter import SQL_Adapter
+from job_orchestration.scheduler.constants import QueryJobStatus, QueryJobType
+from job_orchestration.scheduler.scheduler_data import QueryJobConfig
+
+
+async def run_function_in_process(function, *args, initializer=None, init_args=None):
+    """
+    Runs the given function in a separate process wrapped in a *cancellable*
+    asyncio task. This is necessary because asyncio's multiprocessing process
+    cannot be cancelled once it's started.
+    :param function: Method to run
+    :param args: Arguments for the method
+    :param initializer: Initializer for each process in the pool
+    :param init_args: Arguments for the initializer
+    :return: Return value of the method
+    """
+    pool = multiprocessing.Pool(1, initializer, init_args)
+
+    loop = asyncio.get_event_loop()
+    fut = loop.create_future()
+
+    def process_done_callback(obj):
+        loop.call_soon_threadsafe(fut.set_result, obj)
+
+    def process_error_callback(err):
+        loop.call_soon_threadsafe(fut.set_exception, err)
+
+    pool.apply_async(
+        function, args, callback=process_done_callback, error_callback=process_error_callback
+    )
+
+    try:
+        return await fut
+    except asyncio.CancelledError:
+        pass
+    finally:
+        pool.terminate()
+        pool.close()
+
+
+def submit_query_job(
+    sql_adapter: SQL_Adapter, job_config: QueryJobConfig, job_type: QueryJobType
+) -> int:
+    with closing(sql_adapter.create_connection(True)) as db_conn, closing(
+        db_conn.cursor(dictionary=True)
+    ) as db_cursor:
+        # Create job
+        db_cursor.execute(
+            f"INSERT INTO `{QUERY_JOBS_TABLE_NAME}` (`job_config`, `type`) VALUES (%s, %s)",
+            (msgpack.packb(job_config.dict()), job_type),
+        )
+        db_conn.commit()
+        return db_cursor.lastrowid
+
+
+def wait_for_query_job(sql_adapter: SQL_Adapter, job_id: int) -> QueryJobStatus:
+    with closing(sql_adapter.create_connection(True)) as db_conn, closing(
+        db_conn.cursor(dictionary=True)
+    ) as db_cursor:
+        # Wait for the job to be marked complete
+        while True:
+            db_cursor.execute(
+                f"SELECT `status` FROM `{QUERY_JOBS_TABLE_NAME}` WHERE `id` = {job_id}"
+            )
+            # There will only ever be one row since it's impossible to have more than one job with
+            # the same ID
+            new_status = db_cursor.fetchall()[0]["status"]
+            db_conn.commit()
+            if new_status in (
+                QueryJobStatus.SUCCEEDED,
+                QueryJobStatus.FAILED,
+                QueryJobStatus.CANCELLED,
+            ):
+                return new_status
+
+            time.sleep(0.5)

From a9db5de90eef79c83dcc492022f92ec5c2169556 Mon Sep 17 00:00:00 2001
From: haiqi96 <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 16:04:36 -0400
Subject: [PATCH 05/14] Apply suggestions from code review

Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com>
---
 .../clp_package_utils/general.py               | 18 ++++++++++++++++++
 .../clp_package_utils/scripts/decompress.py    |  1 +
 .../scripts/native/decompress.py               |  8 ++++++++
 .../clp_package_utils/scripts/native/utils.py  | 13 +++++++++++++
 4 files changed, 40 insertions(+)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 644eed22a..a0de61b4e 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -102,6 +102,10 @@ def get_clp_home():
 
 
 def generate_container_name(job_type: JobType) -> str:
+    """
+    :param job_type:
+    :return: A unique container name for the given job type.
+    """
     return f"clp-{job_type}-{str(uuid.uuid4())[-4:]}"
 
 
@@ -260,6 +264,13 @@ def generate_container_config(
 def dump_container_config(
     clp_config: CLPConfig, container_clp_config: CLPConfig, container_name: str
 ) -> Tuple[pathlib.Path, pathlib.Path]:
+    """
+    Writes the given config to the logs directory so that it's accessible in the container.
+    :param clp_config: The corresponding config on the host (used to determine the logs directory).
+    :param container_clp_config: The config to write.
+    :param container_name:
+    :return: The path to the config file in the container and on the host.
+    """
     container_config_filename = f".{container_name}-config.yml"
     config_file_path_on_host = clp_config.logs_directory / container_config_filename
     config_file_path_on_container = container_clp_config.logs_directory / container_config_filename
@@ -272,6 +283,13 @@ def dump_container_config(
 def generate_container_start_cmd(
     container_name: str, container_mounts: List[CLPDockerMounts], execution_container: str
 ) -> List[str]:
+    """
+    Generates the command to start a container with the given mounts and name.
+    :param container_name:
+    :param container_mounts:
+    :param execution_container:
+    :return: The command.
+    """
     clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
     # fmt: off
     container_start_cmd = [
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index 44297e4c7..6b8cbd8b8 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -82,6 +82,7 @@ def main(argv):
     config_file_path_on_container, config_file_path_on_host = dump_container_config(
         clp_config, container_clp_config, container_name
     )
+
     # Set up mounts
     container_extraction_dir = pathlib.Path("/") / "mnt" / "extraction-dir"
     necessary_mounts = [
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index 024a3d761..56404db65 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -30,9 +30,17 @@ def handle_decompression(
     clp_home: pathlib.Path,
     clp_config: CLPConfig,
 ):
+    """
+    Handles the decompression command.
+    :param parsed_args:
+    :param clp_home:
+    :param clp_config:
+    :return: 0 on success, -1 otherwise.
+    """
     # Validate paths were specified using only one method
     if len(parsed_args.paths) > 0 and parsed_args.files_from is not None:
         logger.error("Paths cannot be specified both on the command line and through a file.")
+        return -1
 
     # Validate extraction directory
     extraction_dir = pathlib.Path(parsed_args.extraction_dir)
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/utils.py b/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
index 3de57a330..0ff41cc73 100644
--- a/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
@@ -50,6 +50,13 @@ def process_error_callback(err):
 def submit_query_job(
     sql_adapter: SQL_Adapter, job_config: QueryJobConfig, job_type: QueryJobType
 ) -> int:
+    """
+    Submits a query job.
+    :param sql_adapter:
+    :param job_config:
+    :param job_type:
+    :return: The job's ID.
+    """
     with closing(sql_adapter.create_connection(True)) as db_conn, closing(
         db_conn.cursor(dictionary=True)
     ) as db_cursor:
@@ -63,6 +70,12 @@ def submit_query_job(
 
 
 def wait_for_query_job(sql_adapter: SQL_Adapter, job_id: int) -> QueryJobStatus:
+    """
+    Waits for the query job with the given ID to complete.
+    :param sql_adapter: 
+    :param job_id: 
+    :return: The job's status on completion.
+    """
     with closing(sql_adapter.create_connection(True)) as db_conn, closing(
         db_conn.cursor(dictionary=True)
     ) as db_cursor:

From d2791e29ada3e7da7c5941490be8993fa63fadca Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 16:14:52 -0400
Subject: [PATCH 06/14] address code review comments

---
 components/clp-package-utils/clp_package_utils/general.py  | 7 ++++---
 .../clp_package_utils/scripts/compress.py                  | 6 +++---
 .../clp_package_utils/scripts/decompress.py                | 6 +++---
 .../clp_package_utils/scripts/native/decompress.py         | 6 ++++--
 .../clp_package_utils/scripts/native/utils.py              | 4 ++--
 .../clp-package-utils/clp_package_utils/scripts/search.py  | 6 +++---
 6 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index a0de61b4e..3698733b1 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -203,6 +203,7 @@ def generate_container_config(
 
     :param clp_config:
     :param clp_home:
+    :return: The container config and the mounts.
     """
     container_clp_config = clp_config.copy(deep=True)
 
@@ -281,13 +282,13 @@ def dump_container_config(
 
 
 def generate_container_start_cmd(
-    container_name: str, container_mounts: List[CLPDockerMounts], execution_container: str
+    container_name: str, container_mounts: List[CLPDockerMounts], container_image: str
 ) -> List[str]:
     """
     Generates the command to start a container with the given mounts and name.
     :param container_name:
     :param container_mounts:
-    :param execution_container:
+    :param container_image:
     :return: The command.
     """
     clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
@@ -307,7 +308,7 @@ def generate_container_start_cmd(
         if mount:
             container_start_cmd.append("--mount")
             container_start_cmd.append(str(mount))
-    container_start_cmd.append(execution_container)
+    container_start_cmd.append(container_image)
 
     return container_start_cmd
 
diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index 4a0c98d03..d93a394be 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -70,7 +70,7 @@ def main(argv):
     container_name = generate_container_name(JobType.COMPRESSION)
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
-    config_file_path_on_container, config_file_path_on_host = dump_container_config(
+    generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         clp_config, container_clp_config, container_name
     )
 
@@ -83,7 +83,7 @@ def main(argv):
     compress_cmd = [
         "python3",
         "-m", "clp_package_utils.scripts.native.compress",
-        "--config", str(config_file_path_on_container),
+        "--config", str(generated_config_path_on_container),
         "--remove-path-prefix", str(CONTAINER_INPUT_LOGS_ROOT_DIR),
     ]
     # fmt: on
@@ -122,7 +122,7 @@ def main(argv):
     subprocess.run(cmd, check=True)
 
     # Remove generated files
-    config_file_path_on_host.unlink()
+    generated_config_path_on_host.unlink()
 
     return 0
 
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index 6b8cbd8b8..bf69e8482 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -79,7 +79,7 @@ def main(argv):
 
     container_name = generate_container_name(JobType.DECOMPRESSION)
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
-    config_file_path_on_container, config_file_path_on_host = dump_container_config(
+    generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         clp_config, container_clp_config, container_name
     )
 
@@ -112,7 +112,7 @@ def main(argv):
     decompress_cmd = [
         "python3",
         "-m", "clp_package_utils.scripts.native.decompress",
-        "--config", str(config_file_path_on_container),
+        "--config", str(generated_config_path_on_container),
         "-d", str(container_extraction_dir),
     ]
     # fmt: on
@@ -126,7 +126,7 @@ def main(argv):
     subprocess.run(cmd, check=True)
 
     # Remove generated files
-    config_file_path_on_host.unlink()
+    generated_config_path_on_host.unlink()
 
     return 0
 
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index 56404db65..443d78f7c 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -48,6 +48,9 @@ def handle_decompression(
         logger.error(f"extraction-dir ({extraction_dir}) is not a valid directory.")
         return -1
 
+    paths = parsed_args.paths
+    list_path = parsed_args.files_from
+
     logs_dir = clp_config.logs_directory
     archives_dir = clp_config.archive_output.directory
 
@@ -63,8 +66,7 @@ def handle_decompression(
         "--db-config-file", str(db_config_file_path),
     ]
     # fmt: on
-    paths = parsed_args.paths
-    list_path = parsed_args.files_from
+
     files_to_decompress_list_path = None
     if list_path is not None:
         decompression_cmd.append("-f")
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/utils.py b/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
index 0ff41cc73..6b94e4676 100644
--- a/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/utils.py
@@ -72,8 +72,8 @@ def submit_query_job(
 def wait_for_query_job(sql_adapter: SQL_Adapter, job_id: int) -> QueryJobStatus:
     """
     Waits for the query job with the given ID to complete.
-    :param sql_adapter: 
-    :param job_id: 
+    :param sql_adapter:
+    :param job_id:
     :return: The job's status on completion.
     """
     with closing(sql_adapter.create_connection(True)) as db_conn, closing(
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index 34acbda40..115055101 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -87,7 +87,7 @@ def main(argv):
     container_name = generate_container_name(JobType.SEARCH)
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
-    config_file_path_on_container, config_file_path_on_host = dump_container_config(
+    generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         clp_config, container_clp_config, container_name
     )
 
@@ -100,7 +100,7 @@ def main(argv):
     search_cmd = [
         "python3",
         "-m", "clp_package_utils.scripts.native.search",
-        "--config", str(config_file_path_on_container),
+        "--config", str(generated_config_path_on_container),
         parsed_args.wildcard_query,
     ]
     # fmt: on
@@ -127,7 +127,7 @@ def main(argv):
     subprocess.run(cmd, check=True)
 
     # Remove generated files
-    config_file_path_on_host.unlink()
+    generated_config_path_on_host.unlink()
 
     return 0
 

From 15beecadb8ebd4b9a36bbe61797fd24b689d0bb6 Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 16:52:25 -0400
Subject: [PATCH 07/14] address more code review comments

---
 .../clp_package_utils/general.py              |  2 +-
 .../clp_package_utils/scripts/compress.py     |  2 +-
 .../clp_package_utils/scripts/decompress.py   |  2 +-
 .../scripts/native/decompress.py              | 52 +++++++++++++------
 .../clp_package_utils/scripts/search.py       |  2 +-
 5 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 3698733b1..9d68eeb43 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -263,7 +263,7 @@ def generate_container_config(
 
 
 def dump_container_config(
-    clp_config: CLPConfig, container_clp_config: CLPConfig, container_name: str
+    container_clp_config: CLPConfig, clp_config: CLPConfig, container_name: str
 ) -> Tuple[pathlib.Path, pathlib.Path]:
     """
     Writes the given config to the logs directory so that it's accessible in the container.
diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index d93a394be..d056caf01 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -71,7 +71,7 @@ def main(argv):
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        clp_config, container_clp_config, container_name
+        container_clp_config, clp_config, container_name
     )
 
     necessary_mounts = [mounts.clp_home, mounts.input_logs_dir, mounts.data_dir, mounts.logs_dir]
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index bf69e8482..426cc98b8 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -80,7 +80,7 @@ def main(argv):
     container_name = generate_container_name(JobType.DECOMPRESSION)
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        clp_config, container_clp_config, container_name
+        container_clp_config, clp_config, container_name
     )
 
     # Set up mounts
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index 443d78f7c..0e787cadf 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -4,6 +4,7 @@
 import subprocess
 import sys
 import uuid
+from typing import Optional
 
 import yaml
 from clp_py_utils.clp_config import CLPConfig
@@ -25,16 +26,38 @@
 logger.addHandler(logging_console_handler)
 
 
-def handle_decompression(
-    parsed_args,
+def _validate_and_load_config_file(
     clp_home: pathlib.Path,
-    clp_config: CLPConfig,
+    config_file_path: pathlib.Path,
+    default_config_file_path: pathlib.Path,
+) -> Optional[CLPConfig]:
+    """
+    Validates and loads the config file for container.
+    :param config_file_path:
+    :param clp_home:
+    :param default_config_file_path:
+    :return: clp_config on success, None otherwise.
+    """
+    try:
+        clp_config = validate_and_load_config_file(
+            config_file_path, default_config_file_path, clp_home
+        )
+        clp_config.validate_archive_output_dir()
+        clp_config.validate_logs_dir()
+        return clp_config
+    except:
+        logger.exception("Failed to load config.")
+        return None
+
+
+def handle_decompression_command(
+    parsed_args, clp_home: pathlib.Path, default_config_file_path: pathlib.Path
 ):
     """
     Handles the decompression command.
     :param parsed_args:
     :param clp_home:
-    :param clp_config:
+    :param default_config_file_path:
     :return: 0 on success, -1 otherwise.
     """
     # Validate paths were specified using only one method
@@ -48,6 +71,13 @@ def handle_decompression(
         logger.error(f"extraction-dir ({extraction_dir}) is not a valid directory.")
         return -1
 
+    # Validate and load config file
+    clp_config = _validate_and_load_config_file(
+        clp_home, pathlib.Path(parsed_args.config), default_config_file_path
+    )
+    if not clp_config:
+        return -1
+
     paths = parsed_args.paths
     list_path = parsed_args.files_from
 
@@ -114,19 +144,7 @@ def main(argv):
     )
     parsed_args = args_parser.parse_args(argv[1:])
 
-    # Validate and load config file
-    try:
-        config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
-        clp_config.validate_archive_output_dir()
-        clp_config.validate_logs_dir()
-    except:
-        logger.exception("Failed to load config.")
-        return -1
-
-    return handle_decompression(parsed_args, clp_home, clp_config)
+    return handle_decompression_command(parsed_args, clp_home, default_config_file_path)
 
 
 if "__main__" == __name__:
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index 115055101..c547c9238 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -88,7 +88,7 @@ def main(argv):
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        clp_config, container_clp_config, container_name
+        container_clp_config, clp_config, container_name
     )
 
     necessary_mounts = [mounts.clp_home, mounts.logs_dir]

From 1c5211f9566ec398d2857e718b35ef73db36410a Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 16:53:25 -0400
Subject: [PATCH 08/14] Small touch

---
 .../clp-package-utils/clp_package_utils/scripts/compress.py     | 1 +
 .../clp-package-utils/clp_package_utils/scripts/decompress.py   | 1 +
 .../clp-package-utils/clp_package_utils/scripts/search.py       | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index d056caf01..3d6784fa5 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -62,6 +62,7 @@ def main(argv):
         )
         clp_config.validate_logs_dir()
 
+        # Validate and load necessary credentials
         validate_and_load_db_credentials_file(clp_config, clp_home, False)
     except:
         logger.exception("Failed to load config.")
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index 426cc98b8..a273b42eb 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -59,6 +59,7 @@ def main(argv):
         )
         clp_config.validate_logs_dir()
 
+        # Validate and load necessary credentials
         validate_and_load_db_credentials_file(clp_config, clp_home, False)
     except:
         logger.exception("Failed to load config.")
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index c547c9238..9c1316cf1 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -79,7 +79,7 @@ def main(argv):
         clp_config.validate_logs_dir()
 
         # Validate and load necessary credentials
-        validate_and_load_db_credentials_file(clp_config, clp_home, True)
+        validate_and_load_db_credentials_file(clp_config, clp_home, False)
     except:
         logger.exception("Failed to load config.")
         return -1

From b11ffeb9deff420fd933c49e654787abe7e8cd7e Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 17:42:18 -0400
Subject: [PATCH 09/14] rename functions

---
 components/clp-package-utils/clp_package_utils/general.py | 2 +-
 .../clp_package_utils/scripts/compress.py                 | 4 ++--
 .../clp_package_utils/scripts/decompress.py               | 4 ++--
 .../clp_package_utils/scripts/native/compress.py          | 4 ++--
 .../clp_package_utils/scripts/native/decompress.py        | 8 ++++----
 .../clp_package_utils/scripts/native/search.py            | 4 ++--
 .../clp-package-utils/clp_package_utils/scripts/search.py | 4 ++--
 .../clp_package_utils/scripts/start_clp.py                | 4 ++--
 .../clp_package_utils/scripts/stop_clp.py                 | 4 ++--
 9 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 9d68eeb43..87db23ef1 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -321,7 +321,7 @@ def validate_config_key_existence(config, key):
     return value
 
 
-def validate_and_load_config_file(
+def load_config_file(
     config_file_path: pathlib.Path, default_config_file_path: pathlib.Path, clp_home: pathlib.Path
 ):
     if config_file_path.exists():
diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index 3d6784fa5..53300d007 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -14,7 +14,7 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    validate_and_load_config_file,
+    load_config_file,
     validate_and_load_db_credentials_file,
 )
 
@@ -57,7 +57,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
         clp_config.validate_logs_dir()
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index a273b42eb..e65d7f89c 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -16,7 +16,7 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    validate_and_load_config_file,
+    load_config_file,
     validate_and_load_db_credentials_file,
     validate_path_could_be_dir,
 )
@@ -54,7 +54,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
         clp_config.validate_logs_dir()
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py
index a08602007..cfe0e7625 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py
@@ -20,7 +20,7 @@
 from clp_package_utils.general import (
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     get_clp_home,
-    validate_and_load_config_file,
+    load_config_file,
 )
 
 # Setup logging
@@ -170,7 +170,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
         clp_config.validate_input_logs_dir()
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index 0e787cadf..1eb3faf3b 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -12,7 +12,7 @@
 from clp_package_utils.general import (
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     get_clp_home,
-    validate_and_load_config_file,
+    load_config_file,
 )
 
 # Setup logging
@@ -26,7 +26,7 @@
 logger.addHandler(logging_console_handler)
 
 
-def _validate_and_load_config_file(
+def validate_and_load_config_file(
     clp_home: pathlib.Path,
     config_file_path: pathlib.Path,
     default_config_file_path: pathlib.Path,
@@ -39,7 +39,7 @@ def _validate_and_load_config_file(
     :return: clp_config on success, None otherwise.
     """
     try:
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
         clp_config.validate_archive_output_dir()
@@ -72,7 +72,7 @@ def handle_decompression_command(
         return -1
 
     # Validate and load config file
-    clp_config = _validate_and_load_config_file(
+    clp_config = validate_and_load_config_file(
         clp_home, pathlib.Path(parsed_args.config), default_config_file_path
     )
     if not clp_config:
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py
index df015f25f..ac73298c9 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py
@@ -18,7 +18,7 @@
 from clp_package_utils.general import (
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     get_clp_home,
-    validate_and_load_config_file,
+    load_config_file,
 )
 from clp_package_utils.scripts.native.utils import (
     run_function_in_process,
@@ -270,7 +270,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
         clp_config.validate_logs_dir()
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index 9c1316cf1..3692d753a 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -16,7 +16,7 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    validate_and_load_config_file,
+    load_config_file,
     validate_and_load_db_credentials_file,
 )
 
@@ -73,7 +73,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
         clp_config.validate_logs_dir()
diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
index 5ba5bbe15..e7c67cdd9 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
@@ -44,7 +44,7 @@
     get_clp_home,
     is_container_exited,
     is_container_running,
-    validate_and_load_config_file,
+    load_config_file,
     validate_and_load_db_credentials_file,
     validate_and_load_queue_credentials_file,
     validate_and_load_redis_credentials_file,
@@ -887,7 +887,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
 
diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py
index 00e3f7b6b..299b3d983 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py
@@ -25,7 +25,7 @@
     get_clp_home,
     is_container_exited,
     is_container_running,
-    validate_and_load_config_file,
+    load_config_file,
     validate_and_load_db_credentials_file,
     validate_and_load_queue_credentials_file,
 )
@@ -103,7 +103,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = validate_and_load_config_file(
+        clp_config = load_config_file(
             config_file_path, default_config_file_path, clp_home
         )
 

From f727a79dbfe954a9697a0fdc3560a583aa6e25ba Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 17:53:43 -0400
Subject: [PATCH 10/14] Linter

---
 .../clp_package_utils/scripts/native/compress.py              | 4 +---
 .../clp_package_utils/scripts/native/decompress.py            | 4 +---
 .../clp_package_utils/scripts/native/search.py                | 4 +---
 .../clp-package-utils/clp_package_utils/scripts/start_clp.py  | 4 +---
 .../clp-package-utils/clp_package_utils/scripts/stop_clp.py   | 4 +---
 5 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py
index cfe0e7625..cb495204f 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/compress.py
@@ -170,9 +170,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
         clp_config.validate_input_logs_dir()
         clp_config.validate_logs_dir()
     except:
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index 1eb3faf3b..c30b7b25b 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -39,9 +39,7 @@ def validate_and_load_config_file(
     :return: clp_config on success, None otherwise.
     """
     try:
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
         clp_config.validate_archive_output_dir()
         clp_config.validate_logs_dir()
         return clp_config
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/search.py b/components/clp-package-utils/clp_package_utils/scripts/native/search.py
index ac73298c9..7dd247fa5 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/search.py
@@ -270,9 +270,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
         clp_config.validate_logs_dir()
     except:
         logger.exception("Failed to load config.")
diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
index e7c67cdd9..9dba79886 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
@@ -887,9 +887,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
 
         # Validate and load necessary credentials
         if target in (
diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py
index 299b3d983..7971dd7d8 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py
@@ -103,9 +103,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
 
         # Validate and load necessary credentials
         if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME):

From d3eb238ac18d5d3814b522f20d53bb2df15ebfc3 Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 17:54:22 -0400
Subject: [PATCH 11/14] Last refactoring

---
 .../clp_package_utils/general.py              | 25 +++++++++++++++++--
 .../clp_package_utils/scripts/compress.py     | 18 ++++---------
 .../clp_package_utils/scripts/decompress.py   | 18 ++++---------
 .../clp_package_utils/scripts/search.py       | 18 ++++---------
 4 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 87db23ef1..8b6a5c6ec 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -8,7 +8,8 @@
 import typing
 import uuid
 from enum import auto
-from typing import List, Tuple
+from logging import Logger
+from typing import List, Optional, Tuple
 
 import yaml
 from clp_py_utils.clp_config import (
@@ -321,9 +322,29 @@ def validate_config_key_existence(config, key):
     return value
 
 
+def validate_and_load_config_for_job_submission_script(
+    clp_home: pathlib.Path,
+    config_file_path: pathlib.Path,
+    default_config_file_path: pathlib.Path,
+    logger: Logger,
+) -> Optional[CLPConfig]:
+    # Validate and load config file
+    try:
+        config_file_path = pathlib.Path(config_file_path)
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
+        clp_config.validate_logs_dir()
+
+        # Validate and load necessary credentials
+        validate_and_load_db_credentials_file(clp_config, clp_home, False)
+        return clp_config
+    except:
+        logger.exception("Failed to load config.")
+        return None
+
+
 def load_config_file(
     config_file_path: pathlib.Path, default_config_file_path: pathlib.Path, clp_home: pathlib.Path
-):
+) -> CLPConfig:
     if config_file_path.exists():
         raw_clp_config = read_yaml_config_file(config_file_path)
         if raw_clp_config is None:
diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index 53300d007..2f833b21a 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -14,8 +14,7 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    load_config_file,
-    validate_and_load_db_credentials_file,
+    validate_and_load_config_for_job_submission_script,
 )
 
 # Setup logging
@@ -55,17 +54,10 @@ def main(argv):
     parsed_args = args_parser.parse_args(argv[1:])
 
     # Validate and load config file
-    try:
-        config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
-        clp_config.validate_logs_dir()
-
-        # Validate and load necessary credentials
-        validate_and_load_db_credentials_file(clp_config, clp_home, False)
-    except:
-        logger.exception("Failed to load config.")
+    clp_config = validate_and_load_config_for_job_submission_script(
+        clp_home, pathlib.Path(parsed_args.config), default_config_file_path, logger
+    )
+    if not clp_config:
         return -1
 
     container_name = generate_container_name(JobType.COMPRESSION)
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index e65d7f89c..c1b25a0e3 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -16,8 +16,7 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    load_config_file,
-    validate_and_load_db_credentials_file,
+    validate_and_load_config_for_job_submission_script,
     validate_path_could_be_dir,
 )
 
@@ -52,17 +51,10 @@ def main(argv):
     parsed_args = args_parser.parse_args(argv[1:])
 
     # Validate and load config file
-    try:
-        config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
-        clp_config.validate_logs_dir()
-
-        # Validate and load necessary credentials
-        validate_and_load_db_credentials_file(clp_config, clp_home, False)
-    except:
-        logger.exception("Failed to load config.")
+    clp_config = validate_and_load_config_for_job_submission_script(
+        clp_home, pathlib.Path(parsed_args.config), default_config_file_path, logger
+    )
+    if not clp_config:
         return -1
 
     paths_to_decompress_file_path = None
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index 3692d753a..9d56e3c82 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -16,8 +16,7 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    load_config_file,
-    validate_and_load_db_credentials_file,
+    validate_and_load_config_for_job_submission_script,
 )
 
 # Setup logging
@@ -71,17 +70,10 @@ def main(argv):
     parsed_args = args_parser.parse_args(argv[1:])
 
     # Validate and load config file
-    try:
-        config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
-        clp_config.validate_logs_dir()
-
-        # Validate and load necessary credentials
-        validate_and_load_db_credentials_file(clp_config, clp_home, False)
-    except:
-        logger.exception("Failed to load config.")
+    clp_config = validate_and_load_config_for_job_submission_script(
+        clp_home, pathlib.Path(parsed_args.config), default_config_file_path, logger
+    )
+    if not clp_config:
         return -1
 
     container_name = generate_container_name(JobType.SEARCH)

From b3883e9784a60462793533ba8d4ae733d6dee583 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Fri, 5 Jul 2024 18:54:47 -0400
Subject: [PATCH 12/14] Touch-ups.

---
 components/clp-package-utils/clp_package_utils/general.py | 2 +-
 .../clp_package_utils/scripts/native/decompress.py        | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 8b6a5c6ec..033cac6f2 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -268,8 +268,8 @@ def dump_container_config(
 ) -> Tuple[pathlib.Path, pathlib.Path]:
     """
     Writes the given config to the logs directory so that it's accessible in the container.
-    :param clp_config: The corresponding config on the host (used to determine the logs directory).
     :param container_clp_config: The config to write.
+    :param clp_config: The corresponding config on the host (used to determine the logs directory).
     :param container_name:
     :return: The path to the config file in the container and on the host.
     """
diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
index c30b7b25b..9331492b4 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py
@@ -32,9 +32,9 @@ def validate_and_load_config_file(
     default_config_file_path: pathlib.Path,
 ) -> Optional[CLPConfig]:
     """
-    Validates and loads the config file for container.
-    :param config_file_path:
+    Validates and loads the config file.
     :param clp_home:
+    :param config_file_path:
     :param default_config_file_path:
     :return: clp_config on success, None otherwise.
     """
@@ -43,13 +43,13 @@ def validate_and_load_config_file(
         clp_config.validate_archive_output_dir()
         clp_config.validate_logs_dir()
         return clp_config
-    except:
+    except Exception:
         logger.exception("Failed to load config.")
         return None
 
 
 def handle_decompression_command(
-    parsed_args, clp_home: pathlib.Path, default_config_file_path: pathlib.Path
+    parsed_args: argparse.Namespace, clp_home: pathlib.Path, default_config_file_path: pathlib.Path
 ):
     """
     Handles the decompression command.

From 8d0154393454fa920399b6f8180a085c3eada3e2 Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 19:30:34 -0400
Subject: [PATCH 13/14] Revert "Last refactoring"

This reverts commit d3eb238ac18d5d3814b522f20d53bb2df15ebfc3.
---
 .../clp_package_utils/general.py              | 25 ++-----------------
 .../clp_package_utils/scripts/compress.py     | 18 +++++++++----
 .../clp_package_utils/scripts/decompress.py   | 18 +++++++++----
 .../clp_package_utils/scripts/search.py       | 18 +++++++++----
 4 files changed, 41 insertions(+), 38 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
index 033cac6f2..b84bf298e 100644
--- a/components/clp-package-utils/clp_package_utils/general.py
+++ b/components/clp-package-utils/clp_package_utils/general.py
@@ -8,8 +8,7 @@
 import typing
 import uuid
 from enum import auto
-from logging import Logger
-from typing import List, Optional, Tuple
+from typing import List, Tuple
 
 import yaml
 from clp_py_utils.clp_config import (
@@ -322,29 +321,9 @@ def validate_config_key_existence(config, key):
     return value
 
 
-def validate_and_load_config_for_job_submission_script(
-    clp_home: pathlib.Path,
-    config_file_path: pathlib.Path,
-    default_config_file_path: pathlib.Path,
-    logger: Logger,
-) -> Optional[CLPConfig]:
-    # Validate and load config file
-    try:
-        config_file_path = pathlib.Path(config_file_path)
-        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
-        clp_config.validate_logs_dir()
-
-        # Validate and load necessary credentials
-        validate_and_load_db_credentials_file(clp_config, clp_home, False)
-        return clp_config
-    except:
-        logger.exception("Failed to load config.")
-        return None
-
-
 def load_config_file(
     config_file_path: pathlib.Path, default_config_file_path: pathlib.Path, clp_home: pathlib.Path
-) -> CLPConfig:
+):
     if config_file_path.exists():
         raw_clp_config = read_yaml_config_file(config_file_path)
         if raw_clp_config is None:
diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index 2f833b21a..53300d007 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -14,7 +14,8 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    validate_and_load_config_for_job_submission_script,
+    load_config_file,
+    validate_and_load_db_credentials_file,
 )
 
 # Setup logging
@@ -54,10 +55,17 @@ def main(argv):
     parsed_args = args_parser.parse_args(argv[1:])
 
     # Validate and load config file
-    clp_config = validate_and_load_config_for_job_submission_script(
-        clp_home, pathlib.Path(parsed_args.config), default_config_file_path, logger
-    )
-    if not clp_config:
+    try:
+        config_file_path = pathlib.Path(parsed_args.config)
+        clp_config = load_config_file(
+            config_file_path, default_config_file_path, clp_home
+        )
+        clp_config.validate_logs_dir()
+
+        # Validate and load necessary credentials
+        validate_and_load_db_credentials_file(clp_config, clp_home, False)
+    except:
+        logger.exception("Failed to load config.")
         return -1
 
     container_name = generate_container_name(JobType.COMPRESSION)
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index c1b25a0e3..e65d7f89c 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -16,7 +16,8 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    validate_and_load_config_for_job_submission_script,
+    load_config_file,
+    validate_and_load_db_credentials_file,
     validate_path_could_be_dir,
 )
 
@@ -51,10 +52,17 @@ def main(argv):
     parsed_args = args_parser.parse_args(argv[1:])
 
     # Validate and load config file
-    clp_config = validate_and_load_config_for_job_submission_script(
-        clp_home, pathlib.Path(parsed_args.config), default_config_file_path, logger
-    )
-    if not clp_config:
+    try:
+        config_file_path = pathlib.Path(parsed_args.config)
+        clp_config = load_config_file(
+            config_file_path, default_config_file_path, clp_home
+        )
+        clp_config.validate_logs_dir()
+
+        # Validate and load necessary credentials
+        validate_and_load_db_credentials_file(clp_config, clp_home, False)
+    except:
+        logger.exception("Failed to load config.")
         return -1
 
     paths_to_decompress_file_path = None
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index 9d56e3c82..3692d753a 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -16,7 +16,8 @@
     generate_container_start_cmd,
     get_clp_home,
     JobType,
-    validate_and_load_config_for_job_submission_script,
+    load_config_file,
+    validate_and_load_db_credentials_file,
 )
 
 # Setup logging
@@ -70,10 +71,17 @@ def main(argv):
     parsed_args = args_parser.parse_args(argv[1:])
 
     # Validate and load config file
-    clp_config = validate_and_load_config_for_job_submission_script(
-        clp_home, pathlib.Path(parsed_args.config), default_config_file_path, logger
-    )
-    if not clp_config:
+    try:
+        config_file_path = pathlib.Path(parsed_args.config)
+        clp_config = load_config_file(
+            config_file_path, default_config_file_path, clp_home
+        )
+        clp_config.validate_logs_dir()
+
+        # Validate and load necessary credentials
+        validate_and_load_db_credentials_file(clp_config, clp_home, False)
+    except:
+        logger.exception("Failed to load config.")
         return -1
 
     container_name = generate_container_name(JobType.SEARCH)

From 4aee8e740e334cdc0a9709eb592cb2f79a037be2 Mon Sep 17 00:00:00 2001
From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com>
Date: Fri, 5 Jul 2024 19:53:41 -0400
Subject: [PATCH 14/14] Linter

---
 .../clp-package-utils/clp_package_utils/scripts/compress.py   | 4 +---
 .../clp-package-utils/clp_package_utils/scripts/decompress.py | 4 +---
 .../clp-package-utils/clp_package_utils/scripts/search.py     | 4 +---
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
index 53300d007..d0aa30913 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/compress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -57,9 +57,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
         clp_config.validate_logs_dir()
 
         # Validate and load necessary credentials
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
index e65d7f89c..f700d9721 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -54,9 +54,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
         clp_config.validate_logs_dir()
 
         # Validate and load necessary credentials
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
index 3692d753a..f3f02046d 100755
--- a/components/clp-package-utils/clp_package_utils/scripts/search.py
+++ b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -73,9 +73,7 @@ def main(argv):
     # Validate and load config file
     try:
         config_file_path = pathlib.Path(parsed_args.config)
-        clp_config = load_config_file(
-            config_file_path, default_config_file_path, clp_home
-        )
+        clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
         clp_config.validate_logs_dir()
 
         # Validate and load necessary credentials