Refactor fetch CLI tool logic (#243)

Refactor fetch CLI tool, sharing more functionality with the fetch
Celery task.

AIPscan/Aggregator/task_helpers.py

@@ -158,3 +158,13 @@ def write_mets(http_response, package_uuid, subdir):
     with open(download_file, "wb") as file:
         file.write(http_response.content)
     return download_file
+
+
+def summarize_fetch_job_results(fetch_job):
+    return "aips: '{}'; sips: '{}'; dips: '{}'; deleted: '{}'; replicated: '{}'".format(
+        fetch_job.total_aips,
+        fetch_job.total_sips,
+        fetch_job.total_dips,
+        fetch_job.total_deleted_aips,
+        fetch_job.total_replicas,
+    )

AIPscan/Aggregator/tasks.py

@@ -19,6 +19,7 @@
     format_api_url_with_limit_offset,
     parse_package_list_file,
     process_package_object,
+    summarize_fetch_job_results,
 )
 from AIPscan.extensions import celery
 from AIPscan.helpers import file_sha256_hash
@@ -193,14 +194,7 @@ def workflow_coordinator(
         fetch_job_id, all_packages, total_packages_count
     )
 
-    summary = "aips: '{}'; sips: '{}'; dips: '{}'; deleted: '{}'; replicated: '{}'".format(
-        obj.total_aips,
-        obj.total_sips,
-        obj.total_dips,
-        obj.total_deleted_aips,
-        obj.total_replicas,
-    )
-    logger.info("%s", summary)
+    logger.info("%s", summarize_fetch_job_results(obj))
 
 
 def make_request(request_url, request_url_without_api_key):

AIPscan/Aggregator/tests/test_task_helpers.py

@@ -305,3 +305,23 @@ def test_process_package_object(packages, idx, storage_service_package):
     """
     package_obj = task_helpers.process_package_object(packages[idx])
     assert package_obj == storage_service_package, idx
+
+
+def test_summarize_fetch_job_results():
+    fetch_job = models.FetchJob(
+        total_packages=23,
+        total_aips=1,
+        total_deleted_aips=4,
+        download_start=None,
+        download_end=None,
+        download_directory=None,
+        storage_service_id=None,
+    )
+    fetch_job.total_sips = 2
+    fetch_job.total_dips = 3
+    fetch_job.total_replicas = 5
+
+    assert (
+        "aips: '1'; sips: '2'; dips: '3'; deleted: '4'; replicated: '5'"
+        == task_helpers.summarize_fetch_job_results(fetch_job)
+    )

tools/fetch_aips

@@ -11,7 +11,10 @@ from helpers import fetch
 
 from AIPscan import db
 from AIPscan.Aggregator import database_helpers
-from AIPscan.Aggregator.task_helpers import create_numbered_subdirs
+from AIPscan.Aggregator.task_helpers import (
+    create_numbered_subdirs,
+    summarize_fetch_job_results,
+)
 from AIPscan.models import StorageService
 from config import CONFIGS
 
@@ -142,17 +145,11 @@ def fetch_aips(logger, ss_id, session_id, page, packages_per_page, logfile):
             fetch_job_id, processed_packages, total_packages
         )
 
-        summary = "aips: '{}'; sips: '{}'; dips: '{}'; deleted: '{}'; replicated: '{}'".format(
-            fetch_job.total_aips,
-            fetch_job.total_sips,
-            fetch_job.total_dips,
-            fetch_job.total_deleted_aips,
-            fetch_job.total_replicas,
-        )
-        logger.info("%s", summary)
         logger.info(
             f"Updated fetch job record {fetch_job_id} with package type counts."
         )
+
+        logger.info("%s", summarize_fetch_job_results(fetch_job))
         logger.info("Processing complete.")