From 1eee560026b25e3d62449d10635d5f3fb526f5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20Jer=C5=A1e?= Date: Wed, 4 Dec 2024 14:35:28 +0100 Subject: [PATCH] Make number of download threads configurable and reduce them to 3 by default. --- docs/CHANGELOG.rst | 1 + resolwe/flow/executors/init_container.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index 187ea9670..16982936f 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -20,6 +20,7 @@ Changed ------- - Make processing container startup script ``Python`` 3.12 compatible - Fix stop null executor processing in the preparation phase +- Reduce the maximal number of threads for downolad data in the init container Fixed ----- diff --git a/resolwe/flow/executors/init_container.py b/resolwe/flow/executors/init_container.py index ab9294ac9..6f2d5e1a7 100644 --- a/resolwe/flow/executors/init_container.py +++ b/resolwe/flow/executors/init_container.py @@ -56,6 +56,9 @@ DOWNLOAD_WAITING_TIMEOUT = 60 # in seconds RETRIES = 5 +# Max threads to use for data download. +MAX_DOWNLOAD_THREADS = int(os.environ.get("GENIALIS_MAX_DOWNLOAD_THREADS", 3)) + # Configure container logger. All logs are output to stdout for further # processing. # The log level defaults to debug except for boto and google loggers, which @@ -112,8 +115,8 @@ async def transfer_inputs(communicator: BaseCommunicator, missing_data: dict): try: for connector_name in objects_to_transfer: - min_threads = 5 - max_threads = 20 + min_threads = 1 + max_threads = MAX_DOWNLOAD_THREADS # Use from min_threads to max_threads threads. Assume each thread # can handle at least 100 files in reasonable time. files_count = len(objects_to_transfer[connector_name])