From d8aa3c13e80ac6df8786313365e88641e00b4aa1 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Thu, 12 Oct 2023 00:31:54 +0200 Subject: [PATCH] add index to the folder name --- src/jobflow_remote/jobs/runner.py | 8 +++++--- src/jobflow_remote/remote/data.py | 4 ++-- src/jobflow_remote/utils/data.py | 7 +++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/jobflow_remote/jobs/runner.py b/src/jobflow_remote/jobs/runner.py index f5cdcddd..1089aea5 100644 --- a/src/jobflow_remote/jobs/runner.py +++ b/src/jobflow_remote/jobs/runner.py @@ -336,7 +336,7 @@ def upload(self, doc): except Exception: logging.error(f"error while closing the store {store}", exc_info=True) - remote_path = get_job_path(job.uuid, fw_job_data.worker.work_dir) + remote_path = get_job_path(job.uuid, job.index, fw_job_data.worker.work_dir) # Set the value of the original store for dynamical workflow. Usually it # will be None don't add the serializer, at this stage the default_orjson @@ -436,7 +436,7 @@ def download(self, doc): remote_path = remote_doc["run_dir"] loca_base_dir = Path(self.project.tmp_dir, "download") - local_path = get_job_path(job.uuid, loca_base_dir) + local_path = get_job_path(job.uuid, job.index, loca_base_dir) makedirs_p(local_path) @@ -465,7 +465,9 @@ def complete_launch(self, doc): fw_job_data = self.get_fw_data(doc) loca_base_dir = Path(self.project.tmp_dir, "download") - local_path = get_job_path(fw_job_data.job.uuid, loca_base_dir) + local_path = get_job_path( + fw_job_data.job.uuid, fw_job_data.job.index, loca_base_dir + ) try: remote_data = loadfn(Path(local_path, "FW_offline.json"), cls=None) diff --git a/src/jobflow_remote/remote/data.py b/src/jobflow_remote/remote/data.py index 9945a259..7b347728 100644 --- a/src/jobflow_remote/remote/data.py +++ b/src/jobflow_remote/remote/data.py @@ -11,13 +11,13 @@ from jobflow_remote.utils.data import uuid_to_path -def get_job_path(job_id: str, base_path: str | Path | None = None) -> str: +def get_job_path(job_id: str, index: int, base_path: str | Path | None = None) -> str: if base_path: base_path = Path(base_path) else: base_path = Path() - relative_path = uuid_to_path(job_id) + relative_path = uuid_to_path(job_id, index) return str(base_path / relative_path) diff --git a/src/jobflow_remote/utils/data.py b/src/jobflow_remote/utils/data.py index 89a1dbd0..0acd0cb5 100644 --- a/src/jobflow_remote/utils/data.py +++ b/src/jobflow_remote/utils/data.py @@ -77,7 +77,7 @@ def check_dict_keywords(obj: Any, keywords: list[str]) -> bool: return False -def uuid_to_path(uuid: str, num_subdirs: int = 3, subdir_len: int = 2): +def uuid_to_path(uuid: str, index: int = 1, num_subdirs: int = 3, subdir_len: int = 2): u = UUID(uuid) u_hex = u.hex @@ -87,8 +87,11 @@ def uuid_to_path(uuid: str, num_subdirs: int = 3, subdir_len: int = 2): for i in range(0, num_subdirs * subdir_len, subdir_len) ] + # add the index to the final dir name + dir_name = f"{uuid}_{index}" + # Combine root directory and subdirectories to form the final path - return os.path.join(*subdirs, uuid) + return os.path.join(*subdirs, dir_name) def store_from_dict(store_dict: dict) -> Store: