♻️ adjust rclone defaults ⚠️ (ITISFoundation#6885)

Co-authored-by: Andrei Neagu <[email protected]>
bisgaard-itis · Dec 5, 2024 · e317f6b · e317f6b
1 parent 7b5cba8
commit e317f6b
Show file tree

Hide file tree

Showing 7 changed files with 7 additions and 10 deletions.
diff --git a/.env-devel b/.env-devel
@@ -211,7 +211,7 @@ RESOURCE_USAGE_TRACKER_S3=null
 RESOURCE_USAGE_TRACKER_TRACING={}
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
-R_CLONE_OPTION_BUFFER_SIZE=0M
+R_CLONE_OPTION_BUFFER_SIZE=16M
 R_CLONE_OPTION_RETRIES=3
 R_CLONE_OPTION_TRANSFERS=5
 R_CLONE_PROVIDER=MINIO

diff --git a/packages/settings-library/src/settings_library/r_clone.py b/packages/settings-library/src/settings_library/r_clone.py
@@ -26,6 +26,6 @@ class RCloneSettings(BaseCustomSettings):
     )
     # SEE https://rclone.org/docs/#buffer-size-size
     R_CLONE_OPTION_BUFFER_SIZE: str = Field(
-        default="0M",
+        default="16M",
         description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
     )
diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py
@@ -8,10 +8,9 @@
 from pathlib import Path
 from typing import Final
 
-from common_library.errors_classes import OsparcErrorMixin
-
 from aiocache import cached  # type: ignore[import-untyped]
 from aiofiles import tempfile
+from common_library.errors_classes import OsparcErrorMixin
 from models_library.basic_types import IDStr
 from pydantic import AnyUrl, BaseModel, ByteSize
 from servicelib.progress_bar import ProgressBarData
@@ -207,7 +206,6 @@ async def _sync_sources(
             f"{r_clone_settings.R_CLONE_OPTION_TRANSFERS}",
             # below two options reduce to a minimum the memory footprint
             # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
-            "--use-mmap",  # docs https://rclone.org/docs/#use-mmap
             "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
             r_clone_settings.R_CLONE_OPTION_BUFFER_SIZE,
             "--use-json-log",

diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py
@@ -52,14 +52,14 @@ class SyncProgressLogParser(BaseLogParser):
 
 
     This command:
-    rclone --use-mmap --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress
+    rclone --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress
     generates this but the rclone modifies the terminal printed lines which python does not like so much
     Transferred:        4.666 GiB / 4.666 GiB, 100%, 530.870 MiB/s, ETA 0s
     Transferred:            4 / 4, 100%
     Elapsed time:         9.6s
 
     This other command:
-    rclone --use-mmap --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3
+    rclone --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3
     prints stuff such as:
     {"level":"info","msg":"Copied (new)","object":"README.ipynb","objectType":"*s3.Object","size":5123,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408277+00:00"}
     {"level":"info","msg":"Copied (new)","object":".hidden_do_not_remove","objectType":"*s3.Object","size":219,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408246+00:00"}

diff --git a/services/agent/src/simcore_service_agent/services/backup.py b/services/agent/src/simcore_service_agent/services/backup.py
@@ -161,7 +161,6 @@ async def _store_in_s3(
         f"{settings.AGENT_VOLUMES_CLEANUP_PARALLELISM}",
         # below two options reduce to a minimum the memory footprint
         # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
-        "--use-mmap",  # docs https://rclone.org/docs/#use-mmap
         "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
         "0M",
         "--stats",

diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel
@@ -63,7 +63,7 @@ S3_BUCKET_NAME=simcore
 R_CLONE_PROVIDER=MINIO
 R_CLONE_OPTION_TRANSFERS=5
 R_CLONE_OPTION_RETRIES=3
-R_CLONE_OPTION_BUFFER_SIZE=0M
+R_CLONE_OPTION_BUFFER_SIZE=16M
 
 TRACING_OBSERVABILITY_BACKEND_ENDPOINT=http://jaeger:9411
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
diff --git a/...ices/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/...ices/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py
@@ -286,7 +286,7 @@ def expected_dynamic_sidecar_spec(
                     "RABBIT_PORT": "5672",
                     "RABBIT_USER": "admin",
                     "RABBIT_SECURE": "False",
-                    "R_CLONE_OPTION_BUFFER_SIZE": "0M",
+                    "R_CLONE_OPTION_BUFFER_SIZE": "16M",
                     "R_CLONE_OPTION_RETRIES": "3",
                     "R_CLONE_OPTION_TRANSFERS": "5",
                     "R_CLONE_PROVIDER": "MINIO",