Merge remote-tracking branch 'upstream/master' into pr-osparc-redirec…

…t-remain-dv2-calls-via-dynamic-scheduler
ITISFoundation · Dec 5, 2024 · 7edad2b · 7edad2b
2 parents 2376933 + e317f6b
commit 7edad2b
Show file tree

Hide file tree

Showing 19 changed files with 94 additions and 94 deletions.
diff --git a/.env-devel b/.env-devel
@@ -211,7 +211,7 @@ RESOURCE_USAGE_TRACKER_S3=null
 RESOURCE_USAGE_TRACKER_TRACING={}
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
-R_CLONE_OPTION_BUFFER_SIZE=0M
+R_CLONE_OPTION_BUFFER_SIZE=16M
 R_CLONE_OPTION_RETRIES=3
 R_CLONE_OPTION_TRANSFERS=5
 R_CLONE_PROVIDER=MINIO

diff --git a/...ges/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py b/...ges/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py
@@ -1,8 +1,7 @@
 from datetime import datetime
 from decimal import Decimal
-from typing import Annotated
 
-from pydantic import BaseModel, ConfigDict, PlainSerializer
+from pydantic import BaseModel, ConfigDict
 
 from ..resource_tracker import (
     HardwareInfo,
@@ -19,9 +18,7 @@ class PricingUnitGet(BaseModel):
     pricing_unit_id: PricingUnitId
     unit_name: str
     unit_extra_info: UnitExtraInfo
-    current_cost_per_unit: Annotated[
-        Decimal, PlainSerializer(float, return_type=float, when_used="json")
-    ]
+    current_cost_per_unit: Decimal
     current_cost_per_unit_id: PricingUnitCostId
     default: bool
     specific_info: HardwareInfo

diff --git a/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py b/packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py
@@ -1,8 +1,7 @@
 from datetime import datetime
 from decimal import Decimal
-from typing import Annotated
 
-from pydantic import BaseModel, ConfigDict, Field, PlainSerializer
+from pydantic import BaseModel, ConfigDict, Field
 
 from ..projects import ProjectID
 from ..projects_nodes_io import NodeID
@@ -50,9 +49,7 @@ class PricingUnitGet(OutputSchema):
     pricing_unit_id: PricingUnitId
     unit_name: str
     unit_extra_info: UnitExtraInfo
-    current_cost_per_unit: Annotated[
-        Decimal, PlainSerializer(float, return_type=float, when_used="json")
-    ]
+    current_cost_per_unit: Decimal
     default: bool
 
 

diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py
@@ -1,15 +1,8 @@
 from datetime import datetime
 from decimal import Decimal
-from typing import Annotated, Literal, TypeAlias
+from typing import Literal, TypeAlias
 
-from pydantic import (
-    ConfigDict,
-    Field,
-    HttpUrl,
-    PlainSerializer,
-    ValidationInfo,
-    field_validator,
-)
+from pydantic import ConfigDict, Field, HttpUrl, ValidationInfo, field_validator
 
 from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal
 from ..users import GroupID
@@ -20,9 +13,9 @@
 class WalletGet(OutputSchema):
     wallet_id: WalletID
     name: IDStr
-    description: str | None
+    description: str | None = None
     owner: GroupID
-    thumbnail: str | None
+    thumbnail: str | None = None
     status: WalletStatus
     created: datetime
     modified: datetime
@@ -31,7 +24,7 @@ class WalletGet(OutputSchema):
 
 
 class WalletGetWithAvailableCredits(WalletGet):
-    available_credits: Annotated[Decimal, PlainSerializer(float)]
+    available_credits: Decimal
 
 
 class WalletGetPermissions(WalletGet):

diff --git a/packages/service-library/src/servicelib/redis.py b/packages/service-library/src/servicelib/redis.py
@@ -17,6 +17,7 @@
 from redis.backoff import ExponentialBackoff
 from settings_library.redis import RedisDatabase, RedisSettings
 from tenacity import retry
+from yarl import URL
 
 from .background_task import periodic_task
 from .logging_utils import log_catch, log_context
@@ -94,7 +95,8 @@ def __post_init__(self):
     async def setup(self) -> None:
         if not await self.ping():
             await self.shutdown()
-            raise CouldNotConnectToRedisError(dsn=self.redis_dsn)
+            url_safe: URL = URL(self.redis_dsn).with_password("???")
+            raise CouldNotConnectToRedisError(dsn=f"{url_safe}")
 
         self._health_check_task = asyncio.create_task(
             self._check_health(),

diff --git a/packages/settings-library/src/settings_library/r_clone.py b/packages/settings-library/src/settings_library/r_clone.py
@@ -26,6 +26,6 @@ class RCloneSettings(BaseCustomSettings):
     )
     # SEE https://rclone.org/docs/#buffer-size-size
     R_CLONE_OPTION_BUFFER_SIZE: str = Field(
-        default="0M",
+        default="16M",
         description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
     )
diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py
@@ -8,10 +8,9 @@
 from pathlib import Path
 from typing import Final
 
-from common_library.errors_classes import OsparcErrorMixin
-
 from aiocache import cached  # type: ignore[import-untyped]
 from aiofiles import tempfile
+from common_library.errors_classes import OsparcErrorMixin
 from models_library.basic_types import IDStr
 from pydantic import AnyUrl, BaseModel, ByteSize
 from servicelib.progress_bar import ProgressBarData
@@ -207,7 +206,6 @@ async def _sync_sources(
             f"{r_clone_settings.R_CLONE_OPTION_TRANSFERS}",
             # below two options reduce to a minimum the memory footprint
             # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
-            "--use-mmap",  # docs https://rclone.org/docs/#use-mmap
             "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
             r_clone_settings.R_CLONE_OPTION_BUFFER_SIZE,
             "--use-json-log",

diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py
@@ -52,14 +52,14 @@ class SyncProgressLogParser(BaseLogParser):
 
 
     This command:
-    rclone --use-mmap --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress
+    rclone --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress
     generates this but the rclone modifies the terminal printed lines which python does not like so much
     Transferred:        4.666 GiB / 4.666 GiB, 100%, 530.870 MiB/s, ETA 0s
     Transferred:            4 / 4, 100%
     Elapsed time:         9.6s
 
     This other command:
-    rclone --use-mmap --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3
+    rclone --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3
     prints stuff such as:
     {"level":"info","msg":"Copied (new)","object":"README.ipynb","objectType":"*s3.Object","size":5123,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408277+00:00"}
     {"level":"info","msg":"Copied (new)","object":".hidden_do_not_remove","objectType":"*s3.Object","size":219,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408246+00:00"}

diff --git a/services/agent/src/simcore_service_agent/services/backup.py b/services/agent/src/simcore_service_agent/services/backup.py
@@ -161,7 +161,6 @@ async def _store_in_s3(
         f"{settings.AGENT_VOLUMES_CLEANUP_PARALLELISM}",
         # below two options reduce to a minimum the memory footprint
         # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
-        "--use-mmap",  # docs https://rclone.org/docs/#use-mmap
         "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
         "0M",
         "--stats",

diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json
@@ -6492,7 +6492,7 @@
             "$ref": "#/components/schemas/UnitExtraInfo"
           },
           "currentCostPerUnit": {
-            "type": "number",
+            "type": "string",
             "title": "Currentcostperunit"
           },
           "default": {
@@ -7093,7 +7093,7 @@
             "title": "Modified"
           },
           "availableCredits": {
-            "type": "number",
+            "type": "string",
             "title": "Availablecredits"
           }
         },

diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel
@@ -63,7 +63,7 @@ S3_BUCKET_NAME=simcore
 R_CLONE_PROVIDER=MINIO
 R_CLONE_OPTION_TRANSFERS=5
 R_CLONE_OPTION_RETRIES=3
-R_CLONE_OPTION_BUFFER_SIZE=0M
+R_CLONE_OPTION_BUFFER_SIZE=16M
 
 TRACING_OBSERVABILITY_BACKEND_ENDPOINT=http://jaeger:9411
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
diff --git a/...ices/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/...ices/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py
@@ -286,7 +286,7 @@ def expected_dynamic_sidecar_spec(
                     "RABBIT_PORT": "5672",
                     "RABBIT_USER": "admin",
                     "RABBIT_SECURE": "False",
-                    "R_CLONE_OPTION_BUFFER_SIZE": "0M",
+                    "R_CLONE_OPTION_BUFFER_SIZE": "16M",
                     "R_CLONE_OPTION_RETRIES": "3",
                     "R_CLONE_OPTION_TRANSFERS": "5",
                     "R_CLONE_PROVIDER": "MINIO",

diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile
@@ -187,7 +187,7 @@ RUN chmod +x services/dynamic-sidecar/docker/*.sh
 HEALTHCHECK \
   --interval=10s \
   --timeout=5s \
-  --start-period=30s \
+  --start-period=180s \
   --start-interval=1s \
   --retries=5 \
   CMD ["python3", "services/dynamic-sidecar/docker/healthcheck.py", "http://localhost:8000/health"]

diff --git a/services/dynamic-sidecar/docker/entrypoint.sh b/services/dynamic-sidecar/docker/entrypoint.sh
@@ -93,12 +93,12 @@ fi
 # Change ownership of volumes mount directory
 # directories are empty at this point
 # each individual subdirectory is a unique volume
-chown --verbose --recursive "$SC_USER_NAME":"$GROUPNAME" "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
+chown --recursive "$SC_USER_NAME":"$GROUPNAME" "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
 # Allow owner and group to edit write and execute
 # files from all the subdirectories
 # When the service access files downloaded by the dynamic-sidecar
 # it uses group permissions
-chmod --verbose --recursive 774 "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
+chmod --recursive 774 "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
 
 echo "$INFO Starting $* ..."
 echo "  $SC_USER_NAME rights    : $(id "$SC_USER_NAME")"

diff --git a/services/resource-usage-tracker/tests/unit/api_rest/test__oas_spec.py b/services/resource-usage-tracker/tests/unit/api_rest/test__oas_spec.py
diff --git a/services/static-webserver/client/source/class/osparc/widget/PersistentIframe.js b/services/static-webserver/client/source/class/osparc/widget/PersistentIframe.js
@@ -27,27 +27,7 @@ qx.Class.define("osparc.widget.PersistentIframe", {
   construct: function(source, el) {
     this.base(arguments, source);
 
-    this.themeSwitchHandler = msg => {
-      this.postThemeSwitch(msg.getData());
-    };
-
-    this.postThemeSwitch = theme => {
-      const iframe = this._getIframeElement();
-      if (this._getIframeElement()) {
-        const iframeDomEl = iframe.getDomElement();
-        const iframeSource = iframe.getSource();
-        if (iframeDomEl && iframeSource) {
-          const msg = "osparc;theme=" + theme;
-          try {
-            iframeDomEl.contentWindow.postMessage(msg, iframeSource);
-          } catch (err) {
-            console.log(`Failed posting message ${msg} to iframe ${iframeSource}\n${err.message}`);
-          }
-        }
-      }
-    };
-
-    qx.event.message.Bus.getInstance().subscribe("themeSwitch", this.themeSwitchHandler);
+    this.__attachInterframeMessageHandlers();
   },
 
   statics: {
@@ -265,6 +245,69 @@ qx.Class.define("osparc.widget.PersistentIframe", {
       this.__iframe.setSource(newValue);
     },
 
+    __attachInterframeMessageHandlers: function() {
+      this.__attachTriggerers();
+      this.__attachListeners();
+    },
+
+    __attachTriggerers: function() {
+      this.postThemeSwitch = theme => {
+        const iframe = this._getIframeElement();
+        if (iframe) {
+          const iframeDomEl = iframe.getDomElement();
+          const iframeSource = iframe.getSource();
+          if (iframeDomEl && iframeSource) {
+            const msg = "osparc;theme=" + theme;
+            try {
+              iframeDomEl.contentWindow.postMessage(msg, iframeSource);
+            } catch (err) {
+              console.log(`Failed posting message ${msg} to iframe ${iframeSource}\n${err.message}`);
+            }
+          }
+        }
+      };
+
+      this.themeSwitchHandler = msg => {
+        this.postThemeSwitch(msg.getData());
+      };
+      qx.event.message.Bus.getInstance().subscribe("themeSwitch", this.themeSwitchHandler);
+    },
+
+    __attachListeners: function() {
+      this.__iframe.addListener("load", () => {
+        const iframe = this._getIframeElement();
+        if (iframe) {
+          const iframeDomEl = iframe.getDomElement();
+          if (iframeDomEl) {
+            window.addEventListener("message", message => {
+              const data = message.data;
+              if (data) {
+                this.__handleIframeMessage(data);
+              }
+            });
+          }
+        }
+      }, this);
+    },
+
+    __handleIframeMessage: function(data) {
+      if (data["type"] && data["message"]) {
+        if (data["type"] === "theme") {
+          // switch theme driven by the iframe
+          const message = data["message"];
+          if (message.includes("osparc;theme=")) {
+            const themeName = message.replace("osparc;theme=", "");
+            const validThemes = osparc.ui.switch.ThemeSwitcher.getValidThemes();
+            const themeFound = validThemes.find(theme => theme.basename === themeName);
+            const themeManager = qx.theme.manager.Meta.getInstance();
+            if (themeFound !== themeManager.getTheme()) {
+              themeManager.setTheme(themeFound);
+            }
+          }
+        }
+      }
+    },
+
     // override
     _getIframeElement: function() {
       return this.__iframe._getIframeElement(); // eslint-disable-line no-underscore-dangle

diff --git a/services/web/server/VERSION b/services/web/server/VERSION
@@ -1 +1 @@
-0.46.0
+0.47.0
diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.46.0
+current_version = 0.47.0
 commit = True
 message = services/webserver api version: {current_version} → {new_version}
 tag = False

diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml
@@ -2,7 +2,7 @@ openapi: 3.1.0
 info:
   title: simcore-service-webserver
   description: Main service with an interface (http-API & websockets) to the web front-end
-  version: 0.46.0
+  version: 0.47.0
 servers:
 - url: ''
   description: webserver
@@ -11314,7 +11314,7 @@ components:
         unitExtraInfo:
           $ref: '#/components/schemas/UnitExtraInfo-Output'
         currentCostPerUnit:
-          type: number
+          type: string
           title: Currentcostperunit
         default:
           type: boolean
@@ -11358,7 +11358,7 @@ components:
         unitExtraInfo:
           $ref: '#/components/schemas/UnitExtraInfo-Output'
         currentCostPerUnit:
-          type: number
+          type: string
           title: Currentcostperunit
         default:
           type: boolean
@@ -14079,7 +14079,7 @@ components:
           format: date-time
           title: Modified
         availableCredits:
-          type: number
+          type: string
           title: Availablecredits
       type: object
       required: