From 76c3ba489ac93a140e7e4df838a9ed1acf793347 Mon Sep 17 00:00:00 2001
From: Yury Hrytsuk <50014626+YuryHrytsuk@users.noreply.github.com>
Date: Wed, 4 Dec 2024 09:05:52 +0100
Subject: [PATCH 1/2] Expose dynamic scheduler UI (#899)

---
 services/simcore/docker-compose.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/services/simcore/docker-compose.yml b/services/simcore/docker-compose.yml
index ae919eb7..a836268d 100644
--- a/services/simcore/docker-compose.yml
+++ b/services/simcore/docker-compose.yml
@@ -869,6 +869,7 @@ services:
 
   dynamic-schdlr:
     networks:
+      - public
       - monitored
     deploy:
       replicas: 2
@@ -882,6 +883,16 @@ services:
         reservations:
           memory: 50M
           cpus: '0.1'
+      labels:
+        - traefik.enable=true
+        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.server.port=8000
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/dynamic-scheduler`)
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.entrypoints=https
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.tls=true
+        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.regex=^/dynamic-scheduler/(.*)$$
+        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.replacement=/$${1}
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.middlewares=${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex@swarm, ops_gzip@swarm, ops_auth@swarm
 
 volumes:
   rabbit_data:

From c37a5a67fbd537b84a68b4a2d0e19a6baf3a566d Mon Sep 17 00:00:00 2001
From: Dustin Kaiser <8209087+mrnicegyu11@users.noreply.github.com>
Date: Thu, 5 Dec 2024 10:53:02 +0100
Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20Fix:=20Make=20`osparc.local`?=
 =?UTF-8?q?=20functional=20again=20-=20various=20minor=20fixes=20(#894)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* wip

* Add csi-s3 and have portainer use it

* Change request @hrytsuk 1GB max portainer volume size

* Fix wrong filename

* Fix registry local deploy

* Traefik local deployment fixes

* Fix local deployment graylog provisioning

* Fix j2, double venv

* Add python version

* Idempotency for admin-panels

* Remove faulty command

* Local deploy fixes

* Clean Up Local Minio

* Remove unused code

* Update Minio

---------

Co-authored-by: Dustin Kaiser <mail@dustinkaiser.eu>
---
 .gitignore                                    |  2 +-
 .pylintrc                                     |  2 +-
 Makefile                                      | 11 +--
 scripts/common.Makefile                       | 31 ++-----
 scripts/create-s3-bucket.bash                 |  4 +-
 ...ally.sh => deploy_everything_locally.bash} |  3 +
 .../deployments/start_simcore_locally.bash    | 10 ++-
 scripts/s3-previous-versions/README.md        | 15 ----
 scripts/s3-previous-versions/launch.bash      | 17 ----
 scripts/s3-previous-versions/template.env     |  3 -
 services/graylog/scripts/configure.py         |  8 +-
 services/minio/Makefile                       | 44 ++--------
 services/minio/README.md                      | 27 -------
 .../docker-compose.letsencrypt.dns.yaml.j2    | 11 ---
 .../docker-compose.letsencrypt.http.yaml.j2   | 11 ---
 services/minio/docker-compose.yaml            | 60 ++++++++++++++
 services/minio/docker-compose.yaml.j2         | 81 -------------------
 services/minio/secrets/.gitkeep               |  0
 services/minio/template.env                   |  6 --
 services/registry/Makefile                    |  4 +-
 services/registry/template.env                |  2 +-
 services/simcore/.gitignore                   |  1 +
 services/simcore/Makefile                     |  1 -
 .../simcore/docker-compose.deploy.local.yml   |  6 +-
 services/traefik/docker-compose.local.yml     | 17 ++--
 25 files changed, 114 insertions(+), 263 deletions(-)
 rename scripts/deployments/{deploy_everything_locally.sh => deploy_everything_locally.bash} (97%)
 delete mode 100644 scripts/s3-previous-versions/README.md
 delete mode 100755 scripts/s3-previous-versions/launch.bash
 delete mode 100644 scripts/s3-previous-versions/template.env
 delete mode 100644 services/minio/README.md
 delete mode 100644 services/minio/docker-compose.letsencrypt.dns.yaml.j2
 delete mode 100644 services/minio/docker-compose.letsencrypt.http.yaml.j2
 create mode 100644 services/minio/docker-compose.yaml
 delete mode 100644 services/minio/docker-compose.yaml.j2
 delete mode 100644 services/minio/secrets/.gitkeep

diff --git a/.gitignore b/.gitignore
index 24edb7f6..2b9cef78 100644
--- a/.gitignore
+++ b/.gitignore
@@ -142,7 +142,7 @@ yq
 **/.env-devel
 **/.stack.*.yml
 **/.stack.*.yaml
-./docker-compose.yml
+docker-compose.yml
 stack.yml
 stack_with_prefix.yml
 docker-compose.simcore.yml
diff --git a/.pylintrc b/.pylintrc
index f7317a50..e9682aa1 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -58,7 +58,7 @@ ignore-paths=^.*\\generated_models\\.*$|^.*/generated_models/.*$
 # Files or directories matching the regex patterns are skipped. The regex
 # matches against base names, not paths. The default value ignores Emacs file
 # locks
-ignore-patterns=venv,.venv
+ignore-patterns=venv,.venv,jupyter_server_config.py
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
diff --git a/Makefile b/Makefile
index 91228f23..2e80bf48 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,7 @@ certificates/domain.key:
 	# Done: Creating docker secrets
 
 .PHONY: up-local
-up-local: .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
+up-local: .init .venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
 	@bash scripts/deployments/deploy_everything_locally.bash --stack_target=local --minio_enabled=0 --vcs_check=1
 	@$(MAKE) info-local
 
@@ -71,15 +71,6 @@ down-maintenance: ## Stop the maintenance mode
 	fi \
 	,)
 
-
-.PHONY: venv
-venv: .venv ## Creates a python virtual environment with dev tools (pip, pylint, ...)
-.venv:
-	@python3 -m venv .venv
-	@.venv/bin/pip3 install --upgrade pip wheel setuptools
-	@.venv/bin/pip3 install typer
-	@echo "To activate the venv, execute 'source .venv/bin/activate'"
-
 # Misc: info & clean
 .PHONY: info info-vars info-local
 info: ## Displays some important info
diff --git a/scripts/common.Makefile b/scripts/common.Makefile
index 8e706fc6..c54be7cf 100644
--- a/scripts/common.Makefile
+++ b/scripts/common.Makefile
@@ -7,14 +7,12 @@ VERSION := $(shell uname -a)
 
 # Checks for handling various operating systems
 ifeq ($(filter Windows_NT,$(OS)),)
-IS_WSL  := $(if $(findstring microsoft,$(shell uname -a | tr '[:upper:]' '[:lower:]')),WSL,)
 IS_WSL2 := $(if $(findstring -microsoft-,$(shell uname -a)),WSL2,)
 IS_OSX  := $(filter Darwin,$(shell uname -a))
 IS_LINUX:= $(if $(or $(IS_WSL),$(IS_OSX)),,$(filter Linux,$(shell uname -a)))
 endif
 IS_WIN  := $(strip $(if $(or $(IS_LINUX),$(IS_OSX),$(IS_WSL)),,$(OS)))
 
-$(if $(IS_WIN),$(error Windows is not supported in all recipes. Use WSL2 instead. Follow instructions in README.md),)
 $(if $(IS_WSL2),,$(if $(IS_WSL),$(error WSL1 is not supported in all recipes. Use WSL2 instead. Follow instructions in README.md),))
 
 # Check that a valid location to a config file is set.
@@ -37,13 +35,13 @@ endif
 export DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL:=$(shell set -o allexport; \
 	source $(REPO_CONFIG_LOCATION); \
 	if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
-		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (Host(\`storage.$$MACHINE_FQDN\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))||  (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
 	else \
 		IFS=', ' read -r -a hosts <<< "$${DEPLOYMENT_FQDNS}"; \
-		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (Host(\`storage.$$MACHINE_FQDN\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
 		for element in "$${hosts[@]}"; \
 		do \
-			DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$element\`)) || (Host(\`storage.$$element\`))  || (HostRegexp(\`services.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$element\`) && PathPrefix(\`/\`))";\
+			DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$element\`)) || (HostRegexp(\`services.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$element\`) && PathPrefix(\`/\`))";\
 		done; \
 		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL"; \
 	fi; \
@@ -66,21 +64,6 @@ export DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS:=$(shell set -o allexport; \
 	echo $$DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS; \
 	set +o allexport; )
 
-export DEPLOYMENT_FQDNS_CAPTURE_STORAGE:=$(shell set -o allexport; \
-	source $(REPO_CONFIG_LOCATION); \
-	if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
-		DEPLOYMENT_FQDNS_CAPTURE_STORAGE="(Host(\`storage.$$MACHINE_FQDN\`))"; \
-	else \
-		IFS=', ' read -r -a hosts <<< "$${DEPLOYMENT_FQDNS}"; \
-		DEPLOYMENT_FQDNS_CAPTURE_STORAGE="(Host(\`storage.$$MACHINE_FQDN\`))"; \
-		for element in "$${hosts[@]}"; \
-		do \
-			DEPLOYMENT_FQDNS_CAPTURE_STORAGE="$$DEPLOYMENT_FQDNS_CAPTURE_STORAGE || (Host(\`storage.$$element\`))";\
-		done; \
-		DEPLOYMENT_FQDNS_CAPTURE_STORAGE="$$DEPLOYMENT_FQDNS_CAPTURE_STORAGE"; \
-	fi; \
-	echo $$DEPLOYMENT_FQDNS_CAPTURE_STORAGE; \
-	set +o allexport; )
 
 export DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE:=$(shell set -o allexport; \
 	source $(REPO_CONFIG_LOCATION); \
@@ -207,7 +190,6 @@ clean-default: .check_clean ## Cleans all outputs
 	export DEPLOYMENT_FQDNS_TESTING_CAPTURE_TRAEFIK_RULE='${DEPLOYMENT_FQDNS_TESTING_CAPTURE_TRAEFIK_RULE}'; \
 	export DEPLOYMENT_API_DOMAIN_TESTING_CAPTURE_TRAEFIK_RULE='${DEPLOYMENT_API_DOMAIN_TESTING_CAPTURE_TRAEFIK_RULE}'; \
 	export DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS='${DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS}'; \
-	export DEPLOYMENT_FQDNS_CAPTURE_STORAGE='${DEPLOYMENT_FQDNS_CAPTURE_STORAGE}'; \
 	export DOLLAR='$$'; \
 	set +o allexport; \
 	envsubst < $< > .env
@@ -243,20 +225,21 @@ clean-default: .check_clean ## Cleans all outputs
 	# creating virtual environment with tooling (jinja, etc)
 	@python3 -m venv .venv
 	@.venv/bin/pip3 install --upgrade pip wheel setuptools
-	@.venv/bin/pip3 install jinja2 j2cli[yaml]
+	@.venv/bin/pip3 install jinja2 j2cli[yaml] typer
+	@echo "To activate the venv, execute 'source .venv/bin/activate'"
 
 
 # https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	.venv/bin/j2 --format=env $(1) .env -o $(2)
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2)
 endef
 
 endif
diff --git a/scripts/create-s3-bucket.bash b/scripts/create-s3-bucket.bash
index 1902b1db..88313293 100755
--- a/scripts/create-s3-bucket.bash
+++ b/scripts/create-s3-bucket.bash
@@ -11,5 +11,5 @@ IFS=$'\n\t'
 docker run \
 -v /etc/ssl/certs:/etc/ssl/certs:ro \
 --network host \
---env MC_HOST_local="https://${S3_ACCESS_KEY}:${S3_SECRET_KEY}@${S3_ENDPOINT}" \
-minio/mc:RELEASE.2023-06-19T19-31-19Z mb --ignore-existing local/"$1"
+--env MC_HOST_local="https://${S3_ACCESS_KEY}:${S3_SECRET_KEY}@${STORAGE_DOMAIN}" \
+minio/mc:RELEASE.2023-06-19T19-31-19Z mb --insecure --ignore-existing local/"$1"
diff --git a/scripts/deployments/deploy_everything_locally.sh b/scripts/deployments/deploy_everything_locally.bash
similarity index 97%
rename from scripts/deployments/deploy_everything_locally.sh
rename to scripts/deployments/deploy_everything_locally.bash
index 90e1d21d..878960f5 100755
--- a/scripts/deployments/deploy_everything_locally.sh
+++ b/scripts/deployments/deploy_everything_locally.bash
@@ -235,6 +235,9 @@ if [ "$start_opsstack" -eq 0 ]; then
     # -------------------------------- ADMIN-PANELS -------------------------------
 
     log_info "starting admin-panels..."
+    # Check if the stack 'admin-panels' exists and delete it if it does
+    # shellcheck disable=2015
+    docker stack ls | grep -q admin-panels && docker stack rm admin-panels >/dev/null 2>&1 || true
     # Pushd because a call with call_make trigger a strange behavior
     pushd "${repo_basedir}"/services/admin-panels;
     call_make "." up-"$stack_target";
diff --git a/scripts/deployments/start_simcore_locally.bash b/scripts/deployments/start_simcore_locally.bash
index 22a7e6cc..a69b47f4 100755
--- a/scripts/deployments/start_simcore_locally.bash
+++ b/scripts/deployments/start_simcore_locally.bash
@@ -71,6 +71,7 @@ if [[ "$devel_repo_path" = "0" ]] ; then
     #
     #       IF GETREPO DOESNT EXIST
     if [ ! -d osparc-simcore ]; then
+        export GIT_SIMCORE_REPO_URL="https://github.com/ITISFoundation/osparc-simcore.git"
         git clone "$GIT_SIMCORE_REPO_URL"
     fi
     #       FI
@@ -130,8 +131,12 @@ scripts/deployments/compose_stack_yml.bash
 log_info "Adding prefix $PREFIX_STACK_NAME to all services..."
 ./yq "with(.services; with_entries(.key |= \"${PREFIX_STACK_NAME}_\" + .))" stack.yml > stack_with_prefix.yml
 log_info "Deleting the $SIMCORE_STACK_NAME docker stack if present"
-docker stack rm "$SIMCORE_STACK_NAME" || true
-sleep 3 # Wait for stack to be deleted, the networks often take a while, not waiting might lead to docker network creation issues
+# Wait for stack to be deleted, the networks often take a while, not waiting might lead to docker network creation issues
+# shellcheck disable=2015
+docker stack rm "$SIMCORE_STACK_NAME" && sleep 3 || true
+log_info "Copying dask-certificates into place"
+mkdir -p "$repo_basedir"/services/simcore/dask-sidecar/.dask-certificates
+cp -r "$(dirname "${repo_config}")"/assets/dask-certificates/*.pem "$repo_basedir"/services/simcore/dask-sidecar/.dask-certificates
 log_info "Deploying: Running docker stack deploy for stack $SIMCORE_STACK_NAME..."
 
 # Retry logic via https://unix.stackexchange.com/a/82610
@@ -141,4 +146,5 @@ for i in {1..5}; do docker stack deploy -c stack_with_prefix.yml "$SIMCORE_STACK
 
 ############
 # CLEANUP
+# shellcheck disable=1073
 rm -r "${repo_basedir:?}"/"${tempdirname:?}" 2>/dev/null || true
diff --git a/scripts/s3-previous-versions/README.md b/scripts/s3-previous-versions/README.md
deleted file mode 100644
index 8968fd35..00000000
--- a/scripts/s3-previous-versions/README.md
+++ /dev/null
@@ -1,15 +0,0 @@
-# Goal
-
-This script use [the s3-pit-restore script](https://github.com/angeloc/s3-pit-restore) with a little wrapper to ensure that it can be used with minio.
-
-# Usage
-* Create an .env file from the template.env file and fill it
-* Launch the script with
-```console
-./launch.bash command
-```
-where command is the command you would use with s3-pit-restore.
-
-E.g : if you want to restore the bucket simcore-origin to his  06-17-2021 23:59:50 +2 version in the bucket simcore-new-bucket :
-```console
-./launch.bash --bucket simcore-origin --dest-bucket simcore-new-bucket 06-17-2021 23:59:50 +2 "06-17-2021 23:59:50 +2"
diff --git a/scripts/s3-previous-versions/launch.bash b/scripts/s3-previous-versions/launch.bash
deleted file mode 100755
index d901a14e..00000000
--- a/scripts/s3-previous-versions/launch.bash
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-# This script restore S3 objects to a previous versionized state
-#
-
-set -o nounset
-set -o pipefail
-IFS=$'\n\t'
-
-set -o allexport
-# shellcheck disable=1090,1091
-source .env
-set +o allexport
-git clone https://github.com/angeloc/s3-pit-restore
-pushd s3-pit-restore || exit 1
-s3-pit-restore "$@"
-popd || exit 1
-rm -rf s3-pit-restore
diff --git a/scripts/s3-previous-versions/template.env b/scripts/s3-previous-versions/template.env
deleted file mode 100644
index f005a64f..00000000
--- a/scripts/s3-previous-versions/template.env
+++ /dev/null
@@ -1,3 +0,0 @@
-AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}
diff --git a/services/graylog/scripts/configure.py b/services/graylog/scripts/configure.py
index ff217224..8cb91ed9 100644
--- a/services/graylog/scripts/configure.py
+++ b/services/graylog/scripts/configure.py
@@ -48,7 +48,9 @@
     before=before_log(logger, logging.INFO),
 )
 def wait_graylog_is_online():
-    _r = requests.get(GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH)
+    _r = requests.get(
+        GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH, verify=False
+    )
 
     if _r.status_code == 401:
         raise TypeError(f"Graylog unauthorized HTTP response: {_r}")
@@ -58,7 +60,9 @@ def wait_graylog_is_online():
 
 
 def validate_graylog_version_is_supported():
-    _r = requests.get(GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH)
+    _r = requests.get(
+        GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH, verify=False
+    )
     _r.raise_for_status()
 
     graylog_version = _r.json()["version"]
diff --git a/services/minio/Makefile b/services/minio/Makefile
index f5c260f1..40b0f797 100644
--- a/services/minio/Makefile
+++ b/services/minio/Makefile
@@ -1,9 +1,7 @@
 .DEFAULT_GOAL := help
 
-
-
 # Internal VARIABLES ------------------------------------------------
-# STACK_NAME defaults to name of the current directory. Should not to be changed if you follow GitOps operating procedures.
+# STACK_NAME defaults to name of the current directory.
 STACK_NAME = $(notdir $(shell pwd))
 DOCKER_MINIO_ACCESS_KEY = $(shell docker secret inspect --format {{.Spec.Name}} minio_secret_key 2>/dev/null)
 DOCKER_MINIO_SECRET_KEY = $(shell docker secret inspect --format {{.Spec.Name}} minio_access_key 2>/dev/null)
@@ -14,52 +12,22 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 .PHONY: up
-up: .init .env ${TEMP_COMPOSE} .create-secrets ## Deploys or updates current stack "$(STACK_NAME)" using replicas=X (defaults to 1)
+up: .init .env ${TEMP_COMPOSE} .create-secrets
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} $(STACK_NAME)
-	# "in case you created more than 1 replicas, you need to label the nodes accordingly using"
-	# "'docker node update --label-add minioX=true' with X being from 1 to number of replicas."
-
-.PHONY: up-letsencrypt-http
-up-letsencrypt-http: .init .env ${TEMP_COMPOSE}-letsencrypt-http .create-secrets ## Deploys minio stack using let's encrypt http challenge
-	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME}
-	# "in case you created more than 1 replicas, you need to label the nodes accordingly using"
-	# "'docker node update --label-add minioX=true' with X being from 1 to number of replicas."
-
-.PHONY: up-letsencrypt-dns
-up-letsencrypt-dns: .init .env ${TEMP_COMPOSE}-letsencrypt-dns .create-secrets ## Deploys minio stack using let's encrypt dns challenge
-	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME}
-	# "in case you created more than 1 replicas, you need to label the nodes accordingly using"
-	# "'docker node update --label-add minioX=true' with X being from 1 to number of replicas."
 
 .PHONY: up-dalco
-up-dalco: up ## Deploys minio stack for Dalco Cluster
+up-dalco: up
 
 .PHONY: up-master
-up-master: up ## Deploys minio stack for Master Cluster
+up-master: up
 
 .PHONY: up-local
 up-local: up
 
 .PHONY: ${TEMP_COMPOSE}
-${TEMP_COMPOSE}: docker-compose.yaml.j2	.venv .env
-	$(call jinja, $<, tmp.yaml)
-	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env tmp.yaml > $@
-	@rm tmp.yaml
-
-.PHONY: ${TEMP_COMPOSE}-letsencrypt-http
-${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yaml.j2 docker-compose.letsencrypt.http.yaml.j2 .venv .env
-	$(call jinja, $<, tmp.yaml)
-	$(call jinja, docker-compose.letsencrypt.http.yaml.j2, tmp-letsencrypt.http.yaml)
-	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env tmp.yaml tmp-letsencrypt.http.yaml > $@
-	@rm tmp.yaml tmp-letsencrypt.http.yaml
-
+${TEMP_COMPOSE}: docker-compose.yaml .venv .env
+	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env docker-compose.yaml > $@
 
-.PHONY: ${TEMP_COMPOSE}-letsencrypt-dns
-${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yaml.j2 docker-compose.letsencrypt.dns.yaml.j2 .venv .env
-	$(call jinja, $<, tmp.yaml)
-	$(call jinja, docker-compose.letsencrypt.dns.yaml.j2, tmp-letsencrypt.dns.yaml)
-	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env tmp.yaml tmp-letsencrypt.dns.yaml > $@
-	@rm tmp.yaml tmp-letsencrypt.dns.yaml
 
 .create-secrets:
 	@$(if $(DOCKER_MINIO_ACCESS_KEY), \
diff --git a/services/minio/README.md b/services/minio/README.md
deleted file mode 100644
index 309e52f9..00000000
--- a/services/minio/README.md
+++ /dev/null
@@ -1,27 +0,0 @@
-# Minio (S3) stack
-
-Creates a S3 storage stack using [minio](https://docs.min.io/docs/deploy-minio-on-docker-swarm.html).
-
-## Configuration
-
-### Distributed mode
-
-1. edit .env file
-2. __MINIO_ACCESS_KEY__ and __MINIO_SECRET_KEY__ to be manually defined
-3. define the number of instances __MINIO_NUM_MINIOS__ and the number of partitions in each instance __MINIO_NUM_PARTITIONS__ according to [minio docs](https://docs.min.io/docs/distributed-minio-quickstart-guide.html)
-4. __MINIO_INTERNAL_VOLUME_DATA_PART<index>__ may be defined to mount folders.
-5. Each node where a minio instance shall run shall be labelled with __minio<instance>=true__ using __docker node update --label-add minio<instance>=true <node>__
-
-### Local dev mode
-
-1. edit .env file
-2. __MINIO_ACCESS_KEY__ and __MINIO_SECRET_KEY__ to be manually defined
-3. Defaults will create 1 instance with 1 partition
-
-## Usage
-
-    ```console
-    make help
-    make up
-    make down
-    ```
diff --git a/services/minio/docker-compose.letsencrypt.dns.yaml.j2 b/services/minio/docker-compose.letsencrypt.dns.yaml.j2
deleted file mode 100644
index aeadf7c7..00000000
--- a/services/minio/docker-compose.letsencrypt.dns.yaml.j2
+++ /dev/null
@@ -1,11 +0,0 @@
-version: '3.7'
-services:
-{% set num_minios = MINIO_NUM_MINIOS %}
-{% set num_partitions = MINIO_NUM_PARTITIONS %}
-{%- for i in range(num_minios|int) %}
-  {%- set service_id = i+1 %}
-  minio{{ service_id|string }}:
-    deploy:
-      labels:
-        - traefik.http.routers.minio.tls.certresolver=myresolver
-{% endfor %}
diff --git a/services/minio/docker-compose.letsencrypt.http.yaml.j2 b/services/minio/docker-compose.letsencrypt.http.yaml.j2
deleted file mode 100644
index aeadf7c7..00000000
--- a/services/minio/docker-compose.letsencrypt.http.yaml.j2
+++ /dev/null
@@ -1,11 +0,0 @@
-version: '3.7'
-services:
-{% set num_minios = MINIO_NUM_MINIOS %}
-{% set num_partitions = MINIO_NUM_PARTITIONS %}
-{%- for i in range(num_minios|int) %}
-  {%- set service_id = i+1 %}
-  minio{{ service_id|string }}:
-    deploy:
-      labels:
-        - traefik.http.routers.minio.tls.certresolver=myresolver
-{% endfor %}
diff --git a/services/minio/docker-compose.yaml b/services/minio/docker-compose.yaml
new file mode 100644
index 00000000..92456040
--- /dev/null
+++ b/services/minio/docker-compose.yaml
@@ -0,0 +1,60 @@
+version: '3.7'
+services:
+  minio:
+    image: minio/minio:RELEASE.2024-10-29T16-01-48Z
+    init: true
+    hostname: minio
+    volumes:
+      - minio_data:/data
+    networks:
+      - default
+      - public
+      - monitored
+    ports:
+      - target: 9000
+    environment:
+      - MINIO_ACCESS_KEY_FILE=minio_access_key
+      - MINIO_SECRET_KEY_FILE=minio_secret_key
+    deploy:
+      restart_policy:
+        delay: 10s
+        max_attempts: 10
+        window: 60s
+      labels:
+        - traefik.enable=true
+        - traefik.docker.network=${PUBLIC_NETWORK}
+        # direct access without path (necessary for minio client it does not like /path)
+        - traefik.http.services.minio9000.loadbalancer.server.port=9000
+        - traefik.http.services.minio9000.loadbalancer.healthcheck.path=/minio/health/ready
+        - traefik.http.routers.minio9000.rule=Host(`${STORAGE_DOMAIN}`)
+        - traefik.http.routers.minio9000.entrypoints=https
+        - traefik.http.routers.minio9000.tls=true
+        - traefik.http.routers.minio9000.service=minio9000
+        #
+        - traefik.http.services.minio9001.loadbalancer.server.port=9001
+        - traefik.http.routers.minio9001.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/minio`)
+        - traefik.http.routers.minio9001.entrypoints=https
+        - traefik.http.routers.minio9001.tls=true
+        - traefik.http.routers.minio9001.service=minio9001
+    command: >
+      server /data --console-address ":9001"
+    secrets:
+      - minio_secret_key
+      - minio_access_key
+volumes:
+  minio_data:
+
+
+networks:
+  public:
+    external: true
+    name: ${PUBLIC_NETWORK}
+  monitored:
+    name: ${MONITORED_NETWORK}
+    external: true
+
+secrets:
+  minio_secret_key:
+    external: true
+  minio_access_key:
+    external: true
diff --git a/services/minio/docker-compose.yaml.j2 b/services/minio/docker-compose.yaml.j2
deleted file mode 100644
index 95c8b898..00000000
--- a/services/minio/docker-compose.yaml.j2
+++ /dev/null
@@ -1,81 +0,0 @@
-version: '3.7'
-services:
-{% set num_minios = MINIO_NUM_MINIOS %}
-{% set num_partitions = MINIO_NUM_PARTITIONS %}
-{% set host_folder = MINIO_HOST_DATA_FOLDER %}
-{%- for i in range(num_minios|int) %}
-  {%- set service_id = i+1 %}
-  minio{{ service_id|string }}:
-    image: minio/minio:RELEASE.2023-06-19T19-52-50Z
-    init: true
-    hostname: minio{{ service_id|string }}
-    volumes: # default uses a named volume, option2 is to use the variable to set specific mount path(s)
-      {%- for j in range(num_partitions|int) %}
-      {% if host_folder == 'true' %}
-      - /data/disk{{ j|string }}/minio:/data{{ j|string }}
-      {% else %}
-      - minio{{ service_id|string }}_data_part{{ j|string }}:/data{{ j|string }}
-      {% endif %}
-      {%- endfor %}
-    networks:
-      - default
-      - public
-      - monitored
-    ports:
-      - target: 9000
-    environment:
-      - MINIO_ACCESS_KEY_FILE=minio_access_key
-      - MINIO_SECRET_KEY_FILE=minio_secret_key
-      - MINIO_PROMETHEUS_AUTH_TYPE=public
-    deploy:
-      restart_policy:
-        delay: 10s
-        max_attempts: 10
-        window: 60s
-      {%- if num_minios|int > 1 %}
-      placement:
-        constraints:
-          - {{ "node.labels.minio" ~ service_id|string ~ "==true" }}
-      {%- endif %}
-      labels:
-        - traefik.enable=true
-        - traefik.docker.network=${PUBLIC_NETWORK}
-        # direct access without path (necessary for minio client it does not like /path)
-        - traefik.http.services.minio.loadbalancer.server.port=9000
-        - traefik.http.services.minio.loadbalancer.healthcheck.path=/minio/health/ready
-        - traefik.http.routers.minio.rule=Host(`${STORAGE_DOMAIN}`)
-        - traefik.http.routers.minio.entrypoints=https
-        - traefik.http.routers.minio.tls=true
-        - traefik.http.routers.minio.middlewares=ops_gzip@swarm
-    command: >
-      server
-      {%- for i in range(num_minios|int) %}
-      {%- for j in range(num_partitions|int) %}
-      {% if num_minios|int > 1 %}http://minio{{ (i+1)|string }}{% endif %}/data{{ j|string }}
-      {%- endfor %}
-      {%- endfor %}
-    secrets:
-      - minio_secret_key
-      - minio_access_key
-
-{% endfor %}
-volumes:
-{%- for i in range(num_minios|int) %}
-{%- for j in range(num_partitions|int) %}
-  minio{{ (i+1)|string }}_data_part{{ j|string }}:
-{%- endfor %}
-{%- endfor %}
-
-networks:
-  public:
-    external: true
-    name: ${PUBLIC_NETWORK}
-  monitored:
-    name: ${MONITORED_NETWORK}
-    external: true
-
-secrets:
-  minio_secret_key:
-    external: true
-  minio_access_key:
-    external: true
diff --git a/services/minio/secrets/.gitkeep b/services/minio/secrets/.gitkeep
deleted file mode 100644
index e69de29b..00000000
diff --git a/services/minio/template.env b/services/minio/template.env
index de4fcf46..d9a99f67 100644
--- a/services/minio/template.env
+++ b/services/minio/template.env
@@ -1,10 +1,4 @@
 # define MINIO Access and Secret keys
 MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
 MINIO_SECRET_KEY=${S3_SECRET_KEY}
-# define the number of MINIO replicas
-MINIO_NUM_MINIOS=${MINIO_NUM_MINIOS}
-# define the number of partitions each MINIO may be using
-MINIO_NUM_PARTITIONS=${MINIO_NUM_PARTITIONS}
-# optionally activate the host data folder for each partition (must be /data/diskX/minio on each node where X is the disk number if there is more than one disk per node)
-MINIO_HOST_DATA_FOLDER=${MINIO_HOST_DATA_FOLDER}
 STORAGE_DOMAIN=${STORAGE_DOMAIN}
diff --git a/services/registry/Makefile b/services/registry/Makefile
index b0157b2e..f00ab1b3 100644
--- a/services/registry/Makefile
+++ b/services/registry/Makefile
@@ -24,7 +24,7 @@ endef
 .PHONY: up-local
 up-local: .init .env ${TEMP_COMPOSE}-local  ## Deploys registry stack
 	@$(create-s3-bucket)
-	docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
+	docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME}
 
 .PHONY: up-letsencrypt-http  ## Deploys registry stack using let's encrypt http challenge
 up-letsencrypt-http: .init .env ${TEMP_COMPOSE}-letsencrypt-http
@@ -48,8 +48,6 @@ up-master: up-dalco
 .PHONY: up-public ## Deploys registry on public cluster
 up-public: up-dalco
 
-.PHONY: up-local ## Deploys registry on local deployment
-up-local: up
 
 # Helpers -------------------------------------------------
 
diff --git a/services/registry/template.env b/services/registry/template.env
index 550afb5a..95c9421c 100644
--- a/services/registry/template.env
+++ b/services/registry/template.env
@@ -10,7 +10,7 @@ REGISTRY_S3_ACCESS_KEY=${REGISTRY_S3_ACCESS_KEY}
 REGISTRY_S3_SECRET_KEY=${REGISTRY_S3_SECRET_KEY}
 S3_ACCESS_KEY=${REGISTRY_S3_ACCESS_KEY}
 S3_SECRET_KEY=${REGISTRY_S3_SECRET_KEY}
-S3_ENDPOINT=${S3_ENDPOINT}
+STORAGE_DOMAIN=${STORAGE_DOMAIN}
 S3_BUCKET=${REGISTRY_DOMAIN}
 REGISTRY_S3_BUCKET=${REGISTRY_S3_BUCKET}
 REGISTRY_S3_ENDPOINT=${REGISTRY_S3_ENDPOINT}
diff --git a/services/simcore/.gitignore b/services/simcore/.gitignore
index 8b4d445d..a6cefdd5 100644
--- a/services/simcore/.gitignore
+++ b/services/simcore/.gitignore
@@ -1,2 +1,3 @@
 .env
 docker-compose.deploy.yml
+dask-sidecar/**
diff --git a/services/simcore/Makefile b/services/simcore/Makefile
index 7e001694..311b8a5c 100644
--- a/services/simcore/Makefile
+++ b/services/simcore/Makefile
@@ -70,6 +70,5 @@ ${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.deploy.master.yml dock
 	echo DEPLOYMENT_FQDNS_TESTING_CAPTURE_TRAEFIK_RULE=\''${DEPLOYMENT_FQDNS_TESTING_CAPTURE_TRAEFIK_RULE}'\' >> .env; \
 	echo DEPLOYMENT_API_DOMAIN_TESTING_CAPTURE_TRAEFIK_RULE=\''${DEPLOYMENT_API_DOMAIN_TESTING_CAPTURE_TRAEFIK_RULE}'\' >> .env; \
 	echo DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS=\''${DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS}'\' >> .env; \
-	echo DEPLOYMENT_FQDNS_CAPTURE_STORAGE=\''${DEPLOYMENT_FQDNS_CAPTURE_STORAGE}'\' >> .env; \
 	echo DOLLAR=\'$$\' >> .env; \
 	set +o allexport; \
diff --git a/services/simcore/docker-compose.deploy.local.yml b/services/simcore/docker-compose.deploy.local.yml
index 597ba5e3..8b27ce7d 100644
--- a/services/simcore/docker-compose.deploy.local.yml
+++ b/services/simcore/docker-compose.deploy.local.yml
@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   autoscaling:
     deploy:
@@ -138,3 +137,8 @@ services:
   clusters-keeper:
     deploy:
       replicas: 0
+secrets:
+  rootca.crt:
+    external: true
+  storageca.crt:
+    external: true
diff --git a/services/traefik/docker-compose.local.yml b/services/traefik/docker-compose.local.yml
index 2d8367fd..a1403e91 100644
--- a/services/traefik/docker-compose.local.yml
+++ b/services/traefik/docker-compose.local.yml
@@ -11,6 +11,13 @@ services:
       - "--api.dashboard=true"
       - "--log.level=${OPS_TRAEFIK_LOGLEVEL}"
       - "--accesslog=true"
+      - "--accesslog.format=json"
+      - "--accesslog.fields.defaultmode=keep"
+      - "--accesslog.fields.names.ClientUsername=keep"
+      - "--accesslog.fields.headers.defaultmode=keep"
+      - "--accesslog.fields.headers.names.User-Agent=keep"
+      - "--accesslog.fields.headers.names.Authorization=drop"
+      - "--accesslog.fields.headers.names.Content-Type=keep"
       - "--metrics.prometheus=true"
       - "--metrics.prometheus.addEntryPointsLabels=true"
       - "--metrics.prometheus.addServicesLabels=true"
@@ -26,6 +33,9 @@ services:
       - "--entryPoints.https.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.https.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.master_postgres.address=:5432"
+      - "--entrypoints.http.http.redirections.entrypoint.to=https"
+      - "--entrypoints.http.http.redirections.entrypoint.scheme=https"
+      - "--entrypoints.http.http.redirections.entrypoint.permanent=true"
       - "--providers.swarm.endpoint=unix:///var/run/docker.sock"
       - "--providers.swarm.exposedByDefault=false"
       - "--providers.swarm.constraints=!LabelRegex(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
@@ -42,15 +52,10 @@ services:
         aliases: # This enables the registry to resolve "registry.osparc.local" etc. to minio, via traefik. Necessary to give the registry access to the S3 bucket.
           - "${STORAGE_DOMAIN}"
           - "${REGISTRY_DOMAIN}"
+          - "${MONITORING_DOMAIN}"
     deploy:
       placement:
         constraints: []
-      labels:
-        # redirect http to https
-        - traefik.http.middlewares.http_to_https.redirectScheme.scheme=https
-        - traefik.http.routers.http_to_https.rule=HostRegexp(`(?P<host>.+)`)
-        - traefik.http.routers.http_to_https.entrypoints=http
-        - traefik.http.routers.http_to_https.middlewares=http_to_https
     env_file:
       - .env
     configs: