♻️Maintenance: fixes and refactoring

Makefile

@@ -135,15 +135,7 @@ $(OSPARC_GATEWAY_CONFIG_FILE_HOST): services/osparc-gateway-server/config/defaul
 # they are not 100% compatible with what docker stack deploy command expects
 # some parts have to be modified
 define generate_docker_compose_specs
-    docker --log-level=ERROR compose --env-file .env \
-        $(foreach file,$1,--file=$(file)) \
-        config \
-				| sed '/published:/s/"//g' \
-				| sed '/size:/s/"//g' \
-				| sed '1 { /name:.*/d ; }' \
-				| sed '1 i\version: \"3.9\"' \
-				| sed --regexp-extended "s/cpus: ([0-9\\.]+)/cpus: '\\1'/" \
-				> $@
+	./scripts/docker-compose-config.bash $1 > $@
 endef
 
 

requirements/base.Makefile

@@ -45,7 +45,11 @@ help: ## this colorful help
 #
 %.txt: %.in
 	cd ..; \
-	pip-compile $(UPGRADE_OPTION) --build-isolation --strip-extras --output-file requirements/$@ requirements/$<
+	pip-compile $(UPGRADE_OPTION) \
+		--build-isolation \
+		--strip-extras \
+		--resolver=backtracking \
+		--output-file requirements/$@ requirements/$<
 
 _test.txt: _base.txt
 

requirements/constraints.txt

@@ -27,7 +27,6 @@ urllib3>=1.26.5                               # https://github.com/advisories/GH
 # Bugs
 #
 sqlalchemy<2.0.0 # dask-gateway-server does not appear to be compatible yet
-cloudpickle<2.1.0 # https://github.com/PrefectHQ/prefect/issues/5818
 
 #
 # Compatibility/coordination

scripts/docker-compose-config.bash

@@ -0,0 +1,84 @@
+#!/bin/bash
+# generated using chatgpt
+show_info() {
+    local message="$1"
+    echo -e "\e[37mInfo:\e[0m $message" >&2
+}
+
+show_warning() {
+  local message="$1"
+  echo -e "\e[31mWarning:\e[0m $message" >&2
+}
+
+show_error() {
+  local message="$1"
+  echo -e "\e[31mError:\e[0m $message" >&2
+}
+
+if [[ "$#" -eq 0 ]]; then
+    show_error "No compose files specified!"
+    exit 1
+fi
+
+# REFERENCE: https://github.com/docker/compose/issues/9306
+# composeV2 defines specifications for docker compose to run
+# they are not 100% compatible with what docker stack deploy command expects
+# some parts have to be modified
+
+
+
+
+# check if docker-compose V2 is available
+if docker compose version --short | grep --quiet "^2\." ; then
+  show_info "Running compose V2"
+  # V2 does not write the version anymore, so we take it from the first compose file
+  first_compose_file="${1}"
+  version=$(grep --max-count=1 "^version:" "${first_compose_file}" | cut --delimiter=' ' --fields=2 | tr --delete \"\')
+  if [[ -z "$version" ]]; then
+    version="3.9"  # Default to 3.9 if version is not found in file
+  fi
+
+  docker_command="\
+docker \
+--log-level=ERROR \
+compose \
+--env-file .env"
+
+  for compose_file_path in "$@"
+  do
+    docker_command+=" --file=${compose_file_path}"
+  done
+  docker_command+="\
+ config \
+| sed '/published:/s/\"//g' \
+| sed '/size:/s/\"//g' \
+| sed '1 { /name:.*/d ; }' \
+| sed '1 i version: \"${version}\"' \
+| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'"
+
+  # Execute the command
+  show_info "Executing Docker command: ${docker_command}"
+  eval ${docker_command}
+else
+  show_warning "docker compose V2 is not available, trying V1 instead... please update your docker engine."
+  if docker-compose version --short | grep --quiet "^1\." ; then
+    show_info "Running compose V1"
+    docker_command="\
+docker-compose \
+--log-level=ERROR \
+--env-file .env"
+    for compose_file_path in "$@"
+    do
+      docker_command+=" --file=${compose_file_path}"
+    done
+    docker_command+="\
+config \
+| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'"
+    # Execute the command
+    show_info "Executing Docker command: ${docker_command}"
+    eval ${docker_command}
+  else
+    show_error "docker-compose V1 is not available. It is impossible to run this script!"
+    exit 1
+  fi
+fi

services/osparc-gateway-server/requirements/_test.txt

@@ -36,9 +36,8 @@ click==8.1.3
     #   dask
     #   dask-gateway
     #   distributed
-cloudpickle==2.0.0
+cloudpickle==2.2.1
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   dask
     #   distributed
 codecov==2.1.12

services/osparc-gateway-server/tests/integration/test_clusters.py

@@ -20,8 +20,8 @@
 @pytest.fixture(
     params=[
         "itisfoundation/dask-sidecar:master-github-latest",
-        "itisfoundation/dask-sidecar:staging-github-latest",
-        "itisfoundation/dask-sidecar:release-github-latest",
+        # "itisfoundation/dask-sidecar:staging-github-latest",
+        # "itisfoundation/dask-sidecar:release-github-latest",
     ]
 )
 def minimal_config(
@@ -201,14 +201,12 @@ async def test_cluster_scale(
     gateway_worker_network: dict[str, Any],
     async_docker_client: Docker,
 ):
-
     # No currently running clusters
     clusters = await gateway_client.list_clusters()
     assert clusters == []
 
     # create a cluster
     async with gateway_client.new_cluster() as cluster:
-
         # Cluster is now present in list
         clusters = await gateway_client.list_clusters()
         assert len(clusters)

services/osparc-gateway-server/tests/integration/test_dask_sidecar.py

@@ -101,8 +101,8 @@ async def _wait_for_service_to_be_ready(
     "image_name",
     [
         "itisfoundation/dask-sidecar:master-github-latest",
-        "itisfoundation/dask-sidecar:staging-github-latest",
-        "itisfoundation/dask-sidecar:release-github-latest",
+        # "itisfoundation/dask-sidecar:staging-github-latest",
+        # "itisfoundation/dask-sidecar:release-github-latest",
     ],
 )
 async def test_computational_sidecar_properly_start_stop(

services/osparc-gateway-server/tests/integration/test_gateway.py

@@ -15,8 +15,8 @@
 @pytest.fixture(
     params=[
         "itisfoundation/dask-sidecar:master-github-latest",
-        "itisfoundation/dask-sidecar:staging-github-latest",
-        "itisfoundation/dask-sidecar:release-github-latest",
+        # "itisfoundation/dask-sidecar:staging-github-latest",
+        # "itisfoundation/dask-sidecar:release-github-latest",
     ]
 )
 def minimal_config(

tests/requirements/_test.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in
+#    pip-compile --output-file=requirements/_test.txt --resolver=backtracking --strip-extras requirements/_test.in
 #
 aiohttp==3.8.4
     # via
@@ -29,9 +29,8 @@ click==8.1.3
     #   dask
     #   dask-gateway
     #   distributed
-cloudpickle==2.0.0
+cloudpickle==2.2.1
     # via
-    #   -c requirements/../../requirements/constraints.txt
     #   dask
     #   distributed
 codecov==2.1.12
@@ -88,7 +87,7 @@ markupsafe==2.1.2
     # via jinja2
 mccabe==0.7.0
     # via pylint
-msgpack==1.0.4
+msgpack==1.0.5
     # via distributed
 multidict==6.0.4
     # via

tests/requirements/_tools.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in
+#    pip-compile --output-file=requirements/_tools.txt --resolver=backtracking --strip-extras requirements/_tools.in
 #
 black==23.1.0
     # via -r requirements/../../requirements/devenv.txt
@@ -21,12 +21,10 @@ distlib==0.3.6
     # via virtualenv
 filelock==3.9.0
     # via virtualenv
-identify==2.5.18
+identify==2.5.19
     # via pre-commit
 isort==5.12.0
-    # via
-    #   -c requirements/_test.txt
-    #   -r requirements/../../requirements/devenv.txt
+    # via -r requirements/../../requirements/devenv.txt
 mypy-extensions==1.0.0
     # via black
 nodeenv==1.7.0
@@ -51,14 +49,14 @@ pyproject-hooks==1.0.0
     # via build
 pyyaml==6.0
     # via
-    #   -c requirements/../../requirements/constraints.txt
     #   -c requirements/_test.txt
     #   pre-commit
 tomli==2.0.1
     # via
     #   -c requirements/_test.txt
     #   black
     #   build
+    #   pyproject-hooks
 typing-extensions==4.5.0
     # via
     #   -c requirements/_test.txt

tests/test_deploy.py

@@ -106,7 +106,9 @@ async def dask_gateway_stack_deployed_services(
         cwd=osparc_gateway_server_root_dir,
     )
     stdout, stderr = await process.communicate()
-    assert process.returncode == 0, f"Unexpected error while deploying stack:\n{stderr}"
+    assert (
+        process.returncode == 0
+    ), f"Unexpected error while deploying stack:\nstdout:{stdout.decode()}\n\nstderr:{stderr.decode()}"
     print(f"{stdout}")
     print("--> osparc-dask-gateway stack deployed.")
     healtcheck_endpoint = f"{dask_gateway_entrypoint}/api/health"
@@ -135,7 +137,9 @@ async def dask_gateway_stack_deployed_services(
         cwd=osparc_gateway_server_root_dir,
     )
     stdout, stderr = await process.communicate()
-    assert process.returncode == 0, f"Unexpected error while deploying stack:\n{stderr}"
+    assert (
+        process.returncode == 0
+    ), f"Unexpected error while deploying stack:\nstdout:{stdout.decode()}\n\n{stderr.decode()}"
     print(f"{stdout}")
     print("<-- osparc-dask-gateway stack stopped.")
 
@@ -191,7 +195,6 @@ def neg(x):
             return -x
 
         with cluster.get_client() as client:
-
             square_of_2 = client.submit(square, 2)
             assert square_of_2.result(timeout=10) == 4
             assert not square_of_2.exception(timeout=10)