From d8faf28bb96b7f85cc2ae36369668c1d9f24406b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 7 Mar 2023 17:04:53 +0100 Subject: [PATCH 01/12] reduce warnings --- requirements/base.Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements/base.Makefile b/requirements/base.Makefile index b7c728c2..cdfb1d8f 100644 --- a/requirements/base.Makefile +++ b/requirements/base.Makefile @@ -45,7 +45,11 @@ help: ## this colorful help # %.txt: %.in cd ..; \ - pip-compile $(UPGRADE_OPTION) --build-isolation --strip-extras --output-file requirements/$@ requirements/$< + pip-compile $(UPGRADE_OPTION) \ + --build-isolation \ + --strip-extras \ + --resolver=backtracking \ + --output-file requirements/$@ requirements/$< _test.txt: _base.txt From 470e82e6d5114d7b76d0edd32c6ae6d30eab2199 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 7 Mar 2023 17:05:44 +0100 Subject: [PATCH 02/12] cloudpickle constraint comes from not having the same version everywhere. we might need to synchronise the repos --- requirements/constraints.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 0f867084..546979d4 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -27,7 +27,6 @@ urllib3>=1.26.5 # https://github.com/advisories/GH # Bugs # sqlalchemy<2.0.0 # dask-gateway-server does not appear to be compatible yet -cloudpickle<2.1.0 # https://github.com/PrefectHQ/prefect/issues/5818 # # Compatibility/coordination From f9de16f37d8997a4b81085ac995ea92219731afc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 10:50:40 +0100 Subject: [PATCH 03/12] create bash script for running dokcer compose config --- scripts/docker-compose-config.bash | 85 +++++++++++++++++++ .../requirements/_test.txt | 3 +- 2 files changed, 86 insertions(+), 2 deletions(-) create mode 100755 scripts/docker-compose-config.bash diff --git a/scripts/docker-compose-config.bash b/scripts/docker-compose-config.bash new file mode 100755 index 00000000..ff856f35 --- /dev/null +++ b/scripts/docker-compose-config.bash @@ -0,0 +1,85 @@ +#!/bin/bash +# generated using chatgpt +show_info() { + local message="$1" + echo -e "\e[37mInfo:\e[0m $message" >&2 +} + +show_warning() { + local message="$1" + echo -e "\e[31mWarning:\e[0m $message" >&2 +} + +show_error() { + local message="$1" + echo -e "\e[31mError:\e[0m $message" >&2 +} + +if [[ "$#" -eq 0 ]]; then + show_error "No compose files specified!" + exit 1 +fi + +# REFERENCE: https://github.com/docker/compose/issues/9306 +# composeV2 defines specifications for docker compose to run +# they are not 100% compatible with what docker stack deploy command expects +# some parts have to be modified + + + + +# check if docker-compose V2 is available +if docker compose version --short | grep --quiet "^2\." ; then + # V2 does not write the version anymore, so we take it from the first compose file + first_compose_file="${1}" + version=$(grep --max-count=1 "^version:" "${first_compose_file}" | cut --delimiter=' ' --fields=2) + if [[ -z "$version" ]]; then + version="3.9" # Default to 3.9 if version is not found in file + fi + + docker_command="\ + docker \ + --log-level=ERROR \ + compose \ + --env-file .env" + + for compose_file_path in "$@" + do + docker_command+=" --file=${compose_file_path}" + done + docker_command+="\ + config \ + | sed '/published:/s/\"//g' \ + | sed '/size:/s/\"//g' \ + | sed '1 { /name:.*/d ; }' \ + | sed '1 i\version: \"${version}\"' \ + | sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'" + + # Execute the command + show_info "Executing Docker command: ${docker_command}" + eval ${docker_command} +else + show_warning "docker compose V2 is not available, trying V1 instead... please update your docker engine." + if docker-compose version --short | grep --quiet "^1\." ; then + docker_command="\ +docker-compose \ +--log-level=ERROR \ +--env-file .env" + for compose_file_path in "$@" + do + docker_command+=" --file=${compose_file_path}" + done + docker_command+="\ +config \ +| sed '/published:/s/\"//g' \ +| sed '/size:/s/\"//g' \ +| sed '1 { /name:.*/d ; }' \ +| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'" + # Execute the command + show_info "Executing Docker command: ${docker_command}" + eval ${docker_command} + else + show_error "docker-compose V1 is not available. It is impossible to run this script!" + exit 1 + fi +fi diff --git a/services/osparc-gateway-server/requirements/_test.txt b/services/osparc-gateway-server/requirements/_test.txt index ab43ef68..f3db4354 100644 --- a/services/osparc-gateway-server/requirements/_test.txt +++ b/services/osparc-gateway-server/requirements/_test.txt @@ -36,9 +36,8 @@ click==8.1.3 # dask # dask-gateway # distributed -cloudpickle==2.0.0 +cloudpickle==2.2.1 # via - # -c requirements/../../../requirements/constraints.txt # dask # distributed codecov==2.1.12 From 2fc3b02ba500182e13f64bd47c71cb13c2edaf66 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 10:50:59 +0100 Subject: [PATCH 04/12] use script --- Makefile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Makefile b/Makefile index db8f53d6..a34f5ff1 100644 --- a/Makefile +++ b/Makefile @@ -135,15 +135,7 @@ $(OSPARC_GATEWAY_CONFIG_FILE_HOST): services/osparc-gateway-server/config/defaul # they are not 100% compatible with what docker stack deploy command expects # some parts have to be modified define generate_docker_compose_specs - docker --log-level=ERROR compose --env-file .env \ - $(foreach file,$1,--file=$(file)) \ - config \ - | sed '/published:/s/"//g' \ - | sed '/size:/s/"//g' \ - | sed '1 { /name:.*/d ; }' \ - | sed '1 i\version: \"3.9\"' \ - | sed --regexp-extended "s/cpus: ([0-9\\.]+)/cpus: '\\1'/" \ - > $@ + ./scripts/docker-compose-config.bash $1 > $@ endef From 27946f4080121f487c6a2dad07dccbabac035ba9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 11:55:46 +0100 Subject: [PATCH 05/12] only test against master --- .../tests/integration/test_clusters.py | 6 ++---- .../tests/integration/test_dask_sidecar.py | 4 ++-- .../osparc-gateway-server/tests/integration/test_gateway.py | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/services/osparc-gateway-server/tests/integration/test_clusters.py b/services/osparc-gateway-server/tests/integration/test_clusters.py index c2849249..7a3a1a74 100644 --- a/services/osparc-gateway-server/tests/integration/test_clusters.py +++ b/services/osparc-gateway-server/tests/integration/test_clusters.py @@ -20,8 +20,8 @@ @pytest.fixture( params=[ "itisfoundation/dask-sidecar:master-github-latest", - "itisfoundation/dask-sidecar:staging-github-latest", - "itisfoundation/dask-sidecar:release-github-latest", + # "itisfoundation/dask-sidecar:staging-github-latest", + # "itisfoundation/dask-sidecar:release-github-latest", ] ) def minimal_config( @@ -201,14 +201,12 @@ async def test_cluster_scale( gateway_worker_network: dict[str, Any], async_docker_client: Docker, ): - # No currently running clusters clusters = await gateway_client.list_clusters() assert clusters == [] # create a cluster async with gateway_client.new_cluster() as cluster: - # Cluster is now present in list clusters = await gateway_client.list_clusters() assert len(clusters) diff --git a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py index 57533134..561e8745 100644 --- a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py +++ b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py @@ -101,8 +101,8 @@ async def _wait_for_service_to_be_ready( "image_name", [ "itisfoundation/dask-sidecar:master-github-latest", - "itisfoundation/dask-sidecar:staging-github-latest", - "itisfoundation/dask-sidecar:release-github-latest", + # "itisfoundation/dask-sidecar:staging-github-latest", + # "itisfoundation/dask-sidecar:release-github-latest", ], ) async def test_computational_sidecar_properly_start_stop( diff --git a/services/osparc-gateway-server/tests/integration/test_gateway.py b/services/osparc-gateway-server/tests/integration/test_gateway.py index 7cb91bb0..496a8edf 100644 --- a/services/osparc-gateway-server/tests/integration/test_gateway.py +++ b/services/osparc-gateway-server/tests/integration/test_gateway.py @@ -15,8 +15,8 @@ @pytest.fixture( params=[ "itisfoundation/dask-sidecar:master-github-latest", - "itisfoundation/dask-sidecar:staging-github-latest", - "itisfoundation/dask-sidecar:release-github-latest", + # "itisfoundation/dask-sidecar:staging-github-latest", + # "itisfoundation/dask-sidecar:release-github-latest", ] ) def minimal_config( From 15f1e4315064fd84023589ee0b164402299fabd1 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 13:22:29 +0100 Subject: [PATCH 06/12] in V1 no need to do so many seds --- scripts/docker-compose-config.bash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/docker-compose-config.bash b/scripts/docker-compose-config.bash index ff856f35..ebbf1a1c 100755 --- a/scripts/docker-compose-config.bash +++ b/scripts/docker-compose-config.bash @@ -30,6 +30,7 @@ fi # check if docker-compose V2 is available if docker compose version --short | grep --quiet "^2\." ; then + show_info "Running compose V2" # V2 does not write the version anymore, so we take it from the first compose file first_compose_file="${1}" version=$(grep --max-count=1 "^version:" "${first_compose_file}" | cut --delimiter=' ' --fields=2) @@ -61,6 +62,7 @@ if docker compose version --short | grep --quiet "^2\." ; then else show_warning "docker compose V2 is not available, trying V1 instead... please update your docker engine." if docker-compose version --short | grep --quiet "^1\." ; then + show_info "Running compose V1" docker_command="\ docker-compose \ --log-level=ERROR \ @@ -71,9 +73,6 @@ docker-compose \ done docker_command+="\ config \ -| sed '/published:/s/\"//g' \ -| sed '/size:/s/\"//g' \ -| sed '1 { /name:.*/d ; }' \ | sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'" # Execute the command show_info "Executing Docker command: ${docker_command}" From bb6740dd4636366ce17da8fb1cedf376fa9e3ab2 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 13:32:52 +0100 Subject: [PATCH 07/12] improve errors in tests --- tests/test_deploy.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index da7fe1b9..e976d3d8 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -106,7 +106,9 @@ async def dask_gateway_stack_deployed_services( cwd=osparc_gateway_server_root_dir, ) stdout, stderr = await process.communicate() - assert process.returncode == 0, f"Unexpected error while deploying stack:\n{stderr}" + assert ( + process.returncode == 0 + ), f"Unexpected error while deploying stack:\n{stderr.decode()}" print(f"{stdout}") print("--> osparc-dask-gateway stack deployed.") healtcheck_endpoint = f"{dask_gateway_entrypoint}/api/health" @@ -135,7 +137,9 @@ async def dask_gateway_stack_deployed_services( cwd=osparc_gateway_server_root_dir, ) stdout, stderr = await process.communicate() - assert process.returncode == 0, f"Unexpected error while deploying stack:\n{stderr}" + assert ( + process.returncode == 0 + ), f"Unexpected error while deploying stack:\n{stderr.decode()}" print(f"{stdout}") print("<-- osparc-dask-gateway stack stopped.") @@ -191,7 +195,6 @@ def neg(x): return -x with cluster.get_client() as client: - square_of_2 = client.submit(square, 2) assert square_of_2.result(timeout=10) == 4 assert not square_of_2.exception(timeout=10) From a366de13153e3b44e35eb40e8b939186bb380a67 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 13:34:40 +0100 Subject: [PATCH 08/12] ensure we use the same dependencies --- tests/requirements/_test.txt | 7 +++---- tests/requirements/_tools.txt | 10 ++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/requirements/_test.txt b/tests/requirements/_test.txt index b279789c..26ff1ec0 100644 --- a/tests/requirements/_test.txt +++ b/tests/requirements/_test.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in +# pip-compile --output-file=requirements/_test.txt --resolver=backtracking --strip-extras requirements/_test.in # aiohttp==3.8.4 # via @@ -29,9 +29,8 @@ click==8.1.3 # dask # dask-gateway # distributed -cloudpickle==2.0.0 +cloudpickle==2.2.1 # via - # -c requirements/../../requirements/constraints.txt # dask # distributed codecov==2.1.12 @@ -88,7 +87,7 @@ markupsafe==2.1.2 # via jinja2 mccabe==0.7.0 # via pylint -msgpack==1.0.4 +msgpack==1.0.5 # via distributed multidict==6.0.4 # via diff --git a/tests/requirements/_tools.txt b/tests/requirements/_tools.txt index dd1fb6ca..1c95f3db 100644 --- a/tests/requirements/_tools.txt +++ b/tests/requirements/_tools.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in +# pip-compile --output-file=requirements/_tools.txt --resolver=backtracking --strip-extras requirements/_tools.in # black==23.1.0 # via -r requirements/../../requirements/devenv.txt @@ -21,12 +21,10 @@ distlib==0.3.6 # via virtualenv filelock==3.9.0 # via virtualenv -identify==2.5.18 +identify==2.5.19 # via pre-commit isort==5.12.0 - # via - # -c requirements/_test.txt - # -r requirements/../../requirements/devenv.txt + # via -r requirements/../../requirements/devenv.txt mypy-extensions==1.0.0 # via black nodeenv==1.7.0 @@ -51,7 +49,6 @@ pyproject-hooks==1.0.0 # via build pyyaml==6.0 # via - # -c requirements/../../requirements/constraints.txt # -c requirements/_test.txt # pre-commit tomli==2.0.1 @@ -59,6 +56,7 @@ tomli==2.0.1 # -c requirements/_test.txt # black # build + # pyproject-hooks typing-extensions==4.5.0 # via # -c requirements/_test.txt From e8afa4b87f5782c6a7670a5dedf983e38b606c45 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 13:38:12 +0100 Subject: [PATCH 09/12] fix indentatation --- scripts/docker-compose-config.bash | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/docker-compose-config.bash b/scripts/docker-compose-config.bash index ebbf1a1c..073017cb 100755 --- a/scripts/docker-compose-config.bash +++ b/scripts/docker-compose-config.bash @@ -39,22 +39,22 @@ if docker compose version --short | grep --quiet "^2\." ; then fi docker_command="\ - docker \ - --log-level=ERROR \ - compose \ - --env-file .env" +docker \ +--log-level=ERROR \ +compose \ +--env-file .env" for compose_file_path in "$@" do docker_command+=" --file=${compose_file_path}" done docker_command+="\ - config \ - | sed '/published:/s/\"//g' \ - | sed '/size:/s/\"//g' \ - | sed '1 { /name:.*/d ; }' \ - | sed '1 i\version: \"${version}\"' \ - | sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'" +config \ +| sed '/published:/s/\"//g' \ +| sed '/size:/s/\"//g' \ +| sed '1 { /name:.*/d ; }' \ +| sed '1 i\version: \"${version}\"' \ +| sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'" # Execute the command show_info "Executing Docker command: ${docker_command}" From cd7f777a9b29faa26b39a84c1831292f41876ea8 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 14:14:35 +0100 Subject: [PATCH 10/12] improve bash script --- scripts/docker-compose-config.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/docker-compose-config.bash b/scripts/docker-compose-config.bash index 073017cb..fadf528e 100755 --- a/scripts/docker-compose-config.bash +++ b/scripts/docker-compose-config.bash @@ -33,7 +33,7 @@ if docker compose version --short | grep --quiet "^2\." ; then show_info "Running compose V2" # V2 does not write the version anymore, so we take it from the first compose file first_compose_file="${1}" - version=$(grep --max-count=1 "^version:" "${first_compose_file}" | cut --delimiter=' ' --fields=2) + version=$(grep --max-count=1 "^version:" "${first_compose_file}" | cut --delimiter=' ' --fields=2 | tr --delete \"\') if [[ -z "$version" ]]; then version="3.9" # Default to 3.9 if version is not found in file fi @@ -49,11 +49,11 @@ compose \ docker_command+=" --file=${compose_file_path}" done docker_command+="\ -config \ + config \ | sed '/published:/s/\"//g' \ | sed '/size:/s/\"//g' \ | sed '1 { /name:.*/d ; }' \ -| sed '1 i\version: \"${version}\"' \ +| sed '1 i version: \"${version}\"' \ | sed --regexp-extended 's/cpus: ([0-9\\.]+)/cpus: \"\\1\"/'" # Execute the command From eeae9eb4a6d4a88fb028fb2d478416790703714f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 14:14:47 +0100 Subject: [PATCH 11/12] improve test error reporting --- tests/test_deploy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index e976d3d8..5c2b6d5c 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -108,7 +108,7 @@ async def dask_gateway_stack_deployed_services( stdout, stderr = await process.communicate() assert ( process.returncode == 0 - ), f"Unexpected error while deploying stack:\n{stderr.decode()}" + ), f"Unexpected error while deploying stack:\nstdout:{stdout.decode()}\n\nstderr:{stderr.decode()}" print(f"{stdout}") print("--> osparc-dask-gateway stack deployed.") healtcheck_endpoint = f"{dask_gateway_entrypoint}/api/health" @@ -139,7 +139,7 @@ async def dask_gateway_stack_deployed_services( stdout, stderr = await process.communicate() assert ( process.returncode == 0 - ), f"Unexpected error while deploying stack:\n{stderr.decode()}" + ), f"Unexpected error while deploying stack:\nstdout:{stdout.decode()}\n\n{stderr.decode()}" print(f"{stdout}") print("<-- osparc-dask-gateway stack stopped.") From c466228b4fece13262908db6a32983e1eb5e929d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 9 Mar 2023 14:27:49 +0100 Subject: [PATCH 12/12] @pcrespov review: remove comments --- .../osparc-gateway-server/tests/integration/test_clusters.py | 2 -- .../tests/integration/test_dask_sidecar.py | 2 -- .../osparc-gateway-server/tests/integration/test_gateway.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/services/osparc-gateway-server/tests/integration/test_clusters.py b/services/osparc-gateway-server/tests/integration/test_clusters.py index 7a3a1a74..36d2d529 100644 --- a/services/osparc-gateway-server/tests/integration/test_clusters.py +++ b/services/osparc-gateway-server/tests/integration/test_clusters.py @@ -20,8 +20,6 @@ @pytest.fixture( params=[ "itisfoundation/dask-sidecar:master-github-latest", - # "itisfoundation/dask-sidecar:staging-github-latest", - # "itisfoundation/dask-sidecar:release-github-latest", ] ) def minimal_config( diff --git a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py index 561e8745..d7c479bd 100644 --- a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py +++ b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py @@ -101,8 +101,6 @@ async def _wait_for_service_to_be_ready( "image_name", [ "itisfoundation/dask-sidecar:master-github-latest", - # "itisfoundation/dask-sidecar:staging-github-latest", - # "itisfoundation/dask-sidecar:release-github-latest", ], ) async def test_computational_sidecar_properly_start_stop( diff --git a/services/osparc-gateway-server/tests/integration/test_gateway.py b/services/osparc-gateway-server/tests/integration/test_gateway.py index 496a8edf..93a7142d 100644 --- a/services/osparc-gateway-server/tests/integration/test_gateway.py +++ b/services/osparc-gateway-server/tests/integration/test_gateway.py @@ -15,8 +15,6 @@ @pytest.fixture( params=[ "itisfoundation/dask-sidecar:master-github-latest", - # "itisfoundation/dask-sidecar:staging-github-latest", - # "itisfoundation/dask-sidecar:release-github-latest", ] ) def minimal_config(