Merge remote-tracking branch 'upstream/master' into adding-dynamic-si…

…decar

.github/workflows/ci-testing-deploy.yml

@@ -1767,6 +1767,10 @@ jobs:
           export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
           echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
       - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
       - name: setup python environment
         uses: actions/setup-python@v2
         with:

Makefile

@@ -102,7 +102,7 @@ define _docker_compose_build
 export BUILD_TARGET=$(if $(findstring -devel,$@),development,$(if $(findstring -cache,$@),cache,production));\
 $(if $(findstring -x,$@),\
 	pushd services; docker buildx bake --file docker-compose-build.yml; popd;,\
-	docker-compose -f services/docker-compose-build.yml build $(if $(findstring -nc,$@),--no-cache,) $(if $(target),,--parallel)\
+	docker-compose --file services/docker-compose-build.yml build $(if $(findstring -nc,$@),--no-cache,) $(if $(target),,--parallel)\
 )
 endef
 
@@ -183,21 +183,21 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	@export DOCKER_REGISTRY=local \
 	export DOCKER_IMAGE_TAG=development; \
 	export DEV_PC_CPU_COUNT=${CPU_COUNT}; \
-	docker-compose -f services/docker-compose.yml -f services/docker-compose.local.yml -f services/docker-compose.devel.yml --log-level=ERROR config > $@
+	docker-compose --env-file .env --file services/docker-compose.yml --file services/docker-compose.local.yml --file services/docker-compose.devel.yml --log-level=ERROR config > $@
 
 .stack-simcore-production.yml: .env $(docker-compose-configs)
 	# Creating config for stack with 'local/{service}:production' to $@
 	@export DOCKER_REGISTRY=local;       \
 	export DOCKER_IMAGE_TAG=production; \
-	docker-compose -f services/docker-compose.yml -f services/docker-compose.local.yml --log-level=ERROR config > $@
+	docker-compose --env-file .env --file services/docker-compose.yml --file services/docker-compose.local.yml --log-level=ERROR config > $@
 
 .stack-simcore-version.yml: .env $(docker-compose-configs)
 	# Creating config for stack with '$(DOCKER_REGISTRY)/{service}:${DOCKER_IMAGE_TAG}' to $@
-	@docker-compose -f services/docker-compose.yml -f services/docker-compose.local.yml --log-level=ERROR config > $@
+	@docker-compose --env-file .env --file services/docker-compose.yml --file services/docker-compose.local.yml --log-level=ERROR config > $@
 
 .stack-ops.yml: .env $(docker-compose-configs)
 	# Creating config for ops stack to $@
-	@docker-compose -f services/docker-compose-ops.yml --log-level=ERROR config > $@
+	@docker-compose --env-file .env --file services/docker-compose-ops.yml --log-level=ERROR config > $@
 
 
 .PHONY: up-devel up-prod up-version up-latest .deploy-ops
@@ -234,7 +234,7 @@ ifeq ($(target),)
 	@$(MAKE) .deploy-ops
 else
 	# deploys ONLY $(target) service
-	@docker-compose -f $< up --detach $(target)
+	@docker-compose --file $< up --detach $(target)
 endif
 	@$(_show_endpoints)
 
@@ -312,7 +312,7 @@ pull-cache: .env
 
 pull-version: .env ## pulls images from DOCKER_REGISTRY tagged as DOCKER_IMAGE_TAG
 	# Pulling images '${DOCKER_REGISTRY}/{service}:${DOCKER_IMAGE_TAG}'
-	@docker-compose -f services/docker-compose.yml pull
+	@docker-compose --file services/docker-compose.yml pull
 
 
 .PHONY: push-cache push-version push-latest
@@ -504,11 +504,13 @@ info: ## displays setup information
 	@echo ' STORAGE_API_VERSION   : ${STORAGE_API_VERSION}'
 	@echo ' WEBSERVER_API_VERSION : ${WEBSERVER_API_VERSION}'
 	# dev tools version
-	@echo ' make   : $(shell make --version 2>&1 | head -n 1)'
-	@echo ' jq     : $(shell jq --version)'
-	@echo ' awk    : $(shell awk -W version 2>&1 | head -n 1)'
-	@echo ' python : $(shell python3 --version)'
-	@echo ' node   : $(shell node --version 2> /dev/null || echo ERROR nodejs missing)'
+	@echo ' make          : $(shell make --version 2>&1 | head -n 1)'
+	@echo ' jq            : $(shell jq --version)'
+	@echo ' awk           : $(shell awk -W version 2>&1 | head -n 1)'
+	@echo ' python        : $(shell python3 --version)'
+	@echo ' node          : $(shell node --version 2> /dev/null || echo ERROR nodejs missing)'
+	@echo ' docker        : $(shell docker --version)'
+	@echo ' docker-compose: $(shell docker-compose --version)'
 
 
 define show-meta
@@ -574,14 +576,14 @@ clean: .check-clean ## cleans all unversioned files in project and temp files cr
 
 clean-more: ## cleans containers and unused volumes
 	# stops and deletes running containers
-	@$(if $(_running_containers), docker rm -f $(_running_containers),)
+	@$(if $(_running_containers), docker rm --force $(_running_containers),)
 	# pruning unused volumes
 	docker volume prune --force
 
 clean-images: ## removes all created images
 	# Cleaning all service images
 	-$(foreach service,$(SERVICES_LIST)\
-		,docker image rm -f $(shell docker images */$(service):* -q);)
+		,docker image rm --force $(shell docker images */$(service):* -q);)
 	# Cleaning webclient
 	@$(MAKE_C) services/web/client clean-images
 	# Cleaning postgres maintenance

api/tests/requirements.txt

@@ -26,7 +26,6 @@ idna-ssl==1.1.0
     # via aiohttp
 idna==2.10
     # via
-    #   -c ../../requirements/constraints.txt
     #   idna-ssl
     #   yarl
 importlib-metadata==3.7.0

ci/github/helpers/setup_docker_compose.bash

@@ -6,8 +6,9 @@ set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # when changing the DOCKER_COMPOSE_VERSION please compute the sha256sum on an ubuntu box (macOS has different checksum)
-DOCKER_COMPOSE_VERSION="1.27.4"
-DOCKER_COMPOSE_SHA256SUM="04216d65ce0cd3c27223eab035abfeb20a8bef20259398e3b9d9aa8de633286d"
+DOCKER_COMPOSE_VERSION="1.29.1"
+# Check for sha256 file in Asset section on https://github.com/docker/compose/releases and copy here
+DOCKER_COMPOSE_SHA256SUM="8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7"
 DOCKER_COMPOSE_BIN=/usr/local/bin/docker-compose
 curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o $DOCKER_COMPOSE_BIN
 chmod +x $DOCKER_COMPOSE_BIN

packages/models-library/requirements/_base.txt

@@ -11,9 +11,7 @@ dnspython==2.1.0
 email-validator==1.1.2
     # via pydantic
 idna==2.10
-    # via
-    #   -c requirements/../../../requirements/constraints.txt
-    #   email-validator
+    # via email-validator
 pydantic[email]==1.8.1
     # via -r requirements/_base.in
 typing-extensions==3.7.4.3

packages/models-library/requirements/_test.txt

@@ -35,7 +35,6 @@ idna-ssl==1.1.0
     # via aiohttp
 idna==2.10
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   idna-ssl
     #   requests

packages/postgres-database/requirements/_base.txt

@@ -5,9 +5,7 @@
 #    pip-compile --output-file=requirements/_base.txt requirements/_base.in
 #
 idna==2.10
-    # via
-    #   -c requirements/../../../requirements/constraints.txt
-    #   yarl
+    # via yarl
 multidict==5.1.0
     # via yarl
 psycopg2-binary==2.8.6

packages/postgres-database/requirements/_migration.in

@@ -13,5 +13,4 @@ click
 tenacity
 
 # aiohttp==3.7.3 conflicting with requests==2.25.1 -> docker
-docker # TODO: aiodocker
-chardet<4.0,>=2.0
+docker

packages/postgres-database/requirements/_migration.txt

@@ -9,16 +9,13 @@ alembic==1.5.5
 certifi==2020.12.5
     # via requests
 chardet==3.0.4
-    # via
-    #   -r requirements/_migration.in
-    #   requests
+    # via requests
 click==7.1.2
     # via -r requirements/_migration.in
-docker==4.4.4
+docker==5.0.0
     # via -r requirements/_migration.in
 idna==2.10
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   requests
 mako==1.1.4
@@ -37,7 +34,6 @@ requests==2.25.1
     # via docker
 six==1.15.0
     # via
-    #   docker
     #   python-dateutil
     #   tenacity
     #   websocket-client

packages/postgres-database/requirements/_test.txt

@@ -49,11 +49,11 @@ cryptography==3.4.6
     #   paramiko
 distro==1.5.0
     # via docker-compose
-docker-compose==1.27.4
+docker-compose==1.29.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   pytest-docker
-docker[ssh]==4.4.4
+docker[ssh]==5.0.0
     # via
     #   -c requirements/_migration.txt
     #   docker-compose
@@ -69,7 +69,6 @@ idna-ssl==1.1.0
     # via aiohttp
 idna==2.10
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   -c requirements/_migration.txt
     #   idna-ssl
@@ -157,7 +156,6 @@ six==1.15.0
     # via
     #   -c requirements/_migration.txt
     #   bcrypt
-    #   docker
     #   dockerpty
     #   jsonschema
     #   pynacl

packages/postgres-database/tests/docker-compose.prod.yml

@@ -1,4 +1,4 @@
-version: "3.4"
+version: "3.8"
 services:
   postgres:
     volumes:

packages/postgres-database/tests/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 services:
   postgres:
     image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f"

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -121,6 +121,7 @@ def simcore_docker_compose(
         docker_compose_paths,
         workdir=env_file.parent,
         destination_path=temp_folder / "simcore_docker_compose.yml",
+        env_file_path=env_file,
     )
     print("simcore docker-compose:\n%s", pformat(config))
     return config
@@ -148,6 +149,7 @@ def ops_docker_compose(
         docker_compose_path,
         workdir=env_file.parent,
         destination_path=temp_folder / "ops_docker_compose.yml",
+        env_file_path=env_file,
     )
     print("ops docker-compose:\n%s", pformat(config))
     return config

packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py

@@ -89,6 +89,7 @@ def run_docker_compose_config(
     docker_compose_paths: Union[List[Path], Path],
     workdir: Path,
     destination_path: Optional[Path] = None,
+    env_file_path: Optional[Path] = None,
 ) -> Dict:
     """Runs docker-compose config to validate and resolve a compose file configuration
 
@@ -108,11 +109,15 @@ def run_docker_compose_config(
         destination_path = temp_dir / "docker-compose.yml"
 
     config_paths = [
-        f"-f {os.path.relpath(docker_compose_path, workdir)}"
+        f"--file {os.path.relpath(docker_compose_path, workdir)}"
         for docker_compose_path in docker_compose_paths
     ]
     configs_prefix = " ".join(config_paths)
 
+    if env_file_path:
+        # SEE https://docs.docker.com/compose/env-file/
+        configs_prefix += f" --env-file {env_file_path}"
+
     subprocess.run(
         f"docker-compose {configs_prefix} config > {destination_path}",
         shell=True,

packages/s3wrapper/requirements/_test.txt

@@ -39,11 +39,11 @@ cryptography==3.4.6
     #   paramiko
 distro==1.5.0
     # via docker-compose
-docker-compose==1.27.4
+docker-compose==1.29.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   pytest-docker
-docker[ssh]==4.4.4
+docker[ssh]==5.0.0
     # via docker-compose
 dockerpty==0.4.1
     # via docker-compose
@@ -52,9 +52,7 @@ docopt==0.6.2
     #   coveralls
     #   docker-compose
 idna==2.10
-    # via
-    #   -c requirements/../../../requirements/constraints.txt
-    #   requests
+    # via requests
 importlib-metadata==3.7.0
     # via
     #   jsonschema
@@ -115,7 +113,6 @@ six==1.15.0
     # via
     #   -c requirements/_base.txt
     #   bcrypt
-    #   docker
     #   dockerpty
     #   jsonschema
     #   pynacl

packages/s3wrapper/tests/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3.8"
 services:
   minio:
     image: minio/minio
@@ -7,4 +7,4 @@ services:
       - MINIO_SECRET_KEY=12345678
     ports:
       - "9001:9000"
-    command: server /data
+    command: server /data

packages/service-integration/requirements/_base.txt

@@ -18,14 +18,12 @@ dataclasses==0.8
     # via pydantic
 dnspython==2.1.0
     # via email-validator
-docker==4.4.4
+docker==5.0.0
     # via -r requirements/_base.in
 email-validator==1.1.2
     # via pydantic
 idna==2.10
     # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
     #   email-validator
     #   requests
 importlib-metadata==3.7.0
@@ -60,7 +58,6 @@ requests==2.25.1
     # via docker
 six==1.15.0
     # via
-    #   docker
     #   jsonschema
     #   websocket-client
 toml==0.10.2

packages/service-integration/requirements/_test.txt

@@ -29,7 +29,6 @@ docopt==0.6.2
     # via coveralls
 idna==2.10
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   requests
 importlib-metadata==3.7.0

packages/service-library/requirements/_base.txt

@@ -34,7 +34,6 @@ idna-ssl==1.1.0
     # via aiohttp
 idna==2.10
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   idna-ssl
     #   yarl
 importlib-metadata==3.7.0

packages/service-library/requirements/_test.txt

@@ -50,11 +50,11 @@ cryptography==3.4.6
     #   paramiko
 distro==1.5.0
     # via docker-compose
-docker-compose==1.27.4
+docker-compose==1.29.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   pytest-docker
-docker[ssh]==4.4.4
+docker[ssh]==5.0.0
     # via docker-compose
 dockerpty==0.4.1
     # via docker-compose
@@ -68,7 +68,6 @@ idna-ssl==1.1.0
     #   aiohttp
 idna==2.10
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   idna-ssl
     #   requests
@@ -159,7 +158,6 @@ six==1.15.0
     # via
     #   -c requirements/_base.txt
     #   bcrypt
-    #   docker
     #   dockerpty
     #   jsonschema
     #   pynacl

packages/service-library/tests/with_postgres/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.4"
+version: "3.8"
 services:
   postgres:
     image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f"