Maintenance/weekly upgrades (#1841)

* weekly requirement upgrades for packages: - yarl PR#1836 - pytest PR #1835 - pytest-docker PR #1834 : removes constraints on attrs. Both upgraded in all services as well! - docker-compose to 1.27.4 * Now pip-compile is done from a docker container instead of the developer host. Added draft of dev-container and used to pip-compile reqs:
ITISFoundation · Sep 30, 2020 · 2eafd34 · 2eafd34
1 parent 587a3e8
commit 2eafd34
Show file tree

Hide file tree

Showing 30 changed files with 170 additions and 117 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,13 +1,11 @@
 # root
-.*/
 ancillary/
 docs/
 ops/
 
-# Docker
-**/Dockerfile
-**/docker-compose*
-
 # Byte-compiled / optimized / DLL files
 **/__pycache__
 *.py[cod]
+
+# virtualenv
+.venv
diff --git a/.gitignore b/.gitignore
@@ -158,3 +158,4 @@ prof/
 services/**/.codeclimate.yml
 # WSL
 .fake_hostname_file
+.bash_history
diff --git a/ci/github/helpers/setup_docker_compose.bash b/ci/github/helpers/setup_docker_compose.bash
@@ -6,8 +6,8 @@ set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # when changing the DOCKER_COMPOSE_VERSION please compute the sha256sum on an ubuntu box (macOS has different checksum)
-DOCKER_COMPOSE_VERSION="1.27.3"
-DOCKER_COMPOSE_SHA256SUM="92055c48e1514c0377b76ed3df87f505c50099145d86835b06fa5109811b6a83"
+DOCKER_COMPOSE_VERSION="1.27.4"
+DOCKER_COMPOSE_SHA256SUM="04216d65ce0cd3c27223eab035abfeb20a8bef20259398e3b9d9aa8de633286d"
 DOCKER_COMPOSE_BIN=/usr/local/bin/docker-compose
 curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o $DOCKER_COMPOSE_BIN
 chmod +x $DOCKER_COMPOSE_BIN

diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt
@@ -9,4 +9,4 @@ multidict==4.7.6          # via yarl
 psycopg2-binary==2.8.6    # via sqlalchemy
 sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.in
 typing-extensions==3.7.4.3  # via yarl
-yarl==1.5.1               # via -r requirements/_base.in
+yarl==1.6.0               # via -r requirements/_base.in
diff --git a/packages/postgres-database/requirements/_migration.txt b/packages/postgres-database/requirements/_migration.txt
@@ -23,4 +23,4 @@ tenacity==6.2.0           # via -r requirements/_migration.in
 typing-extensions==3.7.4.3  # via -r requirements/_base.txt, yarl
 urllib3==1.25.10          # via -r requirements/_migration.in, requests
 websocket-client==0.57.0  # via docker
-yarl==1.5.1               # via -r requirements/_base.txt
+yarl==1.6.0               # via -r requirements/_base.txt
diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt
@@ -9,33 +9,32 @@ aiopg[sa]==1.0.0          # via -r requirements/_test.in
 alembic==1.4.3            # via -r requirements/_migration.txt
 astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via aiohttp
-attrs==19.3.0             # via aiohttp, jsonschema, pytest, pytest-docker
+attrs==20.2.0             # via aiohttp, jsonschema, pytest, pytest-docker
 bcrypt==3.2.0             # via paramiko
-cached-property==1.5.1    # via docker-compose
+cached-property==1.5.2    # via docker-compose
 certifi==2020.6.20        # via -r requirements/_migration.txt, requests
 cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements/_migration.txt, aiohttp, requests
 click==7.1.2              # via -r requirements/_migration.txt
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1         # via paramiko
+cryptography==3.1.1       # via paramiko
 distro==1.5.0             # via docker-compose
-docker-compose==1.27.3    # via pytest-docker
+docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via -r requirements/_migration.txt, docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 faker==4.1.3              # via -r requirements/_test.in
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via -r requirements/_migration.txt, idna-ssl, requests, yarl
-importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
+importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
 isort==5.5.3              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mako==1.1.3               # via -r requirements/_migration.txt, alembic
 markupsafe==1.1.1         # via -r requirements/_migration.txt, mako
 mccabe==0.6.1             # via pylint
-more-itertools==8.5.0     # via pytest
 multidict==4.7.6          # via -r requirements/_migration.txt, aiohttp, yarl
 packaging==20.4           # via pytest
 paramiko==2.7.2           # via docker
@@ -49,10 +48,10 @@ pyparsing==2.4.7          # via packaging
 pyrsistent==0.17.3        # via jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
 pytest-cov==2.10.1        # via -r requirements/_test.in
-pytest-docker==0.10.0     # via -r requirements/_test.in
+pytest-docker==0.10.1     # via -r requirements/_test.in
 pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
-pytest==6.0.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail
+pytest==6.1.0             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail
 python-dateutil==2.8.1    # via -r requirements/_migration.txt, alembic, faker
 python-dotenv==0.14.0     # via docker-compose
 python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
@@ -69,8 +68,8 @@ typing-extensions==3.7.4.3  # via -r requirements/_migration.txt, aiohttp, yarl
 urllib3==1.25.10          # via -r requirements/_migration.txt, requests
 websocket-client==0.57.0  # via -r requirements/_migration.txt, docker, docker-compose
 wrapt==1.12.1             # via astroid
-yarl==1.5.1               # via -r requirements/_migration.txt, aiohttp
-zipp==3.1.0               # via importlib-metadata
+yarl==1.6.0               # via -r requirements/_migration.txt, aiohttp
+zipp==3.2.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/packages/s3wrapper/requirements/_test.txt b/packages/s3wrapper/requirements/_test.txt
@@ -5,30 +5,29 @@
 #    pip-compile --output-file=requirements/_test.txt requirements/_test.in
 #
 astroid==2.4.2            # via pylint
-attrs==19.3.0             # via jsonschema, pytest, pytest-docker
+attrs==20.2.0             # via jsonschema, pytest, pytest-docker
 bcrypt==3.2.0             # via paramiko
-cached-property==1.5.1    # via docker-compose
+cached-property==1.5.2    # via docker-compose
 certifi==2020.6.20        # via -r requirements/_base.txt, minio, requests
 cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
 configparser==5.0.0       # via -r requirements/_base.txt, minio
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1         # via paramiko
+cryptography==3.1.1       # via paramiko
 distro==1.5.0             # via docker-compose
-docker-compose==1.27.3    # via pytest-docker
+docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 idna==2.10                # via requests
-importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
+importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
 isort==5.5.3              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
 minio==6.0.0              # via -r requirements/_base.txt
-more-itertools==8.5.0     # via pytest
 packaging==20.4           # via pytest
 paramiko==2.7.2           # via docker
 pluggy==0.13.1            # via pytest
@@ -39,9 +38,9 @@ pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.17.3        # via jsonschema
 pytest-cov==2.10.1        # via -r requirements/_test.in
-pytest-docker==0.10.0     # via -r requirements/_test.in
+pytest-docker==0.10.1     # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
-pytest==6.0.2             # via -r requirements/_test.in, pytest-cov, pytest-docker
+pytest==6.1.0             # via -r requirements/_test.in, pytest-cov, pytest-docker
 python-dateutil==2.8.1    # via -r requirements/_base.txt, minio
 python-dotenv==0.14.0     # via docker-compose
 pytz==2020.1              # via -r requirements/_base.txt, minio
@@ -54,7 +53,7 @@ typed-ast==1.4.1          # via astroid
 urllib3==1.25.10          # via -r requirements/_base.txt, minio, requests
 websocket-client==0.57.0  # via docker, docker-compose
 wrapt==1.12.1             # via astroid
-zipp==3.1.0               # via importlib-metadata
+zipp==3.2.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/packages/service-library/requirements/_base.in b/packages/service-library/requirements/_base.in
@@ -16,6 +16,6 @@ werkzeug
 jsonschema
 prometheus_client
 tenacity
-attrs<20,>=19   # from pytest-docker==0.8.0
+attrs
 trafaret
 aiodebug
diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt
@@ -9,9 +9,11 @@ aiohttp==3.6.2            # via -r requirements/_base.in, aiozipkin
 aiopg[sa]==1.0.0          # via -r requirements/_base.in
 aiozipkin==0.7.1          # via -r requirements/_base.in
 async-timeout==3.0.1      # via aiohttp
-attrs==19.3.0             # via -r requirements/_base.in, aiohttp, jsonschema, openapi-core
+attrs==20.2.0             # via -r requirements/_base.in, aiohttp, jsonschema, openapi-core
 chardet==3.0.4            # via aiohttp
-idna==2.10                # via yarl
+idna-ssl==1.1.0           # via aiohttp
+idna==2.10                # via idna-ssl, yarl
+importlib-metadata==2.0.0  # via jsonschema
 isodate==0.6.0            # via openapi-core
 jsonschema==3.2.0         # via -r requirements/_base.in, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via -r requirements/_base.in, openapi-core
@@ -30,7 +32,8 @@ trafaret==2.1.0           # via -r requirements/_base.in
 typing-extensions==3.7.4.3  # via aiohttp, yarl
 ujson==3.2.0              # via -r requirements/_base.in
 werkzeug==1.0.1           # via -r requirements/_base.in
-yarl==1.5.1               # via aiohttp
+yarl==1.6.0               # via aiohttp
+zipp==3.2.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/packages/service-library/requirements/_test.txt b/packages/service-library/requirements/_test.txt
@@ -10,30 +10,29 @@ aiopg[sa]==1.0.0          # via -r requirements/_base.txt
 aiozipkin==0.7.1          # via -r requirements/_base.txt
 astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via -r requirements/_base.txt, aiohttp
-attrs==19.3.0             # via -r requirements/_base.txt, aiohttp, jsonschema, openapi-core, pytest, pytest-docker
+attrs==20.2.0             # via -r requirements/_base.txt, aiohttp, jsonschema, openapi-core, pytest, pytest-docker
 bcrypt==3.2.0             # via paramiko
-cached-property==1.5.1    # via docker-compose
+cached-property==1.5.2    # via docker-compose
 certifi==2020.6.20        # via requests
 cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements/_base.txt, aiohttp, requests
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1         # via paramiko
+cryptography==3.1.1       # via paramiko
 distro==1.5.0             # via docker-compose
-docker-compose==1.27.3    # via pytest-docker
+docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
-importlib-metadata==1.7.0  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
+importlib-metadata==2.0.0  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
 isodate==0.6.0            # via -r requirements/_base.txt, openapi-core
 isort==5.5.3              # via pylint
 jsonschema==3.2.0         # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
 lazy-object-proxy==1.4.3  # via -r requirements/_base.txt, astroid, openapi-core
 mccabe==0.6.1             # via pylint
-more-itertools==8.5.0     # via pytest
 multidict==4.7.6          # via -r requirements/_base.txt, aiohttp, yarl
 openapi-core==0.12.0      # via -r requirements/_base.txt
 openapi-spec-validator==0.2.9  # via -r requirements/_base.txt, openapi-core
@@ -50,12 +49,12 @@ pyparsing==2.4.7          # via packaging
 pyrsistent==0.17.3        # via -r requirements/_base.txt, jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
 pytest-cov==2.10.1        # via -r requirements/_test.in
-pytest-docker==0.10.0     # via -r requirements/_test.in
+pytest-docker==0.10.1     # via -r requirements/_test.in
 pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-mock==3.3.1        # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
-pytest==6.0.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
+pytest==6.1.0             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
 python-dotenv==0.14.0     # via docker-compose
 pyyaml==5.3.1             # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
 requests==2.24.0          # via coveralls, docker, docker-compose
@@ -74,7 +73,8 @@ urllib3==1.25.10          # via requests
 websocket-client==0.57.0  # via docker, docker-compose
 werkzeug==1.0.1           # via -r requirements/_base.txt
 wrapt==1.12.1             # via astroid
-yarl==1.5.1               # via -r requirements/_base.txt, aiohttp
+yarl==1.6.0               # via -r requirements/_base.txt, aiohttp
+zipp==3.2.0               # via -r requirements/_base.txt, importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in
@@ -10,4 +10,4 @@ pydantic
 tenacity
 trafaret-config
 
-attrs<20,>=19 # from pytest-docker==0.8.0
+attrs
diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt
@@ -25,4 +25,4 @@ tenacity==6.2.0           # via -r requirements/_base.in
 trafaret-config==2.0.2    # via -r requirements/_base.in
 trafaret==2.1.0           # via trafaret-config
 typing-extensions==3.7.4.3  # via aiohttp, yarl
-yarl==1.5.1               # via aiohttp
+yarl==1.6.0               # via aiohttp
diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt
@@ -11,30 +11,29 @@ astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via -r requirements/_base.txt, aiohttp
 attrs==19.3.0             # via -r requirements/_base.txt, aiohttp, jsonschema, pytest, pytest-docker
 bcrypt==3.2.0             # via paramiko
-cached-property==1.5.1    # via docker-compose
+cached-property==1.5.2    # via docker-compose
 certifi==2020.6.20        # via requests
 cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements/_base.txt, aiohttp, requests
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1         # via paramiko
+cryptography==3.1.1       # via paramiko
 dataclasses==0.7          # via -r requirements/_base.txt, pydantic
 decorator==4.4.2          # via -r requirements/_base.txt, networkx
 distro==1.5.0             # via docker-compose
-docker-compose==1.27.3    # via pytest-docker
+docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via -r requirements/_test.in, docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
-importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
+importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
 isort==5.5.3              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
 mock==4.0.2               # via -r requirements/_test.in
-more-itertools==8.5.0     # via pytest
 multidict==4.7.6          # via -r requirements/_base.txt, aiohttp, yarl
 networkx==2.5             # via -r requirements/_base.txt
 packaging==20.4           # via pytest, pytest-sugar
@@ -50,12 +49,12 @@ pyparsing==2.4.7          # via packaging
 pyrsistent==0.17.3        # via jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
 pytest-cov==2.10.1        # via -r requirements/_test.in
-pytest-docker==0.10.0     # via -r requirements/_test.in
+pytest-docker==0.10.1     # via -r requirements/_test.in
 pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-mock==3.3.1        # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
-pytest==6.0.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
+pytest==6.1.0             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
 python-dotenv==0.14.0     # via -r requirements/_test.in, docker-compose
 pyyaml==5.3.1             # via -r requirements/_base.txt, docker-compose, trafaret-config
 requests==2.24.0          # via -r requirements/_test.in, coveralls, docker, docker-compose
@@ -72,8 +71,8 @@ typing-extensions==3.7.4.3  # via -r requirements/_base.txt, aiohttp, yarl
 urllib3==1.25.10          # via requests
 websocket-client==0.57.0  # via docker, docker-compose
 wrapt==1.12.1             # via astroid
-yarl==1.5.1               # via -r requirements/_base.txt, aiohttp
-zipp==3.1.0               # via importlib-metadata
+yarl==1.6.0               # via -r requirements/_base.txt, aiohttp
+zipp==3.2.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/scripts/requirements/Dockerfile b/scripts/requirements/Dockerfile
@@ -1,21 +1,29 @@
+# NOTE: This is a first step towards a devcontainer
+#       to perform operations like pip-compile or auto-formatting
+#       that preserves identical environment across developer machines
+#
 ARG PYTHON_VERSION="3.6.10"
 FROM python:${PYTHON_VERSION}-slim-buster as base
 
 
+RUN apt-get update \
+  && apt-get -y install --no-install-recommends\
+  make \
+  git \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt-get clean
+
+
 RUN pip --no-cache-dir install --upgrade \
   pip~=20.2.2  \
   wheel \
   setuptools
 
 
 # devenv
-COPY requirements.txt .
-RUN pip install -r requirements.txt
-
-
-WORKDIR /work
-
-# .dockerignore to include only target folders
-# mount osparc-simcore -> /work
-# cd script
-# make reqs
+RUN pip install \
+  black \
+  isort \
+  pip-tools \
+  bump2version \
+  rope