From fa89b6be7e001353419edaf074e9f9b45136fb1d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:30:59 +0100 Subject: [PATCH 01/11] update requirements --- .pre-commit-config.yaml | 36 +++- requirements/constraints.txt | 1 - .../requirements/_base.txt | 77 +++++---- .../requirements/_test.txt | 159 +++++++++--------- .../requirements/_tools.txt | 61 ++++--- .../requirements/constraints.txt | 1 - 6 files changed, 184 insertions(+), 151 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d42adb30..b3c53ed5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,18 +5,42 @@ default_language_version: python: python3.9 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.3.0 + rev: v4.2.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - id: check-added-large-files + args: ["--maxkb=1024"] - id: check-merge-conflict + - id: check-yaml + - id: detect-private-key + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: detect-aws-credentials + args: [--allow-missing-credentials] + - id: debug-statements + - id: no-commit-to-branch + - repo: https://github.com/asottile/pyupgrade + rev: v2.34.0 + hooks: + - id: pyupgrade + args: + - "--py39-plus" + name: upgrade code + - repo: https://github.com/hadialqattan/pycln + rev: v1.2.5 + hooks: + - id: pycln + args: [--all, --expand-stars] + name: prune imports - repo: https://github.com/PyCQA/isort - rev: 5.6.4 + rev: 5.12.0 hooks: - id: isort + args: ["--profile", "black"] + name: sort imports - repo: https://github.com/psf/black - rev: 21.9b0 + rev: 22.3.0 hooks: - id: black + name: black format code diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 5398e743..4657a480 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -30,7 +30,6 @@ urllib3>=1.26.5 # https://github.com/advisories/GH # # Compatibility/coordination # -tomli<2.0.0 # this is a constraint with black 21.12b0 # All backports libraries add environ markers # NOTE: If >second dependency, this will annotate a marker in the compiled requirements file diff --git a/services/osparc-gateway-server/requirements/_base.txt b/services/osparc-gateway-server/requirements/_base.txt index ce3cb365..4a1472b3 100644 --- a/services/osparc-gateway-server/requirements/_base.txt +++ b/services/osparc-gateway-server/requirements/_base.txt @@ -1,83 +1,94 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/_base.txt --strip-extras requirements/_base.in # aiodocker==0.21.0 # via -r requirements/_base.in -aiohttp==3.8.1 +aiohttp==3.8.4 # via # aiodocker # dask-gateway-server -aiosignal==1.2.0 +aiosignal==1.3.1 # via aiohttp async-timeout==4.0.2 # via aiohttp -attrs==21.4.0 +attrs==22.2.0 # via aiohttp -bokeh==2.4.2 +bokeh==3.0.3 # via -r requirements/_base.in -cffi==1.15.0 +cffi==1.15.1 # via cryptography -charset-normalizer==2.0.11 +charset-normalizer==3.0.1 # via aiohttp -colorlog==6.6.0 +colorlog==6.7.0 # via dask-gateway-server -cryptography==36.0.1 +contourpy==1.0.7 + # via bokeh +cryptography==39.0.1 # via dask-gateway-server -dask-gateway-server==0.9.0 +dask-gateway-server==2023.1.1 # via -r requirements/_base.in -dnspython==2.2.0 +dnspython==2.3.0 # via email-validator -email-validator==1.1.3 +email-validator==1.3.1 # via pydantic -frozenlist==1.3.0 +frozenlist==1.3.3 # via # aiohttp # aiosignal -greenlet==1.1.2 +greenlet==2.0.2 # via sqlalchemy -idna==3.3 +idna==3.4 # via # email-validator # yarl -jinja2==3.0.3 +jinja2==3.1.2 # via bokeh -markupsafe==2.0.1 +markupsafe==2.1.2 # via jinja2 -multidict==6.0.2 +multidict==6.0.4 # via # aiohttp # yarl -numpy==1.21.5 +numpy==1.24.2 # via - # -c requirements/./constraints.txt # bokeh -packaging==21.3 + # contourpy + # pandas +packaging==23.0 + # via bokeh +pandas==1.5.3 # via bokeh -pillow==9.0.0 +pillow==9.4.0 # via bokeh pycparser==2.21 # via cffi -pydantic==1.9.0 +pydantic==1.10.5 # via -r requirements/_base.in -pyparsing==3.0.7 - # via packaging -python-dotenv==0.19.2 +python-dateutil==2.8.2 + # via pandas +python-dotenv==1.0.0 # via pydantic +pytz==2022.7.1 + # via pandas pyyaml==6.0 # via bokeh -sqlalchemy==1.4.31 +six==1.16.0 + # via python-dateutil +sqlalchemy==2.0.4 # via -r requirements/_base.in -tornado==6.1 +tornado==6.2 # via bokeh -traitlets==5.1.1 +traitlets==5.9.0 # via dask-gateway-server -typing-extensions==4.0.1 +typing-extensions==4.5.0 # via # aiodocker - # bokeh # pydantic -yarl==1.7.2 + # sqlalchemy +xyzservices==2023.2.0 + # via bokeh +yarl==1.8.2 # via aiohttp diff --git a/services/osparc-gateway-server/requirements/_test.txt b/services/osparc-gateway-server/requirements/_test.txt index 143aa608..9fdf35b2 100644 --- a/services/osparc-gateway-server/requirements/_test.txt +++ b/services/osparc-gateway-server/requirements/_test.txt @@ -1,230 +1,231 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in # -aiohttp==3.8.1 +aiohttp==3.8.4 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # dask-gateway # pytest-aiohttp -aiosignal==1.2.0 +aiosignal==1.3.1 # via # -c requirements/_base.txt # aiohttp -astroid==2.9.3 +astroid==2.14.2 # via pylint async-timeout==4.0.2 # via # -c requirements/_base.txt # aiohttp -attrs==21.4.0 +attrs==22.2.0 # via # -c requirements/_base.txt # aiohttp # pytest -certifi==2021.10.8 +certifi==2022.12.7 # via requests -charset-normalizer==2.0.11 +charset-normalizer==3.0.1 # via # -c requirements/_base.txt # aiohttp # requests -click==8.0.3 - # via distributed -cloudpickle==2.0.0 +click==8.1.3 + # via + # dask + # dask-gateway + # distributed +cloudpickle==2.2.1 # via # dask # distributed codecov==2.1.12 # via -r requirements/_test.in -colorama==0.4.4 +colorama==0.4.6 # via pytest-watch -coverage==6.3 +coverage==6.5.0 # via # codecov # coveralls # pytest-cov coveralls==3.3.1 # via -r requirements/_test.in -dask==2022.1.1 +dask==2023.3.0 # via # dask-gateway # distributed -dask-gateway==0.9.0 +dask-gateway==2023.1.1 # via -r requirements/_test.in -debugpy==1.5.1 +debugpy==1.6.6 # via -r requirements/_test.in -distributed==2022.1.1 +dill==0.3.6 + # via pylint +distributed==2023.3.0 # via dask-gateway -docker==5.0.3 +docker==6.0.1 # via -r requirements/_test.in docopt==0.6.2 # via # coveralls # pytest-watch +exceptiongroup==1.1.0 + # via pytest execnet==1.9.0 # via pytest-xdist -faker==11.3.0 +faker==17.5.0 # via -r requirements/_test.in -frozenlist==1.3.0 +frozenlist==1.3.3 # via # -c requirements/_base.txt # aiohttp # aiosignal -fsspec==2022.1.0 +fsspec==2023.1.0 # via dask heapdict==1.0.1 # via zict -icdiff==2.0.4 +icdiff==2.0.6 # via pytest-icdiff -idna==3.3 +idna==3.4 # via # -c requirements/_base.txt # requests # yarl -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest -isort==5.10.1 +isort==5.12.0 # via pylint -jinja2==3.0.3 +jinja2==3.1.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # distributed -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 # via astroid -locket==0.2.1 - # via partd -markupsafe==2.0.1 +locket==1.0.0 + # via + # distributed + # partd +markupsafe==2.1.2 # via # -c requirements/_base.txt # jinja2 -mccabe==0.6.1 +mccabe==0.7.0 # via pylint -msgpack==1.0.3 +msgpack==1.0.4 # via distributed -multidict==6.0.2 +multidict==6.0.4 # via # -c requirements/_base.txt # aiohttp # yarl -packaging==21.3 +packaging==23.0 # via # -c requirements/_base.txt # dask # distributed + # docker # pytest -partd==1.2.0 +partd==1.3.0 # via dask -platformdirs==2.4.1 +platformdirs==3.0.0 # via pylint pluggy==1.0.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff -psutil==5.9.0 +psutil==5.9.4 # via distributed -py==1.11.0 - # via - # pytest - # pytest-forked -pylint==2.12.2 +pylint==2.16.2 # via -r requirements/_test.in -pyparsing==3.0.7 - # via - # -c requirements/_base.txt - # packaging -pytest==6.2.5 +pytest==7.2.1 # via # -r requirements/_test.in # pytest-aiohttp # pytest-asyncio # pytest-cov - # pytest-forked # pytest-icdiff # pytest-mock # pytest-watch # pytest-xdist -pytest-aiohttp==1.0.3 +pytest-aiohttp==1.0.4 # via -r requirements/_test.in -pytest-asyncio==0.17.2 +pytest-asyncio==0.20.3 # via pytest-aiohttp -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/_test.in -pytest-forked==1.4.0 - # via pytest-xdist -pytest-icdiff==0.5 +pytest-icdiff==0.6 # via -r requirements/_test.in -pytest-mock==3.7.0 +pytest-mock==3.10.0 # via -r requirements/_test.in pytest-watch==4.2.0 # via -r requirements/_test.in -pytest-xdist==2.5.0 +pytest-xdist==3.2.0 # via -r requirements/_test.in python-dateutil==2.8.2 - # via faker + # via + # -c requirements/_base.txt + # faker pyyaml==6.0 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # dask + # dask-gateway # distributed -requests==2.27.1 +requests==2.28.2 # via # codecov # coveralls # docker six==1.16.0 - # via python-dateutil + # via + # -c requirements/_base.txt + # python-dateutil sortedcontainers==2.4.0 # via distributed tblib==1.7.0 # via distributed -tenacity==8.0.1 +tenacity==8.2.2 # via -r requirements/_test.in -text-unidecode==1.3 - # via faker -toml==0.10.2 +tomli==2.0.1 # via + # coverage # pylint # pytest -tomli==1.2.3 - # via - # -c requirements/../../../requirements/constraints.txt - # coverage -toolz==0.11.2 +tomlkit==0.11.6 + # via pylint +toolz==0.12.0 # via # dask # distributed # partd -tornado==6.1 +tornado==6.2 # via # -c requirements/_base.txt + # dask-gateway # distributed -typing-extensions==4.0.1 +typing-extensions==4.5.0 # via # -c requirements/_base.txt # astroid # pylint -urllib3==1.26.8 +urllib3==1.26.14 # via # -c requirements/../../../requirements/constraints.txt + # distributed + # docker # requests -watchdog==2.1.6 +watchdog==2.3.1 # via pytest-watch -websocket-client==1.2.3 +websocket-client==1.5.1 # via docker -wrapt==1.13.3 +wrapt==1.15.0 # via astroid -yarl==1.7.2 +yarl==1.8.2 # via # -c requirements/_base.txt # aiohttp -zict==2.0.0 +zict==2.2.0 # via distributed - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/services/osparc-gateway-server/requirements/_tools.txt b/services/osparc-gateway-server/requirements/_tools.txt index e6f122e3..439c0166 100644 --- a/services/osparc-gateway-server/requirements/_tools.txt +++ b/services/osparc-gateway-server/requirements/_tools.txt @@ -1,47 +1,55 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in # -black==22.1.0 +black==23.1.0 # via -r requirements/../../../requirements/devenv.txt +build==0.10.0 + # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt cfgv==3.3.1 # via pre-commit -click==8.0.3 +click==8.1.3 # via # -c requirements/_test.txt # black # pip-tools -distlib==0.3.4 +distlib==0.3.6 # via virtualenv -filelock==3.4.2 +filelock==3.9.0 # via virtualenv -identify==2.4.6 +identify==2.5.18 # via pre-commit -isort==5.10.1 +isort==5.12.0 # via # -c requirements/_test.txt # -r requirements/../../../requirements/devenv.txt -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via black -nodeenv==1.6.0 +nodeenv==1.7.0 # via pre-commit -pathspec==0.9.0 +packaging==23.0 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black + # build +pathspec==0.11.0 # via black -pep517==0.12.0 - # via pip-tools -pip-tools==6.4.0 +pip-tools==6.12.3 # via -r requirements/../../../requirements/devenv.txt -platformdirs==2.4.1 +platformdirs==3.0.0 # via # -c requirements/_test.txt # black # virtualenv -pre-commit==2.17.0 +pre-commit==3.1.1 # via -r requirements/../../../requirements/devenv.txt +pyproject-hooks==1.0.0 + # via build pyyaml==6.0 # via # -c requirements/../../../requirements/constraints.txt @@ -49,32 +57,23 @@ pyyaml==6.0 # -c requirements/_test.txt # pre-commit # watchdog -six==1.16.0 +tomli==2.0.1 # via # -c requirements/_test.txt - # virtualenv -toml==0.10.2 - # via - # -c requirements/_test.txt - # pre-commit -tomli==1.2.3 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/_test.txt # black - # pep517 -typing-extensions==4.0.1 + # build +typing-extensions==4.5.0 # via # -c requirements/_base.txt # -c requirements/_test.txt # black -virtualenv==20.13.0 +virtualenv==20.20.0 # via pre-commit -watchdog==2.1.6 +watchdog==2.3.1 # via # -c requirements/_test.txt # -r requirements/_tools.in -wheel==0.37.1 +wheel==0.38.4 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/services/osparc-gateway-server/requirements/constraints.txt b/services/osparc-gateway-server/requirements/constraints.txt index 0f8cfeaf..e69de29b 100644 --- a/services/osparc-gateway-server/requirements/constraints.txt +++ b/services/osparc-gateway-server/requirements/constraints.txt @@ -1 +0,0 @@ -numpy<1.22.0 # constrain such that it is available on piwheels.org (see https://www.piwheels.org/project/numpy/) until newer version are available From fb2ba6d1bf9126652252ed31375404e83e41b113 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:41:04 +0100 Subject: [PATCH 02/11] linter --- .../src/osparc_gateway_server/remote_debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py b/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py index dbd42bd9..2c8f4c64 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py @@ -16,7 +16,7 @@ def setup_remote_debugging(logger: logging.Logger): # debugpy.wait_for_client() except ImportError as err: - raise Exception( + raise RuntimeError( "Cannot enable remote debugging. Please install debugpy first" ) from err From aa13db9e3a9150ed33903d817b62103e9b3a064f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:42:28 +0100 Subject: [PATCH 03/11] linter rules now same as osparc-simcore --- .pylintrc | 741 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 427 insertions(+), 314 deletions(-) diff --git a/.pylintrc b/.pylintrc index b2fad10e..f23d3956 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,146 +1,216 @@ -[MASTER] +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist=pydantic,orjson,ujson +# run arbitrary code. +extension-pkg-allow-list= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS,migration - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist=docker, + pydantic, + orjson, + ujson + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS, + migration + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths=^.*\\generated_models\\.*$|^.*/generated_models/.*$ + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. The default value ignores Emacs file +# locks ignore-patterns= +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=psycopg2.errors + # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Use multiple processes to speed up Pylint. -jobs=4 +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=0 -# List of plugins (as comma separated values of python modules names) to load, +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. -load-plugins= +load-plugins=pylint.extensions.no_self_use # Pickle collected data for later comparisons. persistent=yes -# Specify a configuration file. -#rcfile= +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.9 + +# Discover python modules and packages in the file system subtree. +recursive=no # When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages +# user-friendly hints instead of false-positive error messages. suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, bad-inline-option, locally-disabled, - locally-enabled, file-ignored, suppressed-message, useless-suppression, deprecated-pragma, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, + use-symbolic-message-instead, too-few-public-methods, trailing-whitespace, - too-few-public-methods, duplicate-code, too-many-locals, - C, + bad-file-encoding, + unnecessary-dunder-call, + unneeded-not, + consider-using-enumerate, + consider-iterating-dictionary, + consider-using-dict-items, + use-maxsplit-arg, + use-sequence-for-iteration, + consider-using-f-string, + use-implicit-booleaness-not-len, + use-implicit-booleaness-not-comparison, + wrong-spelling-in-comment, + wrong-spelling-in-docstring, + invalid-characters-in-docstring, + line-too-long, + too-many-lines, + missing-final-newline, + trailing-newlines, + multiple-statements, + superfluous-parens, + mixed-line-endings, + unexpected-line-ending-format, + non-ascii-name, + non-ascii-module-import, + unnecessary-lambda-assignment, + unnecessary-direct-lambda-call, + multiple-imports, + wrong-import-order, + ungrouped-imports, + wrong-import-position, + useless-import-alias, + import-outside-toplevel, + invalid-name, + disallowed-name, + typevar-name-incorrect-variance, + typevar-double-variance, + typevar-name-mismatch, + empty-docstring, + missing-module-docstring, + missing-class-docstring, + missing-function-docstring, + singleton-comparison, + unidiomatic-typecheck, + bad-classmethod-argument, + bad-mcs-method-argument, + bad-mcs-classmethod-argument, + single-string-used-for-slots, fixme, import-error, - unspecified-encoding - - -# DISABLED unspecified-encoding (W1514): -# explanation: Using the system default implicitly can create problems on other operating systems. See https://www.python.org/dev/peps/pep-0597/. -# disabled: because we use always the same OS in dev-machines or docker + unspecified-encoding, + use-dict-literal # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -149,29 +219,69 @@ disable=print-statement, enable=c-extension-no-member -[REPORTS] +[LOGGING] -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text -# Tells whether to display a full report or only the messages -reports=no +[TYPECHECK] -# Activate the evaluation score. -score=yes +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= [REFACTORING] @@ -186,71 +296,48 @@ max-nested-blocks=5 never-returning-functions=optparse.Values,sys.exit -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes +[SIMILARITIES] -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb +# Comments are removed from the similarity computation +ignore-comments=yes -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ +# Docstrings are removed from the similarity computation +ignore-docstrings=yes -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ +# Imports are removed from the similarity computation +ignore-imports=no -# Tells whether we should check for unused import in __init__ files. -init-import=no +# Signatures are removed from the similarity computation +ignore-signatures=yes -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins +# Minimum lines number of a similarity. +min-similarity-lines=4 [SPELLING] -# Limits count of emitted suggestions for spelling mistakes +# Limits count of emitted suggestions for spelling mistakes. max-spelling-suggestions=4 -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. spelling-dict= +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + # List of comma separated words that should not be checked. spelling-ignore-words= -# A path to a file that contains private dictionary; one word per line. +# A path to a file that contains the private dictionary; one word per line. spelling-private-dict-file= -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. spelling-store-unknown-words=no -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - [FORMAT] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. @@ -259,7 +346,7 @@ expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ -# Number of spaces of indent required inside a hanging or continued line. +# Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 @@ -269,16 +356,9 @@ indent-string=' ' # Maximum number of characters on a single line. max-line-length=150 -# Maximum number of lines in a module +# Maximum number of lines in a module. max-module-lines=1000 -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no @@ -288,79 +368,163 @@ single-line-class-stmt=no single-line-if-stmt=no -[TYPECHECK] +[STRING] -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes +[MISCELLANEOUS] -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# Regular expression of note tags to take in consideration. +notes-rgx= -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 +[VARIABLES] -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes +# List of names allowed to shadow builtins +allowed-redefined-builtins= -[MISCELLANEOUS] +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=10 + +# Maximum number of attributes for a class (see R0902). +max-attributes=9 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=13 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= [BASIC] -# Naming style matching correct argument names +# Naming style matching correct argument names. argument-naming-style=snake_case # Regular expression matching correct argument names. Overrides argument- -# naming-style +# naming-style. If left empty, argument names will be checked with the set +# naming style. #argument-rgx= -# Naming style matching correct attribute names +# Naming style matching correct attribute names. attr-naming-style=snake_case # Regular expression matching correct attribute names. Overrides attr-naming- -# style +# style. If left empty, attribute names will be checked with the set naming +# style. #attr-rgx= -# Bad variable names which should always be refused, separated by a comma +# Bad variable names which should always be refused, separated by a comma. bad-names=foo, bar, baz, @@ -368,38 +532,54 @@ bad-names=foo, tutu, tata -# Naming style matching correct class attribute names +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. class-attribute-naming-style=any # Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. #class-attribute-rgx= -# Naming style matching correct class names +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. class-naming-style=PascalCase -# Regular expression matching correct class names. Overrides class-naming-style +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. #class-rgx= -# Naming style matching correct constant names +# Naming style matching correct constant names. const-naming-style=UPPER_CASE # Regular expression matching correct constant names. Overrides const-naming- -# style +# style. If left empty, constant names will be checked with the set naming +# style. #const-rgx= # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 -# Naming style matching correct function names +# Naming style matching correct function names. function-naming-style=snake_case # Regular expression matching correct function names. Overrides function- -# naming-style +# naming-style. If left empty, function names will be checked with the set +# naming style. #function-rgx= -# Good variable names which should always be accepted, separated by a comma +# Good variable names which should always be accepted, separated by a comma. good-names=i, j, k, @@ -407,28 +587,33 @@ good-names=i, Run, _ -# Include a hint for the correct naming format with invalid-name +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. include-naming-hint=no -# Naming style matching correct inline iteration names +# Naming style matching correct inline iteration names. inlinevar-naming-style=any # Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. #inlinevar-rgx= -# Naming style matching correct method names +# Naming style matching correct method names. method-naming-style=snake_case # Regular expression matching correct method names. Overrides method-naming- -# style +# style. If left empty, method names will be checked with the set naming style. #method-rgx= -# Naming style matching correct module names +# Naming style matching correct module names. module-naming-style=snake_case # Regular expression matching correct module names. Overrides module-naming- -# style +# style. If left empty, module names will be checked with the set naming style. #module-rgx= # Colon-delimited sets of names that determine each other's naming style when @@ -441,25 +626,27 @@ no-docstring-rgx=^_ # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. property-classes=abc.abstractproperty -# Naming style matching correct variable names +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. variable-naming-style=snake_case # Regular expression matching correct variable names. Overrides variable- -# naming-style +# naming-style. If left empty, variable names will be checked with the set +# naming style. #variable-rgx= -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - [CLASSES] +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, @@ -478,77 +665,3 @@ valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[DESIGN] - -# Maximum number of arguments for function / method -# NOTE: due to dependency injection, this number can be quite high -max-args=10 - -# Maximum number of attributes for a class (see R0902). -max-attributes=9 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=13 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception From 206742cfd13a96aaab528c637c39b8fde930d3a7 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:51:00 +0100 Subject: [PATCH 04/11] use correct dependencies --- services/osparc-gateway-server/requirements/_base.in | 3 +-- services/osparc-gateway-server/requirements/_base.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/osparc-gateway-server/requirements/_base.in b/services/osparc-gateway-server/requirements/_base.in index 39ed6955..405ede45 100644 --- a/services/osparc-gateway-server/requirements/_base.in +++ b/services/osparc-gateway-server/requirements/_base.in @@ -2,6 +2,5 @@ aiodocker bokeh -dask-gateway-server +dask-gateway-server[local] pydantic[email,dotenv] -sqlalchemy # required by dask-gateway-server diff --git a/services/osparc-gateway-server/requirements/_base.txt b/services/osparc-gateway-server/requirements/_base.txt index 4a1472b3..223e6c8e 100644 --- a/services/osparc-gateway-server/requirements/_base.txt +++ b/services/osparc-gateway-server/requirements/_base.txt @@ -78,7 +78,7 @@ pyyaml==6.0 six==1.16.0 # via python-dateutil sqlalchemy==2.0.4 - # via -r requirements/_base.in + # via dask-gateway-server tornado==6.2 # via bokeh traitlets==5.9.0 From 625ef229470ff4e5e3f8fb1af3e7e8aba7170e59 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:59:43 +0100 Subject: [PATCH 05/11] constrain sqlalchemy <2.0 since dask-gateway-server is not compatible --- services/osparc-gateway-server/requirements/_base.txt | 7 ++++--- .../osparc-gateway-server/requirements/constraints.txt | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/services/osparc-gateway-server/requirements/_base.txt b/services/osparc-gateway-server/requirements/_base.txt index 223e6c8e..d8c96435 100644 --- a/services/osparc-gateway-server/requirements/_base.txt +++ b/services/osparc-gateway-server/requirements/_base.txt @@ -77,8 +77,10 @@ pyyaml==6.0 # via bokeh six==1.16.0 # via python-dateutil -sqlalchemy==2.0.4 - # via dask-gateway-server +sqlalchemy==1.4.46 + # via + # -c requirements/./constraints.txt + # dask-gateway-server tornado==6.2 # via bokeh traitlets==5.9.0 @@ -87,7 +89,6 @@ typing-extensions==4.5.0 # via # aiodocker # pydantic - # sqlalchemy xyzservices==2023.2.0 # via bokeh yarl==1.8.2 diff --git a/services/osparc-gateway-server/requirements/constraints.txt b/services/osparc-gateway-server/requirements/constraints.txt index e69de29b..f8443fa1 100644 --- a/services/osparc-gateway-server/requirements/constraints.txt +++ b/services/osparc-gateway-server/requirements/constraints.txt @@ -0,0 +1 @@ +sqlalchemy<2.0.0 # dask-gateway-server does not appear to be compatible yet From 90c535ed1381d947f255071126dd81b6eafa15e9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 2 Mar 2023 10:35:13 +0100 Subject: [PATCH 06/11] ensure constraints are passed. --- .../requirements/_base.in | 1 + .../requirements/_base.txt | 20 +++++++++++++++---- .../requirements/_test.in | 3 ++- .../requirements/_test.txt | 3 ++- .../requirements/_tools.in | 1 + .../requirements/constraints.txt | 1 + 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/services/osparc-gateway-server/requirements/_base.in b/services/osparc-gateway-server/requirements/_base.in index 405ede45..61d1db45 100644 --- a/services/osparc-gateway-server/requirements/_base.in +++ b/services/osparc-gateway-server/requirements/_base.in @@ -1,4 +1,5 @@ --constraint ./constraints.txt +--constraint ../../../requirements/constraints.txt aiodocker bokeh diff --git a/services/osparc-gateway-server/requirements/_base.txt b/services/osparc-gateway-server/requirements/_base.txt index d8c96435..f0da1e86 100644 --- a/services/osparc-gateway-server/requirements/_base.txt +++ b/services/osparc-gateway-server/requirements/_base.txt @@ -8,6 +8,7 @@ aiodocker==0.21.0 # via -r requirements/_base.in aiohttp==3.8.4 # via + # -c requirements/../../../requirements/constraints.txt # aiodocker # dask-gateway-server aiosignal==1.3.1 @@ -27,7 +28,9 @@ colorlog==6.7.0 contourpy==1.0.7 # via bokeh cryptography==39.0.1 - # via dask-gateway-server + # via + # -c requirements/../../../requirements/constraints.txt + # dask-gateway-server dask-gateway-server==2023.1.1 # via -r requirements/_base.in dnspython==2.3.0 @@ -45,7 +48,9 @@ idna==3.4 # email-validator # yarl jinja2==3.1.2 - # via bokeh + # via + # -c requirements/../../../requirements/constraints.txt + # bokeh markupsafe==2.1.2 # via jinja2 multidict==6.0.4 @@ -63,10 +68,14 @@ pandas==1.5.3 # via bokeh pillow==9.4.0 # via bokeh +psycopg2-binary==2.9.5 + # via sqlalchemy pycparser==2.21 # via cffi pydantic==1.10.5 - # via -r requirements/_base.in + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_base.in python-dateutil==2.8.2 # via pandas python-dotenv==1.0.0 @@ -74,11 +83,14 @@ python-dotenv==1.0.0 pytz==2022.7.1 # via pandas pyyaml==6.0 - # via bokeh + # via + # -c requirements/../../../requirements/constraints.txt + # bokeh six==1.16.0 # via python-dateutil sqlalchemy==1.4.46 # via + # -c requirements/../../../requirements/constraints.txt # -c requirements/./constraints.txt # dask-gateway-server tornado==6.2 diff --git a/services/osparc-gateway-server/requirements/_test.in b/services/osparc-gateway-server/requirements/_test.in index 3a026367..54dba972 100644 --- a/services/osparc-gateway-server/requirements/_test.in +++ b/services/osparc-gateway-server/requirements/_test.in @@ -1,6 +1,7 @@ # -# Specifies dependencies required to run 'services/director-v2/test' both for unit and integration tests!! +# Specifies dependencies required to run 'services/osparc-gateway-server/test' both for unit and integration tests!! # +--constraint ./constraints.txt --constraint ../../../requirements/constraints.txt # Adds base AS CONSTRAINT specs, not requirement. diff --git a/services/osparc-gateway-server/requirements/_test.txt b/services/osparc-gateway-server/requirements/_test.txt index 9fdf35b2..9fbe5a67 100644 --- a/services/osparc-gateway-server/requirements/_test.txt +++ b/services/osparc-gateway-server/requirements/_test.txt @@ -37,8 +37,9 @@ click==8.1.3 # dask # dask-gateway # distributed -cloudpickle==2.2.1 +cloudpickle==2.0.0 # via + # -c requirements/./constraints.txt # dask # distributed codecov==2.1.12 diff --git a/services/osparc-gateway-server/requirements/_tools.in b/services/osparc-gateway-server/requirements/_tools.in index e095b0fe..6c8b1253 100644 --- a/services/osparc-gateway-server/requirements/_tools.in +++ b/services/osparc-gateway-server/requirements/_tools.in @@ -1,3 +1,4 @@ +--constraint ./constraints.txt --constraint ../../../requirements/constraints.txt --constraint _base.txt diff --git a/services/osparc-gateway-server/requirements/constraints.txt b/services/osparc-gateway-server/requirements/constraints.txt index f8443fa1..383229cd 100644 --- a/services/osparc-gateway-server/requirements/constraints.txt +++ b/services/osparc-gateway-server/requirements/constraints.txt @@ -1 +1,2 @@ sqlalchemy<2.0.0 # dask-gateway-server does not appear to be compatible yet +cloudpickle<2.1.0 # https://github.com/PrefectHQ/prefect/issues/5818 From 2533ca16d5aa683ced74a6da399e2b867621fe38 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 2 Mar 2023 10:37:33 +0100 Subject: [PATCH 07/11] pre-commit all files --- .../osparc-gateway-server/scripts/client.py | 2 +- services/osparc-gateway-server/setup.py | 4 +-- .../osparc_gateway_server/backend/models.py | 8 ++--- .../osparc_gateway_server/backend/osparc.py | 15 ++++---- .../osparc_gateway_server/backend/utils.py | 34 +++++++++---------- .../osparc-gateway-server/tests/conftest.py | 6 ++-- .../tests/integration/conftest.py | 8 ++--- .../tests/integration/test_clusters.py | 12 +++---- .../tests/integration/test_dask_sidecar.py | 26 +++++++------- .../tests/unit/test_utils.py | 22 ++++++------ tests/test_deploy.py | 2 +- 11 files changed, 67 insertions(+), 72 deletions(-) mode change 100644 => 100755 services/osparc-gateway-server/setup.py diff --git a/services/osparc-gateway-server/scripts/client.py b/services/osparc-gateway-server/scripts/client.py index 15404bb1..ffe0d9ec 100644 --- a/services/osparc-gateway-server/scripts/client.py +++ b/services/osparc-gateway-server/scripts/client.py @@ -21,7 +21,7 @@ def square(x): - return x ** 2 + return x**2 def neg(x): diff --git a/services/osparc-gateway-server/setup.py b/services/osparc-gateway-server/setup.py old mode 100644 new mode 100755 index 5b9322e0..650d2221 --- a/services/osparc-gateway-server/setup.py +++ b/services/osparc-gateway-server/setup.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import re import sys from pathlib import Path -from typing import Set -def read_reqs(reqs_path: Path) -> Set[str]: +def read_reqs(reqs_path: Path) -> set[str]: return { r for r in re.findall( diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py index e2a394fd..7fc8bd4e 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/models.py @@ -1,12 +1,10 @@ from ipaddress import IPv4Address from typing import ( Any, - Dict, Generic, ItemsView, Iterator, KeysView, - List, Optional, TypeVar, Union, @@ -24,7 +22,7 @@ class NodeResources(BaseModel): memory: ByteSize cpus: PositiveFloat - others: Dict[ResourceName, ResourceType] = Field(default_factory=dict) + others: dict[ResourceName, ResourceType] = Field(default_factory=dict) class NodeInformation(BaseModel): @@ -38,7 +36,7 @@ class NodeInformation(BaseModel): class DictModel(GenericModel, Generic[DictKey, DictValue]): - __root__: Dict[DictKey, DictValue] + __root__: dict[DictKey, DictValue] def __getitem__(self, k: DictKey) -> DictValue: return self.__root__.__getitem__(k) @@ -70,7 +68,7 @@ def __len__(self) -> int: class ClusterInformation(DictModel[Hostname, NodeInformation]): @staticmethod - def from_docker(nodes_list: List[Dict[str, Any]]) -> "ClusterInformation": + def from_docker(nodes_list: list[dict[str, Any]]) -> "ClusterInformation": return ClusterInformation.parse_obj( { node["Description"]["Hostname"]: { diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py index 125bfd6d..cc66eae1 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py @@ -1,11 +1,10 @@ import asyncio from importlib.metadata import version -from typing import Any, AsyncGenerator, Dict, List, Union +from typing import Any, AsyncGenerator, Union import osparc_gateway_server from aiodocker import Docker from aiodocker.exceptions import DockerContainerError -from dask_gateway_server._version import __version__ from dask_gateway_server.backends.base import PublicException from dask_gateway_server.backends.db_base import ( Cluster, @@ -42,7 +41,7 @@ / _ \/ ___\/ __\/ _ \/ __\/ _\ / _ \/ _ \/ ___\/ |/ / / __// _ \/__ __\/ __// \ /|/ _ \\ \// / ___\/ __// __\/ \ |\/ __// __\ | / \|| \| \/|| / \|| \/|| / _____ | | \|| / \|| \| /_____ | | _| / \| / \ | \ | | ||| / \| \ /_____ | \| \ | \/|| | //| \ | \/| | \_/|\___ || __/| |-||| /| \_\____\| |_/|| |-||\___ || \\____\| |_//| |-|| | | | /_ | |/\||| |-|| / / \____\\___ || /_ | /| \// | /_ | / -\____/\____/\_/ \_/ \|\_/\_\\____/ \____/\_/ \|\____/\_|\_\ \____\\_/ \| \_/ \____\\_/ \|\_/ \|/_/ \____/\____\\_/\_\\__/ \____\\_/\_\ {0} +\____/\____/\_/ \_/ \|\_/\_\\____/ \____/\_/ \|\____/\_|\_\ \____\\_/ \| \_/ \____\\_/ \|\_/ \|/_/ \____/\____\\_/\_\\__/ \____\\_/\_\ {} """.format( @@ -59,7 +58,7 @@ class OsparcBackend(DBBackendBase): settings: AppSettings docker_client: Docker - cluster_secrets: List[DockerSecret] = [] + cluster_secrets: list[DockerSecret] = [] async def do_setup(self) -> None: self.settings = AppSettings() # type: ignore @@ -83,7 +82,7 @@ async def do_cleanup(self) -> None: async def do_start_cluster( self, cluster: Cluster - ) -> AsyncGenerator[Dict[str, Any], None]: + ) -> AsyncGenerator[dict[str, Any], None]: self.log.debug(f"starting {cluster=}") self.cluster_secrets.extend( await create_docker_secrets_from_tls_certs_for_cluster( @@ -124,7 +123,7 @@ async def do_stop_cluster(self, cluster: Cluster): await delete_secrets(self.docker_client, cluster) self.log.debug("<--%s stopped", f"{cluster=}") - async def do_check_clusters(self, clusters: List[Cluster]): + async def do_check_clusters(self, clusters: list[Cluster]): self.log.debug("--> checking statuses of : %s", f"{clusters=}") ok = await asyncio.gather( *[self._check_service_status(c) for c in clusters], return_exceptions=True @@ -134,7 +133,7 @@ async def do_check_clusters(self, clusters: List[Cluster]): async def do_start_worker( self, worker: Worker - ) -> AsyncGenerator[Dict[str, Any], None]: + ) -> AsyncGenerator[dict[str, Any], None]: self.log.debug("--> starting %s", f"{worker=}") node_hostname = None try: @@ -214,7 +213,7 @@ async def _check_service_status( ) return False - async def do_check_workers(self, workers: List[Worker]) -> List[bool]: + async def do_check_workers(self, workers: list[Worker]) -> list[bool]: self.log.debug("--> checking statuses: %s", f"{workers=}") ok = await asyncio.gather( *[self._check_service_status(w) for w in workers], return_exceptions=True diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py index 52cab4de..794288f0 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py @@ -4,7 +4,7 @@ from collections import deque from copy import deepcopy from pathlib import Path -from typing import Any, AsyncGenerator, Dict, Final, List, NamedTuple, Optional +from typing import Any, AsyncGenerator, Final, NamedTuple, Optional import aiodocker from aiodocker import Docker @@ -64,15 +64,15 @@ async def get_network_id( def create_service_config( settings: AppSettings, - service_env: Dict[str, Any], + service_env: dict[str, Any], service_name: str, network_id: str, - service_secrets: List[DockerSecret], - cmd: Optional[List[str]], - labels: Dict[str, str], - placement: Optional[Dict[str, Any]], + service_secrets: list[DockerSecret], + cmd: Optional[list[str]], + labels: dict[str, str], + placement: Optional[dict[str, Any]], **service_kwargs, -) -> Dict[str, Any]: +) -> dict[str, Any]: env = deepcopy(service_env) env.pop("PATH", None) # create the secrets array containing the TLS cert/key pair @@ -188,14 +188,14 @@ async def start_service( settings: AppSettings, logger: logging.Logger, service_name: str, - base_env: Dict[str, str], - cluster_secrets: List[DockerSecret], - cmd: Optional[List[str]], - labels: Dict[str, str], + base_env: dict[str, str], + cluster_secrets: list[DockerSecret], + cmd: Optional[list[str]], + labels: dict[str, str], gateway_api_url: str, - placement: Optional[Dict[str, Any]] = None, + placement: Optional[dict[str, Any]] = None, **service_kwargs, -) -> AsyncGenerator[Dict[str, Any], None]: +) -> AsyncGenerator[dict[str, Any], None]: service_parameters = {} try: assert settings.COMPUTATIONAL_SIDECAR_LOG_LEVEL # nosec @@ -287,7 +287,7 @@ async def stop_service( async def create_docker_secrets_from_tls_certs_for_cluster( docker_client: Docker, backend: DBBackendBase, cluster: Cluster -) -> List[DockerSecret]: +) -> list[DockerSecret]: tls_cert_path, tls_key_path = backend.get_tls_paths(cluster) return [ await create_or_update_secret( @@ -311,7 +311,7 @@ async def create_docker_secrets_from_tls_certs_for_cluster( def get_osparc_scheduler_cmd_modifications( scheduler_service_name: str, -) -> Dict[str, str]: +) -> dict[str, str]: # NOTE: the healthcheck of itisfoundation/dask-sidecar expects the dashboard # to be on port 8787 # (see https://github.com/ITISFoundation/osparc-simcore/blob/f3d98dccdae665d23701b0db4ee917364a0fbd99/services/dask-sidecar/Dockerfile) @@ -323,8 +323,8 @@ def get_osparc_scheduler_cmd_modifications( def modify_cmd_argument( - cmd: List[str], argument_name: str, argument_value: str -) -> List[str]: + cmd: list[str], argument_name: str, argument_value: str +) -> list[str]: modified_cmd = deepcopy(cmd) try: dashboard_address_arg_index = modified_cmd.index(argument_name) diff --git a/services/osparc-gateway-server/tests/conftest.py b/services/osparc-gateway-server/tests/conftest.py index 76ba1adb..4dd85203 100644 --- a/services/osparc-gateway-server/tests/conftest.py +++ b/services/osparc-gateway-server/tests/conftest.py @@ -4,7 +4,7 @@ import asyncio import sys from pathlib import Path -from typing import Any, AsyncIterator, Awaitable, Callable, Dict +from typing import Any, AsyncIterator, Awaitable, Callable import aiodocker import osparc_gateway_server @@ -73,10 +73,10 @@ async def async_docker_client() -> AsyncIterator[aiodocker.Docker]: @pytest.fixture async def docker_network( async_docker_client: aiodocker.Docker, faker: Faker -) -> AsyncIterator[Callable[..., Awaitable[Dict[str, Any]]]]: +) -> AsyncIterator[Callable[..., Awaitable[dict[str, Any]]]]: networks = [] - async def _network_creator(**network_config_kwargs) -> Dict[str, Any]: + async def _network_creator(**network_config_kwargs) -> dict[str, Any]: network = await async_docker_client.networks.create( config={"Name": faker.uuid4(), "Driver": "overlay"} | network_config_kwargs ) diff --git a/services/osparc-gateway-server/tests/integration/conftest.py b/services/osparc-gateway-server/tests/integration/conftest.py index ab081d1c..62e0cf3d 100644 --- a/services/osparc-gateway-server/tests/integration/conftest.py +++ b/services/osparc-gateway-server/tests/integration/conftest.py @@ -4,7 +4,7 @@ import asyncio import json -from typing import Any, AsyncIterator, Awaitable, Callable, Dict, Union +from typing import Any, AsyncIterator, Awaitable, Callable, Union import aiodocker import dask_gateway @@ -27,10 +27,10 @@ @pytest.fixture async def docker_volume( async_docker_client: aiodocker.Docker, -) -> AsyncIterator[Callable[[str], Awaitable[Dict[str, Any]]]]: +) -> AsyncIterator[Callable[[str], Awaitable[dict[str, Any]]]]: volumes = [] - async def _volume_creator(name: str) -> Dict[str, Any]: + async def _volume_creator(name: str) -> dict[str, Any]: volume = await async_docker_client.volumes.create(config={"Name": name}) assert volume print(f"--> created {volume=}") @@ -55,7 +55,7 @@ def gateway_password(faker: Faker) -> str: return faker.password() -def _convert_to_dict(c: Union[traitlets.config.Config, Dict]) -> Dict[str, Any]: +def _convert_to_dict(c: Union[traitlets.config.Config, dict]) -> dict[str, Any]: converted_dict = {} for x, y in c.items(): if isinstance(y, (dict, traitlets.config.Config)): diff --git a/services/osparc-gateway-server/tests/integration/test_clusters.py b/services/osparc-gateway-server/tests/integration/test_clusters.py index 277fa719..c2849249 100644 --- a/services/osparc-gateway-server/tests/integration/test_clusters.py +++ b/services/osparc-gateway-server/tests/integration/test_clusters.py @@ -2,7 +2,7 @@ # pylint: disable=redefined-outer-name import asyncio -from typing import Any, Awaitable, Callable, Dict, List +from typing import Any, Awaitable, Callable import pytest from _dask_helpers import DaskGatewayServer @@ -44,8 +44,8 @@ def minimal_config( @pytest.fixture async def gateway_worker_network( local_dask_gateway_server: DaskGatewayServer, - docker_network: Callable[..., Awaitable[Dict[str, Any]]], -) -> Dict[str, Any]: + docker_network: Callable[..., Awaitable[dict[str, Any]]], +) -> dict[str, Any]: network = await docker_network( **{ "Name": local_dask_gateway_server.server.backend.settings.GATEWAY_WORKERS_NETWORK @@ -77,7 +77,7 @@ async def assert_services_stability(docker_client: Docker, service_name: str): f"--> {service_name} is running, now checking if it is stable during {_SECONDS_STABLE}s..." ) - async def _check_stability(service: Dict[str, Any]): + async def _check_stability(service: dict[str, Any]): inspected_service = await docker_client.services.inspect(service["ID"]) # we ensure the service remains stable for _SECONDS_STABLE seconds (e.g. only one task runs) @@ -103,7 +103,7 @@ async def _wait_for_cluster_services_and_secrets( num_services: int, num_secrets: int, timeout_s: int = 10, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: async for attempt in AsyncRetrying( reraise=True, wait=wait_fixed(1), stop=stop_after_delay(timeout_s) ): @@ -198,7 +198,7 @@ async def test_clusters_start_stop( async def test_cluster_scale( minimal_config: None, gateway_client: Gateway, - gateway_worker_network: Dict[str, Any], + gateway_worker_network: dict[str, Any], async_docker_client: Docker, ): diff --git a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py index 067ff574..57533134 100644 --- a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py +++ b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py @@ -2,7 +2,7 @@ # pylint: disable=redefined-outer-name import asyncio -from typing import Any, AsyncIterator, Awaitable, Callable, Dict, List +from typing import Any, AsyncIterator, Awaitable, Callable import aiodocker import pytest @@ -16,8 +16,8 @@ @pytest.fixture async def sidecar_computational_shared_volume( faker: Faker, - docker_volume: Callable[[str], Awaitable[Dict[str, Any]]], -) -> Dict[str, Any]: + docker_volume: Callable[[str], Awaitable[dict[str, Any]]], +) -> dict[str, Any]: volume = await docker_volume(faker.pystr()) return volume @@ -30,8 +30,8 @@ def computational_sidecar_mounted_folder() -> str: @pytest.fixture def sidecar_envs( computational_sidecar_mounted_folder: str, - sidecar_computational_shared_volume: Dict[str, Any], -) -> Dict[str, str]: + sidecar_computational_shared_volume: dict[str, Any], +) -> dict[str, str]: envs = { "SIDECAR_COMP_SERVICES_SHARED_FOLDER": f"{computational_sidecar_mounted_folder}", "SIDECAR_COMP_SERVICES_SHARED_VOLUME_NAME": f"{sidecar_computational_shared_volume['Name']}", @@ -41,9 +41,9 @@ def sidecar_envs( @pytest.fixture def sidecar_mounts( - sidecar_computational_shared_volume: Dict[str, Any], + sidecar_computational_shared_volume: dict[str, Any], computational_sidecar_mounted_folder: str, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: mounts = [ # docker socket needed to use the docker api { "Source": "/var/run/docker.sock", @@ -65,10 +65,10 @@ def sidecar_mounts( @pytest.fixture async def create_docker_service( async_docker_client: aiodocker.Docker, -) -> AsyncIterator[Callable[..., Awaitable[Dict[str, Any]]]]: +) -> AsyncIterator[Callable[..., Awaitable[dict[str, Any]]]]: services = [] - async def service_creator(**service_kwargs) -> Dict[str, Any]: + async def service_creator(**service_kwargs) -> dict[str, Any]: service = await async_docker_client.services.create(**service_kwargs) assert service assert "ID" in service @@ -107,12 +107,12 @@ async def _wait_for_service_to_be_ready( ) async def test_computational_sidecar_properly_start_stop( docker_swarm: None, - sidecar_computational_shared_volume: Dict[str, Any], + sidecar_computational_shared_volume: dict[str, Any], async_docker_client: aiodocker.Docker, image_name: str, - sidecar_envs: Dict[str, str], - sidecar_mounts: List[Dict[str, Any]], - create_docker_service: Callable[..., Awaitable[Dict[str, Any]]], + sidecar_envs: dict[str, str], + sidecar_mounts: list[dict[str, Any]], + create_docker_service: Callable[..., Awaitable[dict[str, Any]]], ): scheduler_service = await create_docker_service( task_template={ diff --git a/services/osparc-gateway-server/tests/unit/test_utils.py b/services/osparc-gateway-server/tests/unit/test_utils.py index 53c45044..3283bffb 100644 --- a/services/osparc-gateway-server/tests/unit/test_utils.py +++ b/services/osparc-gateway-server/tests/unit/test_utils.py @@ -5,14 +5,14 @@ import socket from copy import deepcopy from pathlib import Path -from typing import Any, AsyncIterator, Awaitable, Callable, Dict, List, Tuple +from typing import Any, AsyncIterator, Awaitable, Callable from unittest import mock import aiodocker import pytest from dask_gateway_server.backends.db_base import Cluster, JobStatus from faker import Faker -from osparc_gateway_server.backend.errors import NoHostFoundError, NoServiceTasksError +from osparc_gateway_server.backend.errors import NoHostFoundError from osparc_gateway_server.backend.settings import AppSettings from osparc_gateway_server.backend.utils import ( _DASK_KEY_CERT_PATH_IN_SIDECAR, @@ -44,10 +44,10 @@ def minimal_config(monkeypatch): @pytest.fixture() async def create_docker_service( docker_swarm, async_docker_client: aiodocker.Docker, faker: Faker -) -> AsyncIterator[Callable[[Dict[str, str]], Awaitable[Dict[str, Any]]]]: +) -> AsyncIterator[Callable[[dict[str, str]], Awaitable[dict[str, Any]]]]: created_services = [] - async def _creator(labels: Dict[str, str]) -> Dict[str, Any]: + async def _creator(labels: dict[str, str]) -> dict[str, Any]: service = await async_docker_client.services.create( task_template={ "ContainerSpec": {"Image": "busybox", "Command": ["sleep", "10000"]} @@ -72,9 +72,9 @@ async def _creator(labels: Dict[str, str]) -> Dict[str, Any]: @pytest.fixture def create_running_service( async_docker_client: aiodocker.Docker, - create_docker_service: Callable[[Dict[str, str]], Awaitable[Dict[str, Any]]], -) -> Callable[[Dict[str, str]], Awaitable[Dict[str, Any]]]: - async def _creator(labels: Dict[str, str]) -> Dict[str, Any]: + create_docker_service: Callable[[dict[str, str]], Awaitable[dict[str, Any]]], +) -> Callable[[dict[str, str]], Awaitable[dict[str, Any]]]: + async def _creator(labels: dict[str, str]) -> dict[str, Any]: service = await create_docker_service(labels) async for attempt in AsyncRetrying( reraise=True, wait=wait_fixed(1), stop=stop_after_delay(60) @@ -103,7 +103,7 @@ async def test_is_task_running( docker_swarm, minimal_config, async_docker_client: aiodocker.Docker, - create_running_service: Callable[[Dict[str, str]], Awaitable[Dict[str, Any]]], + create_running_service: Callable[[dict[str, str]], Awaitable[dict[str, Any]]], mocked_logger: mock.MagicMock, ): service = await create_running_service({}) @@ -125,7 +125,7 @@ async def test_is_task_running( async def test_get_network_id( docker_swarm, async_docker_client: aiodocker.Docker, - docker_network: Callable[..., Awaitable[Dict[str, Any]]], + docker_network: Callable[..., Awaitable[dict[str, Any]]], mocked_logger: mock.MagicMock, ): # wrong name shall raise @@ -347,7 +347,7 @@ async def test_get_cluster_information( @pytest.fixture() -def fake_docker_nodes(faker: Faker) -> List[Dict[str, Any]]: +def fake_docker_nodes(faker: Faker) -> list[dict[str, Any]]: return [ {"ID": f"{faker.uuid4()}", "Description": {"Hostname": f"{faker.hostname()}"}}, {"ID": f"{faker.uuid4()}", "Description": {"Hostname": f"{faker.hostname()}"}}, @@ -365,7 +365,7 @@ def mocked_docker_nodes(mocker: MockerFixture, fake_docker_nodes): async def test_get_empty_node_hostname_rotates_host_names( - fake_docker_nodes: List[Dict[str, Any]], + fake_docker_nodes: list[dict[str, Any]], mocked_docker_nodes, docker_swarm, async_docker_client: aiodocker.Docker, diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 5a57e9a1..da7fe1b9 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -185,7 +185,7 @@ async def test_deployment( # send some work def square(x): - return x ** 2 + return x**2 def neg(x): return -x From 7caa8091d633feb80fde940fdc5bc65be19790b0 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 2 Mar 2023 10:50:58 +0100 Subject: [PATCH 08/11] mypy --- scripts/mypy.bash | 69 +++++++++++-------- scripts/mypy/Dockerfile | 10 +++ scripts/mypy/requirements.txt | 8 +++ .../osparc_gateway_server/backend/models.py | 6 +- .../osparc_gateway_server/backend/osparc.py | 10 +-- .../osparc_gateway_server/backend/settings.py | 9 +-- .../osparc_gateway_server/backend/utils.py | 6 +- .../src/osparc_gateway_server/remote_debug.py | 2 +- 8 files changed, 75 insertions(+), 45 deletions(-) create mode 100644 scripts/mypy/Dockerfile create mode 100644 scripts/mypy/requirements.txt diff --git a/scripts/mypy.bash b/scripts/mypy.bash index 795c4819..54f47e1c 100755 --- a/scripts/mypy.bash +++ b/scripts/mypy.bash @@ -1,41 +1,50 @@ #!/bin/bash # http://redsymbol.net/articles/unofficial-bash-strict-mode/ + set -o errexit set -o nounset set -o pipefail IFS=$'\n\t' +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +IMAGE_NAME="local/scripts-$(basename "$0"):latest" +WORKDIR="$(pwd)" -image_name="$(basename "$0"):latest" - - -docker buildx build --tag "$image_name" &>/dev/null -< KeysView[DictKey]: def values(self) -> ValuesView[DictValue]: return self.__root__.values() - def pop(self, key: DictKey): + def pop(self, key: DictKey) -> DictValue: return self.__root__.pop(key) def __iter__(self) -> Iterator[DictKey]: return self.__root__.__iter__() - def get(self, key: DictKey, default: Optional[DictValue] = None): + def get( + self, key: DictKey, default: Optional[DictValue] = None + ) -> Optional[DictValue]: return self.__root__.get(key, default) def __len__(self) -> int: diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py index cc66eae1..e03cc2e2 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/osparc.py @@ -61,7 +61,7 @@ class OsparcBackend(DBBackendBase): cluster_secrets: list[DockerSecret] = [] async def do_setup(self) -> None: - self.settings = AppSettings() # type: ignore + self.settings = AppSettings() self.log.info( "osparc-gateway-server application settings:\n%s", self.settings.json(indent=2), @@ -116,16 +116,16 @@ async def do_start_cluster( ): yield dask_scheduler_start_result - async def do_stop_cluster(self, cluster: Cluster): + async def do_stop_cluster(self, cluster: Cluster) -> None: self.log.debug("--> stopping %s", f"{cluster=}") dask_scheduler_service_id = cluster.state.get("service_id") await stop_service(self.docker_client, dask_scheduler_service_id, self.log) await delete_secrets(self.docker_client, cluster) self.log.debug("<--%s stopped", f"{cluster=}") - async def do_check_clusters(self, clusters: list[Cluster]): + async def do_check_clusters(self, clusters: list[Cluster]) -> list[bool]: self.log.debug("--> checking statuses of : %s", f"{clusters=}") - ok = await asyncio.gather( + ok: list[bool] = await asyncio.gather( *[self._check_service_status(c) for c in clusters], return_exceptions=True ) self.log.debug("<-- clusters status returned: %s", f"{ok=}") @@ -221,7 +221,7 @@ async def do_check_workers(self, workers: list[Worker]) -> list[bool]: self.log.debug("<-- worker status returned: %s", f"{ok=}") return ok - async def on_cluster_heartbeat(self, cluster_name, msg): + async def on_cluster_heartbeat(self, cluster_name, msg) -> None: # pylint: disable=no-else-continue, unused-variable, too-many-branches # pylint: disable=too-many-statements diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py index 0dc9c729..87eddca6 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/settings.py @@ -18,7 +18,6 @@ class BootModeEnum(str, Enum): class AppSettings(BaseSettings): - COMPUTATIONAL_SIDECAR_IMAGE: str = Field( ..., description="The computational sidecar image in use" ) @@ -49,13 +48,15 @@ class AppSettings(BaseSettings): SC_BOOT_MODE: Optional[BootModeEnum] GATEWAY_SERVER_ONE_WORKER_PER_NODE: bool = Field( - True, + default=True, description="Only one dask-worker is allowed per node (default). If disabled, then scaling must be done manually.", ) GATEWAY_CLUSTER_START_TIMEOUT: float = Field( - 120.0, description="Allowed timeout to define a starting cluster as failed" + default=120.0, + description="Allowed timeout to define a starting cluster as failed", ) GATEWAY_WORKER_START_TIMEOUT: float = Field( - 120.0, description="Allowed timeout to define a starting worker as failed" + default=120.0, + description="Allowed timeout to define a starting worker as failed", ) diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py b/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py index 794288f0..535a02d1 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/backend/utils.py @@ -35,7 +35,7 @@ async def is_service_task_running( "%s current service task states are %s", service_name, f"{tasks_current_state=}" ) num_running = sum(current == "running" for current in tasks_current_state) - return num_running == 1 + return bool(num_running == 1) async def get_network_id( @@ -176,7 +176,7 @@ async def create_or_update_secret( ) -async def delete_secrets(docker_client: aiodocker.Docker, cluster: Cluster): +async def delete_secrets(docker_client: aiodocker.Docker, cluster: Cluster) -> None: secrets = await docker_client.secrets.list( filters={"label": f"cluster_id={cluster.id}"} ) @@ -376,5 +376,5 @@ async def get_next_empty_node_hostname( for node in cluster_nodes: if node["ID"] in used_docker_node_ids: continue - return node["Description"]["Hostname"] + return f"{node['Description']['Hostname']}" raise NoHostFoundError("Could not find any empty host") diff --git a/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py b/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py index 2c8f4c64..53f6dcf8 100644 --- a/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py +++ b/services/osparc-gateway-server/src/osparc_gateway_server/remote_debug.py @@ -5,7 +5,7 @@ import logging -def setup_remote_debugging(logger: logging.Logger): +def setup_remote_debugging(logger: logging.Logger) -> None: try: logger.debug("Attaching debugpy ...") From 40892e650aa7f6a902e38a4db239357e2bafffe9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 2 Mar 2023 11:45:25 +0100 Subject: [PATCH 09/11] fix multiplatform build --- .github/workflows/gateway.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml index 68a1e638..3bf28a66 100644 --- a/.github/workflows/gateway.yml +++ b/.github/workflows/gateway.yml @@ -189,18 +189,18 @@ jobs: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: build and deploy image for linux/amd64,linux/arm/v7,linux/arm64 + - name: build and deploy image for linux/amd64 if: github.event_name == 'push' env: - DOCKER_TARGET_PLATFORMS: linux/amd64,linux/arm/v7,linux/arm64 + DOCKER_TARGET_PLATFORMS: linux/amd64 DOCKER_REGISTRY: ${{ secrets.DOCKER_IMAGE_PATH }} run: | make build push=1 && \ make info-images - - name: build image for linux/amd64,linux/arm/v7,linux/arm64 + - name: build image for linux/amd64 if: github.event_name == 'pull_request' env: - DOCKER_TARGET_PLATFORMS: linux/amd64,linux/arm/v7,linux/arm64 + DOCKER_TARGET_PLATFORMS: linux/amd64 run: | make build && \ make info-images From 3c8bcf1b4d4f29badbe9e4a1003575d92d19811f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 2 Mar 2023 11:55:43 +0100 Subject: [PATCH 10/11] upgrade system tests requirements --- requirements/constraints.txt | 2 + .../requirements/_test.txt | 1 + tests/requirements/_test.txt | 144 +++++++++--------- tests/requirements/_tools.txt | 58 ++++--- 4 files changed, 106 insertions(+), 99 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 4657a480..0f867084 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -26,6 +26,8 @@ urllib3>=1.26.5 # https://github.com/advisories/GH # # Bugs # +sqlalchemy<2.0.0 # dask-gateway-server does not appear to be compatible yet +cloudpickle<2.1.0 # https://github.com/PrefectHQ/prefect/issues/5818 # # Compatibility/coordination diff --git a/services/osparc-gateway-server/requirements/_test.txt b/services/osparc-gateway-server/requirements/_test.txt index 9fbe5a67..06117600 100644 --- a/services/osparc-gateway-server/requirements/_test.txt +++ b/services/osparc-gateway-server/requirements/_test.txt @@ -39,6 +39,7 @@ click==8.1.3 # distributed cloudpickle==2.0.0 # via + # -c requirements/../../../requirements/constraints.txt # -c requirements/./constraints.txt # dask # distributed diff --git a/tests/requirements/_test.txt b/tests/requirements/_test.txt index 40e6bef2..f3be44a6 100644 --- a/tests/requirements/_test.txt +++ b/tests/requirements/_test.txt @@ -1,113 +1,120 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in # -aiohttp==3.8.1 +aiohttp==3.8.4 # via # -c requirements/../../requirements/constraints.txt # dask-gateway # pytest-aiohttp -aiosignal==1.2.0 +aiosignal==1.3.1 # via aiohttp -astroid==2.9.3 +astroid==2.14.2 # via pylint async-timeout==4.0.2 # via aiohttp -attrs==21.4.0 +attrs==22.2.0 # via # aiohttp # pytest -certifi==2021.10.8 +certifi==2022.12.7 # via requests -charset-normalizer==2.0.10 +charset-normalizer==3.0.1 # via # aiohttp # requests -click==8.0.3 - # via distributed +click==8.1.3 + # via + # dask + # dask-gateway + # distributed cloudpickle==2.0.0 # via + # -c requirements/../../requirements/constraints.txt # dask # distributed codecov==2.1.12 # via -r requirements/_test.in -coverage==6.2 +coverage==7.2.1 # via # codecov # pytest-cov -dask==2022.1.0 +dask==2023.3.0 # via # dask-gateway # distributed -dask-gateway==0.9.0 +dask-gateway==2023.1.1 # via -r requirements/_test.in -distributed==2022.1.0 +dill==0.3.6 + # via pylint +distributed==2023.3.0 # via dask-gateway -docker==5.0.3 +docker==6.0.1 # via -r requirements/_test.in -faker==11.3.0 +exceptiongroup==1.1.0 + # via pytest +faker==17.5.0 # via -r requirements/_test.in -frozenlist==1.3.0 +frozenlist==1.3.3 # via # aiohttp # aiosignal -fsspec==2022.1.0 +fsspec==2023.1.0 # via dask heapdict==1.0.1 # via zict -icdiff==2.0.4 +icdiff==2.0.6 # via pytest-icdiff -idna==3.3 +idna==3.4 # via # requests # yarl -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest -isort==5.10.1 +isort==5.12.0 # via pylint -jinja2==3.0.3 +jinja2==3.1.2 # via # -c requirements/../../requirements/constraints.txt # distributed -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 # via astroid -locket==0.2.1 - # via partd -markupsafe==2.0.1 +locket==1.0.0 + # via + # distributed + # partd +markupsafe==2.1.2 # via jinja2 -mccabe==0.6.1 +mccabe==0.7.0 # via pylint -msgpack==1.0.3 +msgpack==1.0.4 # via distributed -multidict==5.2.0 +multidict==6.0.4 # via # aiohttp # yarl -packaging==21.3 +packaging==23.0 # via # dask # distributed + # docker # pytest # pytest-sugar -partd==1.2.0 +partd==1.3.0 # via dask -platformdirs==2.4.1 +platformdirs==3.0.0 # via pylint pluggy==1.0.0 # via pytest pprintpp==0.4.0 # via pytest-icdiff -psutil==5.9.0 +psutil==5.9.4 # via distributed -py==1.11.0 - # via pytest -pylint==2.12.2 +pylint==2.16.2 # via -r requirements/_test.in -pyparsing==3.0.7 - # via packaging -pytest==6.2.5 +pytest==7.2.1 # via # -r requirements/_test.in # pytest-aiohttp @@ -117,21 +124,21 @@ pytest==6.2.5 # pytest-instafail # pytest-mock # pytest-sugar -pytest-aiohttp==1.0.3 +pytest-aiohttp==1.0.4 # via -r requirements/_test.in -pytest-asyncio==0.17.2 +pytest-asyncio==0.20.3 # via pytest-aiohttp -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/_test.in -pytest-icdiff==0.5 +pytest-icdiff==0.6 # via -r requirements/_test.in pytest-instafail==0.4.2 # via -r requirements/_test.in -pytest-mock==3.6.1 +pytest-mock==3.10.0 # via -r requirements/_test.in -pytest-runner==5.3.1 +pytest-runner==6.0.0 # via -r requirements/_test.in -pytest-sugar==0.9.4 +pytest-sugar==0.9.6 # via -r requirements/_test.in python-dateutil==2.8.2 # via faker @@ -139,8 +146,9 @@ pyyaml==6.0 # via # -c requirements/../../requirements/constraints.txt # dask + # dask-gateway # distributed -requests==2.27.1 +requests==2.28.2 # via # codecov # docker @@ -150,43 +158,41 @@ sortedcontainers==2.4.0 # via distributed tblib==1.7.0 # via distributed -tenacity==8.0.1 +tenacity==8.2.2 # via -r requirements/_test.in -termcolor==1.1.0 +termcolor==2.2.0 # via pytest-sugar -text-unidecode==1.3 - # via faker -toml==0.10.2 +tomli==2.0.1 # via + # coverage # pylint # pytest -tomli==1.2.3 - # via - # -c requirements/../../requirements/constraints.txt - # coverage -toolz==0.11.2 +tomlkit==0.11.6 + # via pylint +toolz==0.12.0 # via # dask # distributed # partd -tornado==6.1 - # via distributed -typing-extensions==4.0.1 +tornado==6.2 + # via + # dask-gateway + # distributed +typing-extensions==4.5.0 # via # astroid # pylint -urllib3==1.26.8 +urllib3==1.26.14 # via # -c requirements/../../requirements/constraints.txt + # distributed + # docker # requests -websocket-client==1.2.3 +websocket-client==1.5.1 # via docker -wrapt==1.13.3 +wrapt==1.15.0 # via astroid -yarl==1.7.2 +yarl==1.8.2 # via aiohttp -zict==2.0.0 +zict==2.2.0 # via distributed - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/tests/requirements/_tools.txt b/tests/requirements/_tools.txt index 709f16da..c837d93b 100644 --- a/tests/requirements/_tools.txt +++ b/tests/requirements/_tools.txt @@ -1,73 +1,71 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in # -black==21.12b0 +black==23.1.0 # via -r requirements/../../requirements/devenv.txt +build==0.10.0 + # via pip-tools bump2version==1.0.1 # via -r requirements/../../requirements/devenv.txt cfgv==3.3.1 # via pre-commit -click==8.0.3 +click==8.1.3 # via # -c requirements/_test.txt # black # pip-tools -distlib==0.3.4 +distlib==0.3.6 # via virtualenv -filelock==3.4.2 +filelock==3.9.0 # via virtualenv -identify==2.4.4 +identify==2.5.18 # via pre-commit -isort==5.10.1 +isort==5.12.0 # via # -c requirements/_test.txt # -r requirements/../../requirements/devenv.txt -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via black -nodeenv==1.6.0 +nodeenv==1.7.0 # via pre-commit -pathspec==0.9.0 +packaging==23.0 + # via + # -c requirements/_test.txt + # black + # build +pathspec==0.11.0 # via black -pep517==0.12.0 - # via pip-tools -pip-tools==6.4.0 +pip-tools==6.12.3 # via -r requirements/../../requirements/devenv.txt -platformdirs==2.4.1 +platformdirs==3.0.0 # via # -c requirements/_test.txt # black # virtualenv -pre-commit==2.17.0 +pre-commit==3.1.1 # via -r requirements/../../requirements/devenv.txt +pyproject-hooks==1.0.0 + # via build pyyaml==6.0 # via # -c requirements/../../requirements/constraints.txt # -c requirements/_test.txt # pre-commit -six==1.16.0 - # via - # -c requirements/_test.txt - # virtualenv -toml==0.10.2 - # via - # -c requirements/_test.txt - # pre-commit -tomli==1.2.3 +tomli==2.0.1 # via - # -c requirements/../../requirements/constraints.txt # -c requirements/_test.txt # black - # pep517 -typing-extensions==4.0.1 + # build +typing-extensions==4.5.0 # via # -c requirements/_test.txt # black -virtualenv==20.13.0 +virtualenv==20.20.0 # via pre-commit -wheel==0.37.1 +wheel==0.38.4 # via pip-tools # The following packages are considered to be unsafe in a requirements file: From c155c3a3729b5a9710511c76dae471e20bbbf65f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 2 Mar 2023 11:58:28 +0100 Subject: [PATCH 11/11] moved constraints down to the root --- services/osparc-gateway-server/requirements/_base.txt | 1 - services/osparc-gateway-server/requirements/_test.txt | 1 - services/osparc-gateway-server/requirements/constraints.txt | 2 -- 3 files changed, 4 deletions(-) diff --git a/services/osparc-gateway-server/requirements/_base.txt b/services/osparc-gateway-server/requirements/_base.txt index f0da1e86..30b2b347 100644 --- a/services/osparc-gateway-server/requirements/_base.txt +++ b/services/osparc-gateway-server/requirements/_base.txt @@ -91,7 +91,6 @@ six==1.16.0 sqlalchemy==1.4.46 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/./constraints.txt # dask-gateway-server tornado==6.2 # via bokeh diff --git a/services/osparc-gateway-server/requirements/_test.txt b/services/osparc-gateway-server/requirements/_test.txt index 06117600..26d44f9a 100644 --- a/services/osparc-gateway-server/requirements/_test.txt +++ b/services/osparc-gateway-server/requirements/_test.txt @@ -40,7 +40,6 @@ click==8.1.3 cloudpickle==2.0.0 # via # -c requirements/../../../requirements/constraints.txt - # -c requirements/./constraints.txt # dask # distributed codecov==2.1.12 diff --git a/services/osparc-gateway-server/requirements/constraints.txt b/services/osparc-gateway-server/requirements/constraints.txt index 383229cd..e69de29b 100644 --- a/services/osparc-gateway-server/requirements/constraints.txt +++ b/services/osparc-gateway-server/requirements/constraints.txt @@ -1,2 +0,0 @@ -sqlalchemy<2.0.0 # dask-gateway-server does not appear to be compatible yet -cloudpickle<2.1.0 # https://github.com/PrefectHQ/prefect/issues/5818