Merge branch 'master' into 2024/fix/tracing

.env-devel

@@ -70,11 +70,17 @@ DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
 
+DIRECTOR_DEFAULT_MAX_MEMORY=2147483648
+DIRECTOR_DEFAULT_MAX_NANO_CPUS=1000000000
 DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DIRECTOR_HOST=director
-DIRECTOR_PORT=8080
-DIRECTOR_REGISTRY_CACHING_TTL=900
+DIRECTOR_LOGLEVEL=INFO
+DIRECTOR_MONITORING_ENABLED=True
+DIRECTOR_PORT=8000
+DIRECTOR_PUBLISHED_HOST_NAME="127.0.0.1:9081"
+DIRECTOR_REGISTRY_CACHING_TTL=00:15:00
 DIRECTOR_REGISTRY_CACHING=True
+DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null
 DIRECTOR_TRACING={}
 
 EFS_USER_ID=8006
@@ -186,6 +192,7 @@ REDIS_SECURE=false
 REDIS_USER=null
 
 REGISTRY_AUTH=True
+REGISTRY_PATH=""
 REGISTRY_PW=adminadminadmin
 REGISTRY_SSL=True
 REGISTRY_URL=registry.osparc-master.speag.com

.github/workflows/ci-testing-deploy.yml

@@ -37,8 +37,8 @@ on:
           - all
 
 env:
-  DEFAULT_MAX_NANO_CPUS: 10000000
-  DEFAULT_MAX_MEMORY: 268435456
+  # NOTE: 'COLUMNS' is a shell env var that represents the width (number of columns)
+  #       of the terminal or command-line interface in characters.
   COLUMNS: 120
 
 concurrency:
@@ -189,6 +189,8 @@ jobs:
               - 'packages/**'
               - 'services/director/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             director-v2:
               - 'packages/**'
               - 'services/director-v2/**'
@@ -912,11 +914,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # KEEP 3.6 Development of this service is frozen
-        # KEEP ubuntu 20.04, else no python 3.6
-        python: [3.6]
-        os: [ubuntu-20.04]
-        docker_buildx: [v0.10.4]
+        python: ["3.11"]
+        os: [ubuntu-22.04]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -929,12 +928,27 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
+      - name: install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.x"
+          enable-cache: false
+          cache-dependency-glob: "**/director/requirements/ci.txt"
       - name: show system version
         run: ./ci/helpers/show_system_versions.bash
       - name: install
         run: ./ci/github/unit-testing/director.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/director.bash typecheck
       - name: test
+        if: ${{ !cancelled() }}
         run: ./ci/github/unit-testing/director.bash test
+      - name: upload failed tests logs
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./services/director/test_failures
       - uses: codecov/[email protected]
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -2132,6 +2146,11 @@ jobs:
         python: ["3.11"]
         os: [ubuntu-22.04]
       fail-fast: false
+    env:
+      # NOTE: DIRECTOR_DEFAULT_MAX_* used for integration-tests that include `director` service
+      DIRECTOR_DEFAULT_MAX_MEMORY: 268435456
+      DIRECTOR_DEFAULT_MAX_NANO_CPUS: 10000000
+      DIRECTOR_TRACING: null
     steps:
       - uses: actions/checkout@v4
       - name: setup docker buildx

ci/github/unit-testing/director.bash

@@ -6,44 +6,29 @@ set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-  # Replaces 'bash ci/helpers/ensure_python_pip.bash'
-
-  echo "INFO:" "$(python --version)" "@" "$(command -v python)"
-
-  # installs pip if not in place
-  python -m ensurepip
-
-  echo "INFO:" "$(pip --version)" "@" "$(command -v pip)"
-  # NOTE: pip<22.0 for python 3.6
-  pip3 install --upgrade \
-    pip~=21.0 \
-    wheel \
-    setuptools
-  python3 -m venv .venv
+  make devenv
   # shellcheck source=/dev/null
   source .venv/bin/activate
   pushd services/director
-  pip3 install -r requirements/ci.txt
+  make install-ci
   popd
+  uv pip list
 }
 
 test() {
   # shellcheck source=/dev/null
   source .venv/bin/activate
   pushd services/director
-  pytest \
-    --color=yes \
-    --cov-append \
-    --cov-config=.coveragerc \
-    --cov-report=term-missing \
-    --cov-report=xml \
-    --cov=simcore_service_director \
-    --durations=10 \
-    --keep-docker-up \
-    --log-date-format="%Y-%m-%d %H:%M:%S" \
-    --log-format="%(asctime)s %(levelname)s %(message)s" \
-    --verbose \
-    tests/
+  make test-ci-unit
+  popd
+}
+
+typecheck() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  uv pip install mypy
+  pushd services/director
+  make mypy
   popd
 }
 

packages/aws-library/setup.py

@@ -33,7 +33,7 @@ def read_reqs(reqs_path: Path) -> set[str]:
     "version": Path(CURRENT_DIR / "VERSION").read_text().strip(),
     "author": "Sylvain Anderegg (sanderegg)",
     "description": "Core service library for AWS APIs",
-    "python_requires": "~=3.10",
+    "python_requires": "~=3.11",
     "classifiers": [
         "Development Status :: 2 - Pre-Alpha",
         "Intended Audience :: Developers",

packages/dask-task-models-library/setup.py

@@ -36,7 +36,7 @@ def read_reqs(reqs_path: Path) -> set[str]:
     "version": Path(CURRENT_DIR / "VERSION").read_text().strip(),
     "author": "Sylvain Anderegg (sanderegg)",
     "description": "Core service library for simcore pydantic dask task models",
-    "python_requires": "~=3.10",
+    "python_requires": "~=3.11",
     "classifiers": [
         "Development Status :: 2 - Pre-Alpha",
         "Intended Audience :: Developers",

packages/models-library/setup.py

@@ -34,7 +34,7 @@ def read_reqs(reqs_path: Path) -> set[str]:
     "version": Path(CURRENT_DIR / "VERSION").read_text().strip(),
     "author": "Sylvain Anderegg (sanderegg)",
     "description": "Core service library for simcore pydantic models",
-    "python_requires": "~=3.10",
+    "python_requires": "~=3.11",
     "classifiers": [
         "Development Status :: 2 - Pre-Alpha",
         "Intended Audience :: Developers",

packages/models-library/src/models_library/api_schemas_director/services.py

@@ -0,0 +1,5 @@
+from ..services_metadata_published import ServiceMetaDataPublished
+
+
+class ServiceDataGet(ServiceMetaDataPublished):
+    ...

packages/notifications-library/requirements/_test.in

@@ -8,6 +8,7 @@
 #
 --constraint _base.txt
 
+aiodocker
 coverage
 docker
 faker

packages/notifications-library/requirements/_test.txt

@@ -1,3 +1,17 @@
+aiodocker==0.23.0
+    # via -r requirements/_test.in
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.10.10
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   aiodocker
+aiosignal==1.3.1
+    # via aiohttp
+attrs==24.2.0
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
 certifi==2024.8.30
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -12,6 +26,10 @@ docker==7.1.0
     # via -r requirements/_test.in
 faker==29.0.0
     # via -r requirements/_test.in
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
 greenlet==3.1.1
     # via
     #   -c requirements/_base.txt
@@ -22,8 +40,14 @@ idna==3.10
     # via
     #   -c requirements/_base.txt
     #   requests
+    #   yarl
 iniconfig==2.0.0
     # via pytest
+multidict==6.1.0
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
+    #   yarl
 mypy==1.12.0
     # via sqlalchemy
 mypy-extensions==1.0.0
@@ -101,3 +125,7 @@ urllib3==2.2.3
     #   -c requirements/../../../requirements/constraints.txt
     #   docker
     #   requests
+yarl==1.12.1
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp

packages/notifications-library/setup.py

@@ -38,7 +38,7 @@ def read_reqs(reqs_path: Path) -> set[str]:
     "version": Path(CURRENT_DIR / "VERSION").read_text().strip(),
     "author": "Pedro Crespo-Valero (pcrespov)",
     "description": "simcore library for user notifications e.g. emails, sms, etc",
-    "python_requires": "~=3.10",
+    "python_requires": "~=3.11",
     "classifiers": [
         "Development Status :: 2 - Pre-Alpha",
         "Intended Audience :: Developers",

packages/pytest-simcore/setup.py

@@ -17,7 +17,7 @@
     "description": "pytest plugin with fixtures and test helpers for osparc-simcore repo modules",
     "py_modules": ["pytest_simcore"],
     # WARNING: this is used in frozen services as well !!!!
-    "python_requires": ">=3.6",
+    "python_requires": "~=3.11",
     "install_requires": ["pytest>=3.5.0"],
     "extras_require": {
         "all": [

packages/pytest-simcore/src/pytest_simcore/docker.py

@@ -1,3 +1,9 @@
+# pylint:disable=protected-access
+# pylint:disable=redefined-outer-name
+# pylint:disable=too-many-arguments
+# pylint:disable=unused-argument
+# pylint:disable=unused-variable
+
 import asyncio
 import contextlib
 from collections.abc import AsyncIterator, Callable
@@ -7,6 +13,12 @@
 import pytest
 
 
+@pytest.fixture
+async def async_docker_client() -> AsyncIterator[aiodocker.Docker]:
+    async with aiodocker.Docker() as docker_client:
+        yield docker_client
+
+
 @contextlib.asynccontextmanager
 async def _pause_container(
     async_docker_client: aiodocker.Docker, container_name: str