diff --git a/.github/dependabot.yml b/.github/dependabot.yml index df1b58c0af3..b70d1f17b03 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,9 +29,6 @@ updates: - dependency-name: httpx versions: - 0.17.0 - - dependency-name: minio - versions: - - 7.0.0 - package-ecosystem: pip directory: "/packages/service-library" schedule: diff --git a/Makefile b/Makefile index 6b9371dbc80..f8854e7a206 100644 --- a/Makefile +++ b/Makefile @@ -269,23 +269,15 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l ) .stack-ops.yml: .env $(docker-compose-configs) - # Compiling config file for filestash - $(eval TMP_PATH_TO_FILESTASH_CONFIG=$(shell set -o allexport && \ - source $(CURDIR)/.env && \ - set +o allexport && \ - python3 scripts/filestash/create_config.py)) # Creating config for ops stack to $@ - # -> filestash config at $(TMP_PATH_TO_FILESTASH_CONFIG) ifdef ops_ci @$(shell \ - export TMP_PATH_TO_FILESTASH_CONFIG="${TMP_PATH_TO_FILESTASH_CONFIG}" && \ scripts/docker/docker-compose-config.bash -e .env \ services/docker-compose-ops-ci.yml \ > $@ \ ) else @$(shell \ - export TMP_PATH_TO_FILESTASH_CONFIG="${TMP_PATH_TO_FILESTASH_CONFIG}" && \ scripts/docker/docker-compose-config.bash -e .env \ services/docker-compose-ops.yml \ > $@ \ @@ -299,7 +291,6 @@ endif .deploy-ops: .stack-ops.yml # Deploy stack 'ops' ifndef ops_disabled - # -> filestash config at $(TMP_PATH_TO_FILESTASH_CONFIG) docker stack deploy --with-registry-auth -c $< ops else @echo "Explicitly disabled with ops_disabled flag in CLI" @@ -328,7 +319,6 @@ printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVIT printf "$$rows" "Payments" "http://$(get_my_ip).nip.io:8011/dev/doc" $${PAYMENTS_USERNAME} $${PAYMENTS_PASSWORD};\ printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\ printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\ -printf "$$rows" "Storage S3 Filestash" "http://$(get_my_ip).nip.io:9002" 12345678 12345678;\ printf "$$rows" "Storage S3 Minio" "http://$(get_my_ip).nip.io:9001" 12345678 12345678;\ printf "\n%s\n" "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar";\ diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py b/packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py new file mode 100644 index 00000000000..7281b27d153 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py @@ -0,0 +1,102 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + + +import contextlib +import typing + +import aioboto3 +import pytest +from aiobotocore.session import ClientCreatorContext +from botocore.client import Config +from settings_library.s3 import S3Settings +from types_aiobotocore_s3 import S3Client + + +@pytest.fixture +def s3_settings() -> S3Settings: + return S3Settings.create_from_envs() + + +@pytest.fixture +async def s3_client(s3_settings: S3Settings) -> typing.AsyncIterator[S3Client]: + session = aioboto3.Session() + exit_stack = contextlib.AsyncExitStack() + session_client = session.client( + "s3", + endpoint_url=s3_settings.S3_ENDPOINT, + aws_access_key_id=s3_settings.S3_ACCESS_KEY, + aws_secret_access_key=s3_settings.S3_SECRET_KEY, + aws_session_token=s3_settings.S3_ACCESS_TOKEN, + region_name=s3_settings.S3_REGION, + config=Config(signature_version="s3v4"), + ) + assert isinstance(session_client, ClientCreatorContext) + client = typing.cast(S3Client, await exit_stack.enter_async_context(session_client)) + + yield client + + await exit_stack.aclose() + + +async def _empty_bucket(s3_client: S3Client, bucket_name: str) -> None: + # List object versions + response = await s3_client.list_object_versions(Bucket=bucket_name) + + # Delete all object versions + for version in response.get("Versions", []): + assert "Key" in version + assert "VersionId" in version + await s3_client.delete_object( + Bucket=bucket_name, Key=version["Key"], VersionId=version["VersionId"] + ) + + # Delete all delete markers + for marker in response.get("DeleteMarkers", []): + assert "Key" in marker + assert "VersionId" in marker + await s3_client.delete_object( + Bucket=bucket_name, Key=marker["Key"], VersionId=marker["VersionId"] + ) + + # Delete remaining objects in the bucket + response = await s3_client.list_objects(Bucket=bucket_name) + for obj in response.get("Contents", []): + assert "Key" in obj + await s3_client.delete_object(Bucket=bucket_name, Key=obj["Key"]) + + +@pytest.fixture +async def s3_bucket( + s3_settings: S3Settings, s3_client: S3Client +) -> typing.AsyncIterator[str]: + bucket_name = s3_settings.S3_BUCKET_NAME + + response = await s3_client.list_buckets() + bucket_exists = bucket_name in [ + bucket_struct.get("Name") for bucket_struct in response["Buckets"] + ] + if bucket_exists: + await _empty_bucket(s3_client, bucket_name) + + if not bucket_exists: + await s3_client.create_bucket(Bucket=bucket_name) + response = await s3_client.list_buckets() + assert response["Buckets"] + assert bucket_name in [ + bucket_struct.get("Name") for bucket_struct in response["Buckets"] + ], f"failed creating {bucket_name}" + + yield bucket_name + + await _empty_bucket(s3_client, bucket_name) + + +@pytest.fixture +async def with_bucket_versioning_enabled(s3_client: S3Client, s3_bucket: str) -> str: + await s3_client.put_bucket_versioning( + Bucket=s3_bucket, + VersioningConfiguration={"MFADelete": "Disabled", "Status": "Enabled"}, + ) + return s3_bucket diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_server.py b/packages/pytest-simcore/src/pytest_simcore/aws_server.py index 48baff0e4c2..a1d9b545082 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aws_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/aws_server.py @@ -7,8 +7,10 @@ import pytest import requests from aiohttp.test_utils import unused_port +from faker import Faker from moto.server import ThreadedMotoServer from settings_library.ec2 import EC2Settings +from settings_library.s3 import S3Settings from .helpers.utils_envs import EnvVarsDict, setenvs_from_dict from .helpers.utils_host import get_localhost_ip @@ -71,16 +73,22 @@ def mocked_ec2_server_envs( @pytest.fixture -async def mocked_s3_server_envs( - mocked_aws_server: ThreadedMotoServer, - reset_aws_server_state: None, +def mocked_s3_server_settings( + mocked_aws_server: ThreadedMotoServer, reset_aws_server_state: None, faker: Faker +) -> S3Settings: + return S3Settings( + S3_ACCESS_KEY="xxx", + S3_ENDPOINT=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 + S3_SECRET_KEY="xxx", # noqa: S106 + S3_BUCKET_NAME=f"pytest{faker.pystr().lower()}", + S3_SECURE=False, + ) + + +@pytest.fixture +def mocked_s3_server_envs( + mocked_s3_server_settings: S3Settings, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - changed_envs = { - "S3_SECURE": "false", - "S3_ENDPOINT": f"{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - "S3_ACCESS_KEY": "xxx", - "S3_SECRET_KEY": "xxx", - "S3_BUCKET_NAME": "pytestbucket", - } + changed_envs: EnvVarsDict = mocked_s3_server_settings.dict(exclude_unset=True) return setenvs_from_dict(monkeypatch, changed_envs) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 424f6bb7da2..34372ab6a1a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -13,7 +13,6 @@ import os import re import shutil -import subprocess import sys from collections.abc import Iterator from copy import deepcopy @@ -82,6 +81,9 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict: if "DOCKER_IMAGE_TAG" not in os.environ: env_devel["DOCKER_IMAGE_TAG"] = "production" + # ensure we do not use the bucket of simcore or so + env_devel["S3_BUCKET_NAME"] = "pytestbucket" + return {key: value for key, value in env_devel.items() if value is not None} @@ -166,39 +168,12 @@ def simcore_docker_compose( return compose_specs -@pytest.fixture(scope="module") -def inject_filestash_config_path_env( - osparc_simcore_scripts_dir: Path, - env_file_for_testing: Path, -) -> EnvVarsDict: - create_filestash_config_py = ( - osparc_simcore_scripts_dir / "filestash" / "create_config.py" - ) - - # ensures .env at git_root_dir, which will be used as current directory - assert env_file_for_testing.exists() - env_values = dotenv_values(env_file_for_testing) - - process = subprocess.run( - ["python3", f"{create_filestash_config_py}"], - shell=False, - check=True, - stdout=subprocess.PIPE, - env=env_values, - ) - filestash_config_json_path = Path(process.stdout.decode("utf-8").strip()) - assert filestash_config_json_path.exists() - - return {"TMP_PATH_TO_FILESTASH_CONFIG": f"{filestash_config_json_path}"} - - @pytest.fixture(scope="module") def ops_docker_compose( osparc_simcore_root_dir: Path, osparc_simcore_scripts_dir: Path, env_file_for_testing: Path, temp_folder: Path, - inject_filestash_config_path_env: dict[str, str], ) -> dict[str, Any]: """Filters only services in docker-compose-ops.yml and returns yaml data @@ -219,7 +194,6 @@ def ops_docker_compose( docker_compose_paths=docker_compose_path, env_file_path=env_file_for_testing, destination_path=temp_folder / "ops_docker_compose.yml", - additional_envs=inject_filestash_config_path_env, ) # NOTE: do not add indent. Copy&Paste log into editor instead print( @@ -277,7 +251,7 @@ def ops_docker_compose_file( docker_compose_path = Path(temp_folder / "ops_docker_compose.filtered.yml") # these services are useless when running in the CI - ops_view_only_services = ["adminer", "redis-commander", "portainer", "filestash"] + ops_view_only_services = ["adminer", "redis-commander", "portainer"] if "CI" in os.environ: print( f"WARNING: Services such as {ops_view_only_services!r} are removed from the stack when running in the CI" diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py index 9b187de2348..630dbcdbc92 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py @@ -29,12 +29,17 @@ def setenvs_from_dict( ) -> EnvVarsDict: for key, value in envs.items(): assert isinstance(key, str) - assert value is not None # None keys cannot be is defined w/o value + assert ( + value is not None + ), f"{key=},{value=}" # None keys cannot be is defined w/o value + converted_value = value + if isinstance(value, bool): + converted_value = f"{'true' if value else 'false'}" assert isinstance( - value, str - ), "client MUST explicitly stringify values since some cannot be done automatically e.g. json-like values" + converted_value, str + ), f"client MUST explicitly stringify values since some cannot be done automatically e.g. json-like values. problematic {key=},{value=}" - monkeypatch.setenv(key, value) + monkeypatch.setenv(key, converted_value) return deepcopy(envs) @@ -72,7 +77,7 @@ def delenvs_from_envfile( monkeypatch: pytest.MonkeyPatch, content_or_path: str | Path, raising: bool, - **dotenv_kwags + **dotenv_kwags, ) -> EnvVarsDict: """Batch monkeypatch.delenv(...) on all env vars in an envfile""" envs = load_dotenv(content_or_path, **dotenv_kwags) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index b060ed16d06..b83e9a56c1a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -1,117 +1,36 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - -import logging -from collections.abc import Iterator -from typing import Any - -import pytest -from minio import Minio -from minio.datatypes import Object -from minio.deleteobjects import DeleteError, DeleteObject -from minio.versioningconfig import ENABLED, VersioningConfig -from pydantic import parse_obj_as -from tenacity import Retrying -from tenacity.before_sleep import before_sleep_log -from tenacity.stop import stop_after_attempt -from tenacity.wait import wait_fixed - -from .helpers.utils_docker import get_service_published_port -from .helpers.utils_host import get_localhost_ip - -log = logging.getLogger(__name__) - - -def _ensure_remove_bucket(client: Minio, bucket_name: str): - if client.bucket_exists(bucket_name): - # remove content - objs: Iterator[Object] = client.list_objects( - bucket_name, prefix=None, recursive=True, include_version=True - ) - - # FIXME: minio 7.1.0 does NOT remove all objects!? Added in requirements/constraints.txt - to_delete = [DeleteObject(o.object_name, o.version_id) for o in objs] - errors: Iterator[DeleteError] = client.remove_objects(bucket_name, to_delete) - - list_of_errors = list(errors) - assert not any(list_of_errors), list(list_of_errors) - - # remove bucket - client.remove_bucket(bucket_name) - - assert not client.bucket_exists(bucket_name) - - -@pytest.fixture(scope="module") -def minio_config( - docker_stack: dict, - testing_environ_vars: dict, - monkeypatch_module: pytest.MonkeyPatch, -) -> dict[str, Any]: - assert "pytest-ops_minio" in docker_stack["services"] - - config = { - "client": { - "endpoint": f"{get_localhost_ip()}:{get_service_published_port('minio')}", - "access_key": testing_environ_vars["S3_ACCESS_KEY"], - "secret_key": testing_environ_vars["S3_SECRET_KEY"], - "secure": parse_obj_as(bool, testing_environ_vars["S3_SECURE"]), - }, - "bucket_name": testing_environ_vars["S3_BUCKET_NAME"], - } - - # nodeports takes its configuration from env variables - for key, value in config["client"].items(): - monkeypatch_module.setenv(f"S3_{key.upper()}", str(value)) - - monkeypatch_module.setenv("S3_SECURE", testing_environ_vars["S3_SECURE"]) - monkeypatch_module.setenv("S3_BUCKET_NAME", config["bucket_name"]) - - return config - - -@pytest.fixture(scope="module") -def minio_service(minio_config: dict[str, str]) -> Iterator[Minio]: - client = Minio(**minio_config["client"]) - - for attempt in Retrying( - wait=wait_fixed(5), - stop=stop_after_attempt(60), - before_sleep=before_sleep_log(log, logging.WARNING), - reraise=True, - ): - with attempt: - if not client.bucket_exists("pytest"): - client.make_bucket("pytest") - client.remove_bucket("pytest") - - bucket_name = minio_config["bucket_name"] - - # cleans up in case a failing tests left this bucket - _ensure_remove_bucket(client, bucket_name) - - client.make_bucket(bucket_name) - assert client.bucket_exists(bucket_name) - - yield client - - # cleanup upon tear-down - _ensure_remove_bucket(client, bucket_name) - - -@pytest.fixture -def bucket(minio_config: dict[str, str], minio_service: Minio) -> Iterator[str]: - bucket_name = minio_config["bucket_name"] - - _ensure_remove_bucket(minio_service, bucket_name) - minio_service.make_bucket(bucket_name) - - yield bucket_name - - _ensure_remove_bucket(minio_service, bucket_name) - - -@pytest.fixture -def with_bucket_versioning(minio_service: Minio, bucket: str) -> None: - minio_service.set_bucket_versioning(bucket, VersioningConfig(ENABLED)) +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + +import pytest +from faker import Faker +from pydantic import parse_obj_as +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_docker import get_service_published_port +from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip +from settings_library.s3 import S3Settings + + +@pytest.fixture +def minio_s3_settings( + docker_stack: dict, testing_environ_vars: dict, faker: Faker +) -> S3Settings: + assert "pytest-ops_minio" in docker_stack["services"] + + return S3Settings( + S3_ACCESS_KEY=testing_environ_vars["S3_ACCESS_KEY"], + S3_SECRET_KEY=testing_environ_vars["S3_SECRET_KEY"], + S3_ENDPOINT=f"{get_localhost_ip()}:{get_service_published_port('minio')}", + S3_SECURE=parse_obj_as(bool, testing_environ_vars["S3_SECURE"]), + S3_BUCKET_NAME=testing_environ_vars["S3_BUCKET_NAME"], + ) + + +@pytest.fixture +def minio_s3_settings_envs( + minio_s3_settings: S3Settings, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + changed_envs: EnvVarsDict = minio_s3_settings.dict(exclude_unset=True) + return setenvs_from_dict(monkeypatch, changed_envs) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index 9b0032c51fd..143d85631a0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -8,11 +8,10 @@ import aiohttp import pytest import tenacity -from minio import Minio from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from pydantic import parse_obj_as -from servicelib.minio_utils import MinioRetryPolicyUponInitialization +from servicelib.minio_utils import ServiceRetryPolicyUponInitialization from yarl import URL from .helpers.utils_docker import get_service_published_port @@ -40,16 +39,14 @@ def storage_endpoint(docker_stack: dict, testing_environ_vars: dict) -> Iterable @pytest.fixture() -async def storage_service( - minio_service: Minio, storage_endpoint: URL, docker_stack: dict -) -> URL: +async def storage_service(storage_endpoint: URL, docker_stack: dict) -> URL: await wait_till_storage_responsive(storage_endpoint) return storage_endpoint # TODO: this can be used by ANY of the simcore services! -@tenacity.retry(**MinioRetryPolicyUponInitialization().kwargs) +@tenacity.retry(**ServiceRetryPolicyUponInitialization().kwargs) async def wait_till_storage_responsive(storage_endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get(storage_endpoint.with_path("/v0/")) as resp: diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_service.py index 636de9e6852..fb1699fabd3 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_service.py @@ -6,7 +6,7 @@ import aiohttp import pytest import tenacity -from servicelib.minio_utils import MinioRetryPolicyUponInitialization +from servicelib.minio_utils import ServiceRetryPolicyUponInitialization from yarl import URL from .helpers.utils_docker import get_service_published_port @@ -29,7 +29,7 @@ async def webserver_service(webserver_endpoint: URL, docker_stack: dict) -> URL: # TODO: this can be used by ANY of the simcore services! -@tenacity.retry(**MinioRetryPolicyUponInitialization().kwargs) +@tenacity.retry(**ServiceRetryPolicyUponInitialization().kwargs) async def wait_till_webserver_responsive(webserver_endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get(webserver_endpoint.with_path("/v0/")) as resp: diff --git a/packages/pytest-simcore/src/pytest_simcore/traefik_service.py b/packages/pytest-simcore/src/pytest_simcore/traefik_service.py index b72b26fea9b..3fbbc040c90 100644 --- a/packages/pytest-simcore/src/pytest_simcore/traefik_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/traefik_service.py @@ -6,7 +6,7 @@ import aiohttp import pytest import tenacity -from servicelib.minio_utils import MinioRetryPolicyUponInitialization +from servicelib.minio_utils import ServiceRetryPolicyUponInitialization from yarl import URL from .helpers.utils_docker import get_service_published_port @@ -42,7 +42,7 @@ async def traefik_service( # TODO: this can be used by ANY of the simcore services! -@tenacity.retry(**MinioRetryPolicyUponInitialization().kwargs) +@tenacity.retry(**ServiceRetryPolicyUponInitialization().kwargs) async def wait_till_traefik_responsive(api_endpoint: URL): async with aiohttp.ClientSession() as session: async with session.get(api_endpoint.with_path("/api/http/routers")) as resp: diff --git a/packages/service-library/src/servicelib/minio_utils.py b/packages/service-library/src/servicelib/minio_utils.py index 14e168828fa..53aedaa9e98 100644 --- a/packages/service-library/src/servicelib/minio_utils.py +++ b/packages/service-library/src/servicelib/minio_utils.py @@ -1,20 +1,17 @@ -# FIXME: move to settings-library or refactor - import logging -from typing import Optional from tenacity import before_sleep_log, stop_after_attempt, wait_fixed log = logging.getLogger(__name__) -class MinioRetryPolicyUponInitialization: +class ServiceRetryPolicyUponInitialization: """Retry policy upon service initialization""" WAIT_SECS = 2 ATTEMPTS_COUNT = 40 - def __init__(self, logger: Optional[logging.Logger] = None): + def __init__(self, logger: logging.Logger | None = None): logger = logger or log self.kwargs = dict( diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index e2fa387d5ec..51099e13f50 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -17,7 +17,6 @@ coverage docker faker flaky -minio moto[server] pytest pytest-aiohttp @@ -31,3 +30,4 @@ pytest-sugar pytest-xdist python-dotenv requests +types-aiobotocore[s3] # s3 storage diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index dfb9a64a278..e0d38ccc529 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -9,7 +9,9 @@ aioboto3==9.6.0 # -c requirements/./constraints.txt # -r requirements/_test.in aiobotocore==2.3.0 - # via aioboto3 + # via + # aioboto3 + # aiobotocore aiohttp==3.8.6 # via # -c requirements/../../../requirements/constraints.txt @@ -60,10 +62,11 @@ botocore==1.24.21 # boto3 # moto # s3transfer +botocore-stubs==1.33.6 + # via types-aiobotocore certifi==2023.7.22 # via # -c requirements/../../../requirements/constraints.txt - # minio # requests cffi==1.16.0 # via cryptography @@ -182,10 +185,6 @@ markupsafe==2.1.3 # jinja2 # mako # werkzeug -minio==7.0.4 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_test.in moto==4.2.7 # via -r requirements/_test.in mpmath==1.3.0 @@ -273,7 +272,9 @@ python-dateutil==2.8.2 python-dotenv==1.0.0 # via -r requirements/_test.in python-jose==3.3.0 - # via moto + # via + # moto + # python-jose pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt @@ -336,6 +337,12 @@ tomli==2.0.1 # via # coverage # pytest +types-aiobotocore==2.8.0 + # via -r requirements/_test.in +types-aiobotocore-s3==2.8.0 + # via types-aiobotocore +types-awscrt==0.19.19 + # via botocore-stubs types-pyyaml==6.0.12.12 # via responses typing-extensions==4.8.0 @@ -344,12 +351,13 @@ typing-extensions==4.8.0 # alembic # aws-sam-translator # pydantic + # types-aiobotocore + # types-aiobotocore-s3 urllib3==1.26.18 # via # -c requirements/../../../requirements/constraints.txt # botocore # docker - # minio # requests # responses websocket-client==1.6.4 diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index 62a11df75ac..e02cf703a1f 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -10,6 +10,7 @@ import pytest import simcore_sdk +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_postgres import PostgresTestConfig from simcore_sdk.node_ports_common.file_io_utils import LogRedirectCB @@ -19,11 +20,11 @@ pytest_plugins = [ "pytest_simcore.aws_server", + "pytest_simcore.aws_s3_service", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", "pytest_simcore.file_extra", "pytest_simcore.minio_service", - "pytest_simcore.monkeypatch_extra", "pytest_simcore.postgres_service", "pytest_simcore.pytest_global_environs", "pytest_simcore.repository_paths", @@ -87,7 +88,7 @@ def empty_configuration_file() -> Path: @pytest.fixture def node_ports_config( - postgres_host_config: PostgresTestConfig, minio_config: dict[str, str] + postgres_host_config: PostgresTestConfig, minio_s3_settings_envs: EnvVarsDict ) -> None: ... diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index c1a75279a21..ac0d1693c2b 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -20,6 +20,7 @@ from pydantic import parse_obj_as from pytest_simcore.helpers.rawdata_fakers import random_project, random_user from settings_library.r_clone import RCloneSettings, S3Provider +from settings_library.s3 import S3Settings from simcore_postgres_database.models.comp_pipeline import comp_pipeline from simcore_postgres_database.models.comp_tasks import comp_tasks from simcore_postgres_database.models.file_meta_data import file_meta_data @@ -333,21 +334,11 @@ def _assign_config( @pytest.fixture async def r_clone_settings_factory( - minio_config: dict[str, Any], storage_service: URL + minio_s3_settings: S3Settings, storage_service: URL ) -> Awaitable[RCloneSettings]: async def _factory() -> RCloneSettings: - client = minio_config["client"] - settings = RCloneSettings.parse_obj( - { - "R_CLONE_S3": { - "S3_ENDPOINT": client["endpoint"], - "S3_ACCESS_KEY": client["access_key"], - "S3_SECRET_KEY": client["secret_key"], - "S3_BUCKET_NAME": minio_config["bucket_name"], - "S3_SECURE": client["secure"], - }, - "R_CLONE_PROVIDER": S3Provider.MINIO, - } + settings = RCloneSettings( + R_CLONE_S3=minio_s3_settings, R_CLONE_PROVIDER=S3Provider.MINIO ) if not await is_r_clone_available(settings): pytest.skip("rclone not installed") @@ -374,9 +365,8 @@ def cleanup_file_meta_data(postgres_db: sa.engine.Engine) -> Iterator[None]: @pytest.fixture def node_ports_config( node_ports_config, + with_bucket_versioning_enabled: str, simcore_services_ready, cleanup_file_meta_data: None, - bucket: str, - with_bucket_versioning: None, ) -> None: return None diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 0afceaba480..87a686b853b 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -45,8 +45,6 @@ sqlalchemy<2.0 # Bugs # -# FIXME: minio 7.1.0 does not delete objects. SEE -minio==7.0.4 diff --git a/scripts/filestash/create_config.py b/scripts/filestash/create_config.py deleted file mode 100644 index f776a3c6400..00000000000 --- a/scripts/filestash/create_config.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Parses the configuration template and injects it where the platform expects it -Notes: -- Admin credentials for filestash are admin:adminadmin -- $ must be escaped with $$ in the template file -""" - -import os -import random -import string -import tempfile -from pathlib import Path -from string import Template - -SCRIPT_DIR = Path(__file__).resolve().parent -TEMPLATE_PATH = SCRIPT_DIR / "filestash_config.json.template" -CONFIG_JSON = Path(tempfile.mkdtemp()) / "filestash_config.json" - - -# The distutils package has been deprecated in Python 3.10, so I copied the -# strtobool function from the distutils.util module. -def strtobool(val): - """Convert a string representation of truth to true (1) or false (0). - True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values - are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if - 'val' is anything else. - """ - val = val.lower() - if val in ("y", "yes", "t", "true", "on", "1"): - return 1 - elif val in ("n", "no", "f", "false", "off", "0"): - return 0 - else: - raise ValueError(f"invalid truth value {val}") - - -def random_secret_key(length: int = 16) -> str: - return "".join(random.choice(string.ascii_letters) for _ in range(length)) - - -def patch_env_vars() -> None: - endpoint = os.environ["S3_ENDPOINT"] - if not endpoint.startswith("http"): - protocol = "https" if strtobool(os.environ["S3_SECURE"].lower()) else "http" - endpoint = f"{protocol}://{endpoint}" - - os.environ["S3_ENDPOINT"] = endpoint - - os.environ["REPLACE_SECRET_KEY"] = random_secret_key() - - -def main() -> None: - patch_env_vars() - - assert TEMPLATE_PATH.exists() - - template_content = TEMPLATE_PATH.read_text() - - config_json = Template(template_content).substitute(os.environ) - - assert CONFIG_JSON.parent.exists() - CONFIG_JSON.write_text(config_json) - - # path of configuration file is exported as env var - print(f"{CONFIG_JSON}") - - -if __name__ == "__main__": - main() diff --git a/scripts/filestash/filestash_config.json.template b/scripts/filestash/filestash_config.json.template deleted file mode 100644 index eaaaa2e2695..00000000000 --- a/scripts/filestash/filestash_config.json.template +++ /dev/null @@ -1,90 +0,0 @@ -{ - "general": { - "name": null, - "port": null, - "host": null, - "secret_key": "$REPLACE_SECRET_KEY", - "force_ssl": null, - "editor": null, - "fork_button": null, - "logout": null, - "display_hidden": null, - "refresh_after_upload": null, - "auto_connect": null, - "upload_button": null, - "upload_pool_size": null, - "filepage_default_view": "list", - "filepage_default_sort": "date", - "cookie_timeout": null, - "custom_css": null - }, - "features": { - "share": { - "enable": null, - "default_access": null, - "redirect": null - }, - "protection": { - "zip_timeout": null, - "enable": null, - "disable_svg": null - }, - "office": { - "enable": null, - "onlyoffice_server": null - }, - "server": { - "console_enable": null, - "tor_enable": null, - "tor_url": null - }, - "syncthing": { - "enable": null, - "server_url": null - }, - "image": { - "enable_image": null, - "thumbnail_size": null, - "thumbnail_quality": null, - "thumbnail_caching": null, - "image_quality": null, - "image_caching": null - }, - "search": { - "explore_timeout": null - }, - "video": { - "blacklist_format": null, - "enable_transcoder": null - } - }, - "log": { - "enable": null, - "level": null, - "telemetry": null - }, - "email": { - "server": null, - "port": null, - "username": null, - "password": null, - "from": null - }, - "auth": { - "admin": "$$2a$$10$$viR16hXd35bAaEJFEgpd9OqIzNBb/VoIsgQ8P3SjKxolpEQEHltrW" - }, - "constant": { - "user": "filestash", - "emacs": true, - "pdftotext": true - }, - "connections": [ - { - "label": "S3", - "type": "s3", - "advanced": true, - "path": "$S3_BUCKET_NAME", - "endpoint": "$S3_ENDPOINT" - } - ] -} diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 38861ecdac9..c768ddf2f95 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -34,6 +34,7 @@ aiofiles==23.1.0 aiohttp==3.8.5 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -100,6 +101,7 @@ blosc==1.11.1 certifi==2023.7.22 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -159,6 +161,7 @@ exceptiongroup==1.1.2 fastapi==0.99.1 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -202,6 +205,7 @@ httptools==0.6.0 httpx==0.24.1 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -238,6 +242,7 @@ itsdangerous==2.1.2 jinja2==3.1.2 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -279,6 +284,7 @@ lz4==4.3.2 mako==1.2.4 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -357,6 +363,7 @@ psycopg2-binary==2.9.6 pydantic==1.10.11 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -377,6 +384,7 @@ pydantic==1.10.11 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/dask-task-models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -411,6 +419,7 @@ python-multipart==0.0.6 pyyaml==6.0.1 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -441,6 +450,7 @@ pyyaml==6.0.1 redis==4.6.0 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -473,6 +483,7 @@ referencing==0.29.3 # jsonschema-specifications rich==13.4.2 # via + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -498,6 +509,7 @@ sortedcontainers==2.4.0 sqlalchemy==1.4.49 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -523,6 +535,7 @@ sqlalchemy==1.4.49 starlette==0.27.0 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -575,6 +588,7 @@ tqdm==4.65.0 # -r requirements/../../../packages/simcore-sdk/requirements/_base.in typer==0.9.0 # via + # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -593,6 +607,7 @@ typing-extensions==4.7.1 ujson==5.8.0 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt @@ -615,6 +630,7 @@ ujson==5.8.0 urllib3==1.26.16 # via # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/dask-task-models-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/dask-task-models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt diff --git a/services/director-v2/requirements/_test.in b/services/director-v2/requirements/_test.in index 0dbd8744010..aae4179774b 100644 --- a/services/director-v2/requirements/_test.in +++ b/services/director-v2/requirements/_test.in @@ -19,7 +19,6 @@ dask[distributed,diagnostics] docker Faker flaky -minio pytest pytest-aiohttp pytest-cov diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index 275e9f1fc0f..f022c37fb6a 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -11,7 +11,9 @@ aio-pika==9.1.2 aioboto3==12.0.0 # via -r requirements/_test.in aiobotocore==2.7.0 - # via aioboto3 + # via + # aioboto3 + # aiobotocore aiohttp==3.8.5 # via # -c requirements/../../../requirements/constraints.txt @@ -65,7 +67,6 @@ certifi==2023.7.22 # -c requirements/_base.txt # httpcore # httpx - # minio # requests cffi==1.16.0 # via cryptography @@ -87,7 +88,9 @@ cloudpickle==2.2.1 colorlog==6.7.0 # via dask-gateway-server coverage==7.3.2 - # via pytest-cov + # via + # coverage + # pytest-cov cryptography==41.0.5 # via # -c requirements/../../../requirements/constraints.txt @@ -183,10 +186,6 @@ markupsafe==2.1.3 # -c requirements/_base.txt # jinja2 # mako -minio==7.0.4 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_test.in msgpack==1.0.5 # via # -c requirements/_base.txt @@ -339,7 +338,6 @@ urllib3==1.26.16 # botocore # distributed # docker - # minio # requests websocket-client==1.6.4 # via docker diff --git a/services/director-v2/tests/conftest.py b/services/director-v2/tests/conftest.py index a481d4d2082..281ea08148c 100644 --- a/services/director-v2/tests/conftest.py +++ b/services/director-v2/tests/conftest.py @@ -6,9 +6,10 @@ import json import logging import os +from collections.abc import AsyncIterable, AsyncIterator from copy import deepcopy from pathlib import Path -from typing import Any, AsyncIterable, Iterable +from typing import Any from unittest.mock import AsyncMock import httpx @@ -25,15 +26,14 @@ from starlette.testclient import ASGI3App, TestClient pytest_plugins = [ - "pytest_simcore.db_entries_mocks", "pytest_simcore.dask_gateway", "pytest_simcore.dask_scheduler", + "pytest_simcore.db_entries_mocks", "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", "pytest_simcore.minio_service", - "pytest_simcore.monkeypatch_extra", "pytest_simcore.postgres_service", "pytest_simcore.pydantic_models", "pytest_simcore.pytest_global_environs", @@ -140,7 +140,7 @@ def dynamic_sidecar_docker_image_name() -> str: return f"{registry}/dynamic-sidecar:{image_tag}" -@pytest.fixture() +@pytest.fixture def mock_env( monkeypatch: pytest.MonkeyPatch, dynamic_sidecar_docker_image_name: str ) -> EnvVarsDict: @@ -177,7 +177,7 @@ def mock_env( @pytest.fixture() -async def client(mock_env: EnvVarsDict) -> Iterable[TestClient]: +async def client(mock_env: EnvVarsDict) -> AsyncIterator[TestClient]: settings = AppSettings.create_from_envs() app = init_app(settings) print("Application settings\n", settings.json(indent=2)) @@ -226,8 +226,7 @@ def fake_workbench(fake_workbench_file: Path) -> NodesDict: @pytest.fixture def fake_workbench_as_dict(fake_workbench_file: Path) -> dict[str, Any]: - workbench_dict = json.loads(fake_workbench_file.read_text()) - return workbench_dict + return json.loads(fake_workbench_file.read_text()) @pytest.fixture @@ -236,7 +235,7 @@ def fake_workbench_without_outputs( ) -> dict[str, Any]: workbench = deepcopy(fake_workbench_as_dict) # remove all the outputs from the workbench - for _, data in workbench.items(): + for data in workbench.values(): data["outputs"] = {} return workbench diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index eb0ff1c8947..8889e441906 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -28,7 +28,9 @@ from models_library.projects_pipeline import PipelineDetails from models_library.projects_state import RunningState from models_library.users import UserID +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_postgres import PostgresTestConfig from settings_library.rabbit import RabbitSettings from starlette import status from starlette.testclient import TestClient @@ -49,6 +51,8 @@ @pytest.fixture def mock_env( + mock_env: EnvVarsDict, + minimal_configuration: None, monkeypatch: pytest.MonkeyPatch, dynamic_sidecar_docker_image_name: str, dask_scheduler_service: str, @@ -78,7 +82,7 @@ def minimal_configuration( dask_scheduler_service: str, dask_sidecar_service: None, postgres_db: sa.engine.Engine, - postgres_host_config: dict[str, str], + postgres_host_config: PostgresTestConfig, rabbit_service: RabbitSettings, simcore_services_ready: None, storage_service: URL, @@ -164,7 +168,6 @@ def fake_workbench_computational_pipeline_details_not_started( ], ) def test_invalid_computation( - minimal_configuration: None, client: TestClient, body: dict, exp_response: int, @@ -180,7 +183,6 @@ def test_invalid_computation( async def test_start_empty_computation_is_refused( - minimal_configuration: None, async_client: httpx.AsyncClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -384,7 +386,6 @@ class PartialComputationParams: ) async def test_run_partial_computation( wait_for_catalog_service: Callable[[UserID, str], Awaitable[None]], - minimal_configuration: None, async_client: httpx.AsyncClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -536,7 +537,6 @@ def _convert_to_pipeline_details( async def test_run_computation( wait_for_catalog_service: Callable[[UserID, str], Awaitable[None]], - minimal_configuration: None, async_client: httpx.AsyncClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -655,7 +655,6 @@ async def test_run_computation( async def test_abort_computation( - minimal_configuration: None, async_client: httpx.AsyncClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -734,7 +733,6 @@ async def test_abort_computation( async def test_update_and_delete_computation( - minimal_configuration: None, async_client: httpx.AsyncClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -861,7 +859,6 @@ async def test_update_and_delete_computation( async def test_pipeline_with_no_computational_services_still_create_correct_comp_tasks_in_db( - minimal_configuration: None, async_client: httpx.AsyncClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -905,7 +902,6 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( - minimal_configuration: None, client: TestClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -974,7 +970,6 @@ async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidden( - minimal_configuration: None, client: TestClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], @@ -1056,7 +1051,6 @@ async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidd async def test_burst_create_computations( - minimal_configuration: None, async_client: httpx.AsyncClient, registered_user: Callable, project: Callable[..., Awaitable[ProjectAtDB]], diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py index 32cbcbf5fc5..29788f61017 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py @@ -56,14 +56,11 @@ "rabbit", "redis", ] -pytest_simcore_ops_services_selection = [ - "adminer", -] +pytest_simcore_ops_services_selection = ["adminer", "minio"] @pytest.fixture def minimal_configuration( - mock_env: EnvVarsDict, redis_settings: RedisSettings, postgres_db, postgres_host_config: dict[str, str], @@ -75,10 +72,8 @@ def minimal_configuration( @pytest.fixture -def mock_env(mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: - monkeypatch.setenv("RABBIT_USER", "admin") - monkeypatch.setenv("RABBIT_PASSWORD", "adminadmin") - return mock_env | {"RABBIT_USER": "admin", "RABBIT_PASSWORD": "adminadmin"} +def mock_env(mock_env: EnvVarsDict, minimal_configuration) -> None: + ... @pytest.fixture @@ -150,7 +145,6 @@ def start_request_data( @pytest.fixture async def director_v2_client( - minimal_configuration: None, mock_env: EnvVarsDict, network_name: str, redis_settings: RedisSettings, diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 85cdccb88af..c8f0a4ac70d 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -46,6 +46,7 @@ from models_library.projects_state import RunningState from models_library.users import UserID from pydantic import AnyHttpUrl, parse_obj_as +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.fastapi.long_running_tasks.client import ( @@ -154,6 +155,7 @@ async def minimal_configuration( dask_scheduler_service: str, dask_sidecar_service: None, ensure_swarm_and_networks: None, + minio_s3_settings_envs: EnvVarsDict, current_user: dict[str, Any], osparc_product_name: str, ) -> AsyncIterator[None]: @@ -324,14 +326,12 @@ def dev_feature_r_clone_enabled(request) -> str: @pytest.fixture def mock_env( + mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, - redis_service: RedisSettings, network_name: str, dev_feature_r_clone_enabled: str, - rabbit_service: RabbitSettings, dask_scheduler_service: str, - minio_config: dict[str, Any], - storage_service: URL, + minimal_configuration: None, ) -> None: # Works as below line in docker.compose.yml # ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest} @@ -364,15 +364,10 @@ def mock_env( "RABBIT_HOST": f"{get_localhost_ip()}", "POSTGRES_HOST": f"{get_localhost_ip()}", "R_CLONE_PROVIDER": "MINIO", - "S3_ENDPOINT": minio_config["client"]["endpoint"], - "S3_ACCESS_KEY": minio_config["client"]["access_key"], - "S3_SECRET_KEY": minio_config["client"]["secret_key"], - "S3_BUCKET_NAME": minio_config["bucket_name"], - "S3_SECURE": f"{minio_config['client']['secure']}", "DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED": dev_feature_r_clone_enabled, + "COMPUTATIONAL_BACKEND_ENABLED": "true", + "COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED": "true", "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL": dask_scheduler_service, - "REDIS_HOST": redis_service.REDIS_HOST, - "REDIS_PORT": f"{redis_service.REDIS_PORT}", }, ) monkeypatch.delenv("DYNAMIC_SIDECAR_MOUNT_PATH_DEV", raising=False) @@ -860,7 +855,6 @@ async def _assert_retrieve_completed( @pytest.mark.flaky(max_runs=3) async def test_nodeports_integration( - minimal_configuration: None, cleanup_services_and_networks: None, projects_networks_db: None, mocked_service_awaits_manual_interventions: None, diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py index ca528632c9d..b20994bfb57 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -61,7 +61,7 @@ def mock_env( rabbit_service: RabbitSettings, postgres_db: sa.engine.Engine, postgres_host_config: dict[str, str], - minio_config: dict[str, Any], + minio_s3_settings_envs: EnvVarsDict, storage_service: URL, network_name: str, ) -> EnvVarsDict: diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index 239b0b51d80..2fdc1bbd083 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -1,5 +1,9 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + import asyncio -from typing import AsyncIterator, Awaitable, Callable, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable from unittest.mock import AsyncMock import httpx @@ -9,6 +13,7 @@ from models_library.projects import ProjectAtDB from models_library.users import UserID from pytest_mock import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_postgres_database.models.comp_tasks import comp_tasks from simcore_postgres_database.models.projects import projects from starlette import status @@ -19,10 +24,16 @@ from yarl import URL +@pytest.fixture +def mock_env(mock_env: EnvVarsDict, minio_s3_settings_envs: EnvVarsDict) -> EnvVarsDict: + # overwrite to add minio real settings + return mock_env + + @pytest.fixture def update_project_workbench_with_comp_tasks( postgres_db: sa.engine.Engine, -) -> Iterator[Callable]: +) -> Callable: def updator(project_uuid: str): with postgres_db.connect() as con: result = con.execute( @@ -47,7 +58,7 @@ def updator(project_uuid: str): .where(projects.c.uuid == project_uuid) ) - yield updator + return updator @pytest.fixture(scope="session") diff --git a/services/docker-compose-ops.yml b/services/docker-compose-ops.yml index 19c0d3e517b..3836d72b337 100644 --- a/services/docker-compose-ops.yml +++ b/services/docker-compose-ops.yml @@ -70,20 +70,6 @@ services: timeout: 10s retries: 5 - filestash: - image: machines/filestash:3a01b70 - ports: - - "9002:8334" - volumes: - - ${TMP_PATH_TO_FILESTASH_CONFIG}:/app/data/state/config/config.json - networks: - - simcore_default - - onlyoffice: - image: onlyoffice/documentserver:7.0.0.132 - networks: - - simcore_default - redis-commander: image: rediscommander/redis-commander:latest init: true diff --git a/services/dynamic-sidecar/requirements/_test.in b/services/dynamic-sidecar/requirements/_test.in index ba281c8b6d8..258264e7115 100644 --- a/services/dynamic-sidecar/requirements/_test.in +++ b/services/dynamic-sidecar/requirements/_test.in @@ -9,7 +9,6 @@ async-asgi-testclient # replacement for fastapi.testclient.TestClient [see b) be docker faker flaky -minio pytest pytest-asyncio pytest-cov diff --git a/services/dynamic-sidecar/requirements/_test.txt b/services/dynamic-sidecar/requirements/_test.txt index 46b8e7b0e70..580a3407b74 100644 --- a/services/dynamic-sidecar/requirements/_test.txt +++ b/services/dynamic-sidecar/requirements/_test.txt @@ -7,7 +7,9 @@ aioboto3==12.0.0 # via -r requirements/_test.in aiobotocore==2.7.0 - # via aioboto3 + # via + # aioboto3 + # aiobotocore aiohttp==3.8.5 # via # -c requirements/../../../requirements/constraints.txt @@ -42,7 +44,6 @@ certifi==2023.7.22 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt - # minio # requests charset-normalizer==3.2.0 # via @@ -50,7 +51,9 @@ charset-normalizer==3.2.0 # aiohttp # requests coverage==7.3.2 - # via pytest-cov + # via + # coverage + # pytest-cov docker==6.1.3 # via -r requirements/_test.in exceptiongroup==1.1.2 @@ -81,10 +84,6 @@ jmespath==1.0.1 # via # boto3 # botocore -minio==7.0.4 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_test.in multidict==6.0.4 # via # -c requirements/_base.txt @@ -166,7 +165,6 @@ urllib3==2.0.7 # -c requirements/../../../requirements/constraints.txt # botocore # docker - # minio # requests websocket-client==1.6.4 # via docker diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 80f3a0a85be..6e70de6a80a 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -33,7 +33,7 @@ "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", - "pytest_simcore.monkeypatch_extra", # TODO: remove this dependency + "pytest_simcore.minio_service", "pytest_simcore.pytest_global_environs", "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", diff --git a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py index e08eb9dc0dc..6a53525a8f4 100644 --- a/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/integration/test_modules_long_running_tasks.py @@ -7,7 +7,7 @@ import shutil from collections.abc import AsyncIterable, Iterable from pathlib import Path -from typing import Any, cast +from typing import cast from unittest.mock import AsyncMock import aioboto3 @@ -42,7 +42,6 @@ from yarl import URL pytest_plugins = [ - "pytest_simcore.minio_service", "pytest_simcore.postgres_service", "pytest_simcore.simcore_storage_service", ] @@ -89,7 +88,7 @@ def project_id(user_id: int, postgres_db: sa.engine.Engine) -> Iterable[ProjectI def mock_environment( postgres_host_config: PostgresTestConfig, storage_endpoint: URL, - minio_config: dict[str, Any], + minio_s3_settings_envs: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, base_mock_envs: EnvVarsDict, user_id: UserID, @@ -102,11 +101,6 @@ def mock_environment( "STORAGE_PORT": f"{storage_endpoint.port}", "DY_SIDECAR_USER_ID": f"{user_id}", "DY_SIDECAR_PROJECT_ID": f"{project_id}", - "S3_ENDPOINT": minio_config["client"]["endpoint"], - "S3_ACCESS_KEY": minio_config["client"]["access_key"], - "S3_SECRET_KEY": minio_config["client"]["secret_key"], - "S3_BUCKET_NAME": minio_config["bucket_name"], - "S3_SECURE": f"{minio_config['client']['secure']}", "R_CLONE_PROVIDER": "MINIO", "DY_SIDECAR_CALLBACKS_MAPPING": "{}", **base_mock_envs, diff --git a/tests/swarm-deploy/conftest.py b/tests/swarm-deploy/conftest.py index e5e674f43b3..687157f9381 100644 --- a/tests/swarm-deploy/conftest.py +++ b/tests/swarm-deploy/conftest.py @@ -20,15 +20,13 @@ "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", - "pytest_simcore.minio_service", - "pytest_simcore.monkeypatch_extra", "pytest_simcore.postgres_service", + "pytest_simcore.pytest_global_environs", "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", "pytest_simcore.simcore_webserver_service", "pytest_simcore.tmp_path_extra", "pytest_simcore.traefik_service", - "pytest_simcore.pytest_global_environs", ] log = logging.getLogger(__name__) diff --git a/tests/swarm-deploy/requirements/_test.in b/tests/swarm-deploy/requirements/_test.in index ee92c085e9f..ae4d23b49b5 100644 --- a/tests/swarm-deploy/requirements/_test.in +++ b/tests/swarm-deploy/requirements/_test.in @@ -12,7 +12,6 @@ alembic click docker jsonschema -minio pytest pytest-aiohttp pytest-instafail diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 6592b63016f..eda0fe101cf 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -90,7 +90,6 @@ certifi==2023.7.22 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_migration.txt - # minio # requests charset-normalizer==3.3.2 # via @@ -162,22 +161,6 @@ markupsafe==2.1.3 # mako mdurl==0.1.2 # via markdown-it-py -minio==7.0.4 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_test.in multidict==6.0.4 # via # aiohttp @@ -391,7 +374,6 @@ urllib3==2.0.7 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # docker - # minio # requests websocket-client==1.6.4 # via