diff --git a/.env-devel b/.env-devel index 5fe49b8d58a..051ba5a1ab8 100644 --- a/.env-devel +++ b/.env-devel @@ -38,6 +38,7 @@ CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null CLUSTERS_KEEPER_TASK_INTERVAL=30 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null +CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX="" DASK_SCHEDULER_HOST=dask-scheduler diff --git a/services/clusters-keeper/.env-devel b/services/clusters-keeper/.env-devel index 59467608d03..763f81af99e 100644 --- a/services/clusters-keeper/.env-devel +++ b/services/clusters-keeper/.env-devel @@ -8,6 +8,7 @@ PRIMARY_EC2_INSTANCES_KEY_NAME=XXXXXXXXXX PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS=XXXXXXXXXX PRIMARY_EC2_INSTANCES_SUBNET_ID=XXXXXXXXXX EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY=XXXXXXXXXX +CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX="testing" LOG_FORMAT_LOCAL_DEV_ENABLED=True RABBIT_HOST=rabbit RABBIT_PASSWORD=test diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index c19650ea0c4..c40904c8807 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -219,6 +219,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): auto_default_from_env=True ) + CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX: str = Field( + ..., + description="set a prefix to all machines created (useful for testing)", + ) + CLUSTERS_KEEPER_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True) CLUSTERS_KEEPER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py index c7dab3e615d..3b5be7dff33 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py @@ -25,13 +25,13 @@ def get_cluster_name( wallet_id: WalletID | None, is_manager: bool, ) -> str: - return f"{CLUSTER_NAME_PREFIX}{'manager' if is_manager else 'worker'}-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}" + return f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{CLUSTER_NAME_PREFIX}{'manager' if is_manager else 'worker'}-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}" def _minimal_identification_tag(app_settings: ApplicationSettings) -> EC2Tags: return { - AWSTagKey(".".join([_APPLICATION_TAG_KEY, "deploy"])): AWSTagValue( - app_settings.SWARM_STACK_NAME + AWSTagKey(".".join([_APPLICATION_TAG_KEY, "deploy",])): AWSTagValue( + f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{app_settings.SWARM_STACK_NAME}" ) } diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index ebd7ad17431..fc1973e95ca 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -100,6 +100,7 @@ def app_environment( "CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID": faker.pystr(), "CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY": faker.pystr(), "CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES": "{}", + "CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX": faker.pystr(), "PRIMARY_EC2_INSTANCES_KEY_NAME": faker.pystr(), "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( faker.pylist(allowed_types=(str,)) diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters.py b/services/clusters-keeper/tests/unit/test_modules_clusters.py index 5ce6672b0a3..9cf402e13d4 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters.py @@ -76,9 +76,9 @@ async def _assert_cluster_instance_created( assert all("Value" in x for x in instance_ec2_tags) _EXPECTED_TAGS: dict[str, str] = { - f"{_APPLICATION_TAG_KEY}.deploy": app_settings.SWARM_STACK_NAME, + f"{_APPLICATION_TAG_KEY}.deploy": f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{app_settings.SWARM_STACK_NAME}", f"{_APPLICATION_TAG_KEY}.version": f"{APP_VERSION}", - "Name": f"{CLUSTER_NAME_PREFIX}manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}", + "Name": f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}{CLUSTER_NAME_PREFIX}manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}", "user_id": f"{user_id}", "wallet_id": f"{wallet_id}", "osparc-tag": "the pytest tag is here", diff --git a/services/clusters-keeper/tests/unit/test_utils_ec2.py b/services/clusters-keeper/tests/unit/test_utils_ec2.py index ac400497a15..e1331de628c 100644 --- a/services/clusters-keeper/tests/unit/test_utils_ec2.py +++ b/services/clusters-keeper/tests/unit/test_utils_ec2.py @@ -41,14 +41,14 @@ def test_get_cluster_name( get_cluster_name( app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=True ) - == f"osparc-computational-cluster-manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}" + == f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}osparc-computational-cluster-manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}" ) # worker assert ( get_cluster_name( app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False ) - == f"osparc-computational-cluster-worker-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}" + == f"{app_settings.CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX}osparc-computational-cluster-worker-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}" ) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 5b5fc7358d8..37b5d561a28 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -151,6 +151,7 @@ services: - CLUSTERS_KEEPER_EC2_ENDPOINT=${CLUSTERS_KEEPER_EC2_ENDPOINT} - CLUSTERS_KEEPER_EC2_REGION_NAME=${CLUSTERS_KEEPER_EC2_REGION_NAME} - CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY=${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} + - CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=${CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX} - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} - CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=${CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES} - PRIMARY_EC2_INSTANCES_ALLOWED_TYPES=${PRIMARY_EC2_INSTANCES_ALLOWED_TYPES}