From fedc582915d119c641288c0dcbd0ea4aa7b46adf Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:37:04 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9BAutoscaling=20in=20computational=20?= =?UTF-8?q?clusters=20could=20not=20start=20without=20SSM=20parameters=20?= =?UTF-8?q?=20(#6229)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/auto_scaling_core.py | 5 ++-- ...test_modules_auto_scaling_computational.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index fa01ffeb27f..34c469b254c 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -200,14 +200,13 @@ async def _make_pending_buffer_ec2s_join_cluster( app: FastAPI, cluster: Cluster, ) -> Cluster: - # started buffer instance shall be asked to join the cluster once they are running - ssm_client = get_ssm_client(app) - if buffer_ec2s_pending := [ i.ec2_instance for i in cluster.pending_ec2s if is_buffer_machine(i.ec2_instance.tags) ]: + # started buffer instance shall be asked to join the cluster once they are running + ssm_client = get_ssm_client(app) buffer_ec2_connection_state = await limited_gather( *[ ssm_client.is_instance_connected_to_ssm_server(i.id) diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 1939af97dd2..a1b411fa988 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -202,6 +202,32 @@ async def test_cluster_scaling_with_no_tasks_does_nothing( ) +@pytest.mark.acceptance_test( + "Ensure this does not happen https://github.com/ITISFoundation/osparc-simcore/issues/6227" +) +async def test_cluster_scaling_with_disabled_ssm_does_not_block_autoscaling( + minimal_configuration: None, + disabled_ssm: None, + app_settings: ApplicationSettings, + initialized_app: FastAPI, + mock_launch_instances: mock.Mock, + mock_terminate_instances: mock.Mock, + mock_rabbitmq_post_message: mock.Mock, + dask_spec_local_cluster: distributed.SpecCluster, +): + await auto_scale_cluster( + app=initialized_app, auto_scaling_mode=ComputationalAutoscaling() + ) + mock_launch_instances.assert_not_called() + mock_terminate_instances.assert_not_called() + _assert_rabbit_autoscaling_message_sent( + mock_rabbitmq_post_message, + app_settings, + initialized_app, + dask_spec_local_cluster.scheduler_address, + ) + + async def test_cluster_scaling_with_task_with_too_much_resources_starts_nothing( minimal_configuration: None, app_settings: ApplicationSettings,