Wait for DataScienceCluster to finish reconciling

skodjob · Mar 10, 2024 · 0896726 · 0896726
1 parent 1d8c698
commit 0896726
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 0 deletions.
diff --git a/src/main/java/io/odh/test/OdhConstants.java b/src/main/java/io/odh/test/OdhConstants.java
@@ -55,6 +55,8 @@ private OdhConstants() { }
     private static final String RHOAI_MONITORING_NAMESPACE = "redhat-ods-monitoring";
 
     // Public part
+    public static final String DSC_CREATION_SUCCESSFUL_EVENT_NAME = "DataScienceClusterCreationSuccessful";
+
     public static final String CODEFLARE_DEPLOYMENT_NAME = "codeflare-operator-manager";
     public static final String DS_PIPELINES_OPERATOR = "data-science-pipelines-operator-controller-manager";
     public static final String ETCD = "etcd";

diff --git a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java
@@ -5,6 +5,8 @@
 package io.odh.test.framework.manager.resources;
 
 import io.fabric8.kubernetes.api.model.KubernetesResourceList;
+import io.fabric8.kubernetes.api.model.events.v1.Event;
+import io.fabric8.kubernetes.client.dsl.EventingAPIGroupDSL;
 import io.fabric8.kubernetes.client.dsl.MixedOperation;
 import io.fabric8.kubernetes.client.dsl.Resource;
 import io.odh.test.OdhConstants;
@@ -18,6 +20,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.List;
+import java.util.Objects;
+
 public class DataScienceClusterResource implements ResourceType<DataScienceCluster> {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(DataScienceClusterResource.class);
@@ -105,6 +110,25 @@ public boolean waitForReadiness(DataScienceCluster resource) {
 //                dscReady = dscReady && pipelinesStatus.equals("True");
 //            }
 
+            // Check that DSC reconciliation has been successfully finalized
+            // https://github.com/red-hat-data-services/rhods-operator/blob/rhoai-2.8/controllers/datasciencecluster/datasciencecluster_controller.go#L257
+
+            // Wait for ReconcileComplete condition (for the whole DSC)
+            String reconcileStatus = KubeUtils.getDscConditionByType(dsc.getStatus().getConditions(), "ReconcileComplete").getStatus();
+            LOGGER.debug("DataScienceCluster {} ReconcileComplete status: {}", resource.getMetadata().getName(), reconcileStatus);
+            dscReady = dscReady && reconcileStatus.equals("True");
+
+            // Wait for DataScienceClusterCreationSuccessful event
+            EventingAPIGroupDSL eventsClient = ResourceManager.getKubeClient().getClient().events();
+            List<Event> resourceEvents = eventsClient.v1().events().inAnyNamespace().withNewFilter()
+                    .withField("regarding.name", resource.getMetadata().getName())
+                    .withField("regarding.uid", resource.getMetadata().getUid())
+                    .endFilter().list().getItems();
+            LOGGER.debug("DataScienceCluster {} events: {}", resource.getMetadata().getName(), resourceEvents.stream().map(Event::getReason).toList());
+            boolean hasCreationSuccessfulEvent = resourceEvents.stream()
+                    .anyMatch(resourceEvent -> Objects.equals(resourceEvent.getReason(), OdhConstants.DSC_CREATION_SUCCESSFUL_EVENT_NAME));
+            dscReady = dscReady && hasCreationSuccessfulEvent;
+
             return dscReady;
         }, () -> { });