Add ODH pod readiness because dashboard status is unstable

Signed-off-by: Jakub Stejskal <[email protected]>
skodjob · Jan 11, 2024 · 00bce9b · 00bce9b
1 parent b90eb62
commit 00bce9b
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 0 deletions.
diff --git a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java
@@ -7,11 +7,13 @@
 import io.fabric8.kubernetes.api.model.KubernetesResourceList;
 import io.fabric8.kubernetes.client.dsl.MixedOperation;
 import io.fabric8.kubernetes.client.dsl.Resource;
+import io.odh.test.OdhConstants;
 import io.odh.test.TestConstants;
 import io.odh.test.TestUtils;
 import io.odh.test.framework.manager.ResourceManager;
 import io.odh.test.framework.manager.ResourceType;
 import io.odh.test.platform.KubeUtils;
+import io.odh.test.utils.PodUtils;
 import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,6 +64,11 @@ public boolean waitForReadiness(DataScienceCluster resource) {
 
             return dscReady;
         }, () -> { });
+
+        String namespace = OdhConstants.CONTROLLERS_NAMESPACE;
+        LOGGER.info("Waiting for pods readiness in {}", namespace);
+        PodUtils.waitForPodsReady(namespace, true, () -> { });
+
         return true;
     }
 

diff --git a/src/main/java/io/odh/test/utils/PodUtils.java b/src/main/java/io/odh/test/utils/PodUtils.java
@@ -27,6 +27,35 @@ public class PodUtils {
 
     private PodUtils() { }
 
+    public static void waitForPodsReady(String namespaceName, boolean containers, Runnable onTimeout) {
+        TestUtils.waitFor("readiness of all Pods matching in namespace {} " + namespaceName,
+                TestConstants.GLOBAL_POLL_INTERVAL_MEDIUM, READINESS_TIMEOUT,
+                () -> {
+                    List<Pod> pods = ResourceManager.getClient().listPods(namespaceName);
+                    if (pods.isEmpty()) {
+                        LOGGER.debug("Expected Pods are ready");
+                        return true;
+                    }
+                    for (Pod pod : pods) {
+                        if (!Readiness.isPodReady(pod)) {
+                            LOGGER.debug("Pod not ready: {}/{}", namespaceName, pod.getMetadata().getName());
+                            return false;
+                        } else {
+                            if (containers) {
+                                for (ContainerStatus cs : pod.getStatus().getContainerStatuses()) {
+                                    if (!Boolean.TRUE.equals(cs.getReady())) {
+                                        LOGGER.debug("Container: {} of Pod: {}/{} not ready", namespaceName, pod.getMetadata().getName(), cs.getName());
+                                        return false;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    LOGGER.info("Pods in namespace {} are ready", namespaceName);
+                    return true;
+                }, onTimeout);
+    }
+
     public static void waitForPodsReady(String namespaceName, LabelSelector selector, int expectPods, boolean containers, Runnable onTimeout) {
         TestUtils.waitFor("readiness of all Pods matching: " + selector,
                 TestConstants.GLOBAL_POLL_INTERVAL_MEDIUM, READINESS_TIMEOUT,