From 00bce9bd4d6677fdf02210056e8d5df739fa19ab Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Thu, 11 Jan 2024 10:10:00 +0100 Subject: [PATCH] Add ODH pod readiness because dashboard status is unstable Signed-off-by: Jakub Stejskal --- .../resources/DataScienceClusterResource.java | 7 +++++ src/main/java/io/odh/test/utils/PodUtils.java | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java index dc7a6c14..617b2e18 100644 --- a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java +++ b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java @@ -7,11 +7,13 @@ import io.fabric8.kubernetes.api.model.KubernetesResourceList; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.Resource; +import io.odh.test.OdhConstants; import io.odh.test.TestConstants; import io.odh.test.TestUtils; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.framework.manager.ResourceType; import io.odh.test.platform.KubeUtils; +import io.odh.test.utils.PodUtils; import io.opendatahub.datasciencecluster.v1.DataScienceCluster; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,6 +64,11 @@ public boolean waitForReadiness(DataScienceCluster resource) { return dscReady; }, () -> { }); + + String namespace = OdhConstants.CONTROLLERS_NAMESPACE; + LOGGER.info("Waiting for pods readiness in {}", namespace); + PodUtils.waitForPodsReady(namespace, true, () -> { }); + return true; } diff --git a/src/main/java/io/odh/test/utils/PodUtils.java b/src/main/java/io/odh/test/utils/PodUtils.java index a04ad107..a725740a 100644 --- a/src/main/java/io/odh/test/utils/PodUtils.java +++ b/src/main/java/io/odh/test/utils/PodUtils.java @@ -27,6 +27,35 @@ public class PodUtils { private PodUtils() { } + public static void waitForPodsReady(String namespaceName, boolean containers, Runnable onTimeout) { + TestUtils.waitFor("readiness of all Pods matching in namespace {} " + namespaceName, + TestConstants.GLOBAL_POLL_INTERVAL_MEDIUM, READINESS_TIMEOUT, + () -> { + List pods = ResourceManager.getClient().listPods(namespaceName); + if (pods.isEmpty()) { + LOGGER.debug("Expected Pods are ready"); + return true; + } + for (Pod pod : pods) { + if (!Readiness.isPodReady(pod)) { + LOGGER.debug("Pod not ready: {}/{}", namespaceName, pod.getMetadata().getName()); + return false; + } else { + if (containers) { + for (ContainerStatus cs : pod.getStatus().getContainerStatuses()) { + if (!Boolean.TRUE.equals(cs.getReady())) { + LOGGER.debug("Container: {} of Pod: {}/{} not ready", namespaceName, pod.getMetadata().getName(), cs.getName()); + return false; + } + } + } + } + } + LOGGER.info("Pods in namespace {} are ready", namespaceName); + return true; + }, onTimeout); + } + public static void waitForPodsReady(String namespaceName, LabelSelector selector, int expectPods, boolean containers, Runnable onTimeout) { TestUtils.waitFor("readiness of all Pods matching: " + selector, TestConstants.GLOBAL_POLL_INTERVAL_MEDIUM, READINESS_TIMEOUT,