Improve upgrade checks

src/main/java/io/odh/test/TestConstants.java

@@ -20,7 +20,7 @@ public class TestConstants {
     public static final long GLOBAL_POLL_INTERVAL_MEDIUM = Duration.ofSeconds(5).toMillis();
     public static final long GLOBAL_POLL_INTERVAL_SHORT = Duration.ofSeconds(1).toMillis();
     public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(5).toMillis();
-    public static final long GLOBAL_STABILITY_TIME = Duration.ofSeconds(10).toSeconds();
+    public static final long GLOBAL_STABILITY_TIME = Duration.ofMinutes(2).toSeconds();
 
     private TestConstants() {
     }

src/main/java/io/odh/test/install/BundleInstall.java

@@ -72,11 +72,11 @@ public void create() {
         ResourceManager.getInstance().pushToStack(new ResourceItem(KubeUtils::deleteDefaultDSCI, null));
     }
 
-    public void createWithoutResourceManager() throws IOException {
+    public void createWithoutResourceManager() {
         ResourceManager.getKubeCmdClient().namespace(getNamespace()).apply(installFile.getAbsolutePath());
     }
 
-    public void deleteWithoutResourceManager() throws IOException {
+    public void deleteWithoutResourceManager() {
         KubeUtils.deleteDefaultDSCI();
         ResourceManager.getKubeCmdClient().namespace(getNamespace()).delete(installFile.getAbsolutePath());
     }

src/main/java/io/odh/test/install/OlmInstall.java

@@ -105,8 +105,22 @@ public Subscription prepareSubscription() {
     }
 
     public void deleteCSV() {
-        LOGGER.info("Deleting CSV {}/{}", namespace, olmAppBundlePrefix);
-        ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().clusterServiceVersions().inNamespace(namespace).withName(csvName).delete();
+        ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().clusterServiceVersions().inNamespace(namespace)
+            .list().getItems().stream().filter(csv -> csv.getMetadata().getName().contains(olmAppBundlePrefix)).toList()
+            .forEach(csv -> {
+                LOGGER.info("Deleting CSV {}", csv.getMetadata().getName());
+                ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().clusterServiceVersions().resource(csv).delete();
+            });
+        deleteInstallPlans();
+    }
+
+    public void deleteInstallPlans() {
+        ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespace)
+            .list().getItems().stream().filter(ip -> ip.getSpec().getClusterServiceVersionNames().stream().toList().toString().contains(olmAppBundlePrefix)).toList()
+            .forEach(ip -> {
+                LOGGER.info("Deleting InstallPlan {}", ip.getMetadata().getName());
+                ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().installPlans().resource(ip).delete();
+            });
     }
 
     /**

src/main/java/io/odh/test/platform/KubeClient.java

@@ -27,10 +27,10 @@
 import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlanBuilder;
 import io.fabric8.openshift.client.OpenShiftClient;
 import io.odh.test.Environment;
+import io.odh.test.TestConstants;
+import io.odh.test.TestUtils;
 import io.odh.test.platform.executor.Exec;
-import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
 import io.opendatahub.v1alpha.OdhDashboardConfig;
-import org.kubeflow.v1.Notebook;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -369,15 +369,23 @@ public InstallPlan getInstallPlan(String namespaceName, String installPlanName)
         return client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).get();
     }
 
-    public void approveInstallPlan(String namespaceName, String installPlanName) {
+    public void approveInstallPlan(String namespaceName, String installPlanName) throws InterruptedException {
         InstallPlan installPlan = new InstallPlanBuilder(this.getInstallPlan(namespaceName, installPlanName))
             .editSpec()
             .withApproved()
             .endSpec()
             .build();
 
-        LOGGER.debug("Approving {}", installPlanName);
-        client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).patch(installPlan);
+        LOGGER.debug("Approving InstallPlan {}", installPlanName);
+        TestUtils.waitFor("InstallPlan approval", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, 15_000, () -> {
+            try {
+                client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).patch(installPlan);
+                return true;
+            } catch (Exception ex) {
+                LOGGER.error(String.valueOf(ex));
+                return false;
+            }
+        });
     }
 
     public InstallPlan getNonApprovedInstallPlan(String namespaceName, String csvPrefix) {
@@ -387,15 +395,7 @@ public InstallPlan getNonApprovedInstallPlan(String namespaceName, String csvPre
                 .findFirst().get();
     }
 
-    public MixedOperation<DataScienceCluster, KubernetesResourceList<DataScienceCluster>, Resource<DataScienceCluster>> dataScienceClusterClient() {
-        return client.resources(DataScienceCluster.class);
-    }
-
     public MixedOperation<OdhDashboardConfig, KubernetesResourceList<OdhDashboardConfig>, Resource<OdhDashboardConfig>> dashboardConfigClient() {
         return client.resources(OdhDashboardConfig.class);
     }
-
-    public MixedOperation<Notebook, KubernetesResourceList<Notebook>, Resource<Notebook>> notebookClient() {
-        return client.resources(Notebook.class);
-    }
 }

src/main/java/io/odh/test/platform/KubeUtils.java

@@ -4,6 +4,7 @@
  */
 package io.odh.test.platform;
 
+import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan;
 import io.odh.test.TestConstants;
 import io.odh.test.TestUtils;
 import io.odh.test.framework.manager.ResourceManager;
@@ -45,7 +46,8 @@ public static void deleteDefaultDSCI() {
     public static void waitForInstallPlan(String namespace, String csvName) {
         TestUtils.waitFor("Install paln with new version", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT, () -> {
             try {
-                ResourceManager.getClient().getNonApprovedInstallPlan(namespace, csvName);
+                InstallPlan ip = ResourceManager.getClient().getNonApprovedInstallPlan(namespace, csvName);
+                LOGGER.debug("Found InstallPlan {} - {}", ip.getMetadata().getName(), ip.getSpec().getClusterServiceVersionNames());
                 return true;
             } catch (NoSuchElementException ex) {
                 LOGGER.debug("No new install plan available. Checking again ...");

src/main/java/io/odh/test/utils/UpgradeUtils.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Skodjob authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.odh.test.utils;
+
+import io.odh.test.framework.manager.ResourceManager;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.core.IsNot.not;
+
+public class UpgradeUtils {
+
+    public static void deploymentLogIsErrorEmpty(String namespace, String deploymentName) {
+        // Check that operator doesn't contains errors in logs
+        String operatorLog = ResourceManager.getClient().getClient().apps().deployments()
+                .inNamespace(namespace).withName(deploymentName).getLog();
+
+        assertThat(operatorLog, not(containsString("error")));
+        assertThat(operatorLog, not(containsString("Error")));
+        assertThat(operatorLog, not(containsString("ERROR")));
+    }
+}

src/test/java/io/odh/test/e2e/continuous/DataScienceClusterST.java

@@ -10,6 +10,7 @@
 import io.odh.test.TestConstants;
 import io.odh.test.e2e.Abstract;
 import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.framework.manager.resources.DataScienceClusterResource;
 import io.odh.test.platform.KubeUtils;
 import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
 import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Codeflare;
@@ -40,7 +41,7 @@ public class DataScienceClusterST extends Abstract {
 
     @BeforeAll
     void init() {
-        dataScienceProjectCli = ResourceManager.getClient().dataScienceClusterClient();
+        dataScienceProjectCli = DataScienceClusterResource.dataScienceCLusterClient();
         dashboardConfigCli  = ResourceManager.getClient().dashboardConfigClient();
     }
 

src/test/java/io/odh/test/e2e/continuous/DataScienceProjectST.java

@@ -9,6 +9,7 @@
 import io.fabric8.kubernetes.client.dsl.Resource;
 import io.odh.test.e2e.Abstract;
 import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.framework.manager.resources.NotebookResource;
 import io.odh.test.platform.KubeUtils;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -40,7 +41,7 @@ private static Stream<Arguments> getDsProjects() {
 
     @BeforeAll
     void init() {
-        notebookCli = ResourceManager.getClient().notebookClient();
+        notebookCli = NotebookResource.notebookClient();
     }
 
     @ParameterizedTest(name = "checkDataScienceProjects-{0}")

src/test/java/io/odh/test/e2e/standard/DataScienceClusterST.java

@@ -5,6 +5,7 @@
 package io.odh.test.e2e.standard;
 
 import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.framework.manager.resources.DataScienceClusterResource;
 import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
 import io.opendatahub.datasciencecluster.v1.DataScienceClusterBuilder;
 import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.ComponentsBuilder;
@@ -58,7 +59,7 @@ void createDataScienceCluster() {
 
         ResourceManager.getInstance().createResourceWithWait(c);
 
-        DataScienceCluster cluster = ResourceManager.getClient().dataScienceClusterClient().withName(DS_PROJECT_NAME).get();
+        DataScienceCluster cluster = DataScienceClusterResource.dataScienceCLusterClient().withName(DS_PROJECT_NAME).get();
 
         assertEquals(Kserve.ManagementState.MANAGED, cluster.getSpec().getComponents().getKserve().getManagementState());
         assertEquals(Codeflare.ManagementState.MANAGED, cluster.getSpec().getComponents().getCodeflare().getManagementState());

src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java

@@ -6,64 +6,33 @@
 
 import io.fabric8.kubernetes.api.model.LabelSelector;
 import io.fabric8.kubernetes.api.model.LabelSelectorBuilder;
-import io.fabric8.kubernetes.api.model.Namespace;
-import io.fabric8.kubernetes.api.model.NamespaceBuilder;
-import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
-import io.fabric8.kubernetes.api.model.PersistentVolumeClaimBuilder;
-import io.fabric8.kubernetes.api.model.Quantity;
 import io.odh.test.Environment;
-import io.odh.test.OdhAnnotationsLabels;
 import io.odh.test.OdhConstants;
 import io.odh.test.TestConstants;
-import io.odh.test.e2e.Abstract;
-import io.odh.test.framework.listeners.OdhResourceCleaner;
-import io.odh.test.framework.listeners.ResourceManagerDeleteHandler;
 import io.odh.test.framework.manager.ResourceManager;
-import io.odh.test.framework.manager.resources.NotebookResource;
 import io.odh.test.install.BundleInstall;
 import io.odh.test.utils.DeploymentUtils;
 import io.odh.test.utils.PodUtils;
-import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
-import io.opendatahub.datasciencecluster.v1.DataScienceClusterBuilder;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.ComponentsBuilder;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Codeflare;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.CodeflareBuilder;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Dashboard;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.DashboardBuilder;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Datasciencepipelines;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.DatasciencepipelinesBuilder;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Kserve;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.KserveBuilder;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Workbenches;
-import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.WorkbenchesBuilder;
+import io.odh.test.utils.UpgradeUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.kubeflow.v1.Notebook;
-import org.kubeflow.v1.NotebookBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Map;
 
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.core.IsNot.not;
-
 @Tag("upgrade")
-@ExtendWith(OdhResourceCleaner.class)
-@ExtendWith(ResourceManagerDeleteHandler.class)
-public class BundleUpgradeST extends Abstract {
+public class BundleUpgradeST extends UpgradeAbstract {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(BundleUpgradeST.class);
 
     BundleInstall baseBundle;
     BundleInstall upgradeBundle;
 
     @AfterAll
-    void clean() throws IOException {
+    void clean() {
         if (upgradeBundle != null) {
             upgradeBundle.deleteWithoutResourceManager();
         }
@@ -81,61 +50,8 @@ void testUpgradeBundle() throws IOException {
         String ntbName = "test-odh-notebook";
         String ntbNamespace = "test-odh-notebook-upgrade";
 
-        DataScienceCluster dsc = new DataScienceClusterBuilder()
-                .withNewMetadata()
-                .withName(dsProjectName)
-                .endMetadata()
-                .withNewSpec()
-                .withComponents(
-                        new ComponentsBuilder()
-                                .withWorkbenches(
-                                        new WorkbenchesBuilder().withManagementState(Workbenches.ManagementState.MANAGED).build()
-                                )
-                                .withDashboard(
-                                        new DashboardBuilder().withManagementState(Dashboard.ManagementState.MANAGED).build()
-                                )
-                                .withKserve(
-                                        new KserveBuilder().withManagementState(Kserve.ManagementState.REMOVED).build()
-                                )
-                                .withCodeflare(
-                                        new CodeflareBuilder().withManagementState(Codeflare.ManagementState.REMOVED).build()
-                                )
-                                .withDatasciencepipelines(
-                                        new DatasciencepipelinesBuilder().withManagementState(Datasciencepipelines.ManagementState.REMOVED).build()
-                                )
-                                .build())
-                .endSpec()
-                .build();
-        // Deploy DSC
-        ResourceManager.getInstance().createResourceWithWait(dsc);
-
-        Namespace ns = new NamespaceBuilder()
-                .withNewMetadata()
-                .withName(ntbNamespace)
-                .addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true")
-                .addToAnnotations(OdhAnnotationsLabels.ANNO_SERVICE_MESH, "false")
-                .endMetadata()
-                .build();
-        ResourceManager.getInstance().createResourceWithoutWait(ns);
-
-        PersistentVolumeClaim pvc = new PersistentVolumeClaimBuilder()
-                .withNewMetadata()
-                .withName(ntbName)
-                .withNamespace(ntbNamespace)
-                .addToLabels(OdhAnnotationsLabels.LABEL_DASHBOARD, "true")
-                .endMetadata()
-                .withNewSpec()
-                .addToAccessModes("ReadWriteOnce")
-                .withNewResources()
-                .addToRequests("storage", new Quantity("10Gi"))
-                .endResources()
-                .withVolumeMode("Filesystem")
-                .endSpec()
-                .build();
-        ResourceManager.getInstance().createResourceWithoutWait(pvc);
-
-        Notebook notebook = new NotebookBuilder(NotebookResource.loadDefaultNotebook(ntbNamespace, ntbName)).build();
-        ResourceManager.getInstance().createResourceWithoutWait(notebook);
+        deployDsc(dsProjectName);
+        deployNotebook(ntbNamespace, ntbName);
 
         LabelSelector lblSelector = new LabelSelectorBuilder()
                 .withMatchLabels(Map.of("app", ntbName))
@@ -152,12 +68,9 @@ void testUpgradeBundle() throws IOException {
         LabelSelector labelSelector = ResourceManager.getClient().getDeployment(TestConstants.ODH_NAMESPACE, OdhConstants.ODH_DASHBOARD).getSpec().getSelector();
         PodUtils.verifyThatPodsAreStable(TestConstants.ODH_NAMESPACE, labelSelector);
 
-        // Check that operator doesn't contain errors in logs
-        String operatorLog = ResourceManager.getClient().getClient().apps().deployments()
-                .inNamespace(baseBundle.getNamespace()).withName(baseBundle.getDeploymentName()).getLog();
-
-        assertThat(operatorLog, not(containsString("error")));
-        assertThat(operatorLog, not(containsString("Error")));
-        assertThat(operatorLog, not(containsString("ERROR")));
+        // Verify that NTB pods are stable
+        PodUtils.waitForPodsReady(ntbNamespace, lblSelector, 1, true, () -> { });
+        // Check logs in operator pod
+        UpgradeUtils.deploymentLogIsErrorEmpty(baseBundle.getNamespace(), baseBundle.getDeploymentName());
     }
 }