Improve upgrade checks

src/main/java/io/odh/test/TestConstants.java

@@ -8,6 +8,7 @@
 
 public class TestConstants {
     public static final String ODH_NAMESPACE = "opendatahub";
+    public static final String ODH_DASHBOARD_ROUTE_NAME = "odh-dashboard";
     public static final String DEFAULT_NAMESPACE = "default";
 
     public static final String SUBSCRIPTION = "Subscription";
@@ -20,7 +21,7 @@ public class TestConstants {
     public static final long GLOBAL_POLL_INTERVAL_MEDIUM = Duration.ofSeconds(5).toMillis();
     public static final long GLOBAL_POLL_INTERVAL_SHORT = Duration.ofSeconds(1).toMillis();
     public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(5).toMillis();
-    public static final long GLOBAL_STABILITY_TIME = Duration.ofSeconds(10).toSeconds();
+    public static final long GLOBAL_STABILITY_TIME = Duration.ofMinutes(1).toSeconds();
 
     private TestConstants() {
     }

src/main/java/io/odh/test/framework/manager/resources/NotebookResource.java

@@ -7,6 +7,8 @@
 import io.fabric8.kubernetes.api.model.KubernetesResourceList;
 import io.fabric8.kubernetes.client.dsl.MixedOperation;
 import io.fabric8.kubernetes.client.dsl.Resource;
+import io.fabric8.openshift.client.OpenShiftClient;
+import io.odh.test.TestConstants;
 import io.odh.test.TestUtils;
 import io.odh.test.framework.manager.ResourceManager;
 import io.odh.test.framework.manager.ResourceType;
@@ -58,6 +60,13 @@ public static Notebook loadDefaultNotebook(String namespace, String name) throws
         InputStream is = TestUtils.getFileFromResourceAsStream(NOTEBOOK_TEMPLATE_PATH);
         String notebookString = IOUtils.toString(is, "UTF-8");
         notebookString = notebookString.replace("my-project", namespace).replace("my-workbench", name);
+        // Set new Route url
+        String routeHost = ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).routes().inNamespace(TestConstants.ODH_NAMESPACE).withName(TestConstants.ODH_DASHBOARD_ROUTE_NAME).get().getSpec().getHost();
+        notebookString = notebookString.replace("odh_dashboard_route", "https://" + routeHost);
+        // Set correct username
+        String username = ResourceManager.getKubeCmdClient().getUsername().strip();
+        notebookString = notebookString.replace("odh_user", username);
+
         return TestUtils.configFromYaml(notebookString, Notebook.class);
     }
 }

src/main/java/io/odh/test/install/BundleInstall.java

@@ -72,11 +72,11 @@ public void create() {
         ResourceManager.getInstance().pushToStack(new ResourceItem(KubeUtils::deleteDefaultDSCI, null));
     }
 
-    public void createWithoutResourceManager() throws IOException {
+    public void createWithoutResourceManager() {
         ResourceManager.getKubeCmdClient().namespace(getNamespace()).apply(installFile.getAbsolutePath());
     }
 
-    public void deleteWithoutResourceManager() throws IOException {
+    public void deleteWithoutResourceManager() {
         KubeUtils.deleteDefaultDSCI();
         ResourceManager.getKubeCmdClient().namespace(getNamespace()).delete(installFile.getAbsolutePath());
     }

src/main/java/io/odh/test/install/OlmInstall.java

@@ -105,8 +105,22 @@ public Subscription prepareSubscription() {
     }
 
     public void deleteCSV() {
-        LOGGER.info("Deleting CSV {}/{}", namespace, olmAppBundlePrefix);
-        ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().clusterServiceVersions().inNamespace(namespace).withName(csvName).delete();
+        ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().clusterServiceVersions().inNamespace(namespace)
+            .list().getItems().stream().filter(csv -> csv.getMetadata().getName().contains(olmAppBundlePrefix)).toList()
+            .forEach(csv -> {
+                LOGGER.info("Deleting CSV {}", csv.getMetadata().getName());
+                ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().clusterServiceVersions().resource(csv).delete();
+            });
+        deleteInstallPlans();
+    }
+
+    public void deleteInstallPlans() {
+        ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespace)
+            .list().getItems().stream().filter(ip -> ip.getSpec().getClusterServiceVersionNames().stream().toList().toString().contains(olmAppBundlePrefix)).toList()
+            .forEach(ip -> {
+                LOGGER.info("Deleting InstallPlan {}", ip.getMetadata().getName());
+                ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().installPlans().resource(ip).delete();
+            });
     }
 
     /**

src/main/java/io/odh/test/platform/KubeClient.java

@@ -27,10 +27,10 @@
 import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlanBuilder;
 import io.fabric8.openshift.client.OpenShiftClient;
 import io.odh.test.Environment;
+import io.odh.test.TestConstants;
+import io.odh.test.TestUtils;
 import io.odh.test.platform.executor.Exec;
-import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
 import io.opendatahub.v1alpha.OdhDashboardConfig;
-import org.kubeflow.v1.Notebook;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -369,15 +369,23 @@ public InstallPlan getInstallPlan(String namespaceName, String installPlanName)
         return client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).get();
     }
 
-    public void approveInstallPlan(String namespaceName, String installPlanName) {
-        InstallPlan installPlan = new InstallPlanBuilder(this.getInstallPlan(namespaceName, installPlanName))
-            .editSpec()
-            .withApproved()
-            .endSpec()
-            .build();
+    public void approveInstallPlan(String namespaceName, String installPlanName) throws InterruptedException {
+        LOGGER.debug("Approving InstallPlan {}", installPlanName);
+        TestUtils.waitFor("InstallPlan approval", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, 15_000, () -> {
+            try {
+                InstallPlan installPlan = new InstallPlanBuilder(this.getInstallPlan(namespaceName, installPlanName))
+                        .editSpec()
+                        .withApproved()
+                        .endSpec()
+                        .build();
 
-        LOGGER.debug("Approving {}", installPlanName);
-        client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).patch(installPlan);
+                client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).patch(installPlan);
+                return true;
+            } catch (Exception ex) {
+                LOGGER.error(String.valueOf(ex));
+                return false;
+            }
+        });
     }
 
     public InstallPlan getNonApprovedInstallPlan(String namespaceName, String csvPrefix) {
@@ -387,15 +395,7 @@ public InstallPlan getNonApprovedInstallPlan(String namespaceName, String csvPre
                 .findFirst().get();
     }
 
-    public MixedOperation<DataScienceCluster, KubernetesResourceList<DataScienceCluster>, Resource<DataScienceCluster>> dataScienceClusterClient() {
-        return client.resources(DataScienceCluster.class);
-    }
-
     public MixedOperation<OdhDashboardConfig, KubernetesResourceList<OdhDashboardConfig>, Resource<OdhDashboardConfig>> dashboardConfigClient() {
         return client.resources(OdhDashboardConfig.class);
     }
-
-    public MixedOperation<Notebook, KubernetesResourceList<Notebook>, Resource<Notebook>> notebookClient() {
-        return client.resources(Notebook.class);
-    }
 }

src/main/java/io/odh/test/platform/KubeUtils.java

@@ -4,6 +4,7 @@
  */
 package io.odh.test.platform;
 
+import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan;
 import io.odh.test.TestConstants;
 import io.odh.test.TestUtils;
 import io.odh.test.framework.manager.ResourceManager;
@@ -45,7 +46,8 @@ public static void deleteDefaultDSCI() {
     public static void waitForInstallPlan(String namespace, String csvName) {
         TestUtils.waitFor("Install paln with new version", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT, () -> {
             try {
-                ResourceManager.getClient().getNonApprovedInstallPlan(namespace, csvName);
+                InstallPlan ip = ResourceManager.getClient().getNonApprovedInstallPlan(namespace, csvName);
+                LOGGER.debug("Found InstallPlan {} - {}", ip.getMetadata().getName(), ip.getSpec().getClusterServiceVersionNames());
                 return true;
             } catch (NoSuchElementException ex) {
                 LOGGER.debug("No new install plan available. Checking again ...");

src/main/java/io/odh/test/platform/cmdClient/BaseCmdKubeClient.java

@@ -397,5 +397,4 @@ public K process(Map<String, String> parameters, String file, Consumer<String> c
         c.accept(exec.out());
         return (K) this;
     }
-
 }

src/main/java/io/odh/test/platform/cmdClient/KubeCmdClient.java

@@ -202,4 +202,6 @@ default String logs(String pod) {
     String cmd();
 
     K process(Map<String, String> domain, String file, Consumer<String> c);
+
+    String getUsername();
 }

src/main/java/io/odh/test/platform/cmdClient/Kubectl.java

@@ -48,4 +48,10 @@ public String defaultOlmNamespace() {
     public String cmd() {
         return KUBECTL;
     }
+
+    @Override
+    public String getUsername() {
+        // TODO - implement this!
+        return null;
+    }
 }

src/main/java/io/odh/test/platform/cmdClient/Oc.java

@@ -72,4 +72,9 @@ public Oc newApp(String template, Map<String, String> params) {
     public String cmd() {
         return OC;
     }
+
+    @Override
+    public String getUsername() {
+        return Exec.exec(cmd(), "whoami").out();
+    }
 }

src/main/java/io/odh/test/utils/PodUtils.java

@@ -80,20 +80,23 @@ public static void verifyThatPodsAreStable(String namespaceName, LabelSelector l
         int[] stabilityCounter = {0};
         String phase = "Running";
 
-        List<Pod> runningPods = ResourceManager.getClient().listPods(namespaceName, labelSelector);
-
         TestUtils.waitFor(String.format("Pods in namespace '%s' with LabelSelector %s stability in phase %s", namespaceName, labelSelector, phase), TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT,
                 () -> {
+                    List<Pod> runningPods = ResourceManager.getClient().listPods(namespaceName, labelSelector);
                     List<Pod> actualPods = runningPods.stream().map(p -> ResourceManager.getClient().getPod(namespaceName, p.getMetadata().getName())).toList();
+                    LOGGER.debug("Working with the following pods: {}", actualPods.stream().map(p -> p.getMetadata().getName()).toList());
 
                     for (Pod pod : actualPods) {
+                        if (pod == null) {
+                            continue;
+                        }
                         if (pod.getStatus().getPhase().equals(phase)) {
-                            LOGGER.info("Pod: {}/{} is in the {} state. Remaining seconds Pod to be stable {}",
+                            LOGGER.debug("Pod: {}/{} is in the {} state. Remaining seconds for Pod to be stable {}",
                                     namespaceName, pod.getMetadata().getName(), pod.getStatus().getPhase(),
                                     TestConstants.GLOBAL_STABILITY_TIME - stabilityCounter[0]);
                         } else {
-                            LOGGER.info("Pod: {}/{} is not stable in phase following phase {} reset the stability counter from {} to {}",
-                                    namespaceName, pod.getMetadata().getName(), pod.getStatus().getPhase(), stabilityCounter[0], 0);
+                            LOGGER.warn("Pod: {}/{} is not stable in phase following phase {} ({}) reset the stability counter from {}s to {}s",
+                                    namespaceName, pod.getMetadata().getName(), pod.getStatus().getPhase(), phase, stabilityCounter[0], 0);
                             stabilityCounter[0] = 0;
                             return false;
                         }

src/main/java/io/odh/test/utils/UpgradeUtils.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Skodjob authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.odh.test.utils;
+
+import io.odh.test.framework.manager.ResourceManager;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.core.IsNot.not;
+
+public class UpgradeUtils {
+
+    public static void deploymentLogIsErrorEmpty(String namespace, String deploymentName) {
+        // Check that operator doesn't contains errors in logs
+        String operatorLog = ResourceManager.getClient().getClient().apps().deployments()
+                .inNamespace(namespace).withName(deploymentName).getLog();
+
+        assertThat(operatorLog, not(containsString("error")));
+        assertThat(operatorLog, not(containsString("Error")));
+        assertThat(operatorLog, not(containsString("ERROR")));
+    }
+}

src/main/resources/notebook.yaml

@@ -25,7 +25,7 @@ spec:
                                   --ServerApp.password=''
                                   --ServerApp.base_url=/notebook/my-project/my-workbench
                                   --ServerApp.quit_button=False
-                                  --ServerApp.tornado_settings={"user":"","hub_host":"odh_dashboard_route","hub_prefix":"/projects/my-project"}
+                                  --ServerApp.tornado_settings={"user":"odh_user","hub_host":"odh_dashboard_route","hub_prefix":"/projects/my-project"}
             - name: JUPYTER_IMAGE
               value: image-registry.openshift-image-registry.svc:5000/opendatahub/jupyter-pytorch-notebook:2023.2
           image: image-registry.openshift-image-registry.svc:5000/opendatahub/jupyter-pytorch-notebook:2023.2

src/test/java/io/odh/test/e2e/continuous/DataScienceClusterST.java

@@ -10,6 +10,7 @@
 import io.odh.test.TestConstants;
 import io.odh.test.e2e.Abstract;
 import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.framework.manager.resources.DataScienceClusterResource;
 import io.odh.test.platform.KubeUtils;
 import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
 import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Codeflare;
@@ -40,7 +41,7 @@ public class DataScienceClusterST extends Abstract {
 
     @BeforeAll
     void init() {
-        dataScienceProjectCli = ResourceManager.getClient().dataScienceClusterClient();
+        dataScienceProjectCli = DataScienceClusterResource.dataScienceCLusterClient();
         dashboardConfigCli  = ResourceManager.getClient().dashboardConfigClient();
     }
 

src/test/java/io/odh/test/e2e/continuous/DataScienceProjectST.java

@@ -9,6 +9,7 @@
 import io.fabric8.kubernetes.client.dsl.Resource;
 import io.odh.test.e2e.Abstract;
 import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.framework.manager.resources.NotebookResource;
 import io.odh.test.platform.KubeUtils;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
@@ -40,7 +41,7 @@ private static Stream<Arguments> getDsProjects() {
 
     @BeforeAll
     void init() {
-        notebookCli = ResourceManager.getClient().notebookClient();
+        notebookCli = NotebookResource.notebookClient();
     }
 
     @ParameterizedTest(name = "checkDataScienceProjects-{0}")

src/test/java/io/odh/test/e2e/standard/DataScienceClusterST.java

@@ -5,6 +5,7 @@
 package io.odh.test.e2e.standard;
 
 import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.framework.manager.resources.DataScienceClusterResource;
 import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
 import io.opendatahub.datasciencecluster.v1.DataScienceClusterBuilder;
 import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.ComponentsBuilder;
@@ -58,7 +59,7 @@ void createDataScienceCluster() {
 
         ResourceManager.getInstance().createResourceWithWait(c);
 
-        DataScienceCluster cluster = ResourceManager.getClient().dataScienceClusterClient().withName(DS_PROJECT_NAME).get();
+        DataScienceCluster cluster = DataScienceClusterResource.dataScienceCLusterClient().withName(DS_PROJECT_NAME).get();
 
         assertEquals(Kserve.ManagementState.MANAGED, cluster.getSpec().getComponents().getKserve().getManagementState());
         assertEquals(Codeflare.ManagementState.MANAGED, cluster.getSpec().getComponents().getCodeflare().getManagementState());