Add test for OLM upgrade

src/main/java/io/odh/test/OdhConstants.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Skodjob authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.odh.test;
+
+public class OdhConstants {
+    public static final String CODEFLARE_DEPLOYMENT_NAME = "codeflare-operator-manager";
+    public static final String DS_PIPELINES_OPERATOR = "data-science-pipelines-operator-controller-manager";
+    public static final String ETCD = "etcd";
+    public static final String KSERVE_OPERATOR = "kserve-controller-manager";
+    public static final String KUBERAY_OPERATOR = "kuberay-operator";
+    public static final String MODELMESH_OPERATOR = "modelmesh-controller";
+    public static final String NOTEBOOK_OPERATOR = "notebook-controller-deployment";
+    public static final String ODH_DASHBOARD = "odh-dashboard";
+    public static final String ODH_MODEL_OPERATOR = "odh-model-controller";
+    public static final String ODH_NOTEBOOK_OPERATOR = "odh-notebook-controller-manager";
+    public static final String TRUSTY_AI_OPERATOR = "trustyai-service-operator-controller-manager";
+
+}

src/main/java/io/odh/test/TestConstants.java

@@ -20,6 +20,7 @@ public class TestConstants {
     public static final long GLOBAL_POLL_INTERVAL_MEDIUM = Duration.ofSeconds(5).toMillis();
     public static final long GLOBAL_POLL_INTERVAL_SHORT = Duration.ofSeconds(1).toMillis();
     public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(5).toMillis();
+    public static final long GLOBAL_STABILITY_TIME = Duration.ofSeconds(10).toSeconds();
 
     private TestConstants() {
     }

src/main/java/io/odh/test/install/OlmInstall.java

@@ -34,6 +34,8 @@ public class OlmInstall {
     private String operatorVersion  = Environment.OLM_OPERATOR_VERSION;
     private String csvName = operatorName + ".v" + operatorVersion;
 
+    private String approval = "Automatic";
+
     public void create() {
         createOperatorGroup();
         ResourceManager.getInstance().pushToStack(new ResourceItem(this::deleteCSV));
@@ -43,6 +45,12 @@ public void create() {
         DeploymentUtils.waitForDeploymentReady(namespace, deploymentName);
     }
 
+    public void createManual() {
+        createOperatorGroup();
+        ResourceManager.getInstance().pushToStack(new ResourceItem(this::deleteCSV));
+        createAndModifySubscription();
+    }
+
     /**
      * Creates OperatorGroup in specific namespace
      */
@@ -89,7 +97,7 @@ public Subscription prepareSubscription() {
             .withSourceNamespace(sourceNamespace)
             .withChannel(channel)
             .withStartingCSV(startingCsv)
-            .withInstallPlanApproval("Automatic")
+            .withInstallPlanApproval(approval)
             .editOrNewConfig()
             .endConfig()
             .endSpec()
@@ -155,4 +163,12 @@ public String getCsvName() {
     public void setCsvName(String csvName) {
         this.csvName = csvName;
     }
+
+    public String getApproval() {
+        return approval;
+    }
+
+    public void setApproval(String approval) {
+        this.approval = approval;
+    }
 }

src/main/java/io/odh/test/platform/KubeClient.java

@@ -23,6 +23,8 @@
 import io.fabric8.kubernetes.client.dsl.MixedOperation;
 import io.fabric8.kubernetes.client.dsl.Resource;
 import io.fabric8.kubernetes.client.dsl.RollableScalableResource;
+import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan;
+import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlanBuilder;
 import io.fabric8.openshift.client.OpenShiftClient;
 import io.odh.test.Environment;
 import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
@@ -345,6 +347,28 @@ public String getDeploymentNameByPrefix(String namespace, String namePrefix) {
         }
     }
 
+    public InstallPlan getInstallPlan(String namespaceName, String installPlanName) {
+        return client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).get();
+    }
+
+    public void approveInstallPlan(String namespaceName, String installPlanName) {
+        InstallPlan installPlan = new InstallPlanBuilder(this.getInstallPlan(namespaceName, installPlanName))
+            .editSpec()
+            .withApproved()
+            .endSpec()
+            .build();
+
+        LOGGER.debug("Approving {}", installPlanName);
+        client.adapt(OpenShiftClient.class).operatorHub().installPlans().inNamespace(namespaceName).withName(installPlanName).patch(installPlan);
+    }
+
+    public InstallPlan getNonApprovedInstallPlan(String namespaceName, String csvPrefix) {
+        return client.adapt(OpenShiftClient.class).operatorHub().installPlans()
+                .inNamespace(namespaceName).list().getItems().stream()
+                .filter(installPlan ->  !installPlan.getSpec().getApproved() && installPlan.getSpec().getClusterServiceVersionNames().toString().contains(csvPrefix))
+                .findFirst().get();
+    }
+
     public MixedOperation<DataScienceCluster, KubernetesResourceList<DataScienceCluster>, Resource<DataScienceCluster>> dataScienceClusterClient() {
         return client.resources(DataScienceCluster.class);
     }

src/main/java/io/odh/test/platform/KubeUtils.java

@@ -4,11 +4,14 @@
  */
 package io.odh.test.platform;
 
+import io.odh.test.TestConstants;
+import io.odh.test.TestUtils;
 import io.odh.test.framework.manager.ResourceManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.List;
+import java.util.NoSuchElementException;
 
 public class KubeUtils {
 
@@ -39,6 +42,18 @@ public static void deleteDefaultDSCI() {
         ResourceManager.getKubeCmdClient().exec(false, "delete", "dsci", "--all");
     }
 
+    public static void waitForInstallPlan(String namespace, String csvName) {
+        TestUtils.waitFor("Install paln with new version", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT, () -> {
+            try {
+                ResourceManager.getClient().getNonApprovedInstallPlan(namespace, csvName);
+                return true;
+            } catch (NoSuchElementException ex) {
+                LOGGER.debug("No new install plan available. Checking again ...");
+                return false;
+            }
+        }, () -> { });
+    }
+
     private KubeUtils() {
     }
 }

src/main/java/io/odh/test/utils/DeploymentUtils.java

@@ -4,6 +4,7 @@
  */
 package io.odh.test.utils;
 
+import io.fabric8.kubernetes.api.model.LabelSelector;
 import io.fabric8.kubernetes.api.model.Pod;
 import io.fabric8.kubernetes.api.model.PodCondition;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
@@ -16,6 +17,8 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 
 import static java.util.Arrays.asList;
 
@@ -94,4 +97,46 @@ public static void waitForDeploymentDeletion(String namespaceName, String name)
             });
         LOGGER.debug("Deployment: {}/{} was deleted", namespaceName, name);
     }
+
+    /**
+     * Returns a map of pod name to resource version for the Pods currently in the given deployment.
+     * @param name The Deployment name.
+     * @return A map of pod name to resource version for Pods in the given Deployment.
+     */
+    public static Map<String, String> depSnapshot(String namespaceName, String name) {
+        Deployment deployment = ResourceManager.getClient().getDeployment(namespaceName, name);
+        LabelSelector selector = deployment.getSpec().getSelector();
+        return PodUtils.podSnapshot(namespaceName, selector);
+    }
+
+    /**
+     * Method to check that all Pods for expected Deployment were rolled
+     * @param namespaceName Namespace name
+     * @param name Deployment name
+     * @param snapshot Snapshot of Pods for Deployment before the rolling update
+     * @return true when the Pods for Deployment are recreated
+     */
+    public static boolean depHasRolled(String namespaceName, String name, Map<String, String> snapshot) {
+        LOGGER.debug("Existing snapshot: {}/{}", namespaceName, new TreeMap<>(snapshot));
+        Map<String, String> map = PodUtils.podSnapshot(namespaceName, ResourceManager.getClient().getDeployment(namespaceName, name).getSpec().getSelector());
+        LOGGER.debug("Current  snapshot: {}/{}", namespaceName, new TreeMap<>(map));
+        int current = map.size();
+        map.keySet().retainAll(snapshot.keySet());
+        if (current == snapshot.size() && map.isEmpty()) {
+            LOGGER.debug("All Pods seem to have rolled");
+            return true;
+        } else {
+            LOGGER.debug("Some Pods still need to roll: {}/{}", namespaceName, map);
+            return false;
+        }
+    }
+
+    public static Map<String, String> waitTillDepHasRolled(String namespaceName, String deploymentName, Map<String, String> snapshot) {
+        LOGGER.info("Waiting for Deployment: {}/{} rolling update", namespaceName, deploymentName);
+        TestUtils.waitFor("rolling update of Deployment " + namespaceName + "/" + deploymentName,
+                TestConstants.GLOBAL_POLL_INTERVAL_MEDIUM, TestConstants.GLOBAL_TIMEOUT,
+                () -> depHasRolled(namespaceName, deploymentName, snapshot));
+
+        return depSnapshot(namespaceName, deploymentName);
+    }
 }

src/main/java/io/odh/test/utils/PodUtils.java

@@ -16,6 +16,8 @@
 
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class PodUtils {
 
@@ -61,4 +63,48 @@ public static void waitForPodsReady(String namespaceName, LabelSelector selector
                 return true;
             }, onTimeout);
     }
+
+    /**
+     * Returns a map of resource name to resource version for all the pods in the given {@code namespace}
+     * matching the given {@code selector}.
+     */
+    public static Map<String, String> podSnapshot(String namespaceName, LabelSelector selector) {
+        List<Pod> pods = ResourceManager.getClient().listPods(namespaceName, selector);
+        return pods.stream()
+                .collect(
+                        Collectors.toMap(pod -> pod.getMetadata().getName(),
+                                pod -> pod.getMetadata().getUid()));
+    }
+
+    public static void verifyThatPodsAreStable(String namespaceName, LabelSelector labelSelector) {
+        int[] stabilityCounter = {0};
+        String phase = "Running";
+
+        List<Pod> runningPods = ResourceManager.getClient().listPods(namespaceName, labelSelector);
+
+        TestUtils.waitFor(String.format("Pods in namespace '%s' with LabelSelector %s stability in phase %s", namespaceName, labelSelector, phase), TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT,
+                () -> {
+                    List<Pod> actualPods = runningPods.stream().map(p -> ResourceManager.getClient().getPod(namespaceName, p.getMetadata().getName())).toList();
+
+                    for (Pod pod : actualPods) {
+                        if (pod.getStatus().getPhase().equals(phase)) {
+                            LOGGER.info("Pod: {}/{} is in the {} state. Remaining seconds Pod to be stable {}",
+                                    namespaceName, pod.getMetadata().getName(), pod.getStatus().getPhase(),
+                                    TestConstants.GLOBAL_STABILITY_TIME - stabilityCounter[0]);
+                        } else {
+                            LOGGER.info("Pod: {}/{} is not stable in phase following phase {} reset the stability counter from {} to {}",
+                                    namespaceName, pod.getMetadata().getName(), pod.getStatus().getPhase(), stabilityCounter[0], 0);
+                            stabilityCounter[0] = 0;
+                            return false;
+                        }
+                    }
+                    stabilityCounter[0]++;
+
+                    if (stabilityCounter[0] == TestConstants.GLOBAL_STABILITY_TIME) {
+                        LOGGER.info("All Pods are stable {}", actualPods.stream().map(p -> p.getMetadata().getName()).collect(Collectors.joining(" ,")));
+                        return true;
+                    }
+                    return false;
+                });
+    }
 }

src/test/java/io/odh/test/e2e/standard/StandardAbstract.java

@@ -26,7 +26,7 @@
 @ExtendWith(ResourceManagerDeleteHandler.class)
 public class StandardAbstract extends Abstract {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(Abstract.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(StandardAbstract.class);
 
     @BeforeAll
     void setupEnvironment() throws IOException {

src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright Skodjob authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.odh.test.e2e.upgrade;
+
+import io.fabric8.kubernetes.api.model.LabelSelector;
+import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan;
+import io.odh.test.OdhConstants;
+import io.odh.test.TestConstants;
+import io.odh.test.e2e.Abstract;
+import io.odh.test.framework.listeners.OdhResourceCleaner;
+import io.odh.test.framework.listeners.ResourceManagerDeleteHandler;
+import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.install.OlmInstall;
+import io.odh.test.platform.KubeUtils;
+import io.odh.test.utils.DeploymentUtils;
+import io.odh.test.utils.PodUtils;
+import io.opendatahub.datasciencecluster.v1.DataScienceCluster;
+import io.opendatahub.datasciencecluster.v1.DataScienceClusterBuilder;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.ComponentsBuilder;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Codeflare;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.CodeflareBuilder;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Dashboard;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.DashboardBuilder;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Datasciencepipelines;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.DatasciencepipelinesBuilder;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Kserve;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.KserveBuilder;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Modelmeshserving;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.ModelmeshservingBuilder;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.Workbenches;
+import io.opendatahub.datasciencecluster.v1.datascienceclusterspec.components.WorkbenchesBuilder;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.core.IsNot.not;
+
+import java.util.Map;
+
+@Tag("upgrade")
+@ExtendWith(OdhResourceCleaner.class)
+@ExtendWith(ResourceManagerDeleteHandler.class)
+public class OlmUpgradeST extends Abstract {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(OlmUpgradeST.class);
+    private static final String DS_PROJECT_NAME = "upgrade-dsc";
+
+    private final String startingVersion = "2.3.0";
+
+    @Test
+    void testUpgradeOlm() {
+        OlmInstall olmInstall = new OlmInstall();
+        olmInstall.setApproval("Manual");
+        olmInstall.setStartingCsv(olmInstall.getOperatorName() + ".v" + startingVersion);
+        olmInstall.createManual();
+
+        // Approve install plan created for older version
+        InstallPlan ip = ResourceManager.getClient().getNonApprovedInstallPlan(olmInstall.getNamespace(), olmInstall.getOperatorName());
+        ResourceManager.getClient().approveInstallPlan(olmInstall.getNamespace(), ip.getMetadata().getName());
+        // Wait for old version readiness
+        DeploymentUtils.waitForDeploymentReady(olmInstall.getNamespace(), olmInstall.getDeploymentName());
+
+        // Make snapshot of current operator
+        Map<String, String> operatorSnapshot = DeploymentUtils.depSnapshot(olmInstall.getNamespace(), olmInstall.getDeploymentName());
+
+        // Deploy DSC
+        DataScienceCluster dsc = new DataScienceClusterBuilder()
+            .withNewMetadata()
+            .withName(DS_PROJECT_NAME)
+            .endMetadata()
+            .withNewSpec()
+            .withComponents(
+                new ComponentsBuilder()
+                    .withWorkbenches(
+                            new WorkbenchesBuilder().withManagementState(Workbenches.ManagementState.MANAGED).build()
+                    )
+                    .withDashboard(
+                            new DashboardBuilder().withManagementState(Dashboard.ManagementState.MANAGED).build()
+                    )
+                    .withKserve(
+                            new KserveBuilder().withManagementState(Kserve.ManagementState.REMOVED).build()
+                    )
+                    .withCodeflare(
+                            new CodeflareBuilder().withManagementState(Codeflare.ManagementState.MANAGED).build()
+                    )
+                    .withDatasciencepipelines(
+                            new DatasciencepipelinesBuilder().withManagementState(Datasciencepipelines.ManagementState.MANAGED).build()
+                    )
+                    .withModelmeshserving(
+                            new ModelmeshservingBuilder().withManagementState(Modelmeshserving.ManagementState.REMOVED).build()
+                    )
+                    .build())
+            .endSpec()
+            .build();
+        // Deploy DSC
+        ResourceManager.getInstance().createResourceWithWait(dsc);
+
+        // Approve upgrade to newer version
+        KubeUtils.waitForInstallPlan(olmInstall.getNamespace(), olmInstall.getCsvName());
+
+        ip = ResourceManager.getClient().getNonApprovedInstallPlan(olmInstall.getNamespace(), olmInstall.getCsvName());
+        ResourceManager.getClient().approveInstallPlan(olmInstall.getNamespace(), ip.getMetadata().getName());
+        // Wait for operator RU
+        DeploymentUtils.waitTillDepHasRolled(olmInstall.getNamespace(), olmInstall.getDeploymentName(), operatorSnapshot);
+
+        // Wait for pod stability for Dashboard
+        LabelSelector labelSelector = ResourceManager.getClient().getDeployment(TestConstants.ODH_NAMESPACE, OdhConstants.ODH_DASHBOARD).getSpec().getSelector();
+        PodUtils.verifyThatPodsAreStable(TestConstants.ODH_NAMESPACE, labelSelector);
+
+        // Check that operator doesn't contains errors in logs
+        String operatorLog = ResourceManager.getClient().getClient().apps().deployments()
+                .inNamespace(olmInstall.getNamespace()).withName(olmInstall.getDeploymentName()).getLog();
+
+        assertThat(operatorLog, not(containsString("error")));
+        assertThat(operatorLog, not(containsString("Error")));
+        assertThat(operatorLog, not(containsString("ERROR")));
+    }
+}