Add option to specify starting version for OLM upgrades

README.md

@@ -11,6 +11,8 @@ E2E test suite for opendatahub using fabric8 kubernetes client
 ## List of test suites
 * smoke
 * upgrade
+* olm-upgrade
+* bundle-upgrade
 * standard
 * continuous
 * all (standard + upgrade)
@@ -57,4 +59,4 @@ GITHUB_TOKEN="your_github_read_token" mvn verify -Pupgrade
 ### RUN Unit test of the suite
 ```commandline
 GITHUB_TOKEN="your_github_read_token" mvn test
-```
+```

pom.xml

@@ -343,6 +343,26 @@
             </properties>
         </profile>
 
+        <profile>
+            <id>bundle-upgrade</id>
+            <properties>
+                <it.skip>false</it.skip>
+                <groups>
+                    bundle-upgrade
+                </groups>
+            </properties>
+        </profile>
+
+        <profile>
+            <id>olm-upgrade</id>
+            <properties>
+                <it.skip>false</it.skip>
+                <groups>
+                    olm-upgrade
+                </groups>
+            </properties>
+        </profile>
+
         <profile>
             <id>all</id>
             <properties>

src/main/java/io/odh/test/Environment.java

@@ -61,6 +61,7 @@ public class Environment {
     private static final String OLM_OPERATOR_VERSION_ENV = "OLM_OPERATOR_VERSION";
     private static final String OLM_OPERATOR_CHANNEL_ENV = "OLM_OPERATOR_CHANNEL";
     private static final String OPERATOR_INSTALL_TYPE_ENV = "OPERATOR_INSTALL_TYPE";
+    private static final String OLM_UPGRADE_STARTING_VERSION_ENV = "OLM_UPGRADE_STARTING_VERSION";
 
     public static final String PRODUCT_DEFAULT = "odh";
 
@@ -87,6 +88,7 @@ public class Environment {
     public static final String OLM_SOURCE_NAMESPACE = getOrDefault(OLM_SOURCE_NAMESPACE_ENV, "openshift-marketplace");
     public static final String OLM_OPERATOR_CHANNEL = getOrDefault(OLM_OPERATOR_CHANNEL_ENV, OdhConstants.OLM_OPERATOR_CHANNEL);
     public static final String OLM_OPERATOR_VERSION = getOrDefault(OLM_OPERATOR_VERSION_ENV, OdhConstants.OLM_OPERATOR_VERSION);
+    public static final String OLM_UPGRADE_STARTING_VERSION = getOrDefault(OLM_UPGRADE_STARTING_VERSION_ENV, OdhConstants.OLM_UPGRADE_STARTING_OPERATOR_VERSION);
 
     public static final String OPERATOR_INSTALL_TYPE = getOrDefault(OPERATOR_INSTALL_TYPE_ENV, InstallTypes.BUNDLE.toString());
 

src/main/java/io/odh/test/OdhConstants.java

@@ -4,11 +4,18 @@
  */
 package io.odh.test;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 
 public class OdhConstants {
     private OdhConstants() { }
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(OdhConstants.class);
+    private static final Map<String, String> VALUES = new HashMap<>();
     // ODH
     private static final String ODH_CONTROLLERS_NAMESPACE = "opendatahub";
     private static final String ODH_DASHBOARD_ROUTE_NAME = "odh-dashboard";
@@ -23,7 +30,9 @@ private OdhConstants() { }
     private static final String ODH_OLM_SOURCE_NAME = "community-operators";
     private static final String ODH_OLM_APP_BUNDLE_PREFIX = "opendatahub-operator";
     private static final String ODH_OLM_OPERATOR_CHANNEL = "fast";
+    // TODO - should be changed after 2.5 release
     private static final String ODH_OLM_OPERATOR_VERSION = "v2.4.0";
+    private static final String ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION = "v2.4.0";
 
     // RHOAI
     private static final String RHOAI_CONTROLLERS_NAMESPACE = "redhat-ods-applications";
@@ -37,6 +46,7 @@ private OdhConstants() { }
     private static final String RHOAI_OLM_APP_BUNDLE_PREFIX = "rhods-operator";
     private static final String RHOAI_OLM_OPERATOR_CHANNEL = "stable";
     private static final String RHOAI_OLM_OPERATOR_VERSION = "2.5.0";
+    private static final String RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION = "2.4.0";
 
     // Public part
     public static final String CODEFLARE_DEPLOYMENT_NAME = "codeflare-operator-manager";
@@ -50,24 +60,38 @@ private OdhConstants() { }
     public static final String ODH_NOTEBOOK_OPERATOR = "odh-notebook-controller-manager";
     public static final String TRUSTY_AI_OPERATOR = "trustyai-service-operator-controller-manager";
 
-    public static final String CONTROLLERS_NAMESPACE = getOdhOrRhoai(ODH_CONTROLLERS_NAMESPACE, RHOAI_CONTROLLERS_NAMESPACE);
-    public static final String DASHBOARD_ROUTE_NAME = getOdhOrRhoai(ODH_DASHBOARD_ROUTE_NAME, RHOAI_DASHBOARD_ROUTE_NAME);
-    public static final String DASHBOARD_CONTROLLER = getOdhOrRhoai(ODH_DASHBOARD_CONTROLLER, RHOAI_DASHBOARD_CONTROLLER);
-    public static final String BUNDLE_OPERATOR_NAMESPACE = getOdhOrRhoai(ODH_BUNDLE_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
+    public static final String CONTROLLERS_NAMESPACE = getOdhOrRhoai("CONTROLLERS_NAMESPACE", ODH_CONTROLLERS_NAMESPACE, RHOAI_CONTROLLERS_NAMESPACE);
+    public static final String DASHBOARD_ROUTE_NAME = getOdhOrRhoai("DASHBOARD_ROUTE_NAME", ODH_DASHBOARD_ROUTE_NAME, RHOAI_DASHBOARD_ROUTE_NAME);
+    public static final String DASHBOARD_CONTROLLER = getOdhOrRhoai("DASHBOARD_CONTROLLER", ODH_DASHBOARD_CONTROLLER, RHOAI_DASHBOARD_CONTROLLER);
+    public static final String BUNDLE_OPERATOR_NAMESPACE = getOdhOrRhoai("BUNDLE_OPERATOR_NAMESPACE", ODH_BUNDLE_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
     // OLM env variables
-    public static final String OLM_OPERATOR_NAME = getOdhOrRhoai(ODH_OLM_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
-    public static final String OLM_OPERATOR_NAMESPACE = getOdhOrRhoai(ODH_OLM_OPERATOR_NAMESPACE, RHOAI_OLM_OPERATOR_NAMESPACE);
-    public static final String OLM_OPERATOR_DEPLOYMENT_NAME = getOdhOrRhoai(ODH_OLM_OPERATOR_DEPLOYMENT_NAME, RHOAI_OLM_OPERATOR_DEPLOYMENT_NAME);
-    public static final String OLM_APP_BUNDLE_PREFIX = getOdhOrRhoai(ODH_OLM_APP_BUNDLE_PREFIX, RHOAI_OLM_APP_BUNDLE_PREFIX);
-    public static final String OLM_OPERATOR_VERSION = getOdhOrRhoai(ODH_OLM_OPERATOR_VERSION, RHOAI_OLM_OPERATOR_VERSION);
-    public static final String OLM_SOURCE_NAME = getOdhOrRhoai(ODH_OLM_SOURCE_NAME, RHOAI_OLM_SOURCE_NAME);
-    public static final String OLM_OPERATOR_CHANNEL = getOdhOrRhoai(ODH_OLM_OPERATOR_CHANNEL, RHOAI_OLM_OPERATOR_CHANNEL);
+    public static final String OLM_OPERATOR_NAME = getOdhOrRhoai("OLM_OPERATOR_NAME", ODH_OLM_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
+    public static final String OLM_OPERATOR_NAMESPACE = getOdhOrRhoai("OLM_OPERATOR_NAMESPACE", ODH_OLM_OPERATOR_NAMESPACE, RHOAI_OLM_OPERATOR_NAMESPACE);
+    public static final String OLM_OPERATOR_DEPLOYMENT_NAME = getOdhOrRhoai("OLM_OPERATOR_DEPLOYMENT_NAME", ODH_OLM_OPERATOR_DEPLOYMENT_NAME, RHOAI_OLM_OPERATOR_DEPLOYMENT_NAME);
+    public static final String OLM_APP_BUNDLE_PREFIX = getOdhOrRhoai("OLM_APP_BUNDLE_PREFIX", ODH_OLM_APP_BUNDLE_PREFIX, RHOAI_OLM_APP_BUNDLE_PREFIX);
+    public static final String OLM_OPERATOR_VERSION = getOdhOrRhoai("OLM_OPERATOR_VERSION", ODH_OLM_OPERATOR_VERSION, RHOAI_OLM_OPERATOR_VERSION);
+    public static final String OLM_SOURCE_NAME = getOdhOrRhoai("OLM_SOURCE_NAME", ODH_OLM_SOURCE_NAME, RHOAI_OLM_SOURCE_NAME);
+    public static final String OLM_OPERATOR_CHANNEL = getOdhOrRhoai("OLM_OPERATOR_CHANNEL", ODH_OLM_OPERATOR_CHANNEL, RHOAI_OLM_OPERATOR_CHANNEL);
+    public static final String OLM_UPGRADE_STARTING_OPERATOR_VERSION = getOdhOrRhoai("OLM_UPGRADE_STARTING_OPERATOR_VERSION", ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION, RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION);
 
-    private static <T> T getOdhOrRhoai(T odhValue, T rhoaiValue) {
+    private static <T> T getOdhOrRhoai(String var, T odhValue, T rhoaiValue) {
         T returnValue = odhValue;
         if (!Objects.equals(Environment.PRODUCT, Environment.PRODUCT_DEFAULT)) {
             returnValue = rhoaiValue;
         }
+        VALUES.put(var, String.valueOf(returnValue));
         return returnValue;
     }
+
+    static {
+        String debugFormat = "{}: {}";
+        LOGGER.info("Used OdhConstants:");
+        VALUES.entrySet().stream()
+                .sorted(Map.Entry.comparingByKey())
+                .forEach(entry -> {
+                    if (!Objects.equals(entry.getValue(), "null")) {
+                        LOGGER.info(debugFormat, entry.getKey(), entry.getValue());
+                    }
+                });
+    }
 }

src/main/java/io/odh/test/TestSuite.java

@@ -7,6 +7,8 @@
 public class TestSuite {
     public static final String SMOKE = "smoke";
     public static final String UPGRADE = "upgrade";
+    public static final String OLM_UPGRADE = "olm-upgrade";
+    public static final String BUNDLE_UPGRADE = "bundle-upgrade";
     public static final String STANDARD = "standard";
     public static final String CONTINUOUS = "continuous";
     public static final String UNIT = "unit";

src/main/java/io/odh/test/framework/logs/LogCollector.java

@@ -49,6 +49,16 @@ private static void writeLogsFromPods(Path logpath, Pod pod) {
         });
     }
 
+    private static void writePodsDescription(Path logpath, Pod pod) {
+        try {
+            LOGGER.debug("Get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
+            Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".describe.log"),
+                    ResourceManager.getKubeCmdClient().namespace(pod.getMetadata().getNamespace()).describe(pod.getKind(), pod.getMetadata().getName()));
+        } catch (IOException e) {
+            LOGGER.warn("Cannot get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
+        }
+    }
+
     private static void saveClusterState(Path logpath) throws IOException {
         KubeClient kube = ResourceManager.getClient();
         KubeCmdClient cmdClient = ResourceManager.getKubeCmdClient();
@@ -59,17 +69,20 @@ private static void saveClusterState(Path logpath) throws IOException {
         Files.writeString(logpath.resolve("dsci.yml"), cmdClient.exec(false, false, "get", "dsci", "-o", "yaml").out());
         Files.writeString(logpath.resolve("subscriptions.yml"), cmdClient.exec(false, false, "get", "subscriptions.operators.coreos.com", "--all-namespaces", "-o", "yaml").out());
         Files.writeString(logpath.resolve("notebooks.yml"), cmdClient.exec(false, false, "get", "notebook", "--all-namespaces", "-o", "yaml").out());
+        LOGGER.debug("Listing pods in {}", OdhConstants.BUNDLE_OPERATOR_NAMESPACE);
         kube.listPodsByPrefixInName(OdhConstants.BUNDLE_OPERATOR_NAMESPACE, "opendatahub-operator-controller-manager").forEach(pod -> {
             writeLogsFromPods(logpath, pod);
+            writePodsDescription(logpath, pod);
         });
-        kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, "opendatahub").forEach(pod -> {
-            writeLogsFromPods(logpath, pod);
-        });
-        kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, OdhConstants.OLM_OPERATOR_NAME).forEach(pod -> {
+        LOGGER.debug("Listing pods in {}", OdhConstants.OLM_OPERATOR_NAMESPACE);
+        kube.listPods(OdhConstants.OLM_OPERATOR_NAMESPACE).forEach(pod -> {
             writeLogsFromPods(logpath, pod);
+            writePodsDescription(logpath, pod);
         });
+        LOGGER.debug("Listing pods in {}", OdhConstants.CONTROLLERS_NAMESPACE);
         kube.listPods(OdhConstants.CONTROLLERS_NAMESPACE).forEach(pod -> {
             writeLogsFromPods(logpath, pod);
+            writePodsDescription(logpath, pod);
         });
     }
 }

src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java

@@ -67,7 +67,10 @@ public boolean waitForReadiness(DataScienceCluster resource) {
 
         String namespace = OdhConstants.CONTROLLERS_NAMESPACE;
         LOGGER.info("Waiting for pods readiness in {}", namespace);
-        PodUtils.waitForPodsReady(namespace, true, () -> { });
+        PodUtils.waitForPodsReady(namespace, true, () -> {
+            ResourceManager.getKubeCmdClient().namespace(namespace).exec(false, "oc", "get", "pods");
+            ResourceManager.getKubeCmdClient().namespace(namespace).exec(false, "oc", "get", "events");
+        });
 
         return true;
     }

src/main/java/io/odh/test/install/OlmInstall.java

@@ -40,13 +40,7 @@ public class OlmInstall {
     private String approval = "Automatic";
 
     public void create() {
-        // Create namespace at first because operator-group and subscription could you specific namespace
-        Namespace ns = new NamespaceBuilder()
-                .withNewMetadata()
-                .withName(namespace)
-                .endMetadata()
-                .build();
-        ResourceManager.getInstance().createResourceWithoutWait(ns);
+        createNamespace();
         // Create operator group and subscription
         createOperatorGroup();
         ResourceManager.getInstance().pushToStack(new ResourceItem(this::deleteCSV));
@@ -57,11 +51,25 @@ public void create() {
     }
 
     public void createManual() {
+        createNamespace();
         createOperatorGroup();
         ResourceManager.getInstance().pushToStack(new ResourceItem(this::deleteCSV));
         createAndModifySubscription();
     }
 
+    /**
+     * Creates namespace for operator-group and subscription
+     */
+    private void createNamespace() {
+        // Create namespace at first because operator-group and subscription could you specific namespace
+        Namespace ns = new NamespaceBuilder()
+                .withNewMetadata()
+                .withName(namespace)
+                .endMetadata()
+                .build();
+        ResourceManager.getInstance().createResourceWithoutWait(ns);
+    }
+
     /**
      * Creates OperatorGroup in specific namespace
      */

src/main/java/io/odh/test/platform/KubeUtils.java

@@ -45,7 +45,7 @@ public static void deleteDefaultDSCI() {
     }
 
     public static void waitForInstallPlan(String namespace, String csvName) {
-        TestUtils.waitFor("Install paln with new version", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT, () -> {
+        TestUtils.waitFor("Install plan with new version", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT, () -> {
             try {
                 InstallPlan ip = ResourceManager.getClient().getNonApprovedInstallPlan(namespace, csvName);
                 LOGGER.debug("Found InstallPlan {} - {}", ip.getMetadata().getName(), ip.getSpec().getClusterServiceVersionNames());

src/main/java/io/odh/test/utils/PodUtils.java

@@ -33,8 +33,8 @@ public static void waitForPodsReady(String namespaceName, boolean containers, Ru
                 () -> {
                     List<Pod> pods = ResourceManager.getClient().listPods(namespaceName);
                     if (pods.isEmpty()) {
-                        LOGGER.debug("Expected Pods are ready");
-                        return true;
+                        LOGGER.debug("Expected Pods are not ready!");
+                        return false;
                     }
                     for (Pod pod : pods) {
                         if (!Readiness.isPodReady(pod)) {

src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java

@@ -8,19 +8,22 @@
 import io.fabric8.kubernetes.api.model.LabelSelectorBuilder;
 import io.odh.test.Environment;
 import io.odh.test.OdhConstants;
+import io.odh.test.TestSuite;
 import io.odh.test.framework.manager.ResourceManager;
 import io.odh.test.install.BundleInstall;
 import io.odh.test.utils.DeploymentUtils;
 import io.odh.test.utils.PodUtils;
 import io.odh.test.utils.UpgradeUtils;
 import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Map;
 
+@Tag(TestSuite.BUNDLE_UPGRADE)
 public class BundleUpgradeST extends UpgradeAbstract {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(BundleUpgradeST.class);

src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java

@@ -9,29 +9,30 @@
 import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan;
 import io.odh.test.Environment;
 import io.odh.test.OdhConstants;
+import io.odh.test.TestSuite;
 import io.odh.test.framework.manager.ResourceManager;
 import io.odh.test.install.OlmInstall;
 import io.odh.test.platform.KubeUtils;
 import io.odh.test.utils.DeploymentUtils;
 import io.odh.test.utils.PodUtils;
 import io.odh.test.utils.UpgradeUtils;
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Map;
 
+@Tag(TestSuite.OLM_UPGRADE)
 public class OlmUpgradeST extends UpgradeAbstract {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(OlmUpgradeST.class);
     private static final String DS_PROJECT_NAME = "upgrade-dsc";
 
-    private final String startingVersion = !Environment.PRODUCT.equals(Environment.PRODUCT_DEFAULT) ? "2.4.0" : "v.2.4.0";
+    private final String startingVersion = Environment.OLM_UPGRADE_STARTING_VERSION;
 
     @Test
-    @Disabled("Tested only for upgrades from 2.4+ so we are waiting for 2.5 release")
     void testUpgradeOlm() throws IOException, InterruptedException {
         String ntbName = "test-odh-notebook";
         String ntbNamespace = "test-odh-notebook-upgrade";