From 394e44fc2d930ab33115920b84956db823aa1b37 Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 11:11:26 +0100 Subject: [PATCH 01/10] Add option to specify starting version for OLM upgrades Signed-off-by: Jakub Stejskal --- src/main/java/io/odh/test/Environment.java | 2 ++ src/main/java/io/odh/test/OdhConstants.java | 4 ++++ src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/odh/test/Environment.java b/src/main/java/io/odh/test/Environment.java index 9310a0b6..61398791 100644 --- a/src/main/java/io/odh/test/Environment.java +++ b/src/main/java/io/odh/test/Environment.java @@ -61,6 +61,7 @@ public class Environment { private static final String OLM_OPERATOR_VERSION_ENV = "OLM_OPERATOR_VERSION"; private static final String OLM_OPERATOR_CHANNEL_ENV = "OLM_OPERATOR_CHANNEL"; private static final String OPERATOR_INSTALL_TYPE_ENV = "OPERATOR_INSTALL_TYPE"; + private static final String OLM_UPGRADE_STARTING_VERSION_ENV = "OLM_UPGRADE_STARTING_VERSION"; public static final String PRODUCT_DEFAULT = "odh"; @@ -87,6 +88,7 @@ public class Environment { public static final String OLM_SOURCE_NAMESPACE = getOrDefault(OLM_SOURCE_NAMESPACE_ENV, "openshift-marketplace"); public static final String OLM_OPERATOR_CHANNEL = getOrDefault(OLM_OPERATOR_CHANNEL_ENV, OdhConstants.OLM_OPERATOR_CHANNEL); public static final String OLM_OPERATOR_VERSION = getOrDefault(OLM_OPERATOR_VERSION_ENV, OdhConstants.OLM_OPERATOR_VERSION); + public static final String OLM_UPGRADE_STARTING_VERSION = getOrDefault(OLM_UPGRADE_STARTING_VERSION_ENV, OdhConstants.OLM_UPGRADE_STARTING_OPERATOR_VERSION); public static final String OPERATOR_INSTALL_TYPE = getOrDefault(OPERATOR_INSTALL_TYPE_ENV, InstallTypes.BUNDLE.toString()); diff --git a/src/main/java/io/odh/test/OdhConstants.java b/src/main/java/io/odh/test/OdhConstants.java index 814a19f7..46399a7f 100644 --- a/src/main/java/io/odh/test/OdhConstants.java +++ b/src/main/java/io/odh/test/OdhConstants.java @@ -23,7 +23,9 @@ private OdhConstants() { } private static final String ODH_OLM_SOURCE_NAME = "community-operators"; private static final String ODH_OLM_APP_BUNDLE_PREFIX = "opendatahub-operator"; private static final String ODH_OLM_OPERATOR_CHANNEL = "fast"; + // TODO - should be changed after 2.5 release private static final String ODH_OLM_OPERATOR_VERSION = "v2.4.0"; + private static final String ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION = "v2.4.0"; // RHOAI private static final String RHOAI_CONTROLLERS_NAMESPACE = "redhat-ods-applications"; @@ -37,6 +39,7 @@ private OdhConstants() { } private static final String RHOAI_OLM_APP_BUNDLE_PREFIX = "rhods-operator"; private static final String RHOAI_OLM_OPERATOR_CHANNEL = "stable"; private static final String RHOAI_OLM_OPERATOR_VERSION = "2.5.0"; + private static final String RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION = "2.4.0"; // Public part public static final String CODEFLARE_DEPLOYMENT_NAME = "codeflare-operator-manager"; @@ -62,6 +65,7 @@ private OdhConstants() { } public static final String OLM_OPERATOR_VERSION = getOdhOrRhoai(ODH_OLM_OPERATOR_VERSION, RHOAI_OLM_OPERATOR_VERSION); public static final String OLM_SOURCE_NAME = getOdhOrRhoai(ODH_OLM_SOURCE_NAME, RHOAI_OLM_SOURCE_NAME); public static final String OLM_OPERATOR_CHANNEL = getOdhOrRhoai(ODH_OLM_OPERATOR_CHANNEL, RHOAI_OLM_OPERATOR_CHANNEL); + public static final String OLM_UPGRADE_STARTING_OPERATOR_VERSION = getOdhOrRhoai(ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION, RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION); private static T getOdhOrRhoai(T odhValue, T rhoaiValue) { T returnValue = odhValue; diff --git a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java index 8ee9de4c..775852e7 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java @@ -28,7 +28,7 @@ public class OlmUpgradeST extends UpgradeAbstract { private static final Logger LOGGER = LoggerFactory.getLogger(OlmUpgradeST.class); private static final String DS_PROJECT_NAME = "upgrade-dsc"; - private final String startingVersion = !Environment.PRODUCT.equals(Environment.PRODUCT_DEFAULT) ? "2.4.0" : "v.2.4.0"; + private final String startingVersion = Environment.OLM_UPGRADE_STARTING_VERSION; @Test @Disabled("Tested only for upgrades from 2.4+ so we are waiting for 2.5 release") From 8f1eba1effc58cb50b58897f138ecc3edc8f05af Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 11:56:06 +0100 Subject: [PATCH 02/10] Add assume for upgrade Signed-off-by: Jakub Stejskal --- .../java/io/odh/test/e2e/upgrade/BundleUpgradeST.java | 7 +++++++ src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java index 89ba6a78..36f29aa4 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java @@ -10,6 +10,7 @@ import io.odh.test.OdhConstants; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.install.BundleInstall; +import io.odh.test.install.InstallTypes; import io.odh.test.utils.DeploymentUtils; import io.odh.test.utils.PodUtils; import io.odh.test.utils.UpgradeUtils; @@ -19,7 +20,11 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Locale; import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assumptions.assumeTrue; public class BundleUpgradeST extends UpgradeAbstract { @@ -37,6 +42,8 @@ void clean() { @Test void testUpgradeBundle() throws IOException { + assumeTrue(Objects.equals(Environment.OPERATOR_INSTALL_TYPE.toLowerCase(Locale.ENGLISH), + InstallTypes.BUNDLE.name().toLowerCase(Locale.ENGLISH))); LOGGER.info("Install base version"); baseBundle = new BundleInstall(Environment.INSTALL_FILE_PREVIOUS_PATH); baseBundle.disableModifyOperatorImage(); diff --git a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java index 775852e7..d5e9b386 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java @@ -10,18 +10,22 @@ import io.odh.test.Environment; import io.odh.test.OdhConstants; import io.odh.test.framework.manager.ResourceManager; +import io.odh.test.install.InstallTypes; import io.odh.test.install.OlmInstall; import io.odh.test.platform.KubeUtils; import io.odh.test.utils.DeploymentUtils; import io.odh.test.utils.PodUtils; import io.odh.test.utils.UpgradeUtils; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Locale; import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assumptions.assumeTrue; public class OlmUpgradeST extends UpgradeAbstract { @@ -31,8 +35,9 @@ public class OlmUpgradeST extends UpgradeAbstract { private final String startingVersion = Environment.OLM_UPGRADE_STARTING_VERSION; @Test - @Disabled("Tested only for upgrades from 2.4+ so we are waiting for 2.5 release") void testUpgradeOlm() throws IOException, InterruptedException { + assumeTrue(Objects.equals(Environment.OPERATOR_INSTALL_TYPE.toLowerCase(Locale.ENGLISH), + InstallTypes.OLM.name().toLowerCase(Locale.ENGLISH))); String ntbName = "test-odh-notebook"; String ntbNamespace = "test-odh-notebook-upgrade"; From 200e93a786e2f28d368de92beabdb8788785224e Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 13:03:57 +0100 Subject: [PATCH 03/10] Fix readiness for all pods in ns Signed-off-by: Jakub Stejskal --- src/main/java/io/odh/test/utils/PodUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/odh/test/utils/PodUtils.java b/src/main/java/io/odh/test/utils/PodUtils.java index cd0a6742..b4751fbe 100644 --- a/src/main/java/io/odh/test/utils/PodUtils.java +++ b/src/main/java/io/odh/test/utils/PodUtils.java @@ -32,8 +32,8 @@ public static void waitForPodsReady(String namespaceName, boolean containers, Ru TestConstants.GLOBAL_POLL_INTERVAL_MEDIUM, READINESS_TIMEOUT, () -> { List pods = ResourceManager.getClient().listPods(namespaceName); - if (pods.isEmpty()) { - LOGGER.debug("Expected Pods are ready"); + if (!pods.isEmpty()) { + LOGGER.debug("Expected Pods are not ready!"); return true; } for (Pod pod : pods) { From ff18930e2a4fdf7c5a0bed7c4f7fadd69cbbd63a Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 13:42:55 +0100 Subject: [PATCH 04/10] Fix missing namespace Signed-off-by: Jakub Stejskal --- pom.xml | 20 +++++++++++++++++ src/main/java/io/odh/test/TestSuite.java | 2 ++ .../java/io/odh/test/install/OlmInstall.java | 22 +++++++++++++------ .../odh/test/e2e/upgrade/BundleUpgradeST.java | 9 +++----- .../io/odh/test/e2e/upgrade/OlmUpgradeST.java | 9 +++----- 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 5022db49..db83f9c5 100644 --- a/pom.xml +++ b/pom.xml @@ -343,6 +343,26 @@ + + bundle-upgrade + + false + + bundle-upgrade + + + + + + olm-upgrade + + false + + olm-upgrade + + + + all diff --git a/src/main/java/io/odh/test/TestSuite.java b/src/main/java/io/odh/test/TestSuite.java index f40366c1..f935c5f6 100644 --- a/src/main/java/io/odh/test/TestSuite.java +++ b/src/main/java/io/odh/test/TestSuite.java @@ -7,6 +7,8 @@ public class TestSuite { public static final String SMOKE = "smoke"; public static final String UPGRADE = "upgrade"; + public static final String OLM_UPGRADE = "olm-upgrade"; + public static final String BUNDLE_UPGRADE = "bundle-upgrade"; public static final String STANDARD = "standard"; public static final String CONTINUOUS = "continuous"; public static final String UNIT = "unit"; diff --git a/src/main/java/io/odh/test/install/OlmInstall.java b/src/main/java/io/odh/test/install/OlmInstall.java index e0c4a271..cbbcfe51 100644 --- a/src/main/java/io/odh/test/install/OlmInstall.java +++ b/src/main/java/io/odh/test/install/OlmInstall.java @@ -40,13 +40,7 @@ public class OlmInstall { private String approval = "Automatic"; public void create() { - // Create namespace at first because operator-group and subscription could you specific namespace - Namespace ns = new NamespaceBuilder() - .withNewMetadata() - .withName(namespace) - .endMetadata() - .build(); - ResourceManager.getInstance().createResourceWithoutWait(ns); + createNamespace(); // Create operator group and subscription createOperatorGroup(); ResourceManager.getInstance().pushToStack(new ResourceItem(this::deleteCSV)); @@ -57,11 +51,25 @@ public void create() { } public void createManual() { + createNamespace(); createOperatorGroup(); ResourceManager.getInstance().pushToStack(new ResourceItem(this::deleteCSV)); createAndModifySubscription(); } + /** + * Creates namespace for operator-group and subscription + */ + private void createNamespace() { + // Create namespace at first because operator-group and subscription could you specific namespace + Namespace ns = new NamespaceBuilder() + .withNewMetadata() + .withName(namespace) + .endMetadata() + .build(); + ResourceManager.getInstance().createResourceWithoutWait(ns); + } + /** * Creates OperatorGroup in specific namespace */ diff --git a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java index 36f29aa4..9b8a1f6f 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java @@ -8,6 +8,7 @@ import io.fabric8.kubernetes.api.model.LabelSelectorBuilder; import io.odh.test.Environment; import io.odh.test.OdhConstants; +import io.odh.test.TestSuite; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.install.BundleInstall; import io.odh.test.install.InstallTypes; @@ -15,17 +16,15 @@ import io.odh.test.utils.PodUtils; import io.odh.test.utils.UpgradeUtils; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Locale; import java.util.Map; -import java.util.Objects; - -import static org.junit.jupiter.api.Assumptions.assumeTrue; +@Tag(TestSuite.BUNDLE_UPGRADE) public class BundleUpgradeST extends UpgradeAbstract { private static final Logger LOGGER = LoggerFactory.getLogger(BundleUpgradeST.class); @@ -42,8 +41,6 @@ void clean() { @Test void testUpgradeBundle() throws IOException { - assumeTrue(Objects.equals(Environment.OPERATOR_INSTALL_TYPE.toLowerCase(Locale.ENGLISH), - InstallTypes.BUNDLE.name().toLowerCase(Locale.ENGLISH))); LOGGER.info("Install base version"); baseBundle = new BundleInstall(Environment.INSTALL_FILE_PREVIOUS_PATH); baseBundle.disableModifyOperatorImage(); diff --git a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java index d5e9b386..249d9d65 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java @@ -9,6 +9,7 @@ import io.fabric8.openshift.api.model.operatorhub.v1alpha1.InstallPlan; import io.odh.test.Environment; import io.odh.test.OdhConstants; +import io.odh.test.TestSuite; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.install.InstallTypes; import io.odh.test.install.OlmInstall; @@ -16,17 +17,15 @@ import io.odh.test.utils.DeploymentUtils; import io.odh.test.utils.PodUtils; import io.odh.test.utils.UpgradeUtils; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Locale; import java.util.Map; -import java.util.Objects; - -import static org.junit.jupiter.api.Assumptions.assumeTrue; +@Tag(TestSuite.OLM_UPGRADE) public class OlmUpgradeST extends UpgradeAbstract { private static final Logger LOGGER = LoggerFactory.getLogger(OlmUpgradeST.class); @@ -36,8 +35,6 @@ public class OlmUpgradeST extends UpgradeAbstract { @Test void testUpgradeOlm() throws IOException, InterruptedException { - assumeTrue(Objects.equals(Environment.OPERATOR_INSTALL_TYPE.toLowerCase(Locale.ENGLISH), - InstallTypes.OLM.name().toLowerCase(Locale.ENGLISH))); String ntbName = "test-odh-notebook"; String ntbNamespace = "test-odh-notebook-upgrade"; From d847e89d58cfde16828924e38e6f4c990c5c8c26 Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 13:46:39 +0100 Subject: [PATCH 05/10] Fix build Signed-off-by: Jakub Stejskal --- src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java | 1 - src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java index 9b8a1f6f..0f369074 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/BundleUpgradeST.java @@ -11,7 +11,6 @@ import io.odh.test.TestSuite; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.install.BundleInstall; -import io.odh.test.install.InstallTypes; import io.odh.test.utils.DeploymentUtils; import io.odh.test.utils.PodUtils; import io.odh.test.utils.UpgradeUtils; diff --git a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java index 249d9d65..f163d0b4 100644 --- a/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java +++ b/src/test/java/io/odh/test/e2e/upgrade/OlmUpgradeST.java @@ -11,7 +11,6 @@ import io.odh.test.OdhConstants; import io.odh.test.TestSuite; import io.odh.test.framework.manager.ResourceManager; -import io.odh.test.install.InstallTypes; import io.odh.test.install.OlmInstall; import io.odh.test.platform.KubeUtils; import io.odh.test.utils.DeploymentUtils; From c9daa4350b537d7f3b2ed0a5036bf5506ab11e24 Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 14:22:41 +0100 Subject: [PATCH 06/10] Fix check for pods Signed-off-by: Jakub Stejskal --- src/main/java/io/odh/test/utils/PodUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/odh/test/utils/PodUtils.java b/src/main/java/io/odh/test/utils/PodUtils.java index b4751fbe..02a597f9 100644 --- a/src/main/java/io/odh/test/utils/PodUtils.java +++ b/src/main/java/io/odh/test/utils/PodUtils.java @@ -32,9 +32,9 @@ public static void waitForPodsReady(String namespaceName, boolean containers, Ru TestConstants.GLOBAL_POLL_INTERVAL_MEDIUM, READINESS_TIMEOUT, () -> { List pods = ResourceManager.getClient().listPods(namespaceName); - if (!pods.isEmpty()) { + if (pods.isEmpty()) { LOGGER.debug("Expected Pods are not ready!"); - return true; + return false; } for (Pod pod : pods) { if (!Readiness.isPodReady(pod)) { From 5ac0d178cdab6ae245c4ff74585302f895d0a27e Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 15:24:29 +0100 Subject: [PATCH 07/10] Improve pod logging Signed-off-by: Jakub Stejskal --- .../io/odh/test/framework/logs/LogCollector.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/io/odh/test/framework/logs/LogCollector.java b/src/main/java/io/odh/test/framework/logs/LogCollector.java index 80aba05c..9c6f2765 100644 --- a/src/main/java/io/odh/test/framework/logs/LogCollector.java +++ b/src/main/java/io/odh/test/framework/logs/LogCollector.java @@ -49,6 +49,16 @@ private static void writeLogsFromPods(Path logpath, Pod pod) { }); } + private static void writePodsDescription(Path logpath, Pod pod) { + try { + LOGGER.debug("Get description if pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); + Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".log"), + ResourceManager.getKubeCmdClient().describe(pod.getKind(), pod.getMetadata().getName())); + } catch (IOException e) { + LOGGER.warn("Cannot get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); + } + } + private static void saveClusterState(Path logpath) throws IOException { KubeClient kube = ResourceManager.getClient(); KubeCmdClient cmdClient = ResourceManager.getKubeCmdClient(); @@ -61,15 +71,19 @@ private static void saveClusterState(Path logpath) throws IOException { Files.writeString(logpath.resolve("notebooks.yml"), cmdClient.exec(false, false, "get", "notebook", "--all-namespaces", "-o", "yaml").out()); kube.listPodsByPrefixInName(OdhConstants.BUNDLE_OPERATOR_NAMESPACE, "opendatahub-operator-controller-manager").forEach(pod -> { writeLogsFromPods(logpath, pod); + writePodsDescription(logpath, pod); }); kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, "opendatahub").forEach(pod -> { writeLogsFromPods(logpath, pod); + writePodsDescription(logpath, pod); }); kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, OdhConstants.OLM_OPERATOR_NAME).forEach(pod -> { writeLogsFromPods(logpath, pod); + writePodsDescription(logpath, pod); }); kube.listPods(OdhConstants.CONTROLLERS_NAMESPACE).forEach(pod -> { writeLogsFromPods(logpath, pod); + writePodsDescription(logpath, pod); }); } } From 7116eb473fc22021d49f6b5d93de78d466ee4d8f Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 12 Jan 2024 23:26:26 +0100 Subject: [PATCH 08/10] Significantly improve logging Signed-off-by: Jakub Stejskal --- src/main/java/io/odh/test/OdhConstants.java | 46 +++++++++++++------ .../odh/test/framework/logs/LogCollector.java | 13 +++--- .../resources/DataScienceClusterResource.java | 5 +- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/main/java/io/odh/test/OdhConstants.java b/src/main/java/io/odh/test/OdhConstants.java index 46399a7f..4ee7aaeb 100644 --- a/src/main/java/io/odh/test/OdhConstants.java +++ b/src/main/java/io/odh/test/OdhConstants.java @@ -4,11 +4,18 @@ */ package io.odh.test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; import java.util.Objects; public class OdhConstants { private OdhConstants() { } + private static final Logger LOGGER = LoggerFactory.getLogger(OdhConstants.class); + private static final Map VALUES = new HashMap<>(); // ODH private static final String ODH_CONTROLLERS_NAMESPACE = "opendatahub"; private static final String ODH_DASHBOARD_ROUTE_NAME = "odh-dashboard"; @@ -53,25 +60,38 @@ private OdhConstants() { } public static final String ODH_NOTEBOOK_OPERATOR = "odh-notebook-controller-manager"; public static final String TRUSTY_AI_OPERATOR = "trustyai-service-operator-controller-manager"; - public static final String CONTROLLERS_NAMESPACE = getOdhOrRhoai(ODH_CONTROLLERS_NAMESPACE, RHOAI_CONTROLLERS_NAMESPACE); - public static final String DASHBOARD_ROUTE_NAME = getOdhOrRhoai(ODH_DASHBOARD_ROUTE_NAME, RHOAI_DASHBOARD_ROUTE_NAME); - public static final String DASHBOARD_CONTROLLER = getOdhOrRhoai(ODH_DASHBOARD_CONTROLLER, RHOAI_DASHBOARD_CONTROLLER); - public static final String BUNDLE_OPERATOR_NAMESPACE = getOdhOrRhoai(ODH_BUNDLE_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME); + public static final String CONTROLLERS_NAMESPACE = getOdhOrRhoai("CONTROLLERS_NAMESPACE", ODH_CONTROLLERS_NAMESPACE, RHOAI_CONTROLLERS_NAMESPACE); + public static final String DASHBOARD_ROUTE_NAME = getOdhOrRhoai("DASHBOARD_ROUTE_NAME", ODH_DASHBOARD_ROUTE_NAME, RHOAI_DASHBOARD_ROUTE_NAME); + public static final String DASHBOARD_CONTROLLER = getOdhOrRhoai("DASHBOARD_CONTROLLER", ODH_DASHBOARD_CONTROLLER, RHOAI_DASHBOARD_CONTROLLER); + public static final String BUNDLE_OPERATOR_NAMESPACE = getOdhOrRhoai("BUNDLE_OPERATOR_NAMESPACE", ODH_BUNDLE_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME); // OLM env variables - public static final String OLM_OPERATOR_NAME = getOdhOrRhoai(ODH_OLM_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME); - public static final String OLM_OPERATOR_NAMESPACE = getOdhOrRhoai(ODH_OLM_OPERATOR_NAMESPACE, RHOAI_OLM_OPERATOR_NAMESPACE); - public static final String OLM_OPERATOR_DEPLOYMENT_NAME = getOdhOrRhoai(ODH_OLM_OPERATOR_DEPLOYMENT_NAME, RHOAI_OLM_OPERATOR_DEPLOYMENT_NAME); - public static final String OLM_APP_BUNDLE_PREFIX = getOdhOrRhoai(ODH_OLM_APP_BUNDLE_PREFIX, RHOAI_OLM_APP_BUNDLE_PREFIX); - public static final String OLM_OPERATOR_VERSION = getOdhOrRhoai(ODH_OLM_OPERATOR_VERSION, RHOAI_OLM_OPERATOR_VERSION); - public static final String OLM_SOURCE_NAME = getOdhOrRhoai(ODH_OLM_SOURCE_NAME, RHOAI_OLM_SOURCE_NAME); - public static final String OLM_OPERATOR_CHANNEL = getOdhOrRhoai(ODH_OLM_OPERATOR_CHANNEL, RHOAI_OLM_OPERATOR_CHANNEL); - public static final String OLM_UPGRADE_STARTING_OPERATOR_VERSION = getOdhOrRhoai(ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION, RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION); + public static final String OLM_OPERATOR_NAME = getOdhOrRhoai("OLM_OPERATOR_NAME", ODH_OLM_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME); + public static final String OLM_OPERATOR_NAMESPACE = getOdhOrRhoai("OLM_OPERATOR_NAMESPACE", ODH_OLM_OPERATOR_NAMESPACE, RHOAI_OLM_OPERATOR_NAMESPACE); + public static final String OLM_OPERATOR_DEPLOYMENT_NAME = getOdhOrRhoai("OLM_OPERATOR_DEPLOYMENT_NAME", ODH_OLM_OPERATOR_DEPLOYMENT_NAME, RHOAI_OLM_OPERATOR_DEPLOYMENT_NAME); + public static final String OLM_APP_BUNDLE_PREFIX = getOdhOrRhoai("OLM_APP_BUNDLE_PREFIX", ODH_OLM_APP_BUNDLE_PREFIX, RHOAI_OLM_APP_BUNDLE_PREFIX); + public static final String OLM_OPERATOR_VERSION = getOdhOrRhoai("OLM_OPERATOR_VERSION", ODH_OLM_OPERATOR_VERSION, RHOAI_OLM_OPERATOR_VERSION); + public static final String OLM_SOURCE_NAME = getOdhOrRhoai("OLM_SOURCE_NAME", ODH_OLM_SOURCE_NAME, RHOAI_OLM_SOURCE_NAME); + public static final String OLM_OPERATOR_CHANNEL = getOdhOrRhoai("OLM_OPERATOR_CHANNEL", ODH_OLM_OPERATOR_CHANNEL, RHOAI_OLM_OPERATOR_CHANNEL); + public static final String OLM_UPGRADE_STARTING_OPERATOR_VERSION = getOdhOrRhoai("OLM_UPGRADE_STARTING_OPERATOR_VERSION", ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION, RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION); - private static T getOdhOrRhoai(T odhValue, T rhoaiValue) { + private static T getOdhOrRhoai(String var, T odhValue, T rhoaiValue) { T returnValue = odhValue; if (!Objects.equals(Environment.PRODUCT, Environment.PRODUCT_DEFAULT)) { returnValue = rhoaiValue; } + VALUES.put(var, String.valueOf(returnValue)); return returnValue; } + + static { + String debugFormat = "{}: {}"; + LOGGER.info("Used OdhConstants:"); + VALUES.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { + if (!Objects.equals(entry.getValue(), "null")) { + LOGGER.info(debugFormat, entry.getKey(), entry.getValue()); + } + }); + } } diff --git a/src/main/java/io/odh/test/framework/logs/LogCollector.java b/src/main/java/io/odh/test/framework/logs/LogCollector.java index 9c6f2765..e67364d3 100644 --- a/src/main/java/io/odh/test/framework/logs/LogCollector.java +++ b/src/main/java/io/odh/test/framework/logs/LogCollector.java @@ -51,9 +51,9 @@ private static void writeLogsFromPods(Path logpath, Pod pod) { private static void writePodsDescription(Path logpath, Pod pod) { try { - LOGGER.debug("Get description if pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); + LOGGER.debug("Get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".log"), - ResourceManager.getKubeCmdClient().describe(pod.getKind(), pod.getMetadata().getName())); + ResourceManager.getKubeCmdClient().namespace(pod.getMetadata().getNamespace()).describe(pod.getKind(), pod.getMetadata().getName())); } catch (IOException e) { LOGGER.warn("Cannot get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); } @@ -69,18 +69,17 @@ private static void saveClusterState(Path logpath) throws IOException { Files.writeString(logpath.resolve("dsci.yml"), cmdClient.exec(false, false, "get", "dsci", "-o", "yaml").out()); Files.writeString(logpath.resolve("subscriptions.yml"), cmdClient.exec(false, false, "get", "subscriptions.operators.coreos.com", "--all-namespaces", "-o", "yaml").out()); Files.writeString(logpath.resolve("notebooks.yml"), cmdClient.exec(false, false, "get", "notebook", "--all-namespaces", "-o", "yaml").out()); + LOGGER.debug("Listing pods in {}", OdhConstants.BUNDLE_OPERATOR_NAMESPACE); kube.listPodsByPrefixInName(OdhConstants.BUNDLE_OPERATOR_NAMESPACE, "opendatahub-operator-controller-manager").forEach(pod -> { writeLogsFromPods(logpath, pod); writePodsDescription(logpath, pod); }); - kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, "opendatahub").forEach(pod -> { - writeLogsFromPods(logpath, pod); - writePodsDescription(logpath, pod); - }); - kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, OdhConstants.OLM_OPERATOR_NAME).forEach(pod -> { + LOGGER.debug("Listing pods in {}", OdhConstants.OLM_OPERATOR_NAMESPACE); + kube.listPods(OdhConstants.OLM_OPERATOR_NAMESPACE).forEach(pod -> { writeLogsFromPods(logpath, pod); writePodsDescription(logpath, pod); }); + LOGGER.debug("Listing pods in {}", OdhConstants.CONTROLLERS_NAMESPACE); kube.listPods(OdhConstants.CONTROLLERS_NAMESPACE).forEach(pod -> { writeLogsFromPods(logpath, pod); writePodsDescription(logpath, pod); diff --git a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java index 617b2e18..7428a9e3 100644 --- a/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java +++ b/src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java @@ -67,7 +67,10 @@ public boolean waitForReadiness(DataScienceCluster resource) { String namespace = OdhConstants.CONTROLLERS_NAMESPACE; LOGGER.info("Waiting for pods readiness in {}", namespace); - PodUtils.waitForPodsReady(namespace, true, () -> { }); + PodUtils.waitForPodsReady(namespace, true, () -> { + ResourceManager.getKubeCmdClient().namespace(namespace).exec(false, "oc", "get", "pods"); + ResourceManager.getKubeCmdClient().namespace(namespace).exec(false, "oc", "get", "events"); + }); return true; } From 586cedc84410e629265ece2ec76018ac0ad8ad60 Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Mon, 15 Jan 2024 13:47:34 +0100 Subject: [PATCH 09/10] Some fixes Signed-off-by: Jakub Stejskal --- src/main/java/io/odh/test/framework/logs/LogCollector.java | 2 +- src/main/java/io/odh/test/platform/KubeUtils.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/odh/test/framework/logs/LogCollector.java b/src/main/java/io/odh/test/framework/logs/LogCollector.java index e67364d3..5cb5e37c 100644 --- a/src/main/java/io/odh/test/framework/logs/LogCollector.java +++ b/src/main/java/io/odh/test/framework/logs/LogCollector.java @@ -52,7 +52,7 @@ private static void writeLogsFromPods(Path logpath, Pod pod) { private static void writePodsDescription(Path logpath, Pod pod) { try { LOGGER.debug("Get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); - Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".log"), + Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".describe.log"), ResourceManager.getKubeCmdClient().namespace(pod.getMetadata().getNamespace()).describe(pod.getKind(), pod.getMetadata().getName())); } catch (IOException e) { LOGGER.warn("Cannot get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); diff --git a/src/main/java/io/odh/test/platform/KubeUtils.java b/src/main/java/io/odh/test/platform/KubeUtils.java index 653c8977..04698544 100644 --- a/src/main/java/io/odh/test/platform/KubeUtils.java +++ b/src/main/java/io/odh/test/platform/KubeUtils.java @@ -45,7 +45,7 @@ public static void deleteDefaultDSCI() { } public static void waitForInstallPlan(String namespace, String csvName) { - TestUtils.waitFor("Install paln with new version", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT, () -> { + TestUtils.waitFor("Install plan with new version", TestConstants.GLOBAL_POLL_INTERVAL_SHORT, TestConstants.GLOBAL_TIMEOUT, () -> { try { InstallPlan ip = ResourceManager.getClient().getNonApprovedInstallPlan(namespace, csvName); LOGGER.debug("Found InstallPlan {} - {}", ip.getMetadata().getName(), ip.getSpec().getClusterServiceVersionNames()); From 8d599b48af5778159191328e98b8f52098b53241 Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Mon, 15 Jan 2024 13:49:22 +0100 Subject: [PATCH 10/10] update readme Signed-off-by: Jakub Stejskal --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e0debb38..1d54c488 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ E2E test suite for opendatahub using fabric8 kubernetes client ## List of test suites * smoke * upgrade +* olm-upgrade +* bundle-upgrade * standard * continuous * all (standard + upgrade) @@ -57,4 +59,4 @@ GITHUB_TOKEN="your_github_read_token" mvn verify -Pupgrade ### RUN Unit test of the suite ```commandline GITHUB_TOKEN="your_github_read_token" mvn test -``` \ No newline at end of file +```