From a53076dafda8416891745c9c771fdc5ced40586f Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Mon, 4 Dec 2023 21:42:36 +0100 Subject: [PATCH] Add ResourceManager for OLM handling Signed-off-by: Jakub Stejskal --- src/main/java/io/odh/test/Environment.java | 38 +++- src/main/java/io/odh/test/TestConstants.java | 11 ++ src/main/java/io/odh/test/TestUtils.java | 108 ++++++++++ .../io/odh/test/framework/WaitException.java | 15 ++ .../framework/manager/ResourceCondition.java | 37 ++++ .../test/framework/manager/ResourceItem.java | 29 +++ .../framework/manager/ResourceManager.java | 186 ++++++++++++++++++ .../test/framework/manager/ResourceType.java | 43 ++++ .../framework/manager/ThrowableRunner.java | 10 + .../resources/OperatorGroupResource.java | 50 +++++ .../resources/SubscriptionResource.java | 53 +++++ .../java/io/odh/test/install/OlmInstall.java | 150 ++++++++++++++ .../java/io/odh/test/platform/KubeClient.java | 27 ++- .../io/odh/test/utils/DeploymentUtils.java | 97 +++++++++ src/test/java/io/odh/test/e2e/Abstract.java | 10 + .../io/odh/test/e2e/standard/OdhInstall.java | 25 +++ 16 files changed, 878 insertions(+), 11 deletions(-) create mode 100644 src/main/java/io/odh/test/TestUtils.java create mode 100644 src/main/java/io/odh/test/framework/WaitException.java create mode 100644 src/main/java/io/odh/test/framework/manager/ResourceCondition.java create mode 100644 src/main/java/io/odh/test/framework/manager/ResourceItem.java create mode 100644 src/main/java/io/odh/test/framework/manager/ResourceManager.java create mode 100644 src/main/java/io/odh/test/framework/manager/ResourceType.java create mode 100644 src/main/java/io/odh/test/framework/manager/ThrowableRunner.java create mode 100644 src/main/java/io/odh/test/framework/manager/resources/OperatorGroupResource.java create mode 100644 src/main/java/io/odh/test/framework/manager/resources/SubscriptionResource.java create mode 100644 src/main/java/io/odh/test/utils/DeploymentUtils.java create mode 100644 src/test/java/io/odh/test/e2e/standard/OdhInstall.java diff --git a/src/main/java/io/odh/test/Environment.java b/src/main/java/io/odh/test/Environment.java index 6353a243..6c74fb35 100644 --- a/src/main/java/io/odh/test/Environment.java +++ b/src/main/java/io/odh/test/Environment.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.function.Function; /** @@ -23,6 +24,28 @@ public class Environment { private static final String PASSWORD_ENV = "KUBE_PASSWORD"; private static final String TOKEN_ENV = "KUBE_TOKEN"; private static final String URL_ENV = "KUBE_URL"; + /** + * OLM env variables + */ + private static final String OLM_OPERATOR_NAME_ENV = "OLM_OPERATOR_NAME"; + private static final String OLM_OPERATOR_NAMESPACE_ENV = "OLM_OPERATOR_NAMESPACE"; + private static final String OLM_OPERATOR_DEPLOYMENT_NAME_ENV = "OLM_OPERATOR_DEPLOYMENT_NAME"; + private static final String OLM_SOURCE_NAME_ENV = "OLM_SOURCE_NAME"; + private static final String OLM_SOURCE_NAMESPACE_ENV = "OLM_SOURCE_NAMESPACE"; + private static final String OLM_APP_BUNDLE_PREFIX_ENV = "OLM_APP_BUNDLE_PREFIX"; + private static final String OLM_OPERATOR_VERSION_ENV = "OLM_OPERATOR_VERSION"; + private static final String OLM_OPERATOR_CHANNEL_ENV = "OLM_OPERATOR_CHANNEL"; + + /** + * Defaults + */ + public static final String OLM_OPERATOR_NAME_DEFAULT = "opendatahub-operator"; + public static final String OLM_OPERATOR_NAMESPACE_DEFAULT = "openshift-operators"; + public static final String OLM_OPERATOR_DEPLOYMENT_NAME_DEFAULT = "opendatahub-operator-controller-manager"; + public static final String OLM_SOURCE_NAME_DEFAULT = "community-operators"; + public static final String OLM_APP_BUNDLE_PREFIX_DEFAULT = "opendatahub-operator"; + public static final String OLM_OPERATOR_CHANNEL_DEFAULT = "fast"; + public static final String OLM_OPERATOR_VERSION_DEFAULT = "2.4.0"; /** * Set values @@ -33,6 +56,15 @@ public class Environment { public static final String KUBE_TOKEN = getOrDefault(TOKEN_ENV, null); public static final String KUBE_URL = getOrDefault(URL_ENV, null); + // OLM env variables + public static final String OLM_OPERATOR_NAME = getOrDefault(OLM_OPERATOR_NAME_ENV, OLM_OPERATOR_NAME_DEFAULT); + public static final String OLM_OPERATOR_NAMESPACE = getOrDefault(OLM_OPERATOR_NAMESPACE_ENV, OLM_OPERATOR_NAMESPACE_DEFAULT); + public static final String OLM_OPERATOR_DEPLOYMENT_NAME = getOrDefault(OLM_OPERATOR_DEPLOYMENT_NAME_ENV, OLM_OPERATOR_DEPLOYMENT_NAME_DEFAULT); + public static final String OLM_SOURCE_NAME = getOrDefault(OLM_SOURCE_NAME_ENV, OLM_SOURCE_NAME_DEFAULT); + public static final String OLM_SOURCE_NAMESPACE = getOrDefault(OLM_SOURCE_NAMESPACE_ENV, "openshift-marketplace"); + public static final String OLM_APP_BUNDLE_PREFIX = getOrDefault(OLM_APP_BUNDLE_PREFIX_ENV, OLM_APP_BUNDLE_PREFIX_DEFAULT); + public static final String OLM_OPERATOR_CHANNEL = getOrDefault(OLM_OPERATOR_CHANNEL_ENV, OLM_OPERATOR_CHANNEL_DEFAULT); + public static final String OLM_OPERATOR_VERSION = getOrDefault(OLM_OPERATOR_VERSION_ENV, OLM_OPERATOR_VERSION_DEFAULT); private Environment() { } static { @@ -40,7 +72,11 @@ private Environment() { } LOGGER.info("Used environment variables:"); VALUES.entrySet().stream() .sorted(Map.Entry.comparingByKey()) - .forEach(entry -> LOGGER.info(debugFormat, entry.getKey(), entry.getValue())); + .forEach(entry -> { + if (!Objects.equals(entry.getValue(), "null")) { + LOGGER.info(debugFormat, entry.getKey(), entry.getValue()); + } + }); } public static void print() { } diff --git a/src/main/java/io/odh/test/TestConstants.java b/src/main/java/io/odh/test/TestConstants.java index 8ba90e32..e51b12ae 100644 --- a/src/main/java/io/odh/test/TestConstants.java +++ b/src/main/java/io/odh/test/TestConstants.java @@ -4,8 +4,19 @@ */ package io.odh.test; +import java.time.Duration; + public class TestConstants { public static final String ODH_NAMESPACE = "opendatahub"; + public static final String DEFAULT_NAMESPACE = "default"; + + public static final String SUBSCRIPTION = "Subscription"; + public static final String OPERATOR_GROUP = "OperatorGroup"; + + public static final long GLOBAL_POLL_INTERVAL = Duration.ofSeconds(10).toMillis(); + public static final long GLOBAL_POLL_INTERVAL_MEDIUM = Duration.ofSeconds(5).toMillis(); + public static final long GLOBAL_POLL_INTERVAL_SHORT = Duration.ofSeconds(1).toMillis(); + public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(5).toMillis(); private TestConstants() { } diff --git a/src/main/java/io/odh/test/TestUtils.java b/src/main/java/io/odh/test/TestUtils.java new file mode 100644 index 00000000..448e0889 --- /dev/null +++ b/src/main/java/io/odh/test/TestUtils.java @@ -0,0 +1,108 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test; + +import io.odh.test.framework.WaitException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; + +@SuppressWarnings({"checkstyle:ClassFanOutComplexity"}) +public final class TestUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); + + public static final String USER_PATH = System.getProperty("user.dir"); + + /** + * Default timeout for asynchronous tests. + */ + public static final int DEFAULT_TIMEOUT_DURATION = 30; + + /** + * Default timeout unit for asynchronous tests. + */ + public static final TimeUnit DEFAULT_TIMEOUT_UNIT = TimeUnit.SECONDS; + + private TestUtils() { + // All static methods + } + + /** + * Poll the given {@code ready} function every {@code pollIntervalMs} milliseconds until it returns true, + * or throw a WaitException if it doesn't return true within {@code timeoutMs} milliseconds. + * @return The remaining time left until timeout occurs + * (helpful if you have several calls which need to share a common timeout), + * */ + public static long waitFor(String description, long pollIntervalMs, long timeoutMs, BooleanSupplier ready) { + return waitFor(description, pollIntervalMs, timeoutMs, ready, () -> { }); + } + + public static long waitFor(String description, long pollIntervalMs, long timeoutMs, BooleanSupplier ready, Runnable onTimeout) { + LOGGER.debug("Waiting for {}", description); + long deadline = System.currentTimeMillis() + timeoutMs; + + String exceptionMessage = null; + String previousExceptionMessage = null; + + // in case we are polling every 1s, we want to print exception after x tries, not on the first try + // for minutes poll interval will 2 be enough + int exceptionAppearanceCount = Duration.ofMillis(pollIntervalMs).toMinutes() > 0 ? 2 : Math.max((int) (timeoutMs / pollIntervalMs) / 4, 2); + int exceptionCount = 0; + int newExceptionAppearance = 0; + + StringWriter stackTraceError = new StringWriter(); + + while (true) { + boolean result; + try { + result = ready.getAsBoolean(); + } catch (Exception e) { + exceptionMessage = e.getMessage(); + + if (++exceptionCount == exceptionAppearanceCount && exceptionMessage != null && exceptionMessage.equals(previousExceptionMessage)) { + LOGGER.error("While waiting for {} exception occurred: {}", description, exceptionMessage); + // log the stacktrace + e.printStackTrace(new PrintWriter(stackTraceError)); + } else if (exceptionMessage != null && !exceptionMessage.equals(previousExceptionMessage) && ++newExceptionAppearance == 2) { + previousExceptionMessage = exceptionMessage; + } + + result = false; + } + long timeLeft = deadline - System.currentTimeMillis(); + if (result) { + return timeLeft; + } + if (timeLeft <= 0) { + if (exceptionCount > 1) { + LOGGER.error("Exception waiting for {}, {}", description, exceptionMessage); + + if (!stackTraceError.toString().isEmpty()) { + // printing handled stacktrace + LOGGER.error(stackTraceError.toString()); + } + } + onTimeout.run(); + WaitException waitException = new WaitException("Timeout after " + timeoutMs + " ms waiting for " + description); + waitException.printStackTrace(); + throw waitException; + } + long sleepTime = Math.min(pollIntervalMs, timeLeft); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("{} not ready, will try again in {} ms ({}ms till timeout)", description, sleepTime, timeLeft); + } + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + return deadline - System.currentTimeMillis(); + } + } + } +} diff --git a/src/main/java/io/odh/test/framework/WaitException.java b/src/main/java/io/odh/test/framework/WaitException.java new file mode 100644 index 00000000..bead013c --- /dev/null +++ b/src/main/java/io/odh/test/framework/WaitException.java @@ -0,0 +1,15 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.framework; + +public class WaitException extends RuntimeException { + public WaitException(String message) { + super(message); + } + + public WaitException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/io/odh/test/framework/manager/ResourceCondition.java b/src/main/java/io/odh/test/framework/manager/ResourceCondition.java new file mode 100644 index 00000000..9fa6d7a8 --- /dev/null +++ b/src/main/java/io/odh/test/framework/manager/ResourceCondition.java @@ -0,0 +1,37 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ + +package io.odh.test.framework.manager; + +import io.fabric8.kubernetes.api.model.HasMetadata; + +import java.util.Objects; +import java.util.function.Predicate; + +public class ResourceCondition { + private final Predicate predicate; + private final String conditionName; + + public ResourceCondition(Predicate predicate, String conditionName) { + this.predicate = predicate; + this.conditionName = conditionName; + } + + public String getConditionName() { + return conditionName; + } + + public Predicate getPredicate() { + return predicate; + } + + public static ResourceCondition readiness(ResourceType type) { + return new ResourceCondition<>(type::waitForReadiness, "readiness"); + } + + public static ResourceCondition deletion() { + return new ResourceCondition<>(Objects::isNull, "deletion"); + } +} diff --git a/src/main/java/io/odh/test/framework/manager/ResourceItem.java b/src/main/java/io/odh/test/framework/manager/ResourceItem.java new file mode 100644 index 00000000..c4eafae3 --- /dev/null +++ b/src/main/java/io/odh/test/framework/manager/ResourceItem.java @@ -0,0 +1,29 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.framework.manager; + +import io.fabric8.kubernetes.api.model.HasMetadata; + +public final class ResourceItem { + + ThrowableRunner throwableRunner; + T resource; + + public ResourceItem(ThrowableRunner throwableRunner, T resource) { + this.throwableRunner = throwableRunner; + this.resource = resource; + } + + public ResourceItem(ThrowableRunner throwableRunner) { + this.throwableRunner = throwableRunner; + } + + public ThrowableRunner getThrowableRunner() { + return throwableRunner; + } + public T getResource() { + return resource; + } +} diff --git a/src/main/java/io/odh/test/framework/manager/ResourceManager.java b/src/main/java/io/odh/test/framework/manager/ResourceManager.java new file mode 100644 index 00000000..6275327a --- /dev/null +++ b/src/main/java/io/odh/test/framework/manager/ResourceManager.java @@ -0,0 +1,186 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.framework.manager; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.admissionregistration.v1.ValidatingWebhookConfiguration; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; +import io.fabric8.kubernetes.api.model.rbac.ClusterRole; +import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding; +import io.odh.test.TestConstants; +import io.odh.test.TestUtils; +import io.odh.test.framework.manager.resources.OperatorGroupResource; +import io.odh.test.framework.manager.resources.SubscriptionResource; +import io.odh.test.platform.KubeClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Stack; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ResourceManager { + private static final Logger LOGGER = LoggerFactory.getLogger(ResourceManager.class); + + private static ResourceManager instance; + private static KubeClient client; + + public static final Stack RESOURCE_STACK = new Stack<>(); + + public static synchronized ResourceManager getInstance() { + if (instance == null) { + instance = new ResourceManager(); + client = new KubeClient(TestConstants.DEFAULT_NAMESPACE); + } + return instance; + } + + public static KubeClient getClient() { + return client; + } + + private final ResourceType[] resourceTypes = new ResourceType[]{ + new SubscriptionResource(), + new OperatorGroupResource(), + }; + + @SafeVarargs + public final void createResourceWithoutWait(T... resources) { + createResource(false, resources); + } + + @SafeVarargs + public final void createResourceWithWait(T... resources) { + createResource(true, resources); + } + + @SafeVarargs + private void createResource(boolean waitReady, T... resources) { + for (T resource : resources) { + ResourceType type = findResourceType(resource); + + if (resource.getMetadata().getNamespace() == null) { + LOGGER.info("Creating/Updating {} {}", + resource.getKind(), resource.getMetadata().getName()); + } else { + LOGGER.info("Creating/Updating {} {}/{}", + resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName()); + } + + type.create(resource); + + synchronized (this) { + RESOURCE_STACK.push( + new ResourceItem( + () -> deleteResource(resource), + resource + )); + } + } + + if (waitReady) { + for (T resource : resources) { + ResourceType type = findResourceType(resource); + + assertTrue(waitResourceCondition(resource, ResourceCondition.readiness(type)), + String.format("Timed out waiting for %s %s/%s to be ready", resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName())); + } + } + } + + @SafeVarargs + public final void deleteResource(T... resources) { + for (T resource : resources) { + ResourceType type = findResourceType(resource); + if (type == null) { + LOGGER.warn("Can't find resource type, please delete it manually"); + continue; + } + + if (resource.getMetadata().getNamespace() == null) { + LOGGER.info("Deleting of {} {}", + resource.getKind(), resource.getMetadata().getName()); + } else { + LOGGER.info("Deleting of {} {}/{}", + resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName()); + } + + try { + type.delete(resource); + assertTrue(waitResourceCondition(resource, ResourceCondition.deletion()), + String.format("Timed out deleting %s %s/%s", resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName())); + } catch (Exception e) { + if (resource.getMetadata().getNamespace() == null) { + LOGGER.error("Failed to delete {} {}", resource.getKind(), resource.getMetadata().getName(), e); + } else { + LOGGER.error("Failed to delete {} {}/{}", resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName(), e); + } + } + + } + } + + @SafeVarargs + public final void updateResource(T... resources) { + for (T resource : resources) { + ResourceType type = findResourceType(resource); + type.update(resource); + } + } + + public final boolean waitResourceCondition(T resource, ResourceCondition condition) { + assertNotNull(resource); + assertNotNull(resource.getMetadata()); + assertNotNull(resource.getMetadata().getName()); + + // cluster role binding and custom resource definition does not need namespace... + if (!(resource instanceof ClusterRoleBinding || resource instanceof CustomResourceDefinition || resource instanceof ClusterRole || resource instanceof ValidatingWebhookConfiguration)) { + assertNotNull(resource.getMetadata().getNamespace()); + } + + ResourceType type = findResourceType(resource); + assertNotNull(type); + boolean[] resourceReady = new boolean[1]; + + TestUtils.waitFor("resource condition: " + condition.getConditionName() + " to be fulfilled for resource " + resource.getKind() + ":" + resource.getMetadata().getName(), + TestConstants.GLOBAL_POLL_INTERVAL, TestConstants.GLOBAL_TIMEOUT, + () -> { + T res = type.get(resource.getMetadata().getNamespace(), resource.getMetadata().getName()); + resourceReady[0] = condition.getPredicate().test(res); + if (!resourceReady[0]) { + type.delete(res); + } + return resourceReady[0]; + }); + + return resourceReady[0]; + } + + public void deleteResources() { + LOGGER.info(String.join("", Collections.nCopies(76, "#"))); + + while (!RESOURCE_STACK.empty()) { + try { + ResourceItem resourceItem = RESOURCE_STACK.pop(); + resourceItem.getThrowableRunner().run(); + } catch (Exception e) { + e.printStackTrace(); + } + } + LOGGER.info(String.join("", Collections.nCopies(76, "#"))); + } + + private ResourceType findResourceType(T resource) { + // other no conflicting types + for (ResourceType type : resourceTypes) { + if (type.getKind().equals(resource.getKind())) { + return (ResourceType) type; + } + } + return null; + } +} diff --git a/src/main/java/io/odh/test/framework/manager/ResourceType.java b/src/main/java/io/odh/test/framework/manager/ResourceType.java new file mode 100644 index 00000000..86b50133 --- /dev/null +++ b/src/main/java/io/odh/test/framework/manager/ResourceType.java @@ -0,0 +1,43 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.framework.manager; + +import io.fabric8.kubernetes.api.model.HasMetadata; + +/** + * Providing contract for all resources which must implement REST API methods for create, update (refresh) and so on. + * @param type for all our resources for instance KafkaResource, KafkaConnectResource, OlmResource, ServiceResource etc. + */ +public interface ResourceType { + String getKind(); + + /** + * Retrieve resource using Kubernetes API + * @return specific resource with T type. + */ + T get(String namespace, String name); + + /** + * Creates specific resource based on T type using Kubernetes API + */ + void create(T resource); + + /** + * Delete specific resource based on T type using Kubernetes API + */ + void delete(T resource); + + /** + * Update specific resource based on T type using Kubernetes API + */ + void update(T resource); + + /** + * Check if this resource is marked as ready or not with wait. + * + * @return true if ready. + */ + boolean waitForReadiness(T resource); +} diff --git a/src/main/java/io/odh/test/framework/manager/ThrowableRunner.java b/src/main/java/io/odh/test/framework/manager/ThrowableRunner.java new file mode 100644 index 00000000..9cce54f7 --- /dev/null +++ b/src/main/java/io/odh/test/framework/manager/ThrowableRunner.java @@ -0,0 +1,10 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.framework.manager; + +@FunctionalInterface +public interface ThrowableRunner { + void run() throws Exception; +} diff --git a/src/main/java/io/odh/test/framework/manager/resources/OperatorGroupResource.java b/src/main/java/io/odh/test/framework/manager/resources/OperatorGroupResource.java new file mode 100644 index 00000000..146e5a08 --- /dev/null +++ b/src/main/java/io/odh/test/framework/manager/resources/OperatorGroupResource.java @@ -0,0 +1,50 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.framework.manager.resources; + +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.openshift.api.model.operatorhub.v1.OperatorGroup; +import io.fabric8.openshift.api.model.operatorhub.v1.OperatorGroupList; +import io.fabric8.openshift.client.OpenShiftClient; +import io.odh.test.TestConstants; +import io.odh.test.framework.manager.ResourceManager; +import io.odh.test.framework.manager.ResourceType; + +public class OperatorGroupResource implements ResourceType { + @Override + public String getKind() { + return TestConstants.OPERATOR_GROUP; + } + + @Override + public OperatorGroup get(String namespace, String name) { + return operatorGroupClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(OperatorGroup resource) { + operatorGroupClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(OperatorGroup resource) { + operatorGroupClient().inNamespace(resource.getMetadata().getNamespace()).withName(resource.getMetadata().getName()).delete(); + } + + @Override + public void update(OperatorGroup resource) { + operatorGroupClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(OperatorGroup resource) { + return resource != null; + } + + public static MixedOperation> operatorGroupClient() { + return ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().operatorGroups(); + } +} diff --git a/src/main/java/io/odh/test/framework/manager/resources/SubscriptionResource.java b/src/main/java/io/odh/test/framework/manager/resources/SubscriptionResource.java new file mode 100644 index 00000000..aae371f0 --- /dev/null +++ b/src/main/java/io/odh/test/framework/manager/resources/SubscriptionResource.java @@ -0,0 +1,53 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.framework.manager.resources; + +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.openshift.api.model.operatorhub.v1alpha1.Subscription; +import io.fabric8.openshift.api.model.operatorhub.v1alpha1.SubscriptionList; +import io.fabric8.openshift.client.OpenShiftClient; +import io.odh.test.TestConstants; +import io.odh.test.framework.manager.ResourceManager; +import io.odh.test.framework.manager.ResourceType; + +public class SubscriptionResource implements ResourceType { + + @Override + public String getKind() { + return TestConstants.SUBSCRIPTION; + } + + @Override + public Subscription get(String namespace, String name) { + return subscriptionClient().inNamespace(namespace).withName(name).get(); + } + + @Override + public void create(Subscription resource) { + subscriptionClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).create(); + } + + @Override + public void delete(Subscription resource) { + subscriptionClient().inNamespace(resource.getMetadata().getNamespace()) + .withName(resource.getMetadata().getName()).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete(); + } + + @Override + public void update(Subscription resource) { + subscriptionClient().inNamespace(resource.getMetadata().getNamespace()).resource(resource).update(); + } + + @Override + public boolean waitForReadiness(Subscription resource) { + return resource != null; + } + + public static MixedOperation> subscriptionClient() { + return ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().subscriptions(); + } +} diff --git a/src/main/java/io/odh/test/install/OlmInstall.java b/src/main/java/io/odh/test/install/OlmInstall.java index f84ed032..07f823dc 100644 --- a/src/main/java/io/odh/test/install/OlmInstall.java +++ b/src/main/java/io/odh/test/install/OlmInstall.java @@ -4,5 +4,155 @@ */ package io.odh.test.install; +import io.fabric8.openshift.api.model.operatorhub.v1.OperatorGroupBuilder; +import io.fabric8.openshift.api.model.operatorhub.v1alpha1.Subscription; +import io.fabric8.openshift.api.model.operatorhub.v1alpha1.SubscriptionBuilder; +import io.fabric8.openshift.client.OpenShiftClient; +import io.odh.test.Environment; +import io.odh.test.framework.manager.ResourceItem; +import io.odh.test.framework.manager.ResourceManager; +import io.odh.test.framework.manager.resources.OperatorGroupResource; +import io.odh.test.utils.DeploymentUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + public class OlmInstall { + private static final Logger LOGGER = LoggerFactory.getLogger(ResourceManager.class); + + private String namespace = Environment.OLM_OPERATOR_NAMESPACE; + private String channel = Environment.OLM_OPERATOR_CHANNEL; + private String name = Environment.OLM_OPERATOR_NAME; + private String operatorName = Environment.OLM_OPERATOR_NAME; + private String sourceName = Environment.OLM_SOURCE_NAME; + private String sourceNamespace = Environment.OLM_SOURCE_NAMESPACE; + private String startingCsv; + private String deploymentName = Environment.OLM_OPERATOR_DEPLOYMENT_NAME; + private String olmAppBundlePrefix = Environment.OLM_OPERATOR_NAME; + private String operatorVersion = Environment.OLM_OPERATOR_VERSION; + private String csvName = operatorName + ".v" + operatorVersion; + + public void create() { + createOperatorGroup(); + ResourceManager.RESOURCE_STACK.push(new ResourceItem(this::deleteCSV)); + createAndModifySubscription(); + + // Wait for operator creation + DeploymentUtils.waitForDeploymentReady(namespace, deploymentName); + } + + /** + * Creates OperatorGroup in specific namespace + */ + private void createOperatorGroup() { + if (OperatorGroupResource.operatorGroupClient().inNamespace(namespace).list().getItems().isEmpty()) { + OperatorGroupBuilder operatorGroup = new OperatorGroupBuilder() + .editOrNewMetadata() + .withName("odh-group") + .withNamespace(namespace) + .withLabels(Collections.singletonMap("app", "odh")) + .endMetadata(); + + ResourceManager.getInstance().createResourceWithWait(operatorGroup.build()); + } else { + LOGGER.info("OperatorGroup is already exists."); + } + } + + /** + * Creates Subscription with spec from OlmConfiguration + */ + + private void createAndModifySubscription() { + Subscription subscription = prepareSubscription(); + + ResourceManager.getInstance().createResourceWithWait(subscription); +// ResourceManager.RESOURCE_STACK.push(new ResourceItem(this::deleteCSV)); + + } + public void updateSubscription() { + Subscription subscription = prepareSubscription(); + ResourceManager.getInstance().updateResource(subscription); + } + + public Subscription prepareSubscription() { + return new SubscriptionBuilder() + .editOrNewMetadata() + .withName(name) + .withNamespace(namespace) + .withLabels(Collections.singletonMap("app", "odh")) + .endMetadata() + .editOrNewSpec() + .withName(operatorName) + .withSource(sourceName) + .withSourceNamespace(sourceNamespace) + .withChannel(channel) + .withStartingCSV(startingCsv) + .withInstallPlanApproval("Automatic") + .editOrNewConfig() + .endConfig() + .endSpec() + .build(); + } + + public void deleteCSV() { + LOGGER.info("Deleting CSV {}/{}", namespace, olmAppBundlePrefix); + ResourceManager.getClient().getClient().adapt(OpenShiftClient.class).operatorHub().clusterServiceVersions().inNamespace(namespace).withName(csvName).delete(); + } + + /** + * Useful getters for possible changes in upgrade tests + */ + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getOperatorName() { + return operatorName; + } + + public void setOperatorName(String operatorName) { + this.operatorName = operatorName; + } + + public String getStartingCsv() { + return startingCsv; + } + + public void setStartingCsv(String startingCsv) { + this.startingCsv = startingCsv; + } + + public String getDeploymentName() { + return deploymentName; + } + + public String getNamespace() { + return namespace; + } + + public void setDeploymentName(String deploymentName) { + this.deploymentName = deploymentName; + } + + public String getOperatorVersion() { + return operatorVersion; + } + + public void setOperatorVersion(String operatorVersion) { + this.operatorVersion = operatorVersion; + } + + public String getCsvName() { + return csvName; + } + + public void setCsvName(String csvName) { + this.csvName = csvName; + } } diff --git a/src/main/java/io/odh/test/platform/KubeClient.java b/src/main/java/io/odh/test/platform/KubeClient.java index 71a291a9..f733f28e 100644 --- a/src/main/java/io/odh/test/platform/KubeClient.java +++ b/src/main/java/io/odh/test/platform/KubeClient.java @@ -107,10 +107,6 @@ private Config getConfig() { } } - public String getNamespace() { - return namespace; - } - public Namespace getNamespace(String namespace) { return client.namespaces().withName(namespace).get(); } @@ -139,8 +135,8 @@ public ConfigMap getConfigMap(String configMapName) { } - public boolean getConfigMapStatus(String configMapName) { - return client.configMaps().inNamespace(getNamespace()).withName(configMapName).isReady(); + public boolean getConfigMapStatus(String namespace, String configMapName) { + return client.configMaps().inNamespace(namespace).withName(configMapName).isReady(); } // ========================= @@ -269,8 +265,8 @@ public Job getJob(String jobName) { return client.batch().v1().jobs().inNamespace(namespace).withName(jobName).get(); } - public boolean checkSucceededJobStatus(String jobName) { - return checkSucceededJobStatus(getNamespace(), jobName, 1); + public boolean checkSucceededJobStatus(String namespace, String jobName) { + return checkSucceededJobStatus(namespace, jobName, 1); } public boolean checkSucceededJobStatus(String namespaceName, String jobName, int expectedSucceededPods) { @@ -294,11 +290,22 @@ public JobList getJobList() { return client.batch().v1().jobs().inNamespace(namespace).list(); } - public List listJobs(String namePrefix) { - return client.batch().v1().jobs().inNamespace(getNamespace()).list().getItems().stream() + public List listJobs(String namespace, String namePrefix) { + return client.batch().v1().jobs().inNamespace(namespace).list().getItems().stream() .filter(job -> job.getMetadata().getName().startsWith(namePrefix)).collect(Collectors.toList()); } + public String getDeploymentNameByPrefix(String namespace, String namePrefix) { + List prefixDeployments = client.apps().deployments().inNamespace(namespace).list().getItems().stream().filter( + rs -> rs.getMetadata().getName().startsWith(namePrefix)).toList(); + + if (!prefixDeployments.isEmpty()) { + return prefixDeployments.get(0).getMetadata().getName(); + } else { + return null; + } + } + public MixedOperation, Resource> dataScienceClusterClient() { return client.resources(DataScienceCluster.class); } diff --git a/src/main/java/io/odh/test/utils/DeploymentUtils.java b/src/main/java/io/odh/test/utils/DeploymentUtils.java new file mode 100644 index 00000000..0fb21f7e --- /dev/null +++ b/src/main/java/io/odh/test/utils/DeploymentUtils.java @@ -0,0 +1,97 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.utils; + +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodCondition; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DeploymentCondition; +import io.odh.test.TestConstants; +import io.odh.test.TestUtils; +import io.odh.test.framework.manager.ResourceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static java.util.Arrays.asList; + +public class DeploymentUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DeploymentUtils.class); + private static final long READINESS_TIMEOUT = TestConstants.GLOBAL_TIMEOUT; + private static final long DELETION_TIMEOUT = TestConstants.GLOBAL_TIMEOUT; + + private DeploymentUtils() { } + + /** + * Log actual status of deployment with pods + * @param deployment - every Deployment, that HasMetadata and has status (fabric8 status) + **/ + public static void logCurrentDeploymentStatus(Deployment deployment, String namespaceName) { + if (deployment != null) { + String kind = deployment.getKind(); + String name = deployment.getMetadata().getName(); + + List log = new ArrayList<>(asList("\n", kind, " status:\n", "\nConditions:\n")); + + for (DeploymentCondition deploymentCondition : deployment.getStatus().getConditions()) { + log.add("\tType: " + deploymentCondition.getType() + "\n"); + log.add("\tMessage: " + deploymentCondition.getMessage() + "\n"); + } + + if (!ResourceManager.getClient().listPodsByPrefixInName(namespaceName, name).isEmpty()) { + log.add("\nPods with conditions and messages:\n\n"); + + for (Pod pod : ResourceManager.getClient().listPodsByPrefixInName(namespaceName, name)) { + log.add(pod.getMetadata().getName() + ":"); + for (PodCondition podCondition : pod.getStatus().getConditions()) { + if (podCondition.getMessage() != null) { + log.add("\n\tType: " + podCondition.getType() + "\n"); + log.add("\tMessage: " + podCondition.getMessage() + "\n"); + } + } + log.add("\n\n"); + } + LOGGER.info("{}", String.join("", log)); + } + + LOGGER.info("{}", String.join("", log)); + } + } + + public static boolean waitForDeploymentReady(String namespaceName, String deploymentName) { + LOGGER.info("Waiting for Deployment: {}/{} to be ready", namespaceName, deploymentName); + + TestUtils.waitFor("readiness of Deployment: " + namespaceName + "/" + deploymentName, + TestConstants.GLOBAL_POLL_INTERVAL_SHORT, READINESS_TIMEOUT, + () -> ResourceManager.getClient().getClient().apps().deployments().inNamespace(namespaceName).withName(deploymentName).isReady(), + () -> DeploymentUtils.logCurrentDeploymentStatus(ResourceManager.getClient().getDeployment(namespaceName, deploymentName), namespaceName)); + + LOGGER.info("Deployment: {}/{} is ready", namespaceName, deploymentName); + return true; + } + + /** + * Wait until the given Deployment has been deleted. + * @param namespaceName Namespace name + * @param name The name of the Deployment. + */ + public static void waitForDeploymentDeletion(String namespaceName, String name) { + LOGGER.debug("Waiting for Deployment: {}/{} deletion", namespaceName, name); + TestUtils.waitFor("deletion of Deployment: " + namespaceName + "/" + name, TestConstants.GLOBAL_POLL_INTERVAL, DELETION_TIMEOUT, + () -> { + if (ResourceManager.getClient().getDeployment(namespaceName, name) == null) { + return true; + } else { + LOGGER.warn("Deployment: {}/{} is not deleted yet! Triggering force delete by cmd client!", namespaceName, name); + ResourceManager.getClient().getClient().apps().deployments().inNamespace(namespaceName).withName(name).delete(); + return false; + } + }); + LOGGER.debug("Deployment: {}/{} was deleted", namespaceName, name); + } +} diff --git a/src/test/java/io/odh/test/e2e/Abstract.java b/src/test/java/io/odh/test/e2e/Abstract.java index 58e564bb..c67fef88 100644 --- a/src/test/java/io/odh/test/e2e/Abstract.java +++ b/src/test/java/io/odh/test/e2e/Abstract.java @@ -4,13 +4,23 @@ */ package io.odh.test.e2e; +import io.odh.test.framework.manager.ResourceManager; import io.odh.test.platform.KubeClient; import io.odh.test.TestConstants; import io.odh.test.framework.TestSeparator; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.TestInstance; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class Abstract implements TestSeparator { protected KubeClient kubeClient = new KubeClient(TestConstants.ODH_NAMESPACE); + static { + ResourceManager.getInstance(); + } + + @AfterAll + void teardownEnvironment() { + ResourceManager.getInstance().deleteResources(); + } } diff --git a/src/test/java/io/odh/test/e2e/standard/OdhInstall.java b/src/test/java/io/odh/test/e2e/standard/OdhInstall.java new file mode 100644 index 00000000..0485ff8d --- /dev/null +++ b/src/test/java/io/odh/test/e2e/standard/OdhInstall.java @@ -0,0 +1,25 @@ +/* + * Copyright Skodjob authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.odh.test.e2e.standard; + +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.odh.test.e2e.Abstract; +import io.odh.test.framework.manager.ResourceManager; +import io.odh.test.install.OlmInstall; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class OdhInstall extends Abstract { + + @Test + void testInstallOdh() { + OlmInstall olmInstall = new OlmInstall(); + olmInstall.create(); + + Deployment dep = ResourceManager.getClient().getDeployment(olmInstall.getNamespace(), olmInstall.getDeploymentName()); + assertNotNull(dep); + } +}