From 5638944f49cf74bd6d7c26a437ec7c41057b17a0 Mon Sep 17 00:00:00 2001 From: David Kornel Date: Thu, 18 Jan 2024 17:16:39 +0100 Subject: [PATCH] Add logging of labeled namespaces + increase long duration time --- src/main/java/io/odh/test/TestConstants.java | 3 +- .../odh/test/framework/logs/LogCollector.java | 41 +++++++++++-------- .../framework/manager/ResourceManager.java | 32 +++++++++++++-- .../io/odh/test/e2e/standard/NotebookST.java | 1 - 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/main/java/io/odh/test/TestConstants.java b/src/main/java/io/odh/test/TestConstants.java index 654d38c4..1266a3c7 100644 --- a/src/main/java/io/odh/test/TestConstants.java +++ b/src/main/java/io/odh/test/TestConstants.java @@ -18,8 +18,9 @@ public class TestConstants { public static final long GLOBAL_POLL_INTERVAL_LONG = Duration.ofSeconds(15).toMillis(); public static final long GLOBAL_POLL_INTERVAL_MEDIUM = Duration.ofSeconds(10).toMillis(); public static final long GLOBAL_POLL_INTERVAL_SHORT = Duration.ofSeconds(5).toMillis(); - public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(5).toMillis(); + public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(10).toMillis(); public static final long GLOBAL_STABILITY_TIME = Duration.ofMinutes(1).toSeconds(); + public static final String LOG_COLLECT_LABEL = "io.odh-e2e.collect-logs"; private TestConstants() { } diff --git a/src/main/java/io/odh/test/framework/logs/LogCollector.java b/src/main/java/io/odh/test/framework/logs/LogCollector.java index 5cb5e37c..ea87ac9b 100644 --- a/src/main/java/io/odh/test/framework/logs/LogCollector.java +++ b/src/main/java/io/odh/test/framework/logs/LogCollector.java @@ -7,6 +7,7 @@ import io.fabric8.kubernetes.api.model.Pod; import io.odh.test.Environment; import io.odh.test.OdhConstants; +import io.odh.test.TestConstants; import io.odh.test.TestUtils; import io.odh.test.framework.manager.ResourceManager; import io.odh.test.platform.KubeClient; @@ -28,6 +29,9 @@ public class LogCollector { public static void saveKubernetesState(ExtensionContext extensionContext, Throwable throwable) throws Throwable { Path logPath = TestUtils.getLogPath(Environment.LOG_DIR.resolve("failedTest").toString(), extensionContext); Files.createDirectories(logPath); + ResourceManager.addNamespaceForLogCollect(OdhConstants.BUNDLE_OPERATOR_NAMESPACE); + ResourceManager.addNamespaceForLogCollect(OdhConstants.OLM_OPERATOR_NAMESPACE); + ResourceManager.addNamespaceForLogCollect(OdhConstants.CONTROLLERS_NAMESPACE); LOGGER.info("Storing cluster info into {}", logPath); try { saveClusterState(logPath); @@ -38,10 +42,15 @@ public static void saveKubernetesState(ExtensionContext extensionContext, Throwa } private static void writeLogsFromPods(Path logpath, Pod pod) { + try { + Files.createDirectories(logpath.resolve(pod.getMetadata().getNamespace())); + } catch (IOException e) { + LOGGER.warn("Cannot create logdir in {}", logpath); + } pod.getSpec().getContainers().forEach(container -> { try { LOGGER.debug("Get logs from pod {}/{} container {}", pod.getMetadata().getNamespace(), pod.getMetadata().getName(), container.getName()); - Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + "-" + container.getName() + ".log"), + Files.writeString(logpath.resolve(pod.getMetadata().getNamespace()).resolve(pod.getMetadata().getName() + "-" + container.getName() + ".log"), ResourceManager.getClient().getLogsFromContainer(pod.getMetadata().getNamespace(), pod.getMetadata().getName(), container.getName())); } catch (IOException e) { LOGGER.warn("Cannot get logs for pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); @@ -50,9 +59,14 @@ private static void writeLogsFromPods(Path logpath, Pod pod) { } private static void writePodsDescription(Path logpath, Pod pod) { + try { + Files.createDirectories(logpath.resolve(pod.getMetadata().getNamespace())); + } catch (IOException e) { + LOGGER.warn("Cannot create logdir in {}", logpath); + } try { LOGGER.debug("Get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); - Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".describe.log"), + Files.writeString(logpath.resolve(pod.getMetadata().getNamespace()).resolve(pod.getMetadata().getName() + ".describe.log"), ResourceManager.getKubeCmdClient().namespace(pod.getMetadata().getNamespace()).describe(pod.getKind(), pod.getMetadata().getName())); } catch (IOException e) { LOGGER.warn("Cannot get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName()); @@ -62,6 +76,8 @@ private static void writePodsDescription(Path logpath, Pod pod) { private static void saveClusterState(Path logpath) throws IOException { KubeClient kube = ResourceManager.getClient(); KubeCmdClient cmdClient = ResourceManager.getKubeCmdClient(); + + // Collecting cluster wide resources and CRs Files.writeString(logpath.resolve("describe-cluster-nodes.log"), cmdClient.exec(false, false, "describe", "nodes").out()); Files.writeString(logpath.resolve("all-events.log"), cmdClient.exec(false, false, "get", "events", "--all-namespaces").out()); Files.writeString(logpath.resolve("pvs.log"), cmdClient.exec(false, false, "describe", "pv").out()); @@ -69,20 +85,13 @@ private static void saveClusterState(Path logpath) throws IOException { Files.writeString(logpath.resolve("dsci.yml"), cmdClient.exec(false, false, "get", "dsci", "-o", "yaml").out()); Files.writeString(logpath.resolve("subscriptions.yml"), cmdClient.exec(false, false, "get", "subscriptions.operators.coreos.com", "--all-namespaces", "-o", "yaml").out()); Files.writeString(logpath.resolve("notebooks.yml"), cmdClient.exec(false, false, "get", "notebook", "--all-namespaces", "-o", "yaml").out()); - LOGGER.debug("Listing pods in {}", OdhConstants.BUNDLE_OPERATOR_NAMESPACE); - kube.listPodsByPrefixInName(OdhConstants.BUNDLE_OPERATOR_NAMESPACE, "opendatahub-operator-controller-manager").forEach(pod -> { - writeLogsFromPods(logpath, pod); - writePodsDescription(logpath, pod); - }); - LOGGER.debug("Listing pods in {}", OdhConstants.OLM_OPERATOR_NAMESPACE); - kube.listPods(OdhConstants.OLM_OPERATOR_NAMESPACE).forEach(pod -> { - writeLogsFromPods(logpath, pod); - writePodsDescription(logpath, pod); - }); - LOGGER.debug("Listing pods in {}", OdhConstants.CONTROLLERS_NAMESPACE); - kube.listPods(OdhConstants.CONTROLLERS_NAMESPACE).forEach(pod -> { - writeLogsFromPods(logpath, pod); - writePodsDescription(logpath, pod); + + kube.getClient().namespaces().withLabel(TestConstants.LOG_COLLECT_LABEL).list().getItems().forEach(ns -> { + LOGGER.debug("Listing pods in {}", ns.getMetadata().getNamespace()); + kube.listPodsByPrefixInName(ns.getMetadata().getName(), "opendatahub-operator-controller-manager").forEach(pod -> { + writeLogsFromPods(logpath, pod); + writePodsDescription(logpath, pod); + }); }); } } diff --git a/src/main/java/io/odh/test/framework/manager/ResourceManager.java b/src/main/java/io/odh/test/framework/manager/ResourceManager.java index a37fac82..ee44f18e 100644 --- a/src/main/java/io/odh/test/framework/manager/ResourceManager.java +++ b/src/main/java/io/odh/test/framework/manager/ResourceManager.java @@ -6,6 +6,7 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Namespace; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.admissionregistration.v1.ValidatingWebhookConfiguration; import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; import io.fabric8.kubernetes.api.model.apps.Deployment; @@ -28,7 +29,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Stack; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -46,11 +49,20 @@ public class ResourceManager { static Stack resourceStackPointer = CLASS_RESOURCE_STACK; + static List defaultNamespacesForLogCollect = Arrays.asList( + "openshift-marketplace", + "openshift-operators", + "openshift-operators-redhat" + ); + public static synchronized ResourceManager getInstance() { if (instance == null) { instance = new ResourceManager(); client = new KubeClient(TestConstants.DEFAULT_NAMESPACE); kubeCmdClient = new Oc(client.getKubeconfigPath()); + for (String ns : defaultNamespacesForLogCollect) { + addNamespaceForLogCollect(ns); + } } return instance; } @@ -80,6 +92,17 @@ public final void switchToClassResourceStack() { resourceStackPointer = CLASS_RESOURCE_STACK; } + public static void addNamespaceForLogCollect(String namespace) { + if (getClient().namespaceExists(namespace)) { + getClient().getClient().namespaces().withName(namespace).edit(n -> + new NamespaceBuilder(n) + .editMetadata() + .addToLabels(TestConstants.LOG_COLLECT_LABEL, "true") + .endMetadata() + .build()); + } + } + public final void pushToStack(ResourceItem item) { resourceStackPointer.push(item); } @@ -101,10 +124,10 @@ private void createResource(boolean waitReady, T... reso synchronized (this) { resourceStackPointer.push( - new ResourceItem( - () -> deleteResource(resource), - resource - )); + new ResourceItem( + () -> deleteResource(resource), + resource + )); } if (resource.getMetadata().getNamespace() == null) { @@ -113,6 +136,7 @@ private void createResource(boolean waitReady, T... reso } else { LOGGER.info("Creating/Updating {} {}/{}", resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName()); + addNamespaceForLogCollect(resource.getMetadata().getNamespace()); } if (type == null) { diff --git a/src/test/java/io/odh/test/e2e/standard/NotebookST.java b/src/test/java/io/odh/test/e2e/standard/NotebookST.java index 725f7af9..10745f7c 100644 --- a/src/test/java/io/odh/test/e2e/standard/NotebookST.java +++ b/src/test/java/io/odh/test/e2e/standard/NotebookST.java @@ -85,7 +85,6 @@ void testCreateSimpleNotebook() throws IOException { .build(); PodUtils.waitForPodsReady(NTB_NAMESPACE, lblSelector, 1, true, () -> { }); - } @BeforeAll