Add logging of labeled namespaces + increase long duration time

skodjob · Jan 18, 2024 · 5638944 · 5638944
1 parent c9b43ad
commit 5638944
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 22 deletions.
diff --git a/src/main/java/io/odh/test/TestConstants.java b/src/main/java/io/odh/test/TestConstants.java
@@ -18,8 +18,9 @@ public class TestConstants {
     public static final long GLOBAL_POLL_INTERVAL_LONG = Duration.ofSeconds(15).toMillis();
     public static final long GLOBAL_POLL_INTERVAL_MEDIUM = Duration.ofSeconds(10).toMillis();
     public static final long GLOBAL_POLL_INTERVAL_SHORT = Duration.ofSeconds(5).toMillis();
-    public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(5).toMillis();
+    public static final long GLOBAL_TIMEOUT = Duration.ofMinutes(10).toMillis();
     public static final long GLOBAL_STABILITY_TIME = Duration.ofMinutes(1).toSeconds();
+    public static final String LOG_COLLECT_LABEL = "io.odh-e2e.collect-logs";
 
     private TestConstants() {
     }

diff --git a/src/main/java/io/odh/test/framework/logs/LogCollector.java b/src/main/java/io/odh/test/framework/logs/LogCollector.java
@@ -7,6 +7,7 @@
 import io.fabric8.kubernetes.api.model.Pod;
 import io.odh.test.Environment;
 import io.odh.test.OdhConstants;
+import io.odh.test.TestConstants;
 import io.odh.test.TestUtils;
 import io.odh.test.framework.manager.ResourceManager;
 import io.odh.test.platform.KubeClient;
@@ -28,6 +29,9 @@ public class LogCollector {
     public static void saveKubernetesState(ExtensionContext extensionContext, Throwable throwable) throws Throwable {
         Path logPath = TestUtils.getLogPath(Environment.LOG_DIR.resolve("failedTest").toString(), extensionContext);
         Files.createDirectories(logPath);
+        ResourceManager.addNamespaceForLogCollect(OdhConstants.BUNDLE_OPERATOR_NAMESPACE);
+        ResourceManager.addNamespaceForLogCollect(OdhConstants.OLM_OPERATOR_NAMESPACE);
+        ResourceManager.addNamespaceForLogCollect(OdhConstants.CONTROLLERS_NAMESPACE);
         LOGGER.info("Storing cluster info into {}", logPath);
         try {
             saveClusterState(logPath);
@@ -38,10 +42,15 @@ public static void saveKubernetesState(ExtensionContext extensionContext, Throwa
     }
 
     private static void writeLogsFromPods(Path logpath, Pod pod) {
+        try {
+            Files.createDirectories(logpath.resolve(pod.getMetadata().getNamespace()));
+        } catch (IOException e) {
+            LOGGER.warn("Cannot create logdir in {}", logpath);
+        }
         pod.getSpec().getContainers().forEach(container -> {
             try {
                 LOGGER.debug("Get logs from pod {}/{} container {}", pod.getMetadata().getNamespace(), pod.getMetadata().getName(), container.getName());
-                Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + "-" + container.getName() + ".log"),
+                Files.writeString(logpath.resolve(pod.getMetadata().getNamespace()).resolve(pod.getMetadata().getName() + "-" + container.getName() + ".log"),
                         ResourceManager.getClient().getLogsFromContainer(pod.getMetadata().getNamespace(), pod.getMetadata().getName(), container.getName()));
             } catch (IOException e) {
                 LOGGER.warn("Cannot get logs for pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
@@ -50,9 +59,14 @@ private static void writeLogsFromPods(Path logpath, Pod pod) {
     }
 
     private static void writePodsDescription(Path logpath, Pod pod) {
+        try {
+            Files.createDirectories(logpath.resolve(pod.getMetadata().getNamespace()));
+        } catch (IOException e) {
+            LOGGER.warn("Cannot create logdir in {}", logpath);
+        }
         try {
             LOGGER.debug("Get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
-            Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".describe.log"),
+            Files.writeString(logpath.resolve(pod.getMetadata().getNamespace()).resolve(pod.getMetadata().getName() + ".describe.log"),
                     ResourceManager.getKubeCmdClient().namespace(pod.getMetadata().getNamespace()).describe(pod.getKind(), pod.getMetadata().getName()));
         } catch (IOException e) {
             LOGGER.warn("Cannot get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
@@ -62,27 +76,22 @@ private static void writePodsDescription(Path logpath, Pod pod) {
     private static void saveClusterState(Path logpath) throws IOException {
         KubeClient kube = ResourceManager.getClient();
         KubeCmdClient cmdClient = ResourceManager.getKubeCmdClient();
+
+        // Collecting cluster wide resources and CRs
         Files.writeString(logpath.resolve("describe-cluster-nodes.log"), cmdClient.exec(false, false, "describe", "nodes").out());
         Files.writeString(logpath.resolve("all-events.log"), cmdClient.exec(false, false, "get", "events", "--all-namespaces").out());
         Files.writeString(logpath.resolve("pvs.log"), cmdClient.exec(false, false, "describe", "pv").out());
         Files.writeString(logpath.resolve("dsc.yml"), cmdClient.exec(false, false, "get", "dsc", "-o", "yaml").out());
         Files.writeString(logpath.resolve("dsci.yml"), cmdClient.exec(false, false, "get", "dsci", "-o", "yaml").out());
         Files.writeString(logpath.resolve("subscriptions.yml"), cmdClient.exec(false, false, "get", "subscriptions.operators.coreos.com", "--all-namespaces", "-o", "yaml").out());
         Files.writeString(logpath.resolve("notebooks.yml"), cmdClient.exec(false, false, "get", "notebook", "--all-namespaces", "-o", "yaml").out());
-        LOGGER.debug("Listing pods in {}", OdhConstants.BUNDLE_OPERATOR_NAMESPACE);
-        kube.listPodsByPrefixInName(OdhConstants.BUNDLE_OPERATOR_NAMESPACE, "opendatahub-operator-controller-manager").forEach(pod -> {
-            writeLogsFromPods(logpath, pod);
-            writePodsDescription(logpath, pod);
-        });
-        LOGGER.debug("Listing pods in {}", OdhConstants.OLM_OPERATOR_NAMESPACE);
-        kube.listPods(OdhConstants.OLM_OPERATOR_NAMESPACE).forEach(pod -> {
-            writeLogsFromPods(logpath, pod);
-            writePodsDescription(logpath, pod);
-        });
-        LOGGER.debug("Listing pods in {}", OdhConstants.CONTROLLERS_NAMESPACE);
-        kube.listPods(OdhConstants.CONTROLLERS_NAMESPACE).forEach(pod -> {
-            writeLogsFromPods(logpath, pod);
-            writePodsDescription(logpath, pod);
+
+        kube.getClient().namespaces().withLabel(TestConstants.LOG_COLLECT_LABEL).list().getItems().forEach(ns -> {
+            LOGGER.debug("Listing pods in {}", ns.getMetadata().getNamespace());
+            kube.listPodsByPrefixInName(ns.getMetadata().getName(), "opendatahub-operator-controller-manager").forEach(pod -> {
+                writeLogsFromPods(logpath, pod);
+                writePodsDescription(logpath, pod);
+            });
         });
     }
 }
diff --git a/src/main/java/io/odh/test/framework/manager/ResourceManager.java b/src/main/java/io/odh/test/framework/manager/ResourceManager.java
@@ -6,6 +6,7 @@
 
 import io.fabric8.kubernetes.api.model.HasMetadata;
 import io.fabric8.kubernetes.api.model.Namespace;
+import io.fabric8.kubernetes.api.model.NamespaceBuilder;
 import io.fabric8.kubernetes.api.model.admissionregistration.v1.ValidatingWebhookConfiguration;
 import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
@@ -28,7 +29,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Stack;
 
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -46,11 +49,20 @@ public class ResourceManager {
 
     static Stack<ResourceItem> resourceStackPointer = CLASS_RESOURCE_STACK;
 
+    static List<String> defaultNamespacesForLogCollect = Arrays.asList(
+        "openshift-marketplace",
+        "openshift-operators",
+        "openshift-operators-redhat"
+    );
+
     public static synchronized ResourceManager getInstance() {
         if (instance == null) {
             instance = new ResourceManager();
             client = new KubeClient(TestConstants.DEFAULT_NAMESPACE);
             kubeCmdClient = new Oc(client.getKubeconfigPath());
+            for (String ns : defaultNamespacesForLogCollect) {
+                addNamespaceForLogCollect(ns);
+            }
         }
         return instance;
     }
@@ -80,6 +92,17 @@ public final void switchToClassResourceStack() {
         resourceStackPointer = CLASS_RESOURCE_STACK;
     }
 
+    public static void addNamespaceForLogCollect(String namespace) {
+        if (getClient().namespaceExists(namespace)) {
+            getClient().getClient().namespaces().withName(namespace).edit(n ->
+                    new NamespaceBuilder(n)
+                            .editMetadata()
+                            .addToLabels(TestConstants.LOG_COLLECT_LABEL, "true")
+                            .endMetadata()
+                            .build());
+        }
+    }
+
     public final void pushToStack(ResourceItem item) {
         resourceStackPointer.push(item);
     }
@@ -101,10 +124,10 @@ private <T extends HasMetadata> void createResource(boolean waitReady, T... reso
 
             synchronized (this) {
                 resourceStackPointer.push(
-                    new ResourceItem<T>(
-                        () -> deleteResource(resource),
-                        resource
-                    ));
+                        new ResourceItem<T>(
+                                () -> deleteResource(resource),
+                                resource
+                        ));
             }
 
             if (resource.getMetadata().getNamespace() == null) {
@@ -113,6 +136,7 @@ private <T extends HasMetadata> void createResource(boolean waitReady, T... reso
             } else {
                 LOGGER.info("Creating/Updating {} {}/{}",
                         resource.getKind(), resource.getMetadata().getNamespace(), resource.getMetadata().getName());
+                addNamespaceForLogCollect(resource.getMetadata().getNamespace());
             }
 
             if (type == null) {

diff --git a/src/test/java/io/odh/test/e2e/standard/NotebookST.java b/src/test/java/io/odh/test/e2e/standard/NotebookST.java
@@ -85,7 +85,6 @@ void testCreateSimpleNotebook() throws IOException {
                 .build();
 
         PodUtils.waitForPodsReady(NTB_NAMESPACE, lblSelector, 1, true, () -> { });
-
     }
 
     @BeforeAll
-Original file line number
+Diff line change
@@ Expand Up / @@ -85,7 +85,6 @@ void testCreateSimpleNotebook() throws IOException { @@
                     .build();
             PodUtils.waitForPodsReady(NTB_NAMESPACE, lblSelector, 1, true, () -> { });
         }
         @BeforeAll
@@ Expand Down @@