Add log collector triggered by test failure

skodjob · Dec 13, 2023 · 1858395 · 1858395
1 parent da9abe0
commit 1858395
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 10 deletions.
diff --git a/src/main/java/io/odh/test/Environment.java b/src/main/java/io/odh/test/Environment.java
@@ -8,6 +8,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -21,6 +25,8 @@ public class Environment {
     private static final Logger LOGGER = LoggerFactory.getLogger(Environment.class);
     private static final Map<String, String> VALUES = new HashMap<>();
 
+    public static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm");
+
     public static final String USER_PATH = System.getProperty("user.dir");
 
     private static final String USERNAME_ENV = "KUBE_USERNAME";
@@ -29,6 +35,8 @@ public class Environment {
     private static final String URL_ENV = "KUBE_URL";
     private static final String PRODUCT_ENV = "PRODUCT";
 
+    private static final String LOG_DIR_ENV = "LOG_DIR";
+
     /**
      * Install bundle files
      */
@@ -68,6 +76,8 @@ public class Environment {
     public static final String OPERATOR_INSTALL_TYPE = getOrDefault(OPERATOR_INSTALL_TYPE_ENV, InstallTypes.BUNDLE.toString());
     public static final String PRODUCT = getOrDefault(PRODUCT_ENV, PRODUCT_DEFAULT);
 
+    public static final Path LOG_DIR = getOrDefault(LOG_DIR_ENV, Paths::get, Paths.get(USER_PATH, "target", "logs")).resolve("test-run-" + DATE_FORMAT.format(LocalDateTime.now()));
+
     private Environment() { }
 
     static {

diff --git a/src/main/java/io/odh/test/TestUtils.java b/src/main/java/io/odh/test/TestUtils.java
@@ -8,13 +8,17 @@
 import com.fasterxml.jackson.databind.exc.InvalidFormatException;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import io.odh.test.framework.WaitException;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.ExtensionContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
@@ -189,4 +193,24 @@ public static <T> T configFromYaml(String yamlFile, Class<T> c) {
             throw new RuntimeException(e);
         }
     }
+
+    public static Path getLogPath(String folderName, ExtensionContext context) {
+        String testMethod = context.getDisplayName();
+        String testClassName = context.getTestClass().map(Class::getName).orElse("NOCLASS");
+        return getLogPath(folderName, testClassName, testMethod);
+    }
+
+    public static Path getLogPath(String folderName, TestInfo info) {
+        String testMethod = info.getDisplayName();
+        String testClassName = info.getTestClass().map(Class::getName).orElse("NOCLASS");
+        return getLogPath(folderName, testClassName, testMethod);
+    }
+
+    public static Path getLogPath(String folderName, String testClassName, String testMethod) {
+        Path path = Environment.LOG_DIR.resolve(Paths.get(folderName, testClassName));
+        if (testMethod != null) {
+            path = path.resolve(testMethod.replace("(", "").replace(")", ""));
+        }
+        return path;
+    }
 }
diff --git a/src/main/java/io/odh/test/framework/listeners/TestExceptionCallbackListener.java b/src/main/java/io/odh/test/framework/listeners/TestExceptionCallbackListener.java
@@ -4,6 +4,7 @@
  */
 package io.odh.test.framework.listeners;
 
+import io.odh.test.framework.logs.LogCollector;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.LifecycleMethodExecutionExceptionHandler;
 import org.junit.jupiter.api.extension.TestExecutionExceptionHandler;
@@ -19,35 +20,30 @@ public class TestExceptionCallbackListener implements TestExecutionExceptionHand
     @Override
     public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
         LOGGER.error("Test failed at {} : {}", "Test execution", throwable.getMessage(), throwable);
-        //TODO collect proper logs
-        throw throwable;
+        LogCollector.saveKubernetesState(context, throwable);
     }
 
     @Override
     public void handleBeforeAllMethodExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
         LOGGER.error("Test failed at {} : {}", "Test before all", throwable.getMessage(), throwable);
-        //TODO collect proper logs
-        throw throwable;
+        LogCollector.saveKubernetesState(context, throwable);
     }
 
     @Override
     public void handleBeforeEachMethodExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
         LOGGER.error("Test failed at {} : {}", "Test before each", throwable.getMessage(), throwable);
-        //TODO collect proper logs
-        throw throwable;
+        LogCollector.saveKubernetesState(context, throwable);
     }
 
     @Override
     public void handleAfterEachMethodExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
         LOGGER.error("Test failed at {} : {}", "Test after each", throwable.getMessage(), throwable);
-        //TODO collect proper logs
-        throw throwable;
+        LogCollector.saveKubernetesState(context, throwable);
     }
 
     @Override
     public void handleAfterAllMethodExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
         LOGGER.error("Test failed at {} : {}", "Test after all", throwable.getMessage(), throwable);
-        //TODO collect proper logs
-        throw throwable;
+        LogCollector.saveKubernetesState(context, throwable);
     }
 }
diff --git a/src/main/java/io/odh/test/framework/logs/LogCollector.java b/src/main/java/io/odh/test/framework/logs/LogCollector.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Skodjob authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.odh.test.framework.logs;
+
+import io.fabric8.kubernetes.api.model.PodStatus;
+import io.odh.test.Environment;
+import io.odh.test.TestUtils;
+import io.odh.test.framework.manager.ResourceManager;
+import io.odh.test.platform.cmdClient.KubeCmdClient;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+
+public class LogCollector {
+    private static final Logger LOGGER = LoggerFactory.getLogger(LogCollector.class);
+
+    /**
+     * Calls storing cluster info for connected cluster
+     */
+    public static void saveKubernetesState(ExtensionContext extensionContext, Throwable throwable) throws Throwable {
+        LOGGER.warn("Printing all pods on cluster");
+        ResourceManager.getClient().getClient().pods().inAnyNamespace().list().getItems().forEach(p ->
+                LOGGER.info("Pod: {} in ns: {} with phase: {}",
+                        p.getMetadata().getName(),
+                        p.getMetadata().getNamespace(),
+                        Optional.ofNullable(p.getStatus()).map(PodStatus::getPhase).orElse("null")));
+
+        Path logPath = TestUtils.getLogPath(Environment.LOG_DIR.resolve("failedTest").toString(), extensionContext);
+        Files.createDirectories(logPath);
+        LOGGER.info("Storing cluster info into {}", logPath);
+        try {
+            saveClusterState(logPath);
+        } catch (IOException ex) {
+            LOGGER.warn("Cannot save logs in {}", logPath);
+        }
+        throw throwable;
+    }
+
+    private static void saveClusterState(Path logpath) throws IOException {
+        KubeCmdClient cmdClient = ResourceManager.getKubeCmdClient();
+        Files.writeString(logpath.resolve("describe-cluster-nodes.log"), cmdClient.exec(false, false, "describe", "nodes").out());
+        Files.writeString(logpath.resolve("all-events.log"), cmdClient.exec(false, false, "get", "events", "--all-namespaces").out());
+        Files.writeString(logpath.resolve("pvs.log"), cmdClient.exec(false, false, "describe", "pv").out());
+        Files.writeString(logpath.resolve("dsc.yml"), cmdClient.exec(false, false, "get", "dsc", "-o", "yaml").out());
+        Files.writeString(logpath.resolve("dsci.yml"), cmdClient.exec(false, false, "get", "dsci", "-o", "yaml").out());
+    }
+}