Skip to content

Commit

Permalink
Significantly improve logging
Browse files Browse the repository at this point in the history 
Signed-off-by: Jakub Stejskal <[email protected]>
  • Loading branch information
@Frawless
Frawless committed Jan 12, 2024
1 parent 50b89d9 commit 4faf16a
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 21 deletions.
46 changes: 33 additions & 13 deletions src/main/java/io/odh/test/OdhConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,18 @@
*/
package io.odh.test;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

public class OdhConstants {
private OdhConstants() { }

private static final Logger LOGGER = LoggerFactory.getLogger(OdhConstants.class);
private static final Map<String, String> VALUES = new HashMap<>();
// ODH
private static final String ODH_CONTROLLERS_NAMESPACE = "opendatahub";
private static final String ODH_DASHBOARD_ROUTE_NAME = "odh-dashboard";
Expand Down Expand Up @@ -53,25 +60,38 @@ private OdhConstants() { }
public static final String ODH_NOTEBOOK_OPERATOR = "odh-notebook-controller-manager";
public static final String TRUSTY_AI_OPERATOR = "trustyai-service-operator-controller-manager";

public static final String CONTROLLERS_NAMESPACE = getOdhOrRhoai(ODH_CONTROLLERS_NAMESPACE, RHOAI_CONTROLLERS_NAMESPACE);
public static final String DASHBOARD_ROUTE_NAME = getOdhOrRhoai(ODH_DASHBOARD_ROUTE_NAME, RHOAI_DASHBOARD_ROUTE_NAME);
public static final String DASHBOARD_CONTROLLER = getOdhOrRhoai(ODH_DASHBOARD_CONTROLLER, RHOAI_DASHBOARD_CONTROLLER);
public static final String BUNDLE_OPERATOR_NAMESPACE = getOdhOrRhoai(ODH_BUNDLE_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
public static final String CONTROLLERS_NAMESPACE = getOdhOrRhoai("CONTROLLERS_NAMESPACE", ODH_CONTROLLERS_NAMESPACE, RHOAI_CONTROLLERS_NAMESPACE);
public static final String DASHBOARD_ROUTE_NAME = getOdhOrRhoai("DASHBOARD_ROUTE_NAME", ODH_DASHBOARD_ROUTE_NAME, RHOAI_DASHBOARD_ROUTE_NAME);
public static final String DASHBOARD_CONTROLLER = getOdhOrRhoai("DASHBOARD_CONTROLLER", ODH_DASHBOARD_CONTROLLER, RHOAI_DASHBOARD_CONTROLLER);
public static final String BUNDLE_OPERATOR_NAMESPACE = getOdhOrRhoai("BUNDLE_OPERATOR_NAMESPACE", ODH_BUNDLE_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
// OLM env variables
public static final String OLM_OPERATOR_NAME = getOdhOrRhoai(ODH_OLM_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
public static final String OLM_OPERATOR_NAMESPACE = getOdhOrRhoai(ODH_OLM_OPERATOR_NAMESPACE, RHOAI_OLM_OPERATOR_NAMESPACE);
public static final String OLM_OPERATOR_DEPLOYMENT_NAME = getOdhOrRhoai(ODH_OLM_OPERATOR_DEPLOYMENT_NAME, RHOAI_OLM_OPERATOR_DEPLOYMENT_NAME);
public static final String OLM_APP_BUNDLE_PREFIX = getOdhOrRhoai(ODH_OLM_APP_BUNDLE_PREFIX, RHOAI_OLM_APP_BUNDLE_PREFIX);
public static final String OLM_OPERATOR_VERSION = getOdhOrRhoai(ODH_OLM_OPERATOR_VERSION, RHOAI_OLM_OPERATOR_VERSION);
public static final String OLM_SOURCE_NAME = getOdhOrRhoai(ODH_OLM_SOURCE_NAME, RHOAI_OLM_SOURCE_NAME);
public static final String OLM_OPERATOR_CHANNEL = getOdhOrRhoai(ODH_OLM_OPERATOR_CHANNEL, RHOAI_OLM_OPERATOR_CHANNEL);
public static final String OLM_UPGRADE_STARTING_OPERATOR_VERSION = getOdhOrRhoai(ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION, RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION);
public static final String OLM_OPERATOR_NAME = getOdhOrRhoai("OLM_OPERATOR_NAME", ODH_OLM_OPERATOR_NAME, RHOAI_OLM_OPERATOR_NAME);
public static final String OLM_OPERATOR_NAMESPACE = getOdhOrRhoai("OLM_OPERATOR_NAMESPACE", ODH_OLM_OPERATOR_NAMESPACE, RHOAI_OLM_OPERATOR_NAMESPACE);
public static final String OLM_OPERATOR_DEPLOYMENT_NAME = getOdhOrRhoai("OLM_OPERATOR_DEPLOYMENT_NAME", ODH_OLM_OPERATOR_DEPLOYMENT_NAME, RHOAI_OLM_OPERATOR_DEPLOYMENT_NAME);
public static final String OLM_APP_BUNDLE_PREFIX = getOdhOrRhoai("OLM_APP_BUNDLE_PREFIX", ODH_OLM_APP_BUNDLE_PREFIX, RHOAI_OLM_APP_BUNDLE_PREFIX);
public static final String OLM_OPERATOR_VERSION = getOdhOrRhoai("OLM_OPERATOR_VERSION", ODH_OLM_OPERATOR_VERSION, RHOAI_OLM_OPERATOR_VERSION);
public static final String OLM_SOURCE_NAME = getOdhOrRhoai("OLM_SOURCE_NAME", ODH_OLM_SOURCE_NAME, RHOAI_OLM_SOURCE_NAME);
public static final String OLM_OPERATOR_CHANNEL = getOdhOrRhoai("OLM_OPERATOR_CHANNEL", ODH_OLM_OPERATOR_CHANNEL, RHOAI_OLM_OPERATOR_CHANNEL);
public static final String OLM_UPGRADE_STARTING_OPERATOR_VERSION = getOdhOrRhoai("OLM_UPGRADE_STARTING_OPERATOR_VERSION", ODH_OLM_UPGRADE_STARTING_OPERATOR_VERSION, RHOAI_OLM_UPGRADE_STARTING_OPERATOR_VERSION);

private static <T> T getOdhOrRhoai(T odhValue, T rhoaiValue) {
private static <T> T getOdhOrRhoai(String var, T odhValue, T rhoaiValue) {
T returnValue = odhValue;
if (!Objects.equals(Environment.PRODUCT, Environment.PRODUCT_DEFAULT)) {
returnValue = rhoaiValue;
}
VALUES.put(var, String.valueOf(returnValue));
return returnValue;
}

static {
String debugFormat = "{}: {}";
LOGGER.info("Used OdhConstants:");
VALUES.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.forEach(entry -> {
if (!Objects.equals(entry.getValue(), "null")) {
LOGGER.info(debugFormat, entry.getKey(), entry.getValue());
}
});
}
}
13 changes: 6 additions & 7 deletions src/main/java/io/odh/test/framework/logs/LogCollector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ private static void writeLogsFromPods(Path logpath, Pod pod) {

private static void writePodsDescription(Path logpath, Pod pod) {
try {
LOGGER.debug("Get description if pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
LOGGER.debug("Get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
Files.writeString(logpath.resolve(pod.getMetadata().getNamespace() + "-" + pod.getMetadata().getName() + ".log"),
ResourceManager.getKubeCmdClient().describe(pod.getKind(), pod.getMetadata().getName()));
ResourceManager.getKubeCmdClient().namespace(pod.getMetadata().getNamespace()).describe(pod.getKind(), pod.getMetadata().getName()));
} catch (IOException e) {
LOGGER.warn("Cannot get description of pod {}/{}", pod.getMetadata().getNamespace(), pod.getMetadata().getName());
}
Expand All @@ -69,18 +69,17 @@ private static void saveClusterState(Path logpath) throws IOException {
Files.writeString(logpath.resolve("dsci.yml"), cmdClient.exec(false, false, "get", "dsci", "-o", "yaml").out());
Files.writeString(logpath.resolve("subscriptions.yml"), cmdClient.exec(false, false, "get", "subscriptions.operators.coreos.com", "--all-namespaces", "-o", "yaml").out());
Files.writeString(logpath.resolve("notebooks.yml"), cmdClient.exec(false, false, "get", "notebook", "--all-namespaces", "-o", "yaml").out());
LOGGER.debug("Listing pods in {}", OdhConstants.BUNDLE_OPERATOR_NAMESPACE);
kube.listPodsByPrefixInName(OdhConstants.BUNDLE_OPERATOR_NAMESPACE, "opendatahub-operator-controller-manager").forEach(pod -> {
writeLogsFromPods(logpath, pod);
writePodsDescription(logpath, pod);
});
kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, "opendatahub").forEach(pod -> {
writeLogsFromPods(logpath, pod);
writePodsDescription(logpath, pod);
});
kube.listPodsByPrefixInName(OdhConstants.OLM_OPERATOR_NAMESPACE, OdhConstants.OLM_OPERATOR_NAME).forEach(pod -> {
LOGGER.debug("Listing pods in {}", OdhConstants.OLM_OPERATOR_NAMESPACE);
kube.listPods(OdhConstants.OLM_OPERATOR_NAMESPACE).forEach(pod -> {
writeLogsFromPods(logpath, pod);
writePodsDescription(logpath, pod);
});
LOGGER.debug("Listing pods in {}", OdhConstants.CONTROLLERS_NAMESPACE);
kube.listPods(OdhConstants.CONTROLLERS_NAMESPACE).forEach(pod -> {
writeLogsFromPods(logpath, pod);
writePodsDescription(logpath, pod);
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/io/odh/test/framework/manager/resources/DataScienceClusterResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,10 @@ public boolean waitForReadiness(DataScienceCluster resource) {

String namespace = OdhConstants.CONTROLLERS_NAMESPACE;
LOGGER.info("Waiting for pods readiness in {}", namespace);
PodUtils.waitForPodsReady(namespace, true, () -> { });
PodUtils.waitForPodsReady(namespace, true, () -> {
ResourceManager.getKubeCmdClient().namespace(namespace).exec(false, "oc", "get", "pods");
ResourceManager.getKubeCmdClient().namespace(namespace).exec(false, "oc", "get", "events");
});

return true;
}
Expand Down

0 comments on commit 4faf16a

Please sign in to comment.