diff --git a/microprofile-fault-tolerance/pom.xml b/microprofile-fault-tolerance/pom.xml index 8bb3e0f2..3819af74 100644 --- a/microprofile-fault-tolerance/pom.xml +++ b/microprofile-fault-tolerance/pom.xml @@ -62,6 +62,10 @@ tooling-docker test + + org.jboss.eap.qe + tooling-observability + org.jboss.shrinkwrap.resolver shrinkwrap-resolver-depchain diff --git a/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/UndeployDeployTest.java b/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/UndeployDeployTest.java index ddfe372e..9f4741a9 100644 --- a/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/UndeployDeployTest.java +++ b/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/UndeployDeployTest.java @@ -4,6 +4,8 @@ import static org.hamcrest.Matchers.containsString; import java.net.URL; +import java.util.Arrays; +import java.util.List; import org.jboss.arquillian.container.test.api.Deployer; import org.jboss.arquillian.container.test.api.Deployment; @@ -14,10 +16,14 @@ import org.jboss.arquillian.test.api.ArquillianResource; import org.jboss.eap.qe.microprofile.fault.tolerance.deployments.v10.HelloService; import org.jboss.eap.qe.microprofile.fault.tolerance.util.MicroProfileFaultToleranceServerConfiguration; +import org.jboss.eap.qe.microprofile.fault.tolerance.util.MicroProfileTelemetryServerSetup; import org.jboss.eap.qe.microprofile.tooling.server.configuration.creaper.ManagementClientProvider; import org.jboss.eap.qe.microprofile.tooling.server.configuration.deployment.ConfigurationUtil; import org.jboss.eap.qe.microprofile.tooling.server.log.LogChecker; import org.jboss.eap.qe.microprofile.tooling.server.log.ModelNodeLogChecker; +import org.jboss.eap.qe.observability.containers.OpenTelemetryCollectorContainer; +import org.jboss.eap.qe.observability.prometheus.model.PrometheusMetric; +import org.jboss.eap.qe.ts.common.docker.Docker; import org.jboss.shrinkwrap.api.Archive; import org.jboss.shrinkwrap.api.ShrinkWrap; import org.jboss.shrinkwrap.api.asset.StringAsset; @@ -45,9 +51,11 @@ public class UndeployDeployTest { @ArquillianResource private Deployer deployer; + private OpenTelemetryCollectorContainer otelCollector; + @Deployment(name = FIRST_DEPLOYMENT, managed = false) public static Archive createFirstDeployment() { - String mpConfig = "Timeout/enabled=true"; + String mpConfig = "otel.service.name=UndeployDeployTest-first-deployment\notel.sdk.disabled=false\nTimeout/enabled=true"; return ShrinkWrap.create(WebArchive.class, FIRST_DEPLOYMENT + ".war") .addPackages(true, HelloService.class.getPackage()) @@ -57,7 +65,7 @@ public static Archive createFirstDeployment() { @Deployment(name = SECOND_DEPLOYMENT, managed = false) public static Archive createSecondDeployment() { - String mpConfig = "Timeout/enabled=false"; + String mpConfig = "otel.service.name=UndeployDeployTest-first-deployment\notel.sdk.disabled=false\nTimeout/enabled=false"; return ShrinkWrap.create(WebArchive.class, SECOND_DEPLOYMENT + ".war") .addPackages(true, HelloService.class.getPackage()) @@ -75,6 +83,7 @@ public static Archive createNonMPFTDeployment() { @BeforeClass public static void setup() throws Exception { + // Enable FT MicroProfileFaultToleranceServerConfiguration.enableFaultTolerance(); } @@ -92,6 +101,102 @@ public void deployAll() { deployer.deploy(NO_MP_FT_DEPLOYMENT); } + /** + * @tpTestDetails Deploy the first and then second MP FT application. + * Both of them are the same (same classes/methods). + * @tpPassCrit Verify that the number of total timed out calls is 1, i.e. the one resulting from the request sent + * to the first deployment, where Timeout is enabled. The method also verifies that the total number of + * non-applied + * fallback calls is set to 1, i.e. the one originated from the request sent to the second deployment, where + * Timeout has been disabled. + * + *

+ * Since MP FT 3.0 FT Metrics have been moved to the base scope and hence have different semantic, e.g.: + * {@code application_ft_org_jboss_eap_qe_microprofile_fault_tolerance_deployments_v10_HelloService_timeout_invocations_total} + * which was counting the total number of invocations to method annotated with {@code Timeout} doesn't exist any + * more + *

+ * @tpSince EAP 7.4.0.CD19 + */ + @Test + @InSequence(10) + public void testFaultToleranceMetricsAreTracedWithSameDeployments( + @ArquillianResource @OperateOnDeployment(FIRST_DEPLOYMENT) URL firstDeploymentUlr, + @ArquillianResource @OperateOnDeployment(SECOND_DEPLOYMENT) URL secondDeploymentUlr) throws Exception { + // we need a Docker container for The OTel collector here, so throw an exception if a docker service is not available + try { + Docker.checkDockerPresent(); + } catch (Exception e) { + throw new IllegalStateException("Cannot verify Docker availability: " + e.getMessage()); + } + // start the OTel collector container + otelCollector = OpenTelemetryCollectorContainer.getInstance(); + // Enable MP Telemetry based metrics, which rely on OpenTelemetry subsystem + MicroProfileTelemetryServerSetup.enableOpenTelemetry(); + MicroProfileTelemetryServerSetup.enableMicroProfileTelemetry(); + // manually deploy our deployments + deployer.deploy(FIRST_DEPLOYMENT); + deployer.deploy(SECOND_DEPLOYMENT); + get(firstDeploymentUlr + "?operation=timeout&context=foobar&fail=true").then() + .assertThat() + .body(containsString("Fallback Hello, context = foobar")); + // timeout is not working because 2nd deployment has disabled it + get(secondDeploymentUlr + "?operation=timeout&context=foobar&fail=true").then() + .assertThat() + .body(containsString("Hello from @Timeout method, context = foobar")); + // fetch the collected metrics in prometheus format + List metricsToTest = Arrays.asList( + "ft_timeout_calls_total", + "ft_invocations_total"); + List metrics = OpenTelemetryCollectorContainer.getInstance().fetchMetrics(""); + // assert + metricsToTest.forEach(n -> Assert.assertTrue("Missing metric: " + n, + metrics.stream().anyMatch(m -> m.getKey().startsWith(n)))); + + Assert.assertTrue("\"ft_timeout_calls_total\" not found or not expected", + metrics.stream() + .filter(m -> "ft_timeout_calls_total".equals(m.getKey())) + .filter(m -> m.getTags().entrySet().stream().anyMatch( + t -> "method".equals(t.getKey()) + && "org.jboss.eap.qe.microprofile.fault.tolerance.deployments.v10.HelloService.timeout" + .equals(t.getValue())) + && m.getTags().entrySet().stream().anyMatch( + t -> "timedOut".equals(t.getKey()) && "true".equals(t.getValue()))) + .anyMatch(m -> "1".equals(m.getValue()))); + Assert.assertTrue("\"ft_invocations_total\" (fallback applied) not found or not expected", + metrics.stream() + .filter(m -> "ft_invocations_total".equals(m.getKey())) + .filter(m -> m.getTags().entrySet().stream().anyMatch( + t -> "fallback".equals(t.getKey()) && "applied".equals(t.getValue())) + && m.getTags().entrySet().stream().anyMatch( + t -> "method".equals(t.getKey()) + && "org.jboss.eap.qe.microprofile.fault.tolerance.deployments.v10.HelloService.timeout" + .equals(t.getValue())) + && m.getTags().entrySet().stream().anyMatch( + t -> "result".equals(t.getKey()) && "valueReturned".equals(t.getValue()))) + .anyMatch(m -> "1".equals(m.getValue()))); + Assert.assertTrue("\"ft_invocations_total\" (fallback not applied) not found or not expected", + metrics.stream() + .filter(m -> "ft_invocations_total".equals(m.getKey())) + .filter(m -> m.getTags().entrySet().stream().anyMatch( + t -> "fallback".equals(t.getKey()) && "notApplied".equals(t.getValue())) + && m.getTags().entrySet().stream().anyMatch( + t -> "method".equals(t.getKey()) + && "org.jboss.eap.qe.microprofile.fault.tolerance.deployments.v10.HelloService.timeout" + .equals(t.getValue())) + && m.getTags().entrySet().stream().anyMatch( + t -> "result".equals(t.getKey()) && "valueReturned".equals(t.getValue()))) + .anyMatch(m -> "1".equals(m.getValue()))); + // disable MP Telemetry based metrics + MicroProfileTelemetryServerSetup.disableMicroProfileTelemetry(); + MicroProfileTelemetryServerSetup.disableOpenTelemetry(); + // stop the OTel collector container + otelCollector.stop(); + // undeploy + deployer.undeploy(FIRST_DEPLOYMENT); + deployer.undeploy(SECOND_DEPLOYMENT); + } + /** * @tpTestDetails Enable MP FT in server configuration and deploy application which does not use MP FT. * @tpPassCrit MP FT was not activated. @@ -207,6 +312,7 @@ public void undeploy() { @AfterClass public static void tearDown() throws Exception { + // disable FT MicroProfileFaultToleranceServerConfiguration.disableFaultTolerance(); } } diff --git a/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/util/MicroProfileTelemetryServerSetup.java b/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/util/MicroProfileTelemetryServerSetup.java index 1d00d4f7..f4c6bcbd 100644 --- a/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/util/MicroProfileTelemetryServerSetup.java +++ b/microprofile-fault-tolerance/src/test/java/org/jboss/eap/qe/microprofile/fault/tolerance/util/MicroProfileTelemetryServerSetup.java @@ -1,4 +1,174 @@ package org.jboss.eap.qe.microprofile.fault.tolerance.util; +import org.jboss.eap.qe.microprofile.tooling.server.configuration.creaper.ManagementClientProvider; +import org.wildfly.extras.creaper.core.online.OnlineManagementClient; +import org.wildfly.extras.creaper.core.online.operations.Address; +import org.wildfly.extras.creaper.core.online.operations.Operations; +import org.wildfly.extras.creaper.core.online.operations.admin.Administration; + +/** + * Operations required to set up the server for MicroProfile Telemetry + */ public class MicroProfileTelemetryServerSetup { + private static final Address OPENTELEMETRY_EXTENSION_ADDRESS = Address + .extension("org.wildfly.extension.opentelemetry"); + private static final Address OPENTELEMETRY_SUBSYSTEM_ADDRESS = Address + .subsystem("opentelemetry"); + + private static final Address MICROPROFILE_TELEMETRY_EXTENSION_ADDRESS = Address + .extension("org.wildfly.extension.microprofile.telemetry"); + private static final Address MICROPROFILE_TELEMETRY_SUBSYSTEM_ADDRESS = Address + .subsystem("microprofile-telemetry"); + + /** + * Checks whether "org.wildfly.extension.opentelemetry" extension is present + * + * @return True if extension is already present,false otherwise + * @throws Exception exception thrown by the internal operation executed by {@link Operations} API + */ + public static Boolean openTelemetryExtensionExists(Operations operations) throws Exception { + return operations.exists(OPENTELEMETRY_EXTENSION_ADDRESS); + } + + /** + * Checks whether "opentelemetry" subsystem is present + * + * @return True if extension is already present,false otherwise + * @throws Exception exception thrown by the internal operation executed by {@link Operations} API + */ + public static Boolean openTelemetrySubsystemExists(Operations operations) throws Exception { + return operations.exists(OPENTELEMETRY_SUBSYSTEM_ADDRESS); + } + + /** + * Checks whether "org.wildfly.extension.microprofile.telemetry" extension is present + * + * @return True if extension is already present,false otherwise + * @throws Exception exception thrown by the internal operation executed by {@link Operations} API + */ + public static Boolean microProfileTelemetryExtensionExists(Operations operations) throws Exception { + return operations.exists(MICROPROFILE_TELEMETRY_EXTENSION_ADDRESS); + } + + /** + * Checks whether "microprofile-telemetry" subsystem is present + * + * @return True if extension is already present,false otherwise + * @throws Exception exception thrown by the internal operation executed by {@link Operations} API + */ + public static Boolean microProfileTelemetrySubsystemExists(Operations operations) throws Exception { + return operations.exists(MICROPROFILE_TELEMETRY_SUBSYSTEM_ADDRESS); + } + + /** + * Enable OpenTelemetry extension and subsystem. + * + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void enableOpenTelemetry() throws Exception { + try (OnlineManagementClient client = ManagementClientProvider.onlineStandalone()) { + enableOpenTelemetry(client); + } + } + + /** + * Enable OpenTelemetry extension and subsystem. + * + * @param client {@link OnlineManagementClient} instance used to execute the command + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void enableOpenTelemetry(OnlineManagementClient client) throws Exception { + Operations operations = new Operations(client); + if (!openTelemetryExtensionExists(operations)) { + operations.add(OPENTELEMETRY_EXTENSION_ADDRESS); + } + if (!openTelemetrySubsystemExists(operations)) { + operations.add(OPENTELEMETRY_SUBSYSTEM_ADDRESS); + } + new Administration(client).reloadIfRequired(); + } + + /** + * Disable OpenTelemetry subsystem and extension + * + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void disableOpenTelemetry() throws Exception { + try (OnlineManagementClient client = ManagementClientProvider.onlineStandalone()) { + disableOpenTelemetry(client); + } + } + + /** + * Disable OpenTelemetry subsystem and extension + * + * @param client {@link OnlineManagementClient} instance used to execute the command + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void disableOpenTelemetry(OnlineManagementClient client) throws Exception { + Operations operations = new Operations(client); + if (openTelemetrySubsystemExists(operations)) { + operations.remove(OPENTELEMETRY_SUBSYSTEM_ADDRESS); + } + if (openTelemetryExtensionExists(operations)) { + operations.remove(OPENTELEMETRY_EXTENSION_ADDRESS); + } + new Administration(client).reloadIfRequired(); + } + + /** + * Enable MicroProfile Telemetry extension and subsystem. + * + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void enableMicroProfileTelemetry() throws Exception { + try (OnlineManagementClient client = ManagementClientProvider.onlineStandalone()) { + enableMicroProfileTelemetry(client); + } + } + + /** + * Enable MicroProfile Telemetry extension and subsystem. + * + * @param client {@link OnlineManagementClient} instance used to execute the command + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void enableMicroProfileTelemetry(OnlineManagementClient client) throws Exception { + Operations operations = new Operations(client); + if (!openTelemetryExtensionExists(operations)) { + operations.add(MICROPROFILE_TELEMETRY_EXTENSION_ADDRESS); + } + if (!openTelemetrySubsystemExists(operations)) { + operations.add(MICROPROFILE_TELEMETRY_SUBSYSTEM_ADDRESS); + } + new Administration(client).reloadIfRequired(); + } + + /** + * Disable MicroProfile Telemetry subsystem and extension + * + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void disableMicroProfileTelemetry() throws Exception { + try (OnlineManagementClient client = ManagementClientProvider.onlineStandalone()) { + disableMicroProfileTelemetry(client); + } + } + + /** + * Disable MicroProfile Telemetry subsystem and extension + * + * @param client {@link OnlineManagementClient} instance used to execute the command + * @throws Exception exception thrown by the internal operation executed by {@link OnlineManagementClient} API + */ + public static void disableMicroProfileTelemetry(OnlineManagementClient client) throws Exception { + Operations operations = new Operations(client); + if (microProfileTelemetrySubsystemExists(operations)) { + operations.remove(MICROPROFILE_TELEMETRY_SUBSYSTEM_ADDRESS); + } + if (microProfileTelemetryExtensionExists(operations)) { + operations.remove(MICROPROFILE_TELEMETRY_EXTENSION_ADDRESS); + } + new Administration(client).reloadIfRequired(); + } } diff --git a/microprofile-fault-tolerance/src/test/resources/otel-collector-config.yaml b/microprofile-fault-tolerance/src/test/resources/otel-collector-config.yaml new file mode 100644 index 00000000..4eacec9e --- /dev/null +++ b/microprofile-fault-tolerance/src/test/resources/otel-collector-config.yaml @@ -0,0 +1,43 @@ +extensions: + health_check: + pprof: + endpoint: 0.0.0.0:1777 + zpages: + endpoint: 0.0.0.0:55679 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + logging: + verbosity: detailed + prometheus: + endpoint: "0.0.0.0:49152" + otlp: + endpoint: 0.0.0.0:4217 + tls: + insecure: true + +service: +# telemetry: +# logs: +# level: "debug" + pipelines: + metrics: + receivers: [ otlp ] + processors: [ batch ] + exporters: [ prometheus, logging ] + traces: + receivers: [ otlp ] + processors: [ ] + exporters: [ otlp, logging ] + + extensions: [ health_check, pprof, zpages ] \ No newline at end of file