Merge pull request #234 from solarwinds/cc/NH-57065

NH-57065: automagically set otel log exporting configs when enabled

build.gradle

@@ -44,7 +44,7 @@ subprojects {
                 opentelemetryJavaagent: "2.4.0",
                 bytebuddy             : "1.12.10",
                 guava                 : "30.1-jre",
-                joboe                 : "10.0.4",
+                joboe                 : "10.0.5",
                 agent                 : "2.4.0", // the custom distro agent version
                 autoservice           : "1.0.1",
                 caffeine              : "2.9.3",

custom/src/main/java/com/solarwinds/opentelemetry/extensions/initialize/AutoConfigurationCustomizerProviderImpl.java

@@ -28,6 +28,11 @@
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer;
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
 
+/**
+ * An implementation of {@link AutoConfigurationCustomizer} which serves as the bootstrap for our
+ * distro. It's the first user class loaded by the upstream agent. We use the opportunity to
+ * bootstrap our configuration via static code.
+ */
 @AutoService({AutoConfigurationCustomizerProvider.class})
 public class AutoConfigurationCustomizerProviderImpl
     implements AutoConfigurationCustomizerProvider {

custom/src/main/java/com/solarwinds/opentelemetry/extensions/initialize/ConfigurationLoader.java

@@ -194,6 +194,31 @@ private static void maybeFollowOtelConfigProperties(ConfigContainer configs) {
     }
   }
 
+  static void configOtelLogExport(ConfigContainer container) {
+    Boolean exportLog = (Boolean) container.get(ConfigProperty.AGENT_EXPORT_LOGS_ENABLED);
+    String collectorEndpoint = (String) container.get(ConfigProperty.AGENT_COLLECTOR);
+    if (collectorEndpoint != null && (exportLog == null || exportLog)) {
+      String serviceKey = (String) container.get(ConfigProperty.AGENT_SERVICE_KEY);
+      String apiKey = ServiceKeyUtils.getApiKey(serviceKey);
+
+      String[] fragments = collectorEndpoint.split("\\.");
+      String dataCell = "na-01";
+      if (fragments.length > 2) {
+        // This is based on knowledge of the SWO url format where the third name from the left in
+        // the domain is the data-cell name and assumes this format will stay stable.
+        dataCell = fragments[2];
+      }
+
+      System.setProperty("otel.exporter.otlp.protocol", "grpc");
+      System.setProperty("otel.logs.exporter", "otlp");
+      System.setProperty(
+          "otel.exporter.otlp.logs.headers", String.format("authorization=Bearer %s", apiKey));
+      System.setProperty(
+          "otel.exporter.otlp.logs.endpoint",
+          String.format("https://otel.collector.%s.cloud.solarwinds.com", dataCell));
+    }
+  }
+
   static Map<String, String> mergeEnvWithSysProperties(Map<String, String> env, Properties props) {
     Map<String, String> res = new HashMap<>(env);
 
@@ -245,6 +270,7 @@ private static void loadConfigurations() throws InvalidConfigException {
                 config)); // initialize the logger factory as soon as the config is available
         try {
           processConfigs(configs);
+          configOtelLogExport(configs);
         } catch (InvalidConfigException e) {
           // if there was a config read exception then processConfigs might throw exception due to
           // incomplete config container.

custom/src/test/java/com/solarwinds/opentelemetry/extensions/initialize/ConfigurationLoaderTest.java

@@ -151,4 +151,49 @@ void verifyThatServiceKeyIsUpdatedWithOtelServiceNameWhenSystemPropertyIsSet()
     assertEquals("test", ServiceKeyUtils.getServiceName(serviceKeyAfter));
     assertEquals("token:test", serviceKeyAfter);
   }
+
+  @Test
+  @ClearSystemProperty(key = "otel.logs.exporter")
+  @ClearSystemProperty(key = "otel.exporter.otlp.protocol")
+  @ClearSystemProperty(key = "otel.exporter.otlp.logs.headers")
+  @ClearSystemProperty(key = "otel.exporter.otlp.logs.endpoint")
+  void verifySettingOtelLogExportSystemVariablesWhenEnabled() throws InvalidConfigException {
+    ConfigContainer configContainer = new ConfigContainer();
+    configContainer.putByStringValue(ConfigProperty.AGENT_SERVICE_KEY, "token:service");
+    configContainer.putByStringValue(
+        ConfigProperty.AGENT_COLLECTOR, "apm.collector.na-02.cloud.solarwinds.com");
+
+    configContainer.putByStringValue(ConfigProperty.AGENT_EXPORT_LOGS_ENABLED, "true");
+    ConfigurationLoader.configOtelLogExport(configContainer);
+
+    assertEquals("otlp", System.getProperty("otel.logs.exporter"));
+    assertEquals("grpc", System.getProperty("otel.exporter.otlp.protocol"));
+
+    assertEquals(
+        "https://otel.collector.na-02.cloud.solarwinds.com",
+        System.getProperty("otel.exporter.otlp.logs.endpoint"));
+    assertEquals(
+        "authorization=Bearer token", System.getProperty("otel.exporter.otlp.logs.headers"));
+  }
+
+  @Test
+  @ClearSystemProperty(key = "otel.logs.exporter")
+  @ClearSystemProperty(key = "otel.exporter.otlp.protocol")
+  @ClearSystemProperty(key = "otel.exporter.otlp.logs.headers")
+  @ClearSystemProperty(key = "otel.exporter.otlp.logs.endpoint")
+  void verifyOtelLogExportSystemVariablesAreNotSetWhenDisabled() throws InvalidConfigException {
+    ConfigContainer configContainer = new ConfigContainer();
+    configContainer.putByStringValue(ConfigProperty.AGENT_SERVICE_KEY, "token:service");
+    configContainer.putByStringValue(ConfigProperty.AGENT_EXPORT_LOGS_ENABLED, "false");
+    configContainer.putByStringValue(
+        ConfigProperty.AGENT_COLLECTOR, "apm.collector.na-02.cloud.solarwinds.com");
+
+    ConfigurationLoader.configOtelLogExport(configContainer);
+
+    assertNull(System.getProperty("otel.logs.exporter"));
+    assertNull(System.getProperty("otel.exporter.otlp.protocol"));
+
+    assertNull(System.getProperty("otel.exporter.otlp.logs.endpoint"));
+    assertNull(System.getProperty("otel.exporter.otlp.logs.headers"));
+  }
 }

smoke-tests/README.md

@@ -34,6 +34,12 @@ This directory contains the test machinery code and the machinery is held togeth
 - `SWO_EMAIL`: The swo user email used to get temporary login credentials.
 - `SWO_PWORD`: The swo user password.
 
+## Running the test
+- build docker image for `spring-boot-webmvc` and tag it with `smt:webmvc`
+- set environment variable `LAMBDA=false`
+- From the project root, run `./gradlew test`
+- To execute lambda tests set environment variable `LAMBDA=true`
+
 ## Viewing the data in SWO
 To view test generated data in swo, use service names: `java-apm-smoke-test`, `java-apm-smoke-test-webmvc` and `lambda-e2e`
 

smoke-tests/k6/basic.js

@@ -446,6 +446,105 @@ function verify_transaction_name() {
   }
 }
 
+function getEntityId() {
+  let retryCount = Number.parseInt(`${__ENV.SWO_RETRY_COUNT}`) || 1000;
+  const entityQueryPayload = {
+    "operationName": "getServiceEntitiesQuery",
+    "variables": {
+      "includeKubernetesClusterUid": false,
+      "filter": {
+        "types": [
+          "Service"
+        ]
+      },
+      "timeFilter": {
+        "startTime": "1 hour ago",
+        "endTime": "now"
+      },
+      "sortBy": {
+        "sorts": [
+          {
+            "propertyName": "name",
+            "direction": "DESC"
+          }
+        ]
+      },
+      "pagination": {
+        "first": 50
+      },
+      "bucketSizeInS": 60
+    },
+    "query": "query getServiceEntitiesQuery($filter: EntityFilterInput, $timeFilter: TimeRangeInput!, $sortBy: EntitySortInput, $pagination: PagingInput, $bucketSizeInS: Int!, $includeKubernetesClusterUid: Boolean = false) {\n  entities {\n    search(\n      filter: $filter\n      sortBy: $sortBy\n      paging: $pagination\n      timeRange: $timeFilter\n    ) {\n      totalEntitiesCount\n      pageInfo {\n        endCursor\n        hasNextPage\n        startCursor\n        hasPreviousPage\n        __typename\n      }\n      groups {\n        entities {\n          ... on Service {\n            ...ServiceEntity\n            __typename\n          }\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment ServiceEntity on Service {\n  id\n  name: displayName\n  lastSeenTime\n  language\n  kubernetesPodInstances @include(if: $includeKubernetesClusterUid) {\n    clusterUid\n    __typename\n  }\n  healthScore {\n    scoreV2\n    categoryV2\n    __typename\n  }\n  traceServiceErrorRatio {\n    ...MetricSeriesMeasurementsForServiceEntity\n    __typename\n  }\n  traceServiceErrorRatioValue\n  traceServiceRequestRate {\n    ...MetricSeriesMeasurementsForServiceEntity\n    __typename\n  }\n  traceServiceRequestRateValue\n  responseTime {\n    ...MetricSeriesMeasurementsForServiceEntity\n    __typename\n  }\n  responseTimeValue\n  sumRequests\n  __typename\n}\n\nfragment MetricSeriesMeasurementsForServiceEntity on Metric {\n  measurements(\n    metricInput: {aggregation: {method: AVG, bucketSizeInS: $bucketSizeInS, missingDataPointsHandling: NULL_FILL}, timeRange: $timeFilter}\n  ) {\n    series {\n      measurements {\n        time\n        value\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n  __typename\n}\n"
+  }
+
+  for (; retryCount; retryCount--) {
+    let entityResponse = http.post(`${__ENV.SWO_HOST_URL}/common/graphql`, JSON.stringify(entityQueryPayload),
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            'Cookie': `${__ENV.SWO_COOKIE}`,
+            'X-Csrf-Token': `${__ENV.SWO_XSR_TOKEN}`
+          }
+        });
+
+    entityResponse = JSON.parse(entityResponse.body)
+    if (entityResponse['errors']) {
+      console.log("Error -> Entity id response:", JSON.stringify(entityResponse))
+      continue
+    }
+
+    const {data: {entities: {search:{groups}}}} = entityResponse
+    for (let i = 0; i < groups.length; i++) {
+      const {entities} = groups[i]
+      for (let j = 0; j < entities.length; j++) {
+        const {name, id} = entities[j]
+
+        if(name === `${__ENV.SERVICE_NAME}`){
+          return id
+        }
+      }
+    }
+  }
+}
+
+function verify_logs_export() {
+  let retryCount = Number.parseInt(`${__ENV.SWO_RETRY_COUNT}`) || 1000;
+  const logQueryPayload = {
+    "operationName": "getLogEvents",
+    "variables": {
+      "input": {
+        "direction": "BACKWARD",
+        "searchLimit": 500,
+        "entityIds": [
+          getEntityId()
+        ],
+        "query": ""
+      }
+    },
+    "query": "query getLogEvents($input: LogEventsInput!) {\n  logEvents(input: $input) {\n    events {\n      id\n      facility\n      program\n      message\n      receivedAt\n      severity\n      sourceName\n      isJson\n      positions {\n        length\n        starts\n        __typename\n      }\n      __typename\n    }\n    cursor {\n      maxId\n      maxTimestamp\n      minId\n      minTimestamp\n      __typename\n    }\n    __typename\n  }\n}\n"
+  }
+
+  for (; retryCount; retryCount--) {
+    let logResponse = http.post(`${__ENV.SWO_HOST_URL}/common/graphql`, JSON.stringify(logQueryPayload),
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            'Cookie': `${__ENV.SWO_COOKIE}`,
+            'X-Csrf-Token': `${__ENV.SWO_XSR_TOKEN}`
+          }
+        });
+
+    logResponse = JSON.parse(logResponse.body)
+    if (logResponse['errors']) {
+      console.log("Error -> Log response:", JSON.stringify(logResponse))
+      continue
+    }
+
+    const {data: {logEvents: {events}}} = logResponse
+    check(events, {"logs": events => events.length > 0})
+  }
+}
+
 function silence(fn) {
   try {
     fn()
@@ -480,10 +579,11 @@ export default function () {
     silence(verify_transaction_name)
 
   } else {
+    silence(verify_logs_export)
     silence(verify_that_specialty_path_is_not_sampled)
     silence(verify_that_span_data_is_persisted_0)
-    silence(verify_that_span_data_is_persisted)
 
+    silence(verify_that_span_data_is_persisted)
     silence(verify_that_trace_is_persisted)
     silence(verify_distributed_trace)
   }

smoke-tests/src/test/java/com/solarwinds/SmokeTest.java

@@ -241,4 +241,14 @@ void assertThatJDBCInstrumentationIsApplied() {
     assertTrue(actual, "sw-jdbc instrumentation is not applied");
   }
 
+
+  @Test
+  void assertThatLogsAreExported() throws IOException {
+    String resultJson = new String(
+        Files.readAllBytes(namingConventions.local.k6Results(Configs.E2E.config.agents().get(0))));
+    double passes = ResultsCollector.read(resultJson,
+        "$.root_group.checks.['logs'].passes");
+    assertTrue(passes > 0, "log export is broken");
+  }
+
 }

smoke-tests/src/test/java/com/solarwinds/agents/SwoAgentResolver.java

@@ -16,15 +16,8 @@
 
 package com.solarwinds.agents;
 
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
 import java.util.Optional;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.Response;
 
 public class SwoAgentResolver implements AgentResolver {
   private static final String NH_URL = "https://agent-binaries.global.st-ssp.solarwinds.com/apm/java/latest/solarwinds-apm-agent.jar";

smoke-tests/src/test/java/com/solarwinds/containers/K6Container.java

@@ -59,6 +59,7 @@ public GenericContainer<?> build() {
             .withEnv("SWO_COOKIE", System.getenv("SWO_COOKIE"))
             .withEnv("SWO_XSR_TOKEN", System.getenv("SWO_XSR_TOKEN"))
             .withEnv("LAMBDA", System.getenv("LAMBDA"))
+            .withEnv("SERVICE_NAME", "java-apm-smoke-test")
         .withCommand(
             "run",
             "--summary-export",