Add support for automatic APM tracers configuration

README.md

@@ -226,6 +226,16 @@ From a job specific configuration page:
 | Custom tags                           | Set from a `File` in the job workspace (not compatible with pipeline jobs) or as text `Properties` directly from the configuration page. If set, this overrides the `Global Job Tags` configuration. |
 | Send source control management events | Submits the `Source Control Management Events Type` of events and metrics (enabled by default).                                                                                                         |
 
+### Test Visibility Configuration
+
+The plugin can automatically configure Datadog <a target="_blank" href="https://docs.datadoghq.com/continuous_integration/tests/">Test Visibility</a> for a job or a pipeline.
+
+Before enabling Test Visibility, be sure to properly configure the plugin to submit data to Datadog.
+
+To enable Test Visibility, go to the `Configure` page of the job or pipeline whose tests need to be traced, tick `Enable Datadog Test Visibility` checkbox in the `General` section and save your changes.
+
+Please bear in mind that Test Visibility is a separate Datadog product that is billed separately.
+
 ## Data collected
 
 This plugin is collecting the following [events](#events), [metrics](#metrics), and [service checks](#service-checks):

pom.xml

@@ -167,6 +167,11 @@
       <artifactId>jetty-client</artifactId>
       <version>9.4.51.v20230217</version>
     </dependency>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpg-jdk18on</artifactId>
+      <version>1.72</version>
+    </dependency>
   </dependencies>
 
   <build>

src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java

@@ -214,7 +214,7 @@ private void loadEnvVariables(){
         }
 
         String traceServiceNameVar = System.getenv(TARGET_TRACE_SERVICE_NAME_PROPERTY);
-        if(StringUtils.isNotBlank(targetApiKeyEnvVar)) {
+        if(StringUtils.isNotBlank(traceServiceNameVar)) {
             this.traceServiceName = traceServiceNameVar;
         }
 
@@ -455,7 +455,6 @@ public ListBoxModel doFillTargetCredentialsApiKeyItems(
             .includeCurrentValue(targetCredentialsApiKey);
     }
 
-
     /**
      * Tests the targetCredentialsApiKey field from the configuration screen, to check its' validity.
      *
@@ -477,7 +476,7 @@ public FormValidation doCheckTargetCredentialsApiKey(
         } else {
             if (!item.hasPermission(Item.EXTENDED_READ)
                 && !item.hasPermission(CredentialsProvider.USE_ITEM)) {
-                return FormValidation.ok(); 
+                return FormValidation.ok();
             }
         }
         if (StringUtils.isBlank(targetCredentialsApiKey)) {
@@ -488,7 +487,7 @@ public FormValidation doCheckTargetCredentialsApiKey(
         }
         if (CredentialsProvider.listCredentials(StringCredentials.class,
                         item,
-                        ACL.SYSTEM, 
+                        ACL.SYSTEM,
                         Collections.emptyList(),
                         CredentialsMatchers.withId(targetCredentialsApiKey)).isEmpty()) {
             return FormValidation.error("Cannot find currently selected credentials");
@@ -766,6 +765,7 @@ public boolean configure(final StaplerRequest req, final JSONObject formData) th
 
             final Secret apiKeySecret = findSecret(formData.getString("targetApiKey"), formData.getString("targetCredentialsApiKey"));
             this.setUsedApiKey(apiKeySecret);
+
             //When form is saved....
             DatadogClient client = ClientFactory.getClient(DatadogClient.ClientType.valueOf(this.getReportWith()), this.getTargetApiURL(),
                 this.getTargetLogIntakeURL(), this.getTargetWebhookIntakeURL(), this.getUsedApiKey(), this.getTargetHost(),

src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpClient.java

@@ -1,5 +1,7 @@
 package org.datadog.jenkins.plugins.datadog.clients;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -10,6 +12,7 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.logging.Logger;
@@ -26,6 +29,7 @@
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.util.BufferingResponseListener;
 import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
@@ -152,7 +156,7 @@ public boolean matches(Origin origin) {
                         return false;
                     }
                 }
-                return true;
+                return super.matches(origin);
             }
         });
     }
@@ -182,6 +186,24 @@ public <T> T get(String url, Map<String, String> headers, Function<String, T> re
                 responseParser);
     }
 
+    public void getBinary(String url, Map<String, String> headers, Consumer<InputStream> responseParser) throws ExecutionException, InterruptedException, TimeoutException, IOException {
+        ensureClientIsUpToDate();
+
+        Request request = requestSupplier(url, HttpMethod.GET, headers, null, null).get();
+        InputStreamResponseListener responseListener = new InputStreamResponseListener();
+        request.send(responseListener);
+
+        Response response = responseListener.get(timeoutMillis, TimeUnit.MILLISECONDS);
+        int responseStatus = response.getStatus();
+        if (responseStatus >= 200 && responseStatus < 300) {
+            try (InputStream responseStream = responseListener.getInputStream()) {
+                responseParser.accept(responseStream);
+            }
+        } else {
+            throw new ResponseProcessingException("Received erroneous response " + response);
+        }
+    }
+
     public <T> T post(String url, Map<String, String> headers, String contentType, byte[] body, Function<String, T> responseParser) throws ExecutionException, InterruptedException, TimeoutException {
         return executeSynchronously(
                 requestSupplier(url, HttpMethod.POST, headers, contentType, body),

src/main/java/org/datadog/jenkins/plugins/datadog/tracer/DatadogTracerConfigurator.java

@@ -0,0 +1,174 @@
+package org.datadog.jenkins.plugins.datadog.tracer;
+
+import hudson.FilePath;
+import hudson.model.AbstractBuild;
+import hudson.model.AbstractProject;
+import hudson.model.InvisibleAction;
+import hudson.model.Job;
+import hudson.model.Node;
+import hudson.model.Run;
+import hudson.model.TopLevelItem;
+import hudson.util.Secret;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.datadog.jenkins.plugins.datadog.DatadogClient;
+import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration;
+import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
+
+public class DatadogTracerConfigurator {
+
+    private static final Logger LOGGER = Logger.getLogger(DatadogTracerConfigurator.class.getName());
+
+    private final Map<TracerLanguage, TracerConfigurator> configurators;
+
+    public static final DatadogTracerConfigurator INSTANCE = new DatadogTracerConfigurator();
+
+    public DatadogTracerConfigurator() {
+        configurators = new EnumMap<>(TracerLanguage.class);
+        configurators.put(TracerLanguage.JAVA, new JavaConfigurator());
+        configurators.put(TracerLanguage.JAVASCRIPT, new JavascriptConfigurator());
+        configurators.put(TracerLanguage.PYTHON, new PythonConfigurator());
+    }
+
+    public Map<String, String> configure(Run<?, ?> run, Node node, Map<String, String> envs) {
+        Job<?, ?> job = run.getParent();
+        DatadogTracerJobProperty<?> tracerConfig = job.getProperty(DatadogTracerJobProperty.class);
+        if (tracerConfig == null || !tracerConfig.isOn()) {
+            return Collections.emptyMap();
+        }
+
+        Collection<TracerLanguage> languages = tracerConfig.getLanguages();
+        for (ConfigureTracerAction action : run.getActions(ConfigureTracerAction.class)) {
+            if (action.node == node && languages.equals(action.languages)) {
+                return action.variables;
+            }
+        }
+
+        DatadogGlobalConfiguration datadogConfig = DatadogUtilities.getDatadogGlobalDescriptor();
+        if (datadogConfig == null) {
+            LOGGER.log(Level.WARNING, "Cannot set up tracer: Datadog config not found");
+            return Collections.emptyMap();
+        }
+
+        TopLevelItem topLevelItem = getTopLevelItem(run);
+        FilePath workspacePath = node.getWorkspaceFor(topLevelItem);
+        if (workspacePath == null) {
+            throw new IllegalStateException("Cannot find workspace path for " + topLevelItem + " on " + node);
+        }
+
+        Map<String, String> variables = new HashMap<>(getCommonEnvVariables(datadogConfig, tracerConfig));
+        for (TracerLanguage language : languages) {
+            TracerConfigurator tracerConfigurator = configurators.get(language);
+            if (tracerConfigurator == null) {
+                LOGGER.log(Level.WARNING, "Cannot find tracer configurator for " + language);
+                continue;
+            }
+
+            try {
+                Map<String, String> languageVariables = tracerConfigurator.configure(tracerConfig, node, workspacePath, envs);
+                variables.putAll(languageVariables);
+            } catch (Exception e) {
+                LOGGER.log(Level.WARNING, "Error while configuring " + language + " Datadog Tracer for run " + run + " and node " + node, e);
+                return Collections.emptyMap();
+            }
+        }
+        run.addAction(new ConfigureTracerAction(node, languages, variables));
+        return variables;
+    }
+
+    private static TopLevelItem getTopLevelItem(Run<?, ?> run) {
+        if (run instanceof AbstractBuild) {
+            AbstractBuild<?, ?> build = (AbstractBuild<?, ?>) run;
+            AbstractProject<?, ?> project = build.getProject();
+            if (project instanceof TopLevelItem) {
+                return (TopLevelItem) project;
+            } else {
+                throw new IllegalArgumentException("Unexpected type of project: " + project);
+            }
+        } else {
+            Job<?, ?> parent = run.getParent();
+            if (parent instanceof TopLevelItem) {
+                return (TopLevelItem) parent;
+            } else {
+                throw new IllegalArgumentException("Unexpected type of run parent: " + parent);
+            }
+        }
+    }
+
+    private static Map<String, String> getCommonEnvVariables(DatadogGlobalConfiguration datadogConfig,
+                                                             DatadogTracerJobProperty<?> tracerConfig) {
+        Map<String, String> variables = new HashMap<>();
+        variables.put("DD_CIVISIBILITY_ENABLED", "true");
+        variables.put("DD_ENV", "ci");
+        variables.put("DD_SERVICE", tracerConfig.getServiceName());
+
+        DatadogClient.ClientType clientType = DatadogClient.ClientType.valueOf(datadogConfig.getReportWith());
+        switch (clientType) {
+            case HTTP:
+                variables.put("DD_CIVISIBILITY_AGENTLESS_ENABLED", "true");
+                variables.put("DD_SITE", getSite(datadogConfig.getTargetApiURL()));
+                variables.put("DD_API_KEY", Secret.toString(datadogConfig.getUsedApiKey()));
+                break;
+            case DSD:
+                variables.put("DD_AGENT_HOST", datadogConfig.getTargetHost());
+                variables.put("DD_TRACE_AGENT_PORT", getAgentPort(datadogConfig.getTargetTraceCollectionPort()));
+                break;
+            default:
+                throw new IllegalArgumentException("Unexpected client type: " + clientType);
+        }
+
+        Map<String, String> additionalVariables = tracerConfig.getAdditionalVariables();
+        if (additionalVariables != null) {
+            variables.putAll(additionalVariables);
+        }
+
+        return variables;
+    }
+
+    private static String getSite(String apiUrl) {
+        // what users configure for Pipelines looks like "https://api.datadoghq.com/api/"
+        // while what the tracer needs "datadoghq.com"
+        try {
+            URI uri = new URL(apiUrl).toURI();
+            String host = uri.getHost();
+            if (host == null) {
+                throw new IllegalArgumentException("Cannot find host in Datadog API URL: " + uri);
+            }
+
+            String[] parts = host.split("\\.");
+            return (parts.length >= 2 ? parts[parts.length - 2] + "." : "") + parts[parts.length - 1];
+
+        } catch (MalformedURLException | URISyntaxException e) {
+            throw new IllegalArgumentException("Cannot parse Datadog API URL", e);
+        }
+    }
+
+    private static String getAgentPort(Integer traceCollectionPort) {
+        if (traceCollectionPort == null) {
+            throw new IllegalArgumentException("Traces collection port is not set");
+        } else {
+            return traceCollectionPort.toString();
+        }
+    }
+
+    private static final class ConfigureTracerAction extends InvisibleAction {
+        private final Node node;
+        private final Collection<TracerLanguage> languages;
+        private final Map<String, String> variables;
+
+        private ConfigureTracerAction(Node node, Collection<TracerLanguage> languages, Map<String, String> variables) {
+            this.node = node;
+            this.languages = languages;
+            this.variables = variables;
+        }
+    }
+}

src/main/java/org/datadog/jenkins/plugins/datadog/tracer/DatadogTracerEnvironmentContributor.java

@@ -0,0 +1,33 @@
+package org.datadog.jenkins.plugins.datadog.tracer;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.EnvVars;
+import hudson.Extension;
+import hudson.model.EnvironmentContributor;
+import hudson.model.Executor;
+import hudson.model.Node;
+import hudson.model.Run;
+import hudson.model.TaskListener;
+import java.util.Map;
+import org.jenkinsci.plugins.workflow.job.WorkflowRun;
+
+@Extension
+public class DatadogTracerEnvironmentContributor extends EnvironmentContributor {
+
+    @Override
+    public void buildEnvironmentFor(@NonNull Run run, @NonNull EnvVars envs, @NonNull TaskListener listener) {
+        if (run instanceof WorkflowRun) {
+            // Pipelines are handled by org.datadog.jenkins.plugins.datadog.tracer.DatadogTracerStepEnvironmentContributor
+            return;
+        }
+
+        Executor executor = run.getExecutor();
+        if (executor == null) {
+            return;
+        }
+
+        Node node = executor.getOwner().getNode();
+        Map<String, String> additionalEnvVars = DatadogTracerConfigurator.INSTANCE.configure(run, node, envs);
+        envs.putAll(additionalEnvVars);
+    }
+}