Remove some redundant code; add upstream/downstream pipeline example …

…to provisioned jobs

DEVELOPMENT.md

@@ -36,6 +36,7 @@ To spin up a development environment for the *jenkins-datadog* plugin repository
 
 1. Set the `JENKINS_PLUGIN` environment variable to point to the directory where this repository is cloned/forked.
 1. Set the `JENKINS_PLUGIN_DATADOG_API_KEY` environment variable with your api key.
+1. Set the `JENKINS_PLUGIN_DATADOG_CI_INSTANCE_NAME` to a name you would like your instance to have (makes it easier to identify your pipeline executions by setting `@ci.provider.instance:your-instance-name` filter in the Datadog UI).
 1. Optionally set the `GITHUB_SSH_KEY` and `GITHUB_SSH_KEY_PASSPHRASE` environment variables with the key and passphrase that can be used to access GitHub. This allows to automatically create GitHub credentials in Jenkins.   
 1. Run `mvn clean package -DskipTests` and `docker-compose -p datadog-jenkins-plugin -f docker/docker-compose.yaml up` from the directory where this repository is cloned/forked (if the `docker-compose` command fails with a `path ... not found` error, try updating it to the latest version).
     - NOTE: This spins up the Jenkins docker image and auto mounts the target folder of this repository (the location where the binary is built).

docker/controller-node/Dockerfile

@@ -15,6 +15,9 @@ USER jenkins
 COPY plugin-dependencies.txt /var/jenkins_home/plugin-dependencies.txt
 RUN jenkins-plugin-cli --latest-specified --plugin-file /var/jenkins_home/plugin-dependencies.txt
 
+# this is not a dependency of the Datadog plugin, it is just needed to test upstream/downstream pipeline linking
+RUN jenkins-plugin-cli --latest-specified --plugins pipeline-build-step
+
 COPY jobs /var/jenkins_home/sample-jobs
 
 COPY 10-create-admin-user.groovy /usr/share/jenkins/ref/init.groovy.d/10-create-admin-user.groovy

docker/controller-node/jobs/test-pipeline-downstream.cps

@@ -0,0 +1,12 @@
+pipeline {
+    agent any
+    stages {
+        stage('Build') {
+            steps {
+                sh '''
+                echo test downstream
+                '''
+            }
+        }
+    }
+}

docker/controller-node/jobs/test-pipeline-upstream.cps

@@ -0,0 +1,13 @@
+pipeline {
+    agent any
+    stages {
+        stage('Build') {
+            steps {
+                sh '''
+                echo test
+                '''
+                build job: 'test-pipeline-downstream'
+            }
+        }
+    }
+}

docker/docker-compose.yaml

@@ -19,6 +19,8 @@ services:
       DATADOG_JENKINS_PLUGIN_TARGET_HOST: datadog # `dogstatsd` or `datadog` based on the container you wish to use
       DATADOG_JENKINS_PLUGIN_TARGET_LOG_COLLECTION_PORT: 10518
       DATADOG_JENKINS_PLUGIN_TARGET_API_KEY: $JENKINS_PLUGIN_DATADOG_API_KEY
+      DATADOG_JENKINS_PLUGIN_ENABLE_CI_VISIBILITY: true
+      DATADOG_JENKINS_PLUGIN_CI_VISIBILITY_CI_INSTANCE_NAME: $JENKINS_PLUGIN_DATADOG_CI_INSTANCE_NAME
     volumes:
       - jenkins_shared:/var/jenkins_home/shared
       - $JENKINS_PLUGIN/target/datadog.hpi:/var/jenkins_home/plugins/datadog.hpi

src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListener.java

@@ -410,8 +410,6 @@ public void onFinalized(Run run) {
             traceWriter.submitBuild(buildData, run);
             logger.fine("End DatadogBuildListener#onFinalized");
 
-            BuildSpanManager.get().remove(buildData.getBuildTag(""));
-
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
             DatadogUtilities.severe(logger, e, "Interrupted while processing build finalization");

src/main/java/org/datadog/jenkins/plugins/datadog/model/BuildData.java

@@ -73,7 +73,6 @@ of this software and associated documentation files (the "Software"), to deal
 import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
 import org.datadog.jenkins.plugins.datadog.traces.BuildConfigurationParser;
 import org.datadog.jenkins.plugins.datadog.traces.BuildSpanAction;
-import org.datadog.jenkins.plugins.datadog.traces.BuildSpanManager;
 import org.datadog.jenkins.plugins.datadog.traces.message.TraceSpan;
 import org.datadog.jenkins.plugins.datadog.util.TagsUtil;
 import org.datadog.jenkins.plugins.datadog.util.git.GitUtils;
@@ -150,8 +149,8 @@ public class BuildData implements Serializable {
      * The backend needs version to determine the relative order of these multiple events.
      */
     private Integer version;
-    private String traceId;
-    private String spanId;
+    private Long traceId;
+    private Long spanId;
 
     private String upstreamPipelineUrl;
     private Long upstreamPipelineTraceId;
@@ -172,6 +171,10 @@ public BuildData(Run<?, ?> run, @Nullable TaskListener listener) throws IOExcept
                 this.buildUrl = buildSpanAction.getBuildUrl();
             }
             this.version = buildSpanAction.getAndIncrementVersion();
+
+            TraceSpan.TraceSpanContext buildSpanContext = buildSpanAction.getBuildSpanContext();
+            this.traceId = buildSpanContext.getTraceId();
+            this.spanId = buildSpanContext.getSpanId();
         }
 
         // Populate instance using environment variables.
@@ -268,13 +271,6 @@ public BuildData(Run<?, ?> run, @Nullable TaskListener listener) throws IOExcept
         // Build parameters
         populateBuildParameters(run);
 
-        // Set Tracing IDs
-        TraceSpan.TraceSpanContext buildSpanContext = BuildSpanManager.get().get(getBuildTag(""));
-        if(buildSpanContext !=null) {
-            this.traceId = Long.toUnsignedString(buildSpanContext.getTraceId());
-            this.spanId = Long.toUnsignedString(buildSpanContext.getSpanId());
-        }
-
         populateUpstreamPipelineData(run, envVars);
     }
 
@@ -298,16 +294,13 @@ private void populateUpstreamPipelineData(Run<?, ?> run, EnvVars envVars) {
         String upstreamProject = upstreamCause.getUpstreamProject();
         if (upstreamProject != null) {
             upstreamBuildTag = "jenkins-" + upstreamProject.replace('/', '-') + "-" + upstreamBuild;
-            TraceSpan.TraceSpanContext upstreamPipelineContext = BuildSpanManager.get().get(upstreamBuildTag);
-            if (upstreamPipelineContext == null) {
-                BuildSpanAction buildSpanAction = run.getAction(BuildSpanAction.class);
-                if (buildSpanAction != null) {
-                    upstreamPipelineContext = buildSpanAction.getUpstreamSpanContext();
-                }
-            }
 
-            if (upstreamPipelineContext != null) {
-                upstreamPipelineTraceId = upstreamPipelineContext.getTraceId();
+            BuildSpanAction buildSpanAction = run.getAction(BuildSpanAction.class);
+            if (buildSpanAction != null) {
+                TraceSpan.TraceSpanContext upstreamSpanContext = buildSpanAction.getUpstreamSpanContext();
+                if (upstreamSpanContext != null) {
+                    upstreamPipelineTraceId = upstreamSpanContext.getTraceId();
+                }
             }
         }
     }
@@ -710,6 +703,14 @@ public Integer getVersion() {
         return version;
     }
 
+    public Long getTraceId() {
+        return traceId;
+    }
+
+    public Long getSpanId() {
+        return spanId;
+    }
+
     public String getBuildTag(String value) {
         return defaultIfNull(buildTag, value);
     }
@@ -936,11 +937,11 @@ public JSONObject addLogAttributes(){
             payload.put("hostname", this.hostname);
 
             if(traceId != null){
-                payload.put("dd.trace_id", this.traceId);
+                payload.put("dd.trace_id", Long.toUnsignedString(traceId));
             }
 
             if(spanId != null) {
-                payload.put("dd.span_id", this.spanId);
+                payload.put("dd.span_id",  Long.toUnsignedString(spanId));
             }
             return payload;
         } catch (Exception e){

src/main/java/org/datadog/jenkins/plugins/datadog/traces/BuildSpanManager.java

@@ -8,53 +8,34 @@
 import org.datadog.jenkins.plugins.datadog.traces.message.TraceSpan;
 
 /**
- * Used to propagate the build Span between onStart() and onComplete() methods.
- * This mechanism is needed because the Span object cannot be serialized in a Jenkins Action.
+ * Used to store trace data after the build has finished.
+ * The data is needed to link upstream build to a downstream build.
  */
 public class BuildSpanManager {
 
     private static final Logger LOGGER = Logger.getLogger(BuildSpanManager.class.getName());
 
     private static final BuildSpanManager INSTANCE = new BuildSpanManager();
-    private final Map<String, TraceSpan.TraceSpanContext> inProgress = new ConcurrentHashMap<>();
-
-    /**
-     * The last N finished contexts are stored to be used by the logic that links upstream pipelines to downstream pipelines
-     */
-    private final Map<String, TraceSpan.TraceSpanContext> finished = new ConcurrentHashMap<>();
-    private final BlockingQueue<String> finishedTags = new ArrayBlockingQueue<>(getFinishedContextsCapacity());
+    private final Map<String, TraceSpan.TraceSpanContext> contextByTag = new ConcurrentHashMap<>();
+    private final BlockingQueue<String> tags = new ArrayBlockingQueue<>(getCapacity());
 
     public static BuildSpanManager get() {
         return INSTANCE;
     }
 
     public void put(final String tag, final TraceSpan.TraceSpanContext context) {
-        inProgress.put(tag, context);
-    }
-
-    public TraceSpan.TraceSpanContext get(final String tag) {
-        TraceSpan.TraceSpanContext inProgressContext = inProgress.get(tag);
-        if (inProgressContext != null) {
-            return inProgressContext;
-        } else {
-            return finished.get(tag);
+        while (!tags.offer(tag)) {
+            // drop the oldest tag if the storage is full
+            contextByTag.remove(tags.poll());
         }
+        contextByTag.put(tag, context);
     }
 
-    public void remove(final String tag){
-        TraceSpan.TraceSpanContext context = inProgress.remove(tag);
-        if (context == null) {
-            return;
-        }
-
-        finished.put(tag, context);
-
-        while (!finishedTags.offer(tag)) {
-            finished.remove(finishedTags.poll());
-        }
+    public TraceSpan.TraceSpanContext get(final String tag) {
+        return contextByTag.get(tag);
     }
 
-    private static int getFinishedContextsCapacity() {
+    private static int getCapacity() {
         String maxSize = System.getenv("DD_JENKINS_SPAN_CONTEXT_STORAGE_MAX_SIZE");
         if (maxSize != null) {
             try {

src/main/java/org/datadog/jenkins/plugins/datadog/traces/DatadogTraceBuildLogic.java

@@ -53,11 +53,6 @@ public TraceSpan toSpan(final BuildData buildData, final Run<?,?> run) {
             return null;
         }
 
-        TraceSpan.TraceSpanContext spanContext = BuildSpanManager.get().get(buildData.getBuildTag(""));
-        if(spanContext == null) {
-            return null;
-        }
-
         final BuildSpanAction buildSpanAction = run.getAction(BuildSpanAction.class);
         if(buildSpanAction == null) {
             return null;
@@ -67,6 +62,9 @@ public TraceSpan toSpan(final BuildData buildData, final Run<?,?> run) {
         final String buildLevel = PipelineStepData.StepType.PIPELINE.getBuildLevel();
         final long endTimeMicros = buildData.getEndTime(0L) * 1000;
 
+        long traceId = buildData.getTraceId();
+        long spanId = buildData.getSpanId();
+        TraceSpan.TraceSpanContext spanContext = new TraceSpan.TraceSpanContext(traceId, 0, spanId);
         final TraceSpan buildSpan = new TraceSpan("jenkins.build", TimeUnit.MILLISECONDS.toNanos(buildData.getStartTime(0L)), spanContext);
         buildSpan.setServiceName(DatadogUtilities.getDatadogGlobalDescriptor().getCiInstanceName());
         buildSpan.setType("ci");

src/main/java/org/datadog/jenkins/plugins/datadog/traces/DatadogWebhookBuildLogic.java

@@ -20,7 +20,6 @@
 import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
 import org.datadog.jenkins.plugins.datadog.model.BuildData;
 import org.datadog.jenkins.plugins.datadog.model.PipelineStepData;
-import org.datadog.jenkins.plugins.datadog.traces.message.TraceSpan;
 import org.datadog.jenkins.plugins.datadog.util.TagsUtil;
 
 /**
@@ -38,11 +37,6 @@ public JSONObject toJson(final BuildData buildData, final Run<?,?> run) {
             return null;
         }
 
-        TraceSpan.TraceSpanContext buildSpanContext = BuildSpanManager.get().get(buildData.getBuildTag(""));
-        if(buildSpanContext == null) {
-            return null;
-        }
-
         final BuildSpanAction buildSpanAction = run.getAction(BuildSpanAction.class);
         if(buildSpanAction == null) {
             return null;
@@ -89,8 +83,8 @@ public JSONObject toJson(final BuildData buildData, final Run<?,?> run) {
         payload.put("status", status);
         payload.put("is_manual", isTriggeredManually(run));
 
-        payload.put("trace_id", buildSpanContext.getTraceId());
-        payload.put("span_id", buildSpanContext.getSpanId());
+        payload.put("trace_id", buildData.getTraceId());
+        payload.put("span_id", buildData.getSpanId());
 
         payload.put("pipeline_id", buildData.getBuildTag(""));
         payload.put("unique_id", buildData.getBuildTag(""));