Consider job inclusion/exclusion setting when tracking logs (#432)

README.md

@@ -209,8 +209,8 @@ To customize your global configuration, in Jenkins navigate to `Manage Jenkins -
 | Customization              | Description                                                                                                                                                                                                                                 | Environment variable                          |
 |----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------|
 | Hostname                   | A hostname to use in every event sent to Datadog.                                                                                                                                                                                           | `DATADOG_JENKINS_PLUGIN_HOSTNAME`             |
-| Excluded jobs              | A comma-separated list of regex used to exclude job names from monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`.                                                                                                      | `DATADOG_JENKINS_PLUGIN_EXCLUDED`            |
-| Included jobs              | A comma-separated list of regex used to include job names for monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`.                                                                                                          | `DATADOG_JENKINS_PLUGIN_INCLUDED`            |
+| Excluded jobs              | A comma-separated list of regex used to exclude job names from monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`. This setting affects all aspects of the plugin: events, metrics, logs, CI visibility.               |  `DATADOG_JENKINS_PLUGIN_EXCLUDED`            |
+| Included jobs              | A comma-separated list of regex used to include job names for monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`. This setting affects all aspects of the plugin: events, metrics, logs, CI visibility.                | `DATADOG_JENKINS_PLUGIN_INCLUDED`            |
 | Global tag file            | The path to a workspace file containing a comma separated list of tags (not compatible with pipeline jobs).                                                                                                                                   | `DATADOG_JENKINS_PLUGIN_GLOBAL_TAG_FILE`      |
 | Global tags                | A comma-separated list of tags to apply to all metrics, events, and service checks. Tags can include environment variables that are defined in the master jenkins instance.                                                                                                                                                          | `DATADOG_JENKINS_PLUGIN_GLOBAL_TAGS`          |
 | Global job tags            | A comma separated list of regex to match a job and a list of tags to apply to that job. Tags can include environment variables that are defined in the master jenkins instance. **Note**: Tags can reference match groups in the regex using the `$` symbol, for example: `(.*?)_job_(*?)_release, owner:$1, release_env:$2, optional:Tag3` | `DATADOG_JENKINS_PLUGIN_GLOBAL_JOB_TAGS`      |

src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java

@@ -200,14 +200,24 @@ public static Map<String, Set<String>> getTagsFromPipelineAction(Run run) {
         return result;
     }
 
+    /**
+     * Checks inclusion/exclusion filter settings to see if a run should be tracked by the plugin
+     *
+     * @param run The run to be checked
+     * @return {@code true} if the run should be tracked by the plugin, {@code false} otherwise
+     */
+    public static boolean isJobTracked(Run<?,?> run) {
+        return run != null && isJobTracked(run.getParent().getFullName());
+    }
+
     /**
      * Checks if a jobName is excluded, included, or neither.
      *
      * @param jobName - A String containing the name of some job.
      * @return a boolean to signify if the jobName is or is not excluded or included.
      */
     public static boolean isJobTracked(final String jobName) {
-        return !isJobExcluded(jobName) && isJobIncluded(jobName);
+        return jobName != null && !isJobExcluded(jobName) && isJobIncluded(jobName);
     }
 
     /**
@@ -343,7 +353,7 @@ public static Map<String, Set<String>> getTagsFromGlobalTags() {
      * @param jobName - A String containing the name of some job.
      * @return a boolean to signify if the jobName is or is not excluded.
      */
-    private static boolean isJobExcluded(final String jobName) {
+    private static boolean isJobExcluded(@Nonnull final String jobName) {
         final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor();
         if (datadogGlobalConfig == null) {
             return false;
@@ -367,7 +377,7 @@ private static boolean isJobExcluded(final String jobName) {
      * @param jobName - A String containing the name of some job.
      * @return a boolean to signify if the jobName is or is not included.
      */
-    private static boolean isJobIncluded(final String jobName) {
+    private static boolean isJobIncluded(@Nonnull final String jobName) {
         final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor();
         if (datadogGlobalConfig == null) {
             return true;

src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListener.java

@@ -91,7 +91,7 @@ public class DatadogBuildListener extends RunListener<Run> {
     public void onInitialize(Run run) {
         try {
             // Process only if job is NOT in excluded and is in included
-            if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) {
+            if (!DatadogUtilities.isJobTracked(run)) {
                 return;
             }
             logger.fine("Start DatadogBuildListener#onInitialize");
@@ -190,7 +190,7 @@ public void buildEnvVars(Map<String, String> env) {
     public void onStarted(Run run, TaskListener listener) {
         try {
             // Process only if job is NOT in excluded and is in included
-            if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) {
+            if (!DatadogUtilities.isJobTracked(run)) {
                 return;
             }
             logger.fine("Start DatadogBuildListener#onStarted");
@@ -280,7 +280,7 @@ public void onCompleted(Run run, @Nonnull TaskListener listener) {
         DatadogClient client;
         try {
             // Process only if job in NOT in excluded and is in included
-            if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) {
+            if (!DatadogUtilities.isJobTracked(run)) {
                 return;
             }
 
@@ -394,7 +394,7 @@ public void onCompleted(Run run, @Nonnull TaskListener listener) {
     public void onFinalized(Run run) {
         try {
             // Process only if job in NOT in excluded and is in included
-            if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) {
+            if (!DatadogUtilities.isJobTracked(run)) {
                 return;
             }
             logger.fine("Start DatadogBuildListener#onFinalized");
@@ -436,7 +436,7 @@ public void onFinalized(Run run) {
     public void onDeleted(Run run) {
         try {
             // Process only if job is NOT in excluded and is in included
-            if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) {
+            if (!DatadogUtilities.isJobTracked(run)) {
                 return;
             }
             logger.fine("Start DatadogBuildListener#onDeleted");

src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListener.java

@@ -80,7 +80,7 @@ public class DatadogGraphListener implements GraphListener {
     public void onNewHead(FlowNode flowNode) {
         WorkflowRun run = getRun(flowNode);
         // Filter the node if the job has been excluded from the Datadog plugin configuration.
-        if (run == null || !DatadogUtilities.isJobTracked(run.getParent().getFullName())) {
+        if (run == null || !DatadogUtilities.isJobTracked(run)) {
             return;
         }
 

src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogSCMListener.java

@@ -89,7 +89,7 @@ public void onCheckout(Run<?, ?> build, SCM scm, FilePath workspace, TaskListene
                            File changelogFile, SCMRevisionState pollingBaseline) throws Exception {
         try {
             // Process only if job is NOT in excluded and is in included
-            if (!DatadogUtilities.isJobTracked(build.getParent().getFullName())) {
+            if (!DatadogUtilities.isJobTracked(build)) {
                 return;
             }
 

src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilter.java

@@ -59,10 +59,10 @@ public OutputStream decorateLogger(Run build, OutputStream outputStream) {
                 return outputStream;
             }
 
-            if (build != null) {
+            if (DatadogUtilities.isJobTracked(build)) {
                 DatadogWriter writer = new DatadogWriter(new BuildData(build, null), outputStream);
                 return new DatadogOutputStream(outputStream, writer);
-            } else if (run != null) {
+            } else if (DatadogUtilities.isJobTracked(run)) {
                 DatadogWriter writer = new DatadogWriter(new BuildData(run, null), outputStream);
                 return new DatadogOutputStream(outputStream, writer);
             } else {

src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecorator.java

@@ -32,7 +32,6 @@ of this software and associated documentation files (the "Software"), to deal
 import java.util.logging.Logger;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-
 import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
 import org.datadog.jenkins.plugins.datadog.model.BuildData;
 import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
@@ -82,7 +81,10 @@ public TaskListenerDecorator of(@Nonnull FlowExecutionOwner owner) {
             try {
                 Queue.Executable executable = owner.getExecutable();
                 if (executable instanceof WorkflowRun) {
-                    return new DatadogTaskListenerDecorator((WorkflowRun) executable);
+                    WorkflowRun run = (WorkflowRun) executable;
+                    if (DatadogUtilities.isJobTracked(run)) {
+                        return new DatadogTaskListenerDecorator(run);
+                    }
                 }
             } catch (IOException ex) {
                 LOGGER.log(Level.WARNING, null, ex);

src/main/java/org/datadog/jenkins/plugins/datadog/steps/DatadogOptions.java

@@ -114,6 +114,8 @@ public boolean start() throws Exception {
                 if (DatadogUtilities.getDatadogGlobalDescriptor().isCollectBuildLogs()) {
                     taskLogger.println("[Datadog] Logging is already enabled globally, you do not need to specify 'collectLogs: true'");
                 } else {
+                    // not checking DatadogUtilities#isJobTracked here:
+                    // if Datadog step was added to the job, we assume it should be tracked
                     invoker.withContext(TaskListenerDecorator.merge(
                             context.get(TaskListenerDecorator.class), new DatadogTaskListenerDecorator(workflowRun))
                     );

src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly

@@ -91,11 +91,11 @@
         </f:entry>
         <f:validateButton title="${%Test Hostname}" progress="${%Testing...}" method="testHostname" with="hostname" checkMethod="post" />
 
-        <f:entry title="Excluded Jobs" description="A comma-separated list of job names that should not monitored." >
+        <f:entry title="Excluded Jobs" field="blacklistEntry" description="A comma-separated list of job names that should not monitored." >
           <f:textarea field="blacklist" optional="true" default="${blacklist}" />
         </f:entry>
 
-        <f:entry title="Included Jobs" description="A list of job names that should be monitored. An empty included permits all jobs not excluded." >
+        <f:entry title="Included Jobs" field="whitelistEntry" description="A list of job names that should be monitored. An empty included permits all jobs not excluded." >
           <f:textarea field="whitelist" optional="true" default="${whitelist}" />
         </f:entry>
 

src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-blacklistEntry.html

@@ -0,0 +1,5 @@
+<div>
+    <p>A comma-separated list of regex used to exclude job names from monitoring, for example: "susans-job,johns-.*,prod_folder/prod_release".</p>
+    <p>Please note that for wildcards regular expressions should be used, and not glob patterns (see the example above).</p>
+    <p>This setting affects all aspects of the plugin: events, metrics, logs, CI visibility.</p>
+</div>

src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-whitelistEntry.html

@@ -0,0 +1,5 @@
+<div>
+    <p>A comma-separated list of regex used to include job names for monitoring, for example: "susans-job,johns-.*,prod_folder/prod_release".</p>
+    <p>Please note that for wildcards regular expressions should be used, and not glob patterns (see the example above).</p>
+    <p>This setting affects all aspects of the plugin: events, metrics, logs, CI visibility.</p>
+</div>

src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilterTest.java

@@ -0,0 +1,64 @@
+package org.datadog.jenkins.plugins.datadog.logs;
+
+import static org.junit.Assert.assertSame;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import hudson.model.Run;
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration;
+import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
+import org.jenkinsci.plugins.workflow.job.WorkflowJob;
+import org.jenkinsci.plugins.workflow.job.WorkflowRun;
+import org.jetbrains.annotations.NotNull;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.jvnet.hudson.test.JenkinsRule;
+
+public class DatadogConsoleLogFilterTest {
+
+    @ClassRule
+    public static JenkinsRule JENKINS = new JenkinsRule();
+
+
+    @Test
+    public void testExcludedJobsAreNotDecorated() {
+        String jobName = "my-job";
+        givenJobIsExcludedFromTracking(jobName);
+        WorkflowRun run = givenBuildNamed(jobName);
+
+        OutputStream originalStream = new ByteArrayOutputStream();
+        DatadogConsoleLogFilter filter = new DatadogConsoleLogFilter();
+        OutputStream decoratedStream = filter.decorateLogger(run, originalStream);
+
+        assertSame(originalStream, decoratedStream);
+    }
+
+    @Test
+    public void testExcludedJobsProvidedViaConstructorAreNotDecorated() {
+        String jobName = "my-job";
+        givenJobIsExcludedFromTracking(jobName);
+        WorkflowRun run = givenBuildNamed(jobName);
+
+        OutputStream originalStream = new ByteArrayOutputStream();
+        DatadogConsoleLogFilter filter = new DatadogConsoleLogFilter(run);
+        OutputStream decoratedStream = filter.decorateLogger((Run) null, originalStream);
+
+        assertSame(originalStream, decoratedStream);
+    }
+
+    private static void givenJobIsExcludedFromTracking(String jobName) {
+        DatadogGlobalConfiguration cfg = DatadogUtilities.getDatadogGlobalDescriptor();
+        cfg.setExcluded(jobName);
+    }
+
+    private static @NotNull WorkflowRun givenBuildNamed(String jobName) {
+        WorkflowJob job = mock(WorkflowJob.class);
+        when(job.getFullName()).thenReturn(jobName);
+
+        WorkflowRun run = mock(WorkflowRun.class);
+        when(run.getParent()).thenReturn(job);
+        return run;
+    }
+}

src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecoratorTest.java

@@ -24,6 +24,7 @@ of this software and associated documentation files (the "Software"), to deal
 */
 package org.datadog.jenkins.plugins.datadog.logs;
 
+import static org.junit.Assert.assertNull;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -34,12 +35,22 @@ of this software and associated documentation files (the "Software"), to deal
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.nio.charset.Charset;
+import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration;
+import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
+import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
 import org.jenkinsci.plugins.workflow.job.WorkflowJob;
 import org.jenkinsci.plugins.workflow.job.WorkflowRun;
+import org.jenkinsci.plugins.workflow.log.TaskListenerDecorator;
+import org.jetbrains.annotations.NotNull;
 import org.junit.Before;
+import org.junit.ClassRule;
 import org.junit.Test;
+import org.jvnet.hudson.test.JenkinsRule;
 
 public class DatadogTaskListenerDecoratorTest {
+    @ClassRule
+    public static JenkinsRule JENKINS = new JenkinsRule();
+
     private WorkflowRun workflowRun;   
     private WorkflowJob job; 
 
@@ -80,4 +91,33 @@ public void testSerializeDeserialize() throws Exception {
         // Assert decorate can be called
         datadogTaskListenerDecoratorDes.decorate(System.out);
     }
- }
+
+    @Test
+    public void testExcludedJobsAreNotDecorated() throws Exception {
+        String jobName = "my-job";
+
+        givenJobIsExcludedFromTracking(jobName);
+        WorkflowRun run = givenBuildNamed(jobName);
+
+        FlowExecutionOwner executionOwner = mock(FlowExecutionOwner.class);
+        when(executionOwner.getExecutable()).thenReturn(run);
+        DatadogTaskListenerDecorator.Factory factory = new DatadogTaskListenerDecorator.Factory();
+        TaskListenerDecorator taskListenerDecorator = factory.of(executionOwner);
+
+        assertNull(taskListenerDecorator);
+    }
+
+    private static void givenJobIsExcludedFromTracking(String jobName) {
+        DatadogGlobalConfiguration cfg = DatadogUtilities.getDatadogGlobalDescriptor();
+        cfg.setExcluded(jobName);
+    }
+
+    private static @NotNull WorkflowRun givenBuildNamed(String jobName) {
+        WorkflowJob job = mock(WorkflowJob.class);
+        when(job.getFullName()).thenReturn(jobName);
+
+        WorkflowRun run = mock(WorkflowRun.class);
+        when(run.getParent()).thenReturn(job);
+        return run;
+    }
+}