From aa51fc854135e0335ec4bae701582f0fd6562617 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Fri, 14 Jun 2024 22:14:22 +0200 Subject: [PATCH] Consider job inclusion/exclusion setting when tracking logs (#432) --- README.md | 4 +- .../plugins/datadog/DatadogUtilities.java | 16 ++++- .../listeners/DatadogBuildListener.java | 10 +-- .../listeners/DatadogGraphListener.java | 2 +- .../datadog/listeners/DatadogSCMListener.java | 2 +- .../datadog/logs/DatadogConsoleLogFilter.java | 4 +- .../logs/DatadogTaskListenerDecorator.java | 6 +- .../plugins/datadog/steps/DatadogOptions.java | 2 + .../DatadogGlobalConfiguration/config.jelly | 4 +- .../help-blacklistEntry.html | 5 ++ .../help-whitelistEntry.html | 5 ++ .../logs/DatadogConsoleLogFilterTest.java | 64 +++++++++++++++++++ .../DatadogTaskListenerDecoratorTest.java | 42 +++++++++++- 13 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-blacklistEntry.html create mode 100644 src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-whitelistEntry.html create mode 100644 src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilterTest.java diff --git a/README.md b/README.md index d324928f7..bc78413e0 100644 --- a/README.md +++ b/README.md @@ -209,8 +209,8 @@ To customize your global configuration, in Jenkins navigate to `Manage Jenkins - | Customization | Description | Environment variable | |----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------| | Hostname | A hostname to use in every event sent to Datadog. | `DATADOG_JENKINS_PLUGIN_HOSTNAME` | -| Excluded jobs | A comma-separated list of regex used to exclude job names from monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`. | `DATADOG_JENKINS_PLUGIN_EXCLUDED` | -| Included jobs | A comma-separated list of regex used to include job names for monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`. | `DATADOG_JENKINS_PLUGIN_INCLUDED` | +| Excluded jobs | A comma-separated list of regex used to exclude job names from monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`. This setting affects all aspects of the plugin: events, metrics, logs, CI visibility. | `DATADOG_JENKINS_PLUGIN_EXCLUDED` | +| Included jobs | A comma-separated list of regex used to include job names for monitoring, for example: `susans-job,johns-.*,prod_folder/prod_release`. This setting affects all aspects of the plugin: events, metrics, logs, CI visibility. | `DATADOG_JENKINS_PLUGIN_INCLUDED` | | Global tag file | The path to a workspace file containing a comma separated list of tags (not compatible with pipeline jobs). | `DATADOG_JENKINS_PLUGIN_GLOBAL_TAG_FILE` | | Global tags | A comma-separated list of tags to apply to all metrics, events, and service checks. Tags can include environment variables that are defined in the master jenkins instance. | `DATADOG_JENKINS_PLUGIN_GLOBAL_TAGS` | | Global job tags | A comma separated list of regex to match a job and a list of tags to apply to that job. Tags can include environment variables that are defined in the master jenkins instance. **Note**: Tags can reference match groups in the regex using the `$` symbol, for example: `(.*?)_job_(*?)_release, owner:$1, release_env:$2, optional:Tag3` | `DATADOG_JENKINS_PLUGIN_GLOBAL_JOB_TAGS` | diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java index 221fc735b..1bbc71c4c 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java @@ -200,6 +200,16 @@ public static Map> getTagsFromPipelineAction(Run run) { return result; } + /** + * Checks inclusion/exclusion filter settings to see if a run should be tracked by the plugin + * + * @param run The run to be checked + * @return {@code true} if the run should be tracked by the plugin, {@code false} otherwise + */ + public static boolean isJobTracked(Run run) { + return run != null && isJobTracked(run.getParent().getFullName()); + } + /** * Checks if a jobName is excluded, included, or neither. * @@ -207,7 +217,7 @@ public static Map> getTagsFromPipelineAction(Run run) { * @return a boolean to signify if the jobName is or is not excluded or included. */ public static boolean isJobTracked(final String jobName) { - return !isJobExcluded(jobName) && isJobIncluded(jobName); + return jobName != null && !isJobExcluded(jobName) && isJobIncluded(jobName); } /** @@ -343,7 +353,7 @@ public static Map> getTagsFromGlobalTags() { * @param jobName - A String containing the name of some job. * @return a boolean to signify if the jobName is or is not excluded. */ - private static boolean isJobExcluded(final String jobName) { + private static boolean isJobExcluded(@Nonnull final String jobName) { final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor(); if (datadogGlobalConfig == null) { return false; @@ -367,7 +377,7 @@ private static boolean isJobExcluded(final String jobName) { * @param jobName - A String containing the name of some job. * @return a boolean to signify if the jobName is or is not included. */ - private static boolean isJobIncluded(final String jobName) { + private static boolean isJobIncluded(@Nonnull final String jobName) { final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor(); if (datadogGlobalConfig == null) { return true; diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListener.java b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListener.java index c67b13eef..2d7a4c10d 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListener.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListener.java @@ -91,7 +91,7 @@ public class DatadogBuildListener extends RunListener { public void onInitialize(Run run) { try { // Process only if job is NOT in excluded and is in included - if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) { + if (!DatadogUtilities.isJobTracked(run)) { return; } logger.fine("Start DatadogBuildListener#onInitialize"); @@ -190,7 +190,7 @@ public void buildEnvVars(Map env) { public void onStarted(Run run, TaskListener listener) { try { // Process only if job is NOT in excluded and is in included - if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) { + if (!DatadogUtilities.isJobTracked(run)) { return; } logger.fine("Start DatadogBuildListener#onStarted"); @@ -280,7 +280,7 @@ public void onCompleted(Run run, @Nonnull TaskListener listener) { DatadogClient client; try { // Process only if job in NOT in excluded and is in included - if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) { + if (!DatadogUtilities.isJobTracked(run)) { return; } @@ -394,7 +394,7 @@ public void onCompleted(Run run, @Nonnull TaskListener listener) { public void onFinalized(Run run) { try { // Process only if job in NOT in excluded and is in included - if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) { + if (!DatadogUtilities.isJobTracked(run)) { return; } logger.fine("Start DatadogBuildListener#onFinalized"); @@ -436,7 +436,7 @@ public void onFinalized(Run run) { public void onDeleted(Run run) { try { // Process only if job is NOT in excluded and is in included - if (!DatadogUtilities.isJobTracked(run.getParent().getFullName())) { + if (!DatadogUtilities.isJobTracked(run)) { return; } logger.fine("Start DatadogBuildListener#onDeleted"); diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListener.java b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListener.java index a9a0d6b38..a08071e58 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListener.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListener.java @@ -80,7 +80,7 @@ public class DatadogGraphListener implements GraphListener { public void onNewHead(FlowNode flowNode) { WorkflowRun run = getRun(flowNode); // Filter the node if the job has been excluded from the Datadog plugin configuration. - if (run == null || !DatadogUtilities.isJobTracked(run.getParent().getFullName())) { + if (run == null || !DatadogUtilities.isJobTracked(run)) { return; } diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogSCMListener.java b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogSCMListener.java index 51dc21de0..342d6358e 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogSCMListener.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogSCMListener.java @@ -89,7 +89,7 @@ public void onCheckout(Run build, SCM scm, FilePath workspace, TaskListene File changelogFile, SCMRevisionState pollingBaseline) throws Exception { try { // Process only if job is NOT in excluded and is in included - if (!DatadogUtilities.isJobTracked(build.getParent().getFullName())) { + if (!DatadogUtilities.isJobTracked(build)) { return; } diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilter.java b/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilter.java index bb894fef4..59f8b5693 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilter.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilter.java @@ -59,10 +59,10 @@ public OutputStream decorateLogger(Run build, OutputStream outputStream) { return outputStream; } - if (build != null) { + if (DatadogUtilities.isJobTracked(build)) { DatadogWriter writer = new DatadogWriter(new BuildData(build, null), outputStream); return new DatadogOutputStream(outputStream, writer); - } else if (run != null) { + } else if (DatadogUtilities.isJobTracked(run)) { DatadogWriter writer = new DatadogWriter(new BuildData(run, null), outputStream); return new DatadogOutputStream(outputStream, writer); } else { diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecorator.java b/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecorator.java index 160a8f296..a2bc5f87b 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecorator.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecorator.java @@ -32,7 +32,6 @@ of this software and associated documentation files (the "Software"), to deal import java.util.logging.Logger; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import org.datadog.jenkins.plugins.datadog.DatadogUtilities; import org.datadog.jenkins.plugins.datadog.model.BuildData; import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; @@ -82,7 +81,10 @@ public TaskListenerDecorator of(@Nonnull FlowExecutionOwner owner) { try { Queue.Executable executable = owner.getExecutable(); if (executable instanceof WorkflowRun) { - return new DatadogTaskListenerDecorator((WorkflowRun) executable); + WorkflowRun run = (WorkflowRun) executable; + if (DatadogUtilities.isJobTracked(run)) { + return new DatadogTaskListenerDecorator(run); + } } } catch (IOException ex) { LOGGER.log(Level.WARNING, null, ex); diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/steps/DatadogOptions.java b/src/main/java/org/datadog/jenkins/plugins/datadog/steps/DatadogOptions.java index 7594cebc1..75df2808c 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/steps/DatadogOptions.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/steps/DatadogOptions.java @@ -114,6 +114,8 @@ public boolean start() throws Exception { if (DatadogUtilities.getDatadogGlobalDescriptor().isCollectBuildLogs()) { taskLogger.println("[Datadog] Logging is already enabled globally, you do not need to specify 'collectLogs: true'"); } else { + // not checking DatadogUtilities#isJobTracked here: + // if Datadog step was added to the job, we assume it should be tracked invoker.withContext(TaskListenerDecorator.merge( context.get(TaskListenerDecorator.class), new DatadogTaskListenerDecorator(workflowRun)) ); diff --git a/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly index c3421abf5..ff4a5f1e6 100644 --- a/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly +++ b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly @@ -91,11 +91,11 @@ - + - + diff --git a/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-blacklistEntry.html b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-blacklistEntry.html new file mode 100644 index 000000000..965c90f2a --- /dev/null +++ b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-blacklistEntry.html @@ -0,0 +1,5 @@ +
+

A comma-separated list of regex used to exclude job names from monitoring, for example: "susans-job,johns-.*,prod_folder/prod_release".

+

Please note that for wildcards regular expressions should be used, and not glob patterns (see the example above).

+

This setting affects all aspects of the plugin: events, metrics, logs, CI visibility.

+
diff --git a/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-whitelistEntry.html b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-whitelistEntry.html new file mode 100644 index 000000000..c6d6b505f --- /dev/null +++ b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/help-whitelistEntry.html @@ -0,0 +1,5 @@ +
+

A comma-separated list of regex used to include job names for monitoring, for example: "susans-job,johns-.*,prod_folder/prod_release".

+

Please note that for wildcards regular expressions should be used, and not glob patterns (see the example above).

+

This setting affects all aspects of the plugin: events, metrics, logs, CI visibility.

+
diff --git a/src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilterTest.java b/src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilterTest.java new file mode 100644 index 000000000..77b7dd377 --- /dev/null +++ b/src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogConsoleLogFilterTest.java @@ -0,0 +1,64 @@ +package org.datadog.jenkins.plugins.datadog.logs; + +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import hudson.model.Run; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration; +import org.datadog.jenkins.plugins.datadog.DatadogUtilities; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.jetbrains.annotations.NotNull; +import org.junit.ClassRule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; + +public class DatadogConsoleLogFilterTest { + + @ClassRule + public static JenkinsRule JENKINS = new JenkinsRule(); + + + @Test + public void testExcludedJobsAreNotDecorated() { + String jobName = "my-job"; + givenJobIsExcludedFromTracking(jobName); + WorkflowRun run = givenBuildNamed(jobName); + + OutputStream originalStream = new ByteArrayOutputStream(); + DatadogConsoleLogFilter filter = new DatadogConsoleLogFilter(); + OutputStream decoratedStream = filter.decorateLogger(run, originalStream); + + assertSame(originalStream, decoratedStream); + } + + @Test + public void testExcludedJobsProvidedViaConstructorAreNotDecorated() { + String jobName = "my-job"; + givenJobIsExcludedFromTracking(jobName); + WorkflowRun run = givenBuildNamed(jobName); + + OutputStream originalStream = new ByteArrayOutputStream(); + DatadogConsoleLogFilter filter = new DatadogConsoleLogFilter(run); + OutputStream decoratedStream = filter.decorateLogger((Run) null, originalStream); + + assertSame(originalStream, decoratedStream); + } + + private static void givenJobIsExcludedFromTracking(String jobName) { + DatadogGlobalConfiguration cfg = DatadogUtilities.getDatadogGlobalDescriptor(); + cfg.setExcluded(jobName); + } + + private static @NotNull WorkflowRun givenBuildNamed(String jobName) { + WorkflowJob job = mock(WorkflowJob.class); + when(job.getFullName()).thenReturn(jobName); + + WorkflowRun run = mock(WorkflowRun.class); + when(run.getParent()).thenReturn(job); + return run; + } +} diff --git a/src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecoratorTest.java b/src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecoratorTest.java index 13c39cd48..f4f5a4dfd 100644 --- a/src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecoratorTest.java +++ b/src/test/java/org/datadog/jenkins/plugins/datadog/logs/DatadogTaskListenerDecoratorTest.java @@ -24,6 +24,7 @@ of this software and associated documentation files (the "Software"), to deal */ package org.datadog.jenkins.plugins.datadog.logs; +import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -34,12 +35,22 @@ of this software and associated documentation files (the "Software"), to deal import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.nio.charset.Charset; +import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration; +import org.datadog.jenkins.plugins.datadog.DatadogUtilities; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jenkinsci.plugins.workflow.job.WorkflowJob; import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.jenkinsci.plugins.workflow.log.TaskListenerDecorator; +import org.jetbrains.annotations.NotNull; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; public class DatadogTaskListenerDecoratorTest { + @ClassRule + public static JenkinsRule JENKINS = new JenkinsRule(); + private WorkflowRun workflowRun; private WorkflowJob job; @@ -80,4 +91,33 @@ public void testSerializeDeserialize() throws Exception { // Assert decorate can be called datadogTaskListenerDecoratorDes.decorate(System.out); } - } + + @Test + public void testExcludedJobsAreNotDecorated() throws Exception { + String jobName = "my-job"; + + givenJobIsExcludedFromTracking(jobName); + WorkflowRun run = givenBuildNamed(jobName); + + FlowExecutionOwner executionOwner = mock(FlowExecutionOwner.class); + when(executionOwner.getExecutable()).thenReturn(run); + DatadogTaskListenerDecorator.Factory factory = new DatadogTaskListenerDecorator.Factory(); + TaskListenerDecorator taskListenerDecorator = factory.of(executionOwner); + + assertNull(taskListenerDecorator); + } + + private static void givenJobIsExcludedFromTracking(String jobName) { + DatadogGlobalConfiguration cfg = DatadogUtilities.getDatadogGlobalDescriptor(); + cfg.setExcluded(jobName); + } + + private static @NotNull WorkflowRun givenBuildNamed(String jobName) { + WorkflowJob job = mock(WorkflowJob.class); + when(job.getFullName()).thenReturn(jobName); + + WorkflowRun run = mock(WorkflowRun.class); + when(run.getParent()).thenReturn(job); + return run; + } +}