From 9b6d2c22d8b80b7bfa202623af3af6ebe4caad90 Mon Sep 17 00:00:00 2001 From: Sarah Witt Date: Thu, 15 Jun 2023 10:06:04 -0400 Subject: [PATCH 01/16] Add option to use AWS instance ID as hostname (#345) * Add option to use AWS instance ID as hostname * Change name of method --- .../datadog/DatadogGlobalConfiguration.java | 27 +++++++++ .../plugins/datadog/DatadogUtilities.java | 57 +++++++++++++++++++ .../DatadogGlobalConfiguration/config.jelly | 6 ++ .../plugins/datadog/DatadogUtilitiesTest.java | 43 ++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java index c8d772f3c..7a4968997 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java @@ -109,6 +109,7 @@ public class DatadogGlobalConfiguration extends GlobalConfiguration { private static final String RETRY_LOGS_PROPERTY = "DATADOG_JENKINS_PLUGIN_RETRY_LOGS"; private static final String REFRESH_DOGSTATSD_CLIENT_PROPERTY = "DATADOG_REFRESH_STATSD_CLIENT"; private static final String CACHE_BUILD_RUNS_PROPERTY = "DATADOG_CACHE_BUILD_RUNS"; + private static final String USE_AWS_INSTANCE_HOSTNAME_PROPERTY = "DATADOG_USE_AWS_INSTANCE_HOSTNAME"; private static final String ENABLE_CI_VISIBILITY_PROPERTY = "DATADOG_JENKINS_PLUGIN_ENABLE_CI_VISIBILITY"; private static final String CI_VISIBILITY_CI_INSTANCE_NAME_PROPERTY = "DATADOG_JENKINS_PLUGIN_CI_VISIBILITY_CI_INSTANCE_NAME"; @@ -130,6 +131,7 @@ public class DatadogGlobalConfiguration extends GlobalConfiguration { private static final boolean DEFAULT_RETRY_LOGS_VALUE = true; private static final boolean DEFAULT_REFRESH_DOGSTATSD_CLIENT_VALUE = false; private static final boolean DEFAULT_CACHE_BUILD_RUNS_VALUE = true; + private static final boolean DEFAULT_USE_AWS_INSTANCE_HOSTNAME_VALUE = false; private String reportWith = DEFAULT_REPORT_WITH_VALUE; private String targetApiURL = DEFAULT_TARGET_API_URL_VALUE; @@ -157,6 +159,7 @@ public class DatadogGlobalConfiguration extends GlobalConfiguration { private boolean retryLogs = DEFAULT_RETRY_LOGS_VALUE; private boolean refreshDogstatsdClient = DEFAULT_REFRESH_DOGSTATSD_CLIENT_VALUE; private boolean cacheBuildRuns = DEFAULT_CACHE_BUILD_RUNS_VALUE; + private boolean useAwsInstanceHostname = DEFAULT_USE_AWS_INSTANCE_HOSTNAME_VALUE; @DataBoundConstructor public DatadogGlobalConfiguration() { @@ -292,6 +295,11 @@ private void loadEnvVariables(){ this.cacheBuildRuns = Boolean.valueOf(cacheBuildRunsEnvVar); } + String useAwsInstanceHostnameEnvVar = System.getenv(USE_AWS_INSTANCE_HOSTNAME_PROPERTY); + if(StringUtils.isNotBlank(useAwsInstanceHostnameEnvVar)){ + this.useAwsInstanceHostname = Boolean.valueOf(useAwsInstanceHostnameEnvVar); + } + String enableCiVisibilityVar = System.getenv(ENABLE_CI_VISIBILITY_PROPERTY); if(StringUtils.isNotBlank(enableCiVisibilityVar)) { this.collectBuildTraces = Boolean.valueOf(enableCiVisibilityVar); @@ -746,6 +754,7 @@ public boolean configure(final StaplerRequest req, final JSONObject formData) th this.setRetryLogs(formData.getBoolean("retryLogs")); this.setRefreshDogstatsdClient(formData.getBoolean("refreshDogstatsdClient")); this.setCacheBuildRuns(formData.getBoolean("cacheBuildRuns")); + this.setUseAwsInstanceHostname(formData.getBoolean("useAwsInstanceHostname")); this.setEmitSystemEvents(formData.getBoolean("emitSystemEvents")); this.setEmitConfigChangeEvents(formData.getBoolean("emitConfigChangeEvents")); @@ -1267,6 +1276,24 @@ public void setCacheBuildRuns(boolean cacheBuildRuns) { this.cacheBuildRuns = cacheBuildRuns; } + /** + * @return - A {@link Boolean} indicating if the user has configured Datadog to use AWS instance as hostname + */ + public boolean isUseAwsInstanceHostname() { + return useAwsInstanceHostname; + } + + + /** + * Set the checkbox in the UI, used for Jenkins data binding + * + * @param useAwsInstanceHostname - The checkbox status (checked/unchecked) + */ + @DataBoundSetter + public void setUseAwsInstanceHostname(boolean useAwsInstanceHostname) { + this.useAwsInstanceHostname = useAwsInstanceHostname; + } + /** * @return - A {@link Boolean} indicating if the user has configured Datadog to emit System related events. */ diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java index 9317cea73..ee82d9aae 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java @@ -469,6 +469,40 @@ public static Map> computeTagListFromVarList(EnvVars envVars return result; } + public static String getAwsInstanceID() throws IOException { + String metadataUrl = "http://169.254.169.254/latest/meta-data/instance-id"; + HttpURLConnection conn = null; + String instance_id = null; + // Make request + conn = getHttpURLConnection(new URL(metadataUrl), 300); + conn.setRequestMethod("GET"); + + // Get response + BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8")); + StringBuilder result = new StringBuilder(); + String line; + while ((line = rd.readLine()) != null) { + result.append(line); + } + rd.close(); + + // Validate + instance_id = result.toString(); + try { + if (conn.getResponseCode() == 404) { + logger.fine("Could not retrieve AWS instance ID"); + } + conn.disconnect(); + } catch (IOException e) { + logger.info("Failed to inspect HTTP response when getting AWS Instance ID"); + } + + if (instance_id.equals("")) { + return null; + } + return instance_id; + } + /** * Getter function to return either the saved hostname global configuration, * or the hostname that is set in the Jenkins host itself. Returns null if no @@ -477,6 +511,8 @@ public static Map> computeTagListFromVarList(EnvVars envVars * Tries, in order: * Jenkins configuration * Jenkins hostname environment variable + * AWS instance ID, if enabled + * System hostname environment variable * Unix hostname via `/bin/hostname -f` * Localhost hostname * @@ -506,6 +542,27 @@ public static String getHostname(EnvVars envVars) { return hostname; } } + + final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor(); + if (datadogGlobalConfig != null){ + if (datadogGlobalConfig.isUseAwsInstanceHostname()) { + try { + hostname = getAwsInstanceID(); + } catch (IOException e) { + logger.fine("Error retrieving AWS hostname: " + e); + } + if (hostname != null) { + logger.fine("Using AWS instance ID as hostname. Hostname: " + hostname); + return hostname; + } + } + } + + if (isValidHostname(hostname)) { + logger.fine("Using hostname found in $HOSTNAME controller environment variable. Hostname: " + hostname); + return hostname; + } + hostname = System.getenv("HOSTNAME"); if (isValidHostname(hostname)) { logger.fine("Using hostname found in $HOSTNAME controller environment variable. Hostname: " + hostname); diff --git a/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly index ae07b04b7..e972a9fb9 100644 --- a/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly +++ b/src/main/resources/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration/config.jelly @@ -124,10 +124,16 @@ + + + + + + diff --git a/src/test/java/org/datadog/jenkins/plugins/datadog/DatadogUtilitiesTest.java b/src/test/java/org/datadog/jenkins/plugins/datadog/DatadogUtilitiesTest.java index c0c7c0684..4da146b94 100644 --- a/src/test/java/org/datadog/jenkins/plugins/datadog/DatadogUtilitiesTest.java +++ b/src/test/java/org/datadog/jenkins/plugins/datadog/DatadogUtilitiesTest.java @@ -26,7 +26,11 @@ of this software and associated documentation files (the "Software"), to deal package org.datadog.jenkins.plugins.datadog; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.when; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + import hudson.model.Result; import org.apache.commons.math3.exception.NullArgumentException; @@ -38,13 +42,29 @@ of this software and associated documentation files (the "Software"), to deal import org.jenkinsci.plugins.workflow.graph.BlockEndNode; import org.jenkinsci.plugins.workflow.graph.BlockStartNode; import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.jvnet.hudson.test.JenkinsRule; + import org.junit.Assert; import org.junit.Test; +import org.junit.Before; +import org.junit.ClassRule; import java.util.*; +import java.io.IOException; +import java.net.HttpURLConnection; public class DatadogUtilitiesTest { + public DatadogGlobalConfiguration cfg; + + @ClassRule + public static JenkinsRule jenkinsRule = new JenkinsRule(); + + @Before + public void setUpMocks() { + cfg = DatadogUtilities.getDatadogGlobalDescriptor(); + } + @Test public void testCstrToList(){ Assert.assertTrue(DatadogUtilities.cstrToList(null).isEmpty()); @@ -170,4 +190,27 @@ public void testToJsonMap() { Assert.assertEquals("{\"itemKey1\":\"itemValue1\",\"itemKey2\":\"itemValue2\",\"itemKey3\":\"itemValue3\"}", DatadogUtilities.toJson(multipleItems)); } + @Test + public void testGetHostname() throws IOException { + try (MockedStatic datadogUtilities = Mockito.mockStatic(DatadogUtilities.class)) { + datadogUtilities.when(() -> DatadogUtilities.getDatadogGlobalDescriptor()).thenReturn(cfg); + HttpURLConnection mockHTTP = mock(HttpURLConnection.class); + + datadogUtilities.when(() -> DatadogUtilities.getAwsInstanceID()).thenReturn("test"); + datadogUtilities.when(() -> DatadogUtilities.getHostname(null)).thenCallRealMethod(); + + String hostname = DatadogUtilities.getHostname(null); + Assert.assertNotEquals("test", hostname); + + cfg.setUseAwsInstanceHostname(true); + + hostname = DatadogUtilities.getHostname(null); + Assert.assertEquals("test", hostname); + + cfg.setUseAwsInstanceHostname(false); + hostname = DatadogUtilities.getHostname(null); + Assert.assertNotEquals("test", hostname); + } + } + } From c6fcd0ad321948a5dba5fcfe2e5b415ff07f897e Mon Sep 17 00:00:00 2001 From: dawitgirm <137215481+dawitgirm@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:30:56 -0400 Subject: [PATCH 02/16] changed upper bound on timeout to allow for small discrepancy (#348) --- .../plugins/datadog/listeners/DatadogGraphListenerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListenerTest.java b/src/test/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListenerTest.java index 58ba65005..581cb6227 100644 --- a/src/test/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListenerTest.java +++ b/src/test/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogGraphListenerTest.java @@ -186,7 +186,7 @@ public void testIntegration() throws Exception { // we test it's at least 10s. double pauseValue = clientStub.assertMetricGetValue("jenkins.job.stage_pause_duration", hostname, expectedTags); assertTrue(pauseValue > 10000); - assertTrue(pauseValue <= 11000); + assertTrue(pauseValue <= 11100); } else { clientStub.assertMetric("jenkins.job.stage_pause_duration", 0.0, hostname, expectedTags); } From 2f75987c628939b32e8d63f0d47dbbb1c9b285b7 Mon Sep 17 00:00:00 2001 From: Daniel Beck <1831569+daniel-beck@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:56:57 +0200 Subject: [PATCH 03/16] Merge 5.4.2 back into default branch (#350) * SECURITY-3130 * [maven-release-plugin] prepare release datadog-5.4.2 * [maven-release-plugin] prepare for next development iteration --------- Co-authored-by: Sarah Witt Co-authored-by: Yaroslav Afenkin --- pom.xml | 2 +- .../jenkins/plugins/datadog/DatadogGlobalConfiguration.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5952a426c..2e5d35358 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.4.2-SNAPSHOT + 5.4.3-SNAPSHOT hpi diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java index 7a4968997..0c5d9a0fd 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java @@ -414,7 +414,7 @@ public FormValidation doTestConnection( @QueryParameter("targetCredentialsApiKey") final String targetCredentialsApiKey, @QueryParameter("targetApiURL") final String targetApiURL) throws IOException, ServletException { - + Jenkins.getInstance().checkPermission(Jenkins.ADMINISTER); final Secret secret = findSecret(targetApiKey, targetCredentialsApiKey); if (DatadogHttpClient.validateDefaultIntakeConnection(targetApiURL, secret)) { return FormValidation.ok("Great! Your API key is valid."); From e74d7aa857efc8cde0bd4b32137e7e9d9176b87e Mon Sep 17 00:00:00 2001 From: Sarah Witt Date: Mon, 21 Aug 2023 10:05:17 -0400 Subject: [PATCH 04/16] Add changelog for 5.4.2 (#351) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8ee3214f..8449d17b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ Changes ======= + +## 5.4.2 / 2023-07-12 +### Details +https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.1...datadog-5.4.2 + +* [Fixed] Fix [CVE-2023-37944](https://www.jenkins.io/security/advisory/2023-07-12/#SECURITY-3130) and require Overall/Administer permission to access the affected HTTP endpoint. See [#350](https://github.com/jenkinsci/datadog-plugin/pull/350). + ## 5.4.1 / 2023-05-24 ### Details https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.0...datadog-5.4.1 From 804d34ca2e89dfd49b7c1224b296bb1fe49ed668 Mon Sep 17 00:00:00 2001 From: Sarah Witt Date: Tue, 22 Aug 2023 10:54:33 -0400 Subject: [PATCH 05/16] Add changelog for 5.5.0 release (#355) * Update changelog * Add details link --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8449d17b1..b0b5ca7e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ Changes ======= +## 5.5.0 / 2023-08-22 +### Details +https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.2...datadog-5.5.0 + +***Added***: + +* Add option to use AWS instance ID as hostname. See [#345](https://github.com/jenkinsci/datadog-plugin/pull/345). + +***Fixed***: + +* Fix error status propagation to take into account catch/catchError/warnError blocks. See [#343](https://github.com/jenkinsci/datadog-plugin/pull/343). +* Look up hostname from controller environment. See [#340](https://github.com/jenkinsci/datadog-plugin/pull/340). Thanks [Vlatombe](https://github.com/Vlatombe). + ## 5.4.2 / 2023-07-12 ### Details https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.1...datadog-5.4.2 From 94a5fc1c7f312acd58ec387e3460c906c3482eaf Mon Sep 17 00:00:00 2001 From: "ci.jenkins-plugin" Date: Wed, 23 Aug 2023 14:26:39 +0000 Subject: [PATCH 06/16] [maven-release-plugin] prepare release datadog-5.5.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2e5d35358..dbc6275ca 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.4.3-SNAPSHOT + 5.5.0 hpi @@ -46,7 +46,7 @@ scm:git:git://github.com/jenkinsci/datadog-plugin.git scm:git:git@github.com:jenkinsci/datadog-plugin.git https://github.com/jenkinsci/datadog-plugin - datadog-3.0.0 + datadog-5.5.0 From fa7d83ffe112bad357e95ee09f33472b03c88b6c Mon Sep 17 00:00:00 2001 From: "ci.jenkins-plugin" Date: Wed, 23 Aug 2023 14:26:40 +0000 Subject: [PATCH 07/16] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index dbc6275ca..391bab814 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.5.0 + 5.5.1-SNAPSHOT hpi @@ -46,7 +46,7 @@ scm:git:git://github.com/jenkinsci/datadog-plugin.git scm:git:git@github.com:jenkinsci/datadog-plugin.git https://github.com/jenkinsci/datadog-plugin - datadog-5.5.0 + datadog-3.0.0 From 7258f7e228dce3b0bb3f6d7c21556c5e6b7abdf4 Mon Sep 17 00:00:00 2001 From: Sarah Witt Date: Thu, 24 Aug 2023 11:22:22 -0400 Subject: [PATCH 08/16] Revert failed release commits (#357) * Revert "[maven-release-plugin] prepare for next development iteration" This reverts commit fa7d83ffe112bad357e95ee09f33472b03c88b6c. * Revert "[maven-release-plugin] prepare release datadog-5.5.0" This reverts commit 94a5fc1c7f312acd58ec387e3460c906c3482eaf. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 391bab814..2e5d35358 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.5.1-SNAPSHOT + 5.4.3-SNAPSHOT hpi From 068c3930582149e2b9e17409dee537aef0dfe6b7 Mon Sep 17 00:00:00 2001 From: "ci.jenkins-plugin" Date: Thu, 24 Aug 2023 19:35:23 +0000 Subject: [PATCH 09/16] [maven-release-plugin] prepare release datadog-5.5.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2e5d35358..dbc6275ca 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.4.3-SNAPSHOT + 5.5.0 hpi @@ -46,7 +46,7 @@ scm:git:git://github.com/jenkinsci/datadog-plugin.git scm:git:git@github.com:jenkinsci/datadog-plugin.git https://github.com/jenkinsci/datadog-plugin - datadog-3.0.0 + datadog-5.5.0 From e86a1417ba22e338fa49080d551778c98449fb39 Mon Sep 17 00:00:00 2001 From: "ci.jenkins-plugin" Date: Thu, 24 Aug 2023 19:35:25 +0000 Subject: [PATCH 10/16] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index dbc6275ca..391bab814 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.5.0 + 5.5.1-SNAPSHOT hpi @@ -46,7 +46,7 @@ scm:git:git://github.com/jenkinsci/datadog-plugin.git scm:git:git@github.com:jenkinsci/datadog-plugin.git https://github.com/jenkinsci/datadog-plugin - datadog-5.5.0 + datadog-3.0.0 From 0720f2bf12e45611013540c3c2a2b99adc2fe2c1 Mon Sep 17 00:00:00 2001 From: Sarah Witt Date: Fri, 25 Aug 2023 09:43:36 -0400 Subject: [PATCH 11/16] Revert failed release (#358) * Revert "[maven-release-plugin] prepare for next development iteration" This reverts commit e86a1417ba22e338fa49080d551778c98449fb39. * Revert "[maven-release-plugin] prepare release datadog-5.5.0" This reverts commit 068c3930582149e2b9e17409dee537aef0dfe6b7. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 391bab814..2e5d35358 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.5.1-SNAPSHOT + 5.4.3-SNAPSHOT hpi From 6120057cc1fd57c7e0e00a7665dff657b19edffc Mon Sep 17 00:00:00 2001 From: "ci.jenkins-plugin" Date: Fri, 25 Aug 2023 14:50:03 +0000 Subject: [PATCH 12/16] [maven-release-plugin] prepare release datadog-5.5.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2e5d35358..dbc6275ca 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.4.3-SNAPSHOT + 5.5.0 hpi @@ -46,7 +46,7 @@ scm:git:git://github.com/jenkinsci/datadog-plugin.git scm:git:git@github.com:jenkinsci/datadog-plugin.git https://github.com/jenkinsci/datadog-plugin - datadog-3.0.0 + datadog-5.5.0 From 78a150f1371a608e56e01e0edee112a9961ea90b Mon Sep 17 00:00:00 2001 From: "ci.jenkins-plugin" Date: Fri, 25 Aug 2023 14:50:06 +0000 Subject: [PATCH 13/16] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index dbc6275ca..391bab814 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ https://github.com/jenkinsci/datadog-plugin org.datadog.jenkins.plugins datadog - 5.5.0 + 5.5.1-SNAPSHOT hpi @@ -46,7 +46,7 @@ scm:git:git://github.com/jenkinsci/datadog-plugin.git scm:git:git@github.com:jenkinsci/datadog-plugin.git https://github.com/jenkinsci/datadog-plugin - datadog-5.5.0 + datadog-3.0.0 From 32f2ec76c44fbd8128376dc857dcca5f97ad450b Mon Sep 17 00:00:00 2001 From: Sarah Witt Date: Fri, 25 Aug 2023 11:27:59 -0400 Subject: [PATCH 14/16] update changelog (#359) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0b5ca7e5..a1ce57cb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Changes ======= -## 5.5.0 / 2023-08-22 +## 5.5.0 / 2023-08-25 ### Details https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.2...datadog-5.5.0 From 55103b97578fc94a22f6987196e1aeeaaeba798a Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Mon, 28 Aug 2023 10:42:20 +0200 Subject: [PATCH 15/16] When creating a BuildData instance, use Git values from BuildData populated during previous pipeline stages as fallback (#356) --- .../plugins/datadog/model/BuildData.java | 64 ++++++++++++++++--- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/model/BuildData.java b/src/main/java/org/datadog/jenkins/plugins/datadog/model/BuildData.java index 6a4709ba4..9acdc0388 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/model/BuildData.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/model/BuildData.java @@ -55,16 +55,6 @@ of this software and associated documentation files (the "Software"), to deal import hudson.triggers.SCMTrigger; import hudson.triggers.TimerTrigger; import hudson.util.LogTaskListener; -import net.sf.json.JSONObject; -import org.apache.commons.lang.StringUtils; -import org.datadog.jenkins.plugins.datadog.DatadogUtilities; -import org.datadog.jenkins.plugins.datadog.traces.BuildSpanManager; -import org.datadog.jenkins.plugins.datadog.traces.message.TraceSpan; -import org.datadog.jenkins.plugins.datadog.util.SuppressFBWarnings; -import org.datadog.jenkins.plugins.datadog.util.TagsUtil; -import org.datadog.jenkins.plugins.datadog.util.git.GitUtils; -import org.jenkinsci.plugins.gitclient.GitClient; - import java.io.IOException; import java.io.Serializable; import java.nio.charset.Charset; @@ -74,6 +64,16 @@ of this software and associated documentation files (the "Software"), to deal import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; +import org.datadog.jenkins.plugins.datadog.DatadogUtilities; +import org.datadog.jenkins.plugins.datadog.traces.BuildSpanAction; +import org.datadog.jenkins.plugins.datadog.traces.BuildSpanManager; +import org.datadog.jenkins.plugins.datadog.traces.message.TraceSpan; +import org.datadog.jenkins.plugins.datadog.util.SuppressFBWarnings; +import org.datadog.jenkins.plugins.datadog.util.TagsUtil; +import org.datadog.jenkins.plugins.datadog.util.git.GitUtils; +import org.jenkinsci.plugins.gitclient.GitClient; public class BuildData implements Serializable { @@ -229,6 +229,50 @@ public BuildData(Run run, TaskListener listener) throws IOException, Interrupted setTraceId(Long.toUnsignedString(buildSpan.context().getTraceId())); setSpanId(Long.toUnsignedString(buildSpan.context().getSpanId())); } + + BuildSpanAction buildSpanAction = run.getAction(BuildSpanAction.class); + if (buildSpanAction != null) { + getMissingGitValuesFrom(buildSpanAction.getBuildData()); + } + } + + private void getMissingGitValuesFrom(BuildData previousData) { + if (branch == null) { + branch = previousData.branch; + } + if (gitUrl == null) { + gitUrl = previousData.gitUrl; + } + if (gitCommit == null) { + gitCommit = previousData.gitCommit; + } + if (gitMessage == null) { + gitMessage = previousData.gitMessage; + } + if (gitAuthorName == null) { + gitAuthorName = previousData.gitAuthorName; + } + if (gitAuthorEmail == null) { + gitAuthorEmail = previousData.gitAuthorEmail; + } + if (gitAuthorDate == null) { + gitAuthorDate = previousData.gitAuthorDate; + } + if (gitCommitterName == null) { + gitCommitterName = previousData.gitCommitterName; + } + if (gitCommitterEmail == null) { + gitCommitterEmail = previousData.gitCommitterEmail; + } + if (gitCommitterDate == null) { + gitCommitterDate = previousData.gitCommitterDate; + } + if (gitDefaultBranch == null) { + gitDefaultBranch = previousData.gitDefaultBranch; + } + if (gitTag == null) { + gitTag = previousData.gitTag; + } } private void populateBuildParameters(Run run) { From 0c0fea3366045900edfbc36fa92a5309e3c1ac6b Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Mon, 28 Aug 2023 10:43:38 +0200 Subject: [PATCH 16/16] Move all HTTP calls to a dedicated class and use Jetty HTTP client instead of raw HttpUrlConnection (#346) --- pom.xml | 5 + .../datadog/DatadogGlobalConfiguration.java | 4 +- .../plugins/datadog/DatadogUtilities.java | 248 +++++------ .../clients/ConcurrentMetricCounters.java | 2 +- .../datadog/clients/DatadogAgentClient.java | 162 +++----- .../datadog/clients/DatadogHttpClient.java | 386 ++++++------------ .../plugins/datadog/clients/HttpClient.java | 333 +++++++++++++++ .../datadog/clients/HttpRetryPolicy.java | 134 ++++++ .../datadog/transport/HttpMessage.java | 9 + .../plugins/datadog/transport/HttpSender.java | 70 +--- .../transport/NonBlockingHttpClient.java | 3 +- .../datadog/clients/DatadogClientTest.java | 6 +- .../datadog/transport/FakeHttpSender.java | 2 +- 13 files changed, 790 insertions(+), 574 deletions(-) create mode 100644 src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpClient.java create mode 100644 src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpRetryPolicy.java diff --git a/pom.xml b/pom.xml index 391bab814..37ae20295 100644 --- a/pom.xml +++ b/pom.xml @@ -162,6 +162,11 @@ 2.13.3-285.vc03c0256d517 test + + org.eclipse.jetty + jetty-client + 9.4.51.v20230217 + diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java index 0c5d9a0fd..344c95c2f 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java @@ -40,9 +40,9 @@ of this software and associated documentation files (the "Software"), to deal import com.cloudbees.plugins.credentials.CredentialsMatchers; import com.cloudbees.plugins.credentials.CredentialsProvider; -import com.cloudbees.plugins.credentials.Credentials; import com.cloudbees.plugins.credentials.common.StandardListBoxModel; import com.cloudbees.plugins.credentials.domains.URIRequirementBuilder; +import org.datadog.jenkins.plugins.datadog.clients.HttpClient; import org.jenkinsci.plugins.plaincredentials.StringCredentials; import net.sf.json.JSONObject; @@ -416,7 +416,7 @@ public FormValidation doTestConnection( throws IOException, ServletException { Jenkins.getInstance().checkPermission(Jenkins.ADMINISTER); final Secret secret = findSecret(targetApiKey, targetCredentialsApiKey); - if (DatadogHttpClient.validateDefaultIntakeConnection(targetApiURL, secret)) { + if (DatadogHttpClient.validateDefaultIntakeConnection(new HttpClient(60_000), targetApiURL, secret)) { return FormValidation.ok("Great! Your API key is valid."); } else { return FormValidation.error("Hmmm, your API key seems to be invalid."); diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java index ee82d9aae..ee04d1961 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java @@ -27,7 +27,6 @@ of this software and associated documentation files (the "Software"), to deal import hudson.EnvVars; import hudson.ExtensionList; -import hudson.ProxyConfiguration; import hudson.XmlFile; import hudson.model.Computer; import hudson.model.Item; @@ -36,6 +35,34 @@ of this software and associated documentation files (the "Software"), to deal import hudson.model.User; import hudson.model.labels.LabelAtom; import hudson.util.LogTaskListener; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.Inet4Address; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TimeZone; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nonnull; import jenkins.model.Jenkins; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; @@ -68,37 +95,6 @@ of this software and associated documentation files (the "Software"), to deal import org.jenkinsci.plugins.workflow.graph.FlowEndNode; import org.jenkinsci.plugins.workflow.graph.FlowNode; -import javax.annotation.Nonnull; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.net.HttpURLConnection; -import java.net.Inet4Address; -import java.net.MalformedURLException; -import java.net.Proxy; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.charset.Charset; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.TimeZone; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - public class DatadogUtilities { private static final Logger logger = Logger.getLogger(DatadogUtilities.class.getName()); @@ -125,7 +121,7 @@ public static DatadogGlobalConfiguration getDatadogGlobalDescriptor() { public static DatadogJobProperty getDatadogJobProperties(@Nonnull Run r) { try { return (DatadogJobProperty) r.getParent().getProperty(DatadogJobProperty.class); - } catch(NullPointerException e){ + } catch (NullPointerException e) { // It can only throw a NullPointerException when running tests return null; } @@ -134,37 +130,37 @@ public static DatadogJobProperty getDatadogJobProperties(@Nonnull Run r) { /** * Builds extraTags if any are configured in the Job. * - * @param run - Current build - * @param envVars - Environment Variables + * @param run - Current build + * @param envVars - Environment Variables * @return A {@link HashMap} containing the key,value pairs of tags if any. */ public static Map> getBuildTags(Run run, EnvVars envVars) { Map> result = new HashMap<>(); - if(run == null){ + if (run == null) { return result; } String jobName; try { jobName = run.getParent().getFullName(); - } catch (NullPointerException e){ + } catch (NullPointerException e) { // It can only throw a NullPointerException when running tests return result; } final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor(); - if (datadogGlobalConfig == null){ + if (datadogGlobalConfig == null) { return result; } final String globalJobTags = datadogGlobalConfig.getGlobalJobTags(); String workspaceTagFile = null; String tagProperties = null; final DatadogJobProperty property = DatadogUtilities.getDatadogJobProperties(run); - if(property != null){ + if (property != null) { workspaceTagFile = property.readTagFile(run); tagProperties = property.getTagProperties(); } // If job doesn't have a workspace Tag File set we check if one has been defined globally - if(workspaceTagFile == null){ + if (workspaceTagFile == null) { workspaceTagFile = datadogGlobalConfig.getGlobalTagFile(); } if (workspaceTagFile != null) { @@ -182,24 +178,24 @@ public static Map> getBuildTags(Run run, EnvVars envVars) { /** * Pipeline extraTags if any are configured in the Job from DatadogPipelineAction. * - * @param run - Current build + * @param run - Current build * @return A {@link HashMap} containing the key,value pairs of tags if any. */ public static Map> getTagsFromPipelineAction(Run run) { // pipeline defined tags final Map> result = new HashMap<>(); DatadogPipelineAction action = run.getAction(DatadogPipelineAction.class); - if(action != null) { + if (action != null) { List pipelineTags = action.getTags(); for (int i = 0; i < pipelineTags.size(); i++) { String[] tagItem = pipelineTags.get(i).replaceAll(" ", "").split(":", 2); - if(tagItem.length == 2) { + if (tagItem.length == 2) { String tagName = tagItem[0]; String tagValue = tagItem[1]; Set tagValues = result.containsKey(tagName) ? result.get(tagName) : new HashSet(); tagValues.add(tagValue.toLowerCase()); result.put(tagName, tagValues); - } else if(tagItem.length == 1) { + } else if (tagItem.length == 1) { String tagName = tagItem[0]; Set tagValues = result.containsKey(tagName) ? result.get(tagName) : new HashSet(); tagValues.add(""); // no values @@ -237,7 +233,7 @@ private static String getOS() { /** * Retrieve the list of tags from the globalJobTagsLines param for jobName * - * @param jobName - JobName to retrieve and process tags from. + * @param jobName - JobName to retrieve and process tags from. * @param globalJobTags - globalJobTags string * @return - A Map of values containing the key and values of each Datadog tag to apply to the metric/event */ @@ -270,20 +266,19 @@ private static Map> getTagsFromGlobalJobTags(String jobName, } catch (IndexOutOfBoundsException e) { String tagNameEnvVar = tagValue.substring(1); - if (EnvVars.masterEnvVars.containsKey(tagNameEnvVar)){ + if (EnvVars.masterEnvVars.containsKey(tagNameEnvVar)) { tagValue = EnvVars.masterEnvVars.get(tagNameEnvVar); - } - else { + } else { logger.fine(String.format( - "Specified a capture group or environment variable that doesn't exist, not applying tag: %s Exception: %s", - Arrays.toString(tagItem), e)); + "Specified a capture group or environment variable that doesn't exist, not applying tag: %s Exception: %s", + Arrays.toString(tagItem), e)); } } } Set tagValues = tags.containsKey(tagName) ? tags.get(tagName) : new HashSet(); tagValues.add(tagValue.toLowerCase()); tags.put(tagName, tagValues); - } else if(tagItem.length == 1) { + } else if (tagItem.length == 1) { String tagName = tagItem[0]; Set tagValues = tags.containsKey(tagName) ? tags.get(tagName) : new HashSet(); tagValues.add(""); // no values @@ -308,7 +303,7 @@ public static Map> getTagsFromGlobalTags() { Map> tags = new HashMap<>(); final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor(); - if (datadogGlobalConfig == null){ + if (datadogGlobalConfig == null) { return tags; } @@ -323,22 +318,21 @@ public static Map> getTagsFromGlobalTags() { for (int i = 0; i < tagList.size(); i++) { String[] tagItem = tagList.get(i).replaceAll(" ", "").split(":", 2); - if(tagItem.length == 2) { + if (tagItem.length == 2) { String tagName = tagItem[0]; String tagValue = tagItem[1]; Set tagValues = tags.containsKey(tagName) ? tags.get(tagName) : new HashSet(); // Apply environment variables if specified. ie (custom_tag:$ENV_VAR) - if (tagValue.startsWith("$") && EnvVars.masterEnvVars.containsKey(tagValue.substring(1))){ + if (tagValue.startsWith("$") && EnvVars.masterEnvVars.containsKey(tagValue.substring(1))) { tagValue = EnvVars.masterEnvVars.get(tagValue.substring(1)); - } - else { + } else { logger.fine(String.format( - "Specified an environment variable that doesn't exist, not applying tag: %s", - Arrays.toString(tagItem))); + "Specified an environment variable that doesn't exist, not applying tag: %s", + Arrays.toString(tagItem))); } tagValues.add(tagValue.toLowerCase()); tags.put(tagName, tagValues); - } else if(tagItem.length == 1) { + } else if (tagItem.length == 1) { String tagName = tagItem[0]; Set tagValues = tags.containsKey(tagName) ? tags.get(tagName) : new HashSet(); tagValues.add(""); // no values @@ -360,12 +354,12 @@ public static Map> getTagsFromGlobalTags() { */ private static boolean isJobExcluded(final String jobName) { final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor(); - if (datadogGlobalConfig == null){ + if (datadogGlobalConfig == null) { return false; } final String excludedProp = datadogGlobalConfig.getExcluded(); List excluded = cstrToList(excludedProp); - for (String excludedJob : excluded){ + for (String excludedJob : excluded) { Pattern excludedJobPattern = Pattern.compile(excludedJob); Matcher jobNameMatcher = excludedJobPattern.matcher(jobName); if (jobNameMatcher.matches()) { @@ -384,12 +378,12 @@ private static boolean isJobExcluded(final String jobName) { */ private static boolean isJobIncluded(final String jobName) { final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor(); - if (datadogGlobalConfig == null){ + if (datadogGlobalConfig == null) { return true; } final String includedProp = datadogGlobalConfig.getIncluded(); final List included = cstrToList(includedProp); - for (String includedJob : included){ + for (String includedJob : included) { Pattern includedJobPattern = Pattern.compile(includedJob); Matcher jobNameMatcher = includedJobPattern.matcher(jobName); if (jobNameMatcher.matches()) { @@ -422,7 +416,7 @@ public static List linesToList(final String str) { /** * Converts a string List into a List Object * - * @param str - A String containing a comma separated list of items. + * @param str - A String containing a comma separated list of items. * @param regex - Regex to use to split the string list * @return a String List with all items */ @@ -456,7 +450,7 @@ public static Map> computeTagListFromVarList(EnvVars envVars values.add(value); result.put(name, values); logger.fine(String.format("Emitted tag %s:%s", name, value)); - } else if(expanded.length == 1) { + } else if (expanded.length == 1) { String name = expanded[0]; Set values = result.containsKey(name) ? result.get(name) : new HashSet(); values.add(""); // no values @@ -507,7 +501,7 @@ public static String getAwsInstanceID() throws IOException { * Getter function to return either the saved hostname global configuration, * or the hostname that is set in the Jenkins host itself. Returns null if no * valid hostname is found. - * + *

* Tries, in order: * Jenkins configuration * Jenkins hostname environment variable @@ -526,7 +520,7 @@ public static String getHostname(EnvVars envVars) { String hostname = null; try { hostname = getDatadogGlobalDescriptor().getHostname(); - } catch (NullPointerException e){ + } catch (NullPointerException e) { // noop } if (isValidHostname(hostname)) { @@ -624,7 +618,7 @@ public static String getHostname(EnvVars envVars) { * Fetches the environment variables from the worker and returns the value * of DD_CI_HOSTNAME if set. * - * @param run - Current build + * @param run - Current build * @return the specified hostname or an empty Optional if not set */ public static Optional getHostnameFromWorkerEnv(Run run) { @@ -634,7 +628,8 @@ public static Optional getHostnameFromWorkerEnv(Run run) { if (StringUtils.isNotEmpty(envHostname)) { return Optional.of(envHostname); } - } catch (IOException | InterruptedException e) { } + } catch (IOException | InterruptedException e) { + } return Optional.empty(); } @@ -683,7 +678,7 @@ public static Map> getComputerTags(Computer computer) { Set labels = null; try { labels = computer.getNode().getAssignedLabels(); - } catch (NullPointerException e){ + } catch (NullPointerException e) { logger.fine("Could not retrieve labels"); } String nodeHostname = null; @@ -697,14 +692,14 @@ public static Map> getComputerTags(Computer computer) { Set nodeNameValues = new HashSet<>(); nodeNameValues.add(nodeName); result.put("node_name", nodeNameValues); - if(nodeHostname != null){ + if (nodeHostname != null) { Set nodeHostnameValues = new HashSet<>(); nodeHostnameValues.add(nodeHostname); result.put("node_hostname", nodeHostnameValues); } - if(labels != null){ + if (labels != null) { Set nodeLabelsValues = new HashSet<>(); - for (LabelAtom label: labels){ + for (LabelAtom label : labels) { nodeLabelsValues.add(label.getName()); } result.put("node_label", nodeLabelsValues); @@ -713,8 +708,8 @@ public static Map> getComputerTags(Computer computer) { return result; } - public static String getNodeName(Computer computer){ - if(computer == null){ + public static String getNodeName(Computer computer) { + if (computer == null) { return null; } if (computer instanceof Jenkins.MasterComputer) { @@ -724,7 +719,7 @@ public static String getNodeName(Computer computer){ } } - public static boolean isMainNode(String nodeName){ + public static boolean isMainNode(String nodeName) { return "master".equalsIgnoreCase(nodeName) || "built-in".equalsIgnoreCase(nodeName); } @@ -734,13 +729,13 @@ public static Set getNodeLabels(Computer computer) { Set labels; try { labels = computer.getNode().getAssignedLabels(); - } catch (Exception e){ + } catch (Exception e) { logger.fine("Could not retrieve labels: " + e.getMessage()); return Collections.emptySet(); } final Set labelsStr = new HashSet<>(); - for(final LabelAtom label : labels) { + for (final LabelAtom label : labels) { labelsStr.add(label.getName()); } @@ -768,13 +763,13 @@ public static Long getRunStartTimeInMillis(Run run) { return run.getStartTimeInMillis(); } - public static long currentTimeMillis(){ + public static long currentTimeMillis() { // This method exist so we can mock System.currentTimeMillis in unit tests return System.currentTimeMillis(); } public static String getFileName(XmlFile file) { - if(file == null || file.getFile() == null || file.getFile().getName().isEmpty()){ + if (file == null || file.getFile() == null || file.getFile().getName().isEmpty()) { return "unknown"; } else { return file.getFile().getName(); @@ -783,12 +778,12 @@ public static String getFileName(XmlFile file) { public static String getJenkinsUrl() { Jenkins jenkins = Jenkins.getInstance(); - if(jenkins == null){ + if (jenkins == null) { return "unknown"; - }else{ + } else { try { return jenkins.getRootUrl(); - }catch(Exception e){ + } catch (Exception e) { return "unknown"; } } @@ -796,12 +791,12 @@ public static String getJenkinsUrl() { public static String getResultTag(@Nonnull FlowNode node) { if (StageStatus.isSkippedStage(node)) { - return "SKIPPED"; + return "SKIPPED"; } if (node instanceof BlockEndNode) { BlockStartNode startNode = ((BlockEndNode) node).getStartNode(); if (StageStatus.isSkippedStage(startNode)) { - return "SKIPPED"; + return "SKIPPED"; } } ErrorAction error = node.getError(); @@ -827,6 +822,7 @@ public static String getResultTag(@Nonnull FlowNode node) { /** * Returns true if a {@code FlowNode} is a Stage node. + * * @param flowNode the flow node to evaluate * @return flag indicating if a flowNode is a Stage node. */ @@ -848,6 +844,7 @@ public static boolean isStageNode(BlockStartNode flowNode) { /** * Returns true if a {@code FlowNode} is a Pipeline node. + * * @param flowNode the flow node to evaluate * @return flag indicating if a flowNode is a Pipeline node. */ @@ -857,6 +854,7 @@ public static boolean isPipelineNode(FlowNode flowNode) { /** * Returns a normalized result for traces. + * * @param result (success, failure, error, aborted, not_build, canceled, skipped, unknown) * @return the normalized result for the traces based on the jenkins result */ @@ -875,14 +873,14 @@ public static String statusFromResult(String result) { } @SuppressFBWarnings("NP_NULL_ON_SOME_PATH") - public static void severe(Logger logger, Throwable e, String message){ - if(message == null){ - message = e != null ? "An unexpected error occurred": ""; + public static void severe(Logger logger, Throwable e, String message) { + if (message == null) { + message = e != null ? "An unexpected error occurred" : ""; } - if(!message.isEmpty()) { + if (!message.isEmpty()) { logger.severe(message); } - if(e != null) { + if (e != null) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); logger.info(message + ": " + sw.toString()); @@ -896,11 +894,12 @@ public static int toInt(boolean b) { /** * Returns a date as String in the ISO8601 format + * * @param date the date object to transform * @return date as String in the ISO8601 format */ public static String toISO8601(Date date) { - if(date == null) { + if (date == null) { return null; } @@ -911,11 +910,12 @@ public static String toISO8601(Date date) { /** * Returns a JSON array string based on the set. + * * @param set the set to transform into a JSON * @return json array string */ public static String toJson(final Set set) { - if(set == null || set.isEmpty()) { + if (set == null || set.isEmpty()) { return null; } @@ -924,10 +924,10 @@ public static String toJson(final Set set) { final StringBuilder sb = new StringBuilder(); sb.append("["); int index = 1; - for(String val : set) { + for (String val : set) { final String escapedValue = StringEscapeUtils.escapeJavaScript(val); sb.append("\"").append(escapedValue).append("\""); - if(index < set.size()) { + if (index < set.size()) { sb.append(","); } index += 1; @@ -939,11 +939,12 @@ public static String toJson(final Set set) { /** * Returns a JSON object string based on the map. + * * @param map the map to transform into a JSON * @return json object string */ public static String toJson(final Map map) { - if(map == null || map.isEmpty()) { + if (map == null || map.isEmpty()) { return null; } @@ -952,11 +953,11 @@ public static String toJson(final Map map) { final StringBuilder sb = new StringBuilder(); sb.append("{"); int index = 1; - for(Map.Entry entry : map.entrySet()) { + for (Map.Entry entry : map.entrySet()) { final String escapedKey = StringEscapeUtils.escapeJavaScript(entry.getKey()); final String escapedValue = StringEscapeUtils.escapeJavaScript(entry.getValue()); sb.append(String.format("\"%s\":\"%s\"", escapedKey, escapedValue)); - if(index < map.size()) { + if (index < map.size()) { sb.append(","); } index += 1; @@ -968,10 +969,11 @@ public static String toJson(final Map map) { /** * Removes all actions related to traces for Jenkins pipelines. + * * @param run the current run. */ public static void cleanUpTraceActions(final Run run) { - if(run != null) { + if (run != null) { run.removeActions(BuildSpanAction.class); run.removeActions(StepDataAction.class); run.removeActions(CIGlobalTagsAction.class); @@ -988,6 +990,7 @@ public static void cleanUpTraceActions(final Run run) { /** * Check if a run is from a Jenkins pipeline. * This action is added if the run is based on FlowNodes. + * * @param run the current run. * @return true if is a Jenkins pipeline. */ @@ -995,56 +998,17 @@ public static boolean isPipeline(final Run run) { return run != null && run.getAction(IsPipelineAction.class) != null; } - /** - * Returns an HTTP url connection given a url object. Supports jenkins configured proxy. - * - * @param url - a URL object containing the URL to open a connection to. - * @param timeoutMS - the timeout in MS - * @return a HttpURLConnection object. - * @throws IOException if HttpURLConnection fails to open connection - */ - public static HttpURLConnection getHttpURLConnection(final URL url, final int timeoutMS) throws IOException { - HttpURLConnection conn = null; - ProxyConfiguration proxyConfig = null; - - Jenkins jenkins = Jenkins.getInstance(); - if(jenkins != null){ - proxyConfig = jenkins.proxy; - } - - /* Attempt to use proxy */ - if (proxyConfig != null) { - Proxy proxy = proxyConfig.createProxy(url.getHost()); - if (proxy != null && proxy.type() == Proxy.Type.HTTP) { - logger.fine("Attempting to use the Jenkins proxy configuration"); - conn = (HttpURLConnection) url.openConnection(proxy); - } - } else { - logger.fine("Jenkins proxy configuration not found"); - } - - /* If proxy fails, use HttpURLConnection */ - if (conn == null) { - conn = (HttpURLConnection) url.openConnection(); - logger.fine("Using HttpURLConnection, without proxy"); - } - - conn.setConnectTimeout(timeoutMS); - conn.setReadTimeout(timeoutMS); - - return conn; - } - /** * Returns an HTTP URL + * * @param hostname - the Hostname - * @param port - the port to use - * @param path - the path + * @param port - the port to use + * @param path - the path * @return the HTTP URL * @throws MalformedURLException if the URL is not in a valid format */ public static URL buildHttpURL(final String hostname, final Integer port, final String path) throws MalformedURLException { - return new URL(String.format("http://%s:%d"+path, hostname, port)); + return new URL(String.format("http://%s:%d" + path, hostname, port)); } public static String getCatchErrorResult(BlockStartNode startNode) { @@ -1064,4 +1028,4 @@ public static String getCatchErrorResult(BlockStartNode startNode) { } return null; } -} +} \ No newline at end of file diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/ConcurrentMetricCounters.java b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/ConcurrentMetricCounters.java index 549f1f628..8d8fd05a2 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/ConcurrentMetricCounters.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/ConcurrentMetricCounters.java @@ -36,7 +36,7 @@ of this software and associated documentation files (the "Software"), to deal public class ConcurrentMetricCounters { private static final Logger logger = Logger.getLogger(ConcurrentMetricCounters.class.getName()); - private static ConcurrentMetricCounters instance; + private static volatile ConcurrentMetricCounters instance; private static ConcurrentMap counters = new ConcurrentHashMap<>(); private ConcurrentMetricCounters(){} diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogAgentClient.java b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogAgentClient.java index e98166810..0d86cb6e3 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogAgentClient.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogAgentClient.java @@ -26,7 +26,6 @@ of this software and associated documentation files (the "Software"), to deal package org.datadog.jenkins.plugins.datadog.clients; import static org.datadog.jenkins.plugins.datadog.DatadogUtilities.buildHttpURL; -import static org.datadog.jenkins.plugins.datadog.DatadogUtilities.getHttpURLConnection; import static org.datadog.jenkins.plugins.datadog.transport.LoggerHttpErrorHandler.LOGGER_HTTP_ERROR_HANDLER; import com.timgroup.statsd.Event; @@ -35,9 +34,25 @@ of this software and associated documentation files (the "Software"), to deal import com.timgroup.statsd.StatsDClient; import hudson.model.Run; import hudson.util.Secret; +import java.io.IOException; +import java.net.ConnectException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.logging.Handler; +import java.util.logging.Logger; +import java.util.logging.SocketHandler; import org.apache.commons.lang.StringUtils; import org.datadog.jenkins.plugins.datadog.DatadogClient; import org.datadog.jenkins.plugins.datadog.DatadogEvent; +import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration; import org.datadog.jenkins.plugins.datadog.DatadogUtilities; import org.datadog.jenkins.plugins.datadog.model.BuildData; import org.datadog.jenkins.plugins.datadog.traces.DatadogBaseBuildLogic; @@ -47,7 +62,6 @@ of this software and associated documentation files (the "Software"), to deal import org.datadog.jenkins.plugins.datadog.traces.DatadogWebhookBuildLogic; import org.datadog.jenkins.plugins.datadog.traces.DatadogWebhookPipelineLogic; import org.datadog.jenkins.plugins.datadog.traces.mapper.JsonTraceSpanMapper; -import org.datadog.jenkins.plugins.datadog.transport.HttpClient; import org.datadog.jenkins.plugins.datadog.transport.HttpMessage; import org.datadog.jenkins.plugins.datadog.transport.HttpMessageFactory; import org.datadog.jenkins.plugins.datadog.transport.NonBlockingHttpClient; @@ -55,47 +69,27 @@ of this software and associated documentation files (the "Software"), to deal import org.datadog.jenkins.plugins.datadog.util.SuppressFBWarnings; import org.datadog.jenkins.plugins.datadog.util.TagsUtil; import org.jenkinsci.plugins.workflow.graph.FlowNode; -import org.jfree.util.Log; import org.json.JSONArray; import org.json.JSONObject; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.ConnectException; -import java.net.HttpURLConnection; -import java.net.InetAddress; -import java.net.Socket; -import java.net.URL; -import java.net.UnknownHostException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.logging.Handler; -import java.util.logging.Logger; -import java.util.logging.SocketHandler; - /** * This class is used to collect all methods that has to do with transmitting * data to Datadog. */ public class DatadogAgentClient implements DatadogClient { - private static DatadogAgentClient instance = null; + private static volatile DatadogAgentClient instance = null; // Used to determine if the instance failed last validation last time, so // we do not keep retrying to create the instance and logging the same error private static boolean failedLastValidation = false; private static final Logger logger = Logger.getLogger(DatadogAgentClient.class.getName()); - private static final Integer BAD_REQUEST = 400; - private static final Integer NOT_FOUND = 404; @SuppressFBWarnings(value="MS_SHOULD_BE_FINAL") public static boolean enableValidations = true; - private HttpClient agentHttpClient; + private org.datadog.jenkins.plugins.datadog.transport.HttpClient agentHttpClient; private DatadogBaseBuildLogic traceBuildLogic; private DatadogBasePipelineLogic tracePipelineLogic; @@ -113,6 +107,8 @@ public class DatadogAgentClient implements DatadogClient { private boolean evpProxySupported = false; private long lastEvpProxyCheckTimeMs = 0L; + private final HttpClient client; + /** * How often to check the /info endpoint in case the Agent got updated. */ @@ -178,6 +174,7 @@ protected DatadogAgentClient(String hostname, Integer port, Integer logCollectio this.port = port; this.logCollectionPort = logCollectionPort; this.traceCollectionPort = traceCollectionPort; + this.client = new HttpClient(HTTP_TIMEOUT_EVP_PROXY_MS); } public static ConnectivityResult checkConnectivity(final String host, final int port) { @@ -327,48 +324,26 @@ private boolean reinitializeLogger(boolean force) { public Set fetchAgentSupportedEndpoints() { logger.fine("Fetching Agent info"); - HashSet endpoints = new HashSet<>(); - - HttpURLConnection conn = null; + String url = String.format("http://%s:%d/info", hostname, traceCollectionPort); try { - logger.fine("Setting up HttpURLConnection..."); - final URL traceAgentUrl = buildHttpURL(this.hostname, this.traceCollectionPort, "/info"); - conn = getHttpURLConnection(traceAgentUrl, HTTP_TIMEOUT_INFO_MS); - conn.setRequestProperty("Content-Type", "application/json"); - conn.setUseCaches(false); - - // Get response - BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8")); - StringBuilder result = new StringBuilder(); - String line; - while ((line = rd.readLine()) != null) { - result.append(line); - } - rd.close(); + return client.get(url, Collections.emptyMap(), s -> { + JSONObject jsonResponse = new JSONObject(s); + JSONArray jsonEndpoints = jsonResponse.getJSONArray("endpoints"); - JSONObject jsonResponse = new JSONObject(result.toString()); - JSONArray jsonEndpoints = jsonResponse.getJSONArray("endpoints"); - - // Iterate jsonArray using for loop - for (int i = 0; i < jsonEndpoints.length(); i++) { - endpoints.add(jsonEndpoints.getString(i)); - } - } catch (Exception e) { - try { - if (conn != null && conn.getResponseCode() == NOT_FOUND) { - logger.info("Agent /info returned 404. Requires Agent v6.27+ or v7.27+."); - } else { - DatadogUtilities.severe(logger, e, "Unknown client error, please check your config"); + Set endpoints = new HashSet<>(); + for (int i = 0; i < jsonEndpoints.length(); i++) { + endpoints.add(jsonEndpoints.getString(i)); } - } catch (IOException ex) { - DatadogUtilities.severe(logger, ex, "Failed to inspect HTTP response"); - } - } finally { - if (conn != null) { - conn.disconnect(); - } + return endpoints; + }); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + DatadogUtilities.severe(logger, e, "Could not get the list of agent endpoints"); + return Collections.emptySet(); + } catch (Exception e) { + DatadogUtilities.severe(logger, e, "Could not get the list of agent endpoints"); + return Collections.emptySet(); } - return endpoints; } /** @@ -387,60 +362,23 @@ public boolean postWebhook(String payload) { return false; } - HttpURLConnection conn = null; + DatadogGlobalConfiguration datadogGlobalDescriptor = DatadogUtilities.getDatadogGlobalDescriptor(); + String urlParameters = datadogGlobalDescriptor != null ? "?service=" + datadogGlobalDescriptor.getCiInstanceName() : ""; + String url = String.format("http://%s:%d/evp_proxy/v1/api/v2/webhook/%s", hostname, traceCollectionPort, urlParameters); + + Map headers = new HashMap<>(); + headers.put("X-Datadog-EVP-Subdomain", "webhook-intake"); + headers.put("DD-CI-PROVIDER-NAME", "jenkins"); + + byte[] body = payload.getBytes(StandardCharsets.UTF_8); + try { - logger.fine("Setting up HttpURLConnection..."); - String urlParameters = "?service=" + DatadogUtilities.getDatadogGlobalDescriptor().getCiInstanceName(); - final URL traceAgentUrl = buildHttpURL(this.hostname, this.traceCollectionPort, "/evp_proxy/v1/api/v2/webhook/"+urlParameters); - conn = getHttpURLConnection(traceAgentUrl, HTTP_TIMEOUT_EVP_PROXY_MS); - conn.setRequestMethod("POST"); - conn.setRequestProperty("Content-Type", "application/json"); - conn.setRequestProperty("X-Datadog-EVP-Subdomain", "webhook-intake"); - conn.setRequestProperty("DD-CI-PROVIDER-NAME", "jenkins"); - conn.setUseCaches(false); - conn.setDoInput(true); - conn.setDoOutput(true); - - OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream(), "utf-8"); - logger.fine("Writing to OutputStreamWriter..."); - wr.write(payload); - wr.close(); - - // Get response - BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8")); - StringBuilder result = new StringBuilder(); - String line; - while ((line = rd.readLine()) != null) { - result.append(line); - } - rd.close(); - - if ("{}".equals(result.toString())) { - logger.fine(String.format("Webhook API call through Agent proxy was sent successfully!")); - logger.fine(String.format("Payload: %s", payload)); - } else { - logger.severe(String.format("Webhook API call through Agent proxy failed!")); - logger.fine(String.format("Payload: %s", payload)); - logger.fine(String.format("Response: %s", result)); - return false; - } + client.postAsynchronously(url, headers, "application/json", body); + return true; } catch (Exception e) { - try { - if (conn != null && conn.getResponseCode() == BAD_REQUEST) { - logger.severe("We received a 400 from the Agent EVP Proxy."); - } else { - DatadogUtilities.severe(logger, e, "Unknown client error, please check your config"); - } - } catch (IOException ex) { - DatadogUtilities.severe(logger, ex, "Failed to inspect HTTP response"); - } + DatadogUtilities.severe(logger, e, "Error while posting webhook"); return false; - } finally { - if (conn != null) { - conn.disconnect(); - } } - return true; } /** diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogHttpClient.java b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogHttpClient.java index 9789883be..11a73eb5c 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogHttpClient.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogHttpClient.java @@ -25,17 +25,25 @@ of this software and associated documentation files (the "Software"), to deal package org.datadog.jenkins.plugins.datadog.clients; -import static org.datadog.jenkins.plugins.datadog.DatadogUtilities.getHttpURLConnection; - import hudson.model.Run; import hudson.util.Secret; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Logger; import jenkins.model.Jenkins; +import net.sf.json.JSON; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import net.sf.json.JSONSerializer; import org.apache.commons.lang.StringUtils; import org.datadog.jenkins.plugins.datadog.DatadogClient; import org.datadog.jenkins.plugins.datadog.DatadogEvent; +import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration; import org.datadog.jenkins.plugins.datadog.DatadogUtilities; import org.datadog.jenkins.plugins.datadog.model.BuildData; import org.datadog.jenkins.plugins.datadog.traces.DatadogWebhookBuildLogic; @@ -44,25 +52,13 @@ of this software and associated documentation files (the "Software"), to deal import org.datadog.jenkins.plugins.datadog.util.TagsUtil; import org.jenkinsci.plugins.workflow.graph.FlowNode; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.logging.Logger; - /** * This class is used to collect all methods that has to do with transmitting * data to Datadog. */ public class DatadogHttpClient implements DatadogClient { - private static DatadogHttpClient instance = null; + private static volatile DatadogHttpClient instance = null; // Used to determine if the instance failed last validation last time, so // we do not keep retrying to create the instance and logging the same error private static boolean failedLastValidation = false; @@ -74,8 +70,6 @@ public class DatadogHttpClient implements DatadogClient { private static final String SERVICECHECK = "v1/check_run"; private static final String VALIDATE = "v1/validate"; - private static final Integer HTTP_FORBIDDEN = 403; - private static final Integer BAD_REQUEST = 400; /* Timeout of 1 minutes for connecting and reading. * this prevents this plugin from causing jobs to hang in case of @@ -100,6 +94,8 @@ public class DatadogHttpClient implements DatadogClient { private DatadogWebhookBuildLogic webhookBuildLogic; private DatadogWebhookPipelineLogic webhookPipelineLogic; + private final HttpClient httpClient; + /** * NOTE: Use ClientFactory.getClient method to instantiate the client in the Jenkins Plugin * This method is not recommended to be used because it misses some validations. @@ -143,6 +139,7 @@ private DatadogHttpClient(String url, String logIntakeUrl, String webhookIntakeU this.webhookIntakeUrl = webhookIntakeUrl; this.webhookBuildLogic = new DatadogWebhookBuildLogic(this); this.webhookPipelineLogic = new DatadogWebhookPipelineLogic(this); + this.httpClient = new HttpClient(HTTP_TIMEOUT_MS); } public void validateConfiguration() throws IllegalArgumentException { @@ -184,14 +181,8 @@ public void validateConfiguration() throws IllegalArgumentException { } } - try { - boolean intakeConnection = validateDefaultIntakeConnection(url, apiKey); - if (!intakeConnection) { - instance.setDefaultIntakeConnectionBroken(true); - - throw new IllegalArgumentException("Connection broken, please double check both your API URL and Key"); - } - } catch (IOException e) { + boolean intakeConnection = validateDefaultIntakeConnection(httpClient, url, apiKey); + if (!intakeConnection) { instance.setDefaultIntakeConnectionBroken(true); throw new IllegalArgumentException("Connection broken, please double check both your API URL and Key"); } @@ -322,7 +313,8 @@ public boolean event(DatadogEvent event) { payload.put("source_type_name", "jenkins"); payload.put("priority", event.getPriority().name().toLowerCase()); payload.put("alert_type", event.getAlertType().name().toLowerCase()); - return postApi(payload, EVENT); + postApi(payload, EVENT); + return true; } catch (Exception e) { DatadogUtilities.severe(logger, e, "Failed to send event"); return false; @@ -345,27 +337,38 @@ public void flushCounters() { logger.fine("Run flushCounters method"); // Submit all metrics as gauge - for(final Iterator> iter = counters.entrySet().iterator(); iter.hasNext();){ - Map.Entry entry = iter.next(); + for (Map.Entry entry : counters.entrySet()) { CounterMetric counterMetric = entry.getKey(); int count = entry.getValue(); logger.fine("Flushing: " + counterMetric.getMetricName() + " - " + count); - // Since we submit a rate we need to divide the submitted value by the interval (10) - this.postMetric(counterMetric.getMetricName(), count, counterMetric.getHostname(), - counterMetric.getTags(), "rate"); + try { + // Since we submit a rate we need to divide the submitted value by the interval (10) + this.postMetric( + counterMetric.getMetricName(), count, + counterMetric.getHostname(), + counterMetric.getTags(), + "rate"); + } catch (IOException e) { + DatadogUtilities.severe(logger, e, "Failed to flush counters"); + } } } @Override public boolean gauge(String name, long value, String hostname, Map> tags) { - return postMetric(name, value, hostname, tags, "gauge"); + try { + postMetric(name, value, hostname, tags, "gauge"); + return true; + } catch (IOException e) { + DatadogUtilities.severe(logger, e, "Failed to send gauge"); + return false; + } } - private boolean postMetric(String name, float value, String hostname, Map> tags, String type) { - if(this.isDefaultIntakeConnectionBroken()){ - logger.severe("Your client is not initialized properly"); - return false; + private void postMetric(String name, float value, String hostname, Map> tags, String type) throws IOException { + if (this.isDefaultIntakeConnectionBroken()) { + throw new IOException("Your client is not initialized properly"); } int INTERVAL = 10; @@ -405,15 +408,7 @@ private boolean postMetric(String name, float value, String hostname, Map headers = new HashMap<>(); + headers.put("DD-API-KEY", Secret.toString(apiKey)); + headers.put("User-Agent", String.format("Datadog/%s/jenkins Java/%s Jenkins/%s", + getDatadogPluginVersion(), + getJavaRuntimeVersion(), + getJenkinsVersion())); + + byte[] body = payload.toString().getBytes(StandardCharsets.UTF_8); + + httpClient.postAsynchronously(url, headers, "application/json", body); } /** @@ -547,50 +492,24 @@ private boolean postLogs(String payload) { return true; } - HttpURLConnection conn = null; + String url = getLogIntakeUrl(); + + Map headers = new HashMap<>(); + headers.put("DD-API-KEY", Secret.toString(apiKey)); + headers.put("User-Agent", String.format("Datadog/%s/jenkins Java/%s Jenkins/%s", + getDatadogPluginVersion(), + getJavaRuntimeVersion(), + getJenkinsVersion())); + + byte[] body = payload.getBytes(StandardCharsets.UTF_8); + try { - logger.fine("Setting up HttpURLConnection..."); - conn = createApiPostConnection(new URL(this.getLogIntakeUrl())); - - OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream(), "utf-8"); - logger.fine("Writing to OutputStreamWriter..."); - wr.write(payload); - wr.close(); - - // Get response - BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8")); - StringBuilder result = new StringBuilder(); - String line; - while ((line = rd.readLine()) != null) { - result.append(line); - } - rd.close(); - - if ("{}".equals(result.toString())) { - logger.fine(String.format("Logs API call was sent successfully!")); - logger.fine(String.format("Payload: %s", payload)); - } else { - logger.severe(String.format("Logs API call failed!")); - logger.fine(String.format("Payload: %s", payload)); - return false; - } + httpClient.postAsynchronously(url, headers, "application/json", body); + return true; } catch (Exception e) { - try { - if (conn != null && conn.getResponseCode() == BAD_REQUEST) { - logger.severe("Hmmm, your API key or your Log Intake URL may be invalid. We received a 400 in response."); - } else { - DatadogUtilities.severe(logger, e, "Unknown client error, please check your config"); - } - } catch (IOException ex) { - DatadogUtilities.severe(logger, ex, "Failed to inspect HTTP response"); - } + DatadogUtilities.severe(logger, e, "Failed to post logs"); return false; - } finally { - if (conn != null) { - conn.disconnect(); - } } - return true; } /** @@ -608,90 +527,44 @@ public boolean postWebhook(String payload) { return false; } - HttpURLConnection conn = null; + DatadogGlobalConfiguration datadogGlobalDescriptor = DatadogUtilities.getDatadogGlobalDescriptor(); + String urlParameters = datadogGlobalDescriptor != null ? "?service=" + datadogGlobalDescriptor.getCiInstanceName() : ""; + String url = getWebhookIntakeUrl() + urlParameters; + + Map headers = new HashMap<>(); + headers.put("DD-API-KEY", Secret.toString(apiKey)); + headers.put("DD-CI-PROVIDER-NAME", "jenkins"); + headers.put("User-Agent", String.format("Datadog/%s/jenkins Java/%s Jenkins/%s", + getDatadogPluginVersion(), + getJavaRuntimeVersion(), + getJenkinsVersion())); + + byte[] body = payload.getBytes(StandardCharsets.UTF_8); + try { - logger.fine("Setting up HttpURLConnection..."); - String urlParameters = "?service=" + DatadogUtilities.getDatadogGlobalDescriptor().getCiInstanceName(); - conn = createApiPostConnection(new URL(this.getWebhookIntakeUrl() + urlParameters)); - conn.setRequestProperty("DD-CI-PROVIDER-NAME", "jenkins"); - - OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream(), "utf-8"); - logger.fine("Writing to OutputStreamWriter..."); - wr.write(payload); - wr.close(); - - // Get response - BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8")); - StringBuilder result = new StringBuilder(); - String line; - while ((line = rd.readLine()) != null) { - result.append(line); - } - rd.close(); - - if ("{}".equals(result.toString())) { - logger.fine(String.format("Webhook API call was sent successfully!")); - logger.fine(String.format("Payload: %s", payload)); - } else { - logger.severe(String.format("Webhook API call failed!")); - logger.fine(String.format("Payload: %s", payload)); - return false; - } + httpClient.postAsynchronously(url, headers, "application/json", body); + return true; } catch (Exception e) { - try { - if (conn != null && conn.getResponseCode() == BAD_REQUEST) { - logger.severe("Hmmm, your API key or your Webhook Intake URL may be invalid. We received a 400 in response."); - } else { - DatadogUtilities.severe(logger, e, "Unknown client error, please check your config"); - } - } catch (IOException ex) { - DatadogUtilities.severe(logger, ex, "Failed to inspect HTTP response"); - } + DatadogUtilities.severe(logger, e, "Failed to post webhook"); return false; - } finally { - if (conn != null) { - conn.disconnect(); - } } - return true; } - public static boolean validateDefaultIntakeConnection(String url, Secret apiKey) throws IOException { + public static boolean validateDefaultIntakeConnection(HttpClient client, String validatedUrl, Secret apiKey) { String urlParameters = "?api_key=" + Secret.toString(apiKey); - HttpURLConnection conn = null; - boolean status = true; - try { - // Make request - conn = getHttpURLConnection(new URL(url + VALIDATE + urlParameters), HTTP_TIMEOUT_MS); - conn.setRequestMethod("GET"); - - // Get response - BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8")); - StringBuilder result = new StringBuilder(); - String line; - while ((line = rd.readLine()) != null) { - result.append(line); - } - rd.close(); + String url = validatedUrl + VALIDATE + urlParameters; - // Validate - JSONObject json = (JSONObject) JSONSerializer.toJSON(result.toString()); - if (!json.getBoolean("valid")) { - status = false; - } + try { + JSONObject json = (JSONObject) client.get(url, Collections.emptyMap(), JSONSerializer::toJSON); + return json.getBoolean("valid"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + DatadogUtilities.severe(logger, e, "Failed to validate webhook connection"); + return false; } catch (Exception e) { - if (conn != null && conn.getResponseCode() == HTTP_FORBIDDEN) { - logger.severe("Hmmm, your API key may be invalid. We received a 403 error."); - } else { - DatadogUtilities.severe(logger, e, "Unknown client error, please check your config"); - } - status = false; - } finally { - if (conn != null) { - conn.disconnect(); - } + DatadogUtilities.severe(logger, e, "Failed to validate webhook connection"); + return false; } - return status; } public boolean validateLogIntakeConnection() throws IOException { @@ -700,41 +573,31 @@ public boolean validateLogIntakeConnection() throws IOException { "\"hostname\":\""+DatadogUtilities.getHostname(null)+"\"}"); } + @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE") public boolean validateWebhookIntakeConnection() throws IOException { - HttpURLConnection conn = null; - boolean status = true; - try { - // Make request - conn = createApiPostConnection(new URL(this.getWebhookIntakeUrl())); - - OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream(), "utf-8"); - wr.write("{}"); - wr.close(); - - // Get response - BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8")); - StringBuilder result = new StringBuilder(); - String line; - while ((line = rd.readLine()) != null) { - result.append(line); - } - rd.close(); + String url = getWebhookIntakeUrl(); + + Map headers = new HashMap<>(); + headers.put("DD-API-KEY", Secret.toString(apiKey)); + headers.put("User-Agent", String.format("Datadog/%s/jenkins Java/%s Jenkins/%s", + getDatadogPluginVersion(), + getJavaRuntimeVersion(), + getJenkinsVersion())); + + byte[] body = "{}".getBytes(StandardCharsets.UTF_8); - // Validate response - JSONSerializer.toJSON(result.toString()); // throws if response is not json + try { + JSON jsonResponse = httpClient.post(url, headers, "application/json", body, JSONSerializer::toJSON); + // consider test successful if JSON was parsed without errors + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + DatadogUtilities.severe(logger, e, "Failed to validate webhook connection"); + return false; } catch (Exception e) { - if (conn != null && conn.getResponseCode() == HTTP_FORBIDDEN) { - logger.severe("Hmmm, your API key may be invalid. We received a 403 error."); - } else { - DatadogUtilities.severe(logger, e, "Unknown client error, please check your config"); - } - status = false; - } finally { - if (conn != null) { - conn.disconnect(); - } + DatadogUtilities.severe(logger, e, "Failed to validate webhook connection"); + return false; } - return status; } private String getJavaRuntimeVersion(){ @@ -805,5 +668,4 @@ public boolean sendPipelineTrace(Run run, FlowNode flowNode) { return false; } } - } diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpClient.java b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpClient.java new file mode 100644 index 000000000..16e7032af --- /dev/null +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpClient.java @@ -0,0 +1,333 @@ +package org.datadog.jenkins.plugins.datadog.clients; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import jenkins.model.Jenkins; +import org.datadog.jenkins.plugins.datadog.DatadogUtilities; +import org.datadog.jenkins.plugins.datadog.transport.HttpMessage; +import org.eclipse.jetty.client.HttpProxy; +import org.eclipse.jetty.client.Origin; +import org.eclipse.jetty.client.ProxyConfiguration; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.client.api.Result; +import org.eclipse.jetty.client.util.BufferingResponseListener; +import org.eclipse.jetty.client.util.BytesContentProvider; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.util.thread.QueuedThreadPool; + +public class HttpClient { + + private static final org.eclipse.jetty.client.HttpClient CLIENT = buildHttpClient(); + + private static final String MAX_THREADS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_MAX_THREADS"; + private static final String MIN_THREADS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_MIN_THREADS"; + private static final String IDLE_THREAD_TIMEOUT_MILLIS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_IDLE_THREAD_TIMEOUT"; + private static final String RESERVED_THREADS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_RESERVED_THREADS"; + private static final String MAX_REQUEST_RETRIES_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_REQUEST_RETRIES"; + private static final String INITIAL_RETRY_DELAY_MILLIS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_INITIAL_RETRY_DELAY"; + private static final String RETRY_DELAY_FACTOR_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_RETRY_DELAY_FACTOR"; + private static final String MAX_RESPONSE_LENGTH_BYTES_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_MAX_RESPONSE_LENGTH"; + private static final int MAX_THREADS_DEFAULT = 64; + private static final int MIN_THREADS_DEFAULT = 1; + private static final int IDLE_THREAD_TIMEOUT_MILLIS = 60_000; + private static final int RESERVED_THREADS_DEFAULT = -1; + private static final int MAX_REQUEST_RETRIES_DEFAULT = 5; + private static final int INITIAL_RETRY_DELAY_MILLIS_DEFAULT = 100; + private static final double RETRY_DELAY_FACTOR_DEFAULT = 2.0; + private static final int MAX_RESPONSE_LENGTH_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB + private static volatile hudson.ProxyConfiguration EFFECTIVE_PROXY_CONFIGURATION; + + private static final Logger logger = Logger.getLogger(HttpClient.class.getName()); + + private static org.eclipse.jetty.client.HttpClient buildHttpClient() { + BlockingQueue queue = new ArrayBlockingQueue<>(1024); + ThreadFactory threadFactory = new ThreadFactory() { + final ThreadFactory delegate = Executors.defaultThreadFactory(); + + @Override + public Thread newThread(final Runnable r) { + final Thread result = delegate.newThread(r); + result.setName("dd-http-client-" + result.getName()); + result.setDaemon(true); + return result; + } + }; + + QueuedThreadPool threadPool = new QueuedThreadPool( + getEnv(MAX_THREADS_ENV_VAR, MAX_THREADS_DEFAULT), + getEnv(MIN_THREADS_ENV_VAR, MIN_THREADS_DEFAULT), + getEnv(IDLE_THREAD_TIMEOUT_MILLIS_ENV_VAR, IDLE_THREAD_TIMEOUT_MILLIS), + getEnv(RESERVED_THREADS_ENV_VAR, RESERVED_THREADS_DEFAULT), + queue, + null, + threadFactory + ); + threadPool.setName("dd-http-client-thread-pool"); + + SslContextFactory.Client sslContextFactory = new SslContextFactory.Client(); + org.eclipse.jetty.client.HttpClient httpClient = new org.eclipse.jetty.client.HttpClient(sslContextFactory); + httpClient.setExecutor(threadPool); + try { + httpClient.start(); + } catch (Exception e) { + throw new RuntimeException("Could not start HTTP client", e); + } + return httpClient; + } + + private static void ensureProxyConfiguration() { + Jenkins jenkins = Jenkins.getInstanceOrNull(); + if (jenkins == null) { + return; + } + hudson.ProxyConfiguration jenkinsProxyConfiguration = jenkins.getProxy(); + if (jenkinsProxyConfiguration == null) { + return; + } + + if (EFFECTIVE_PROXY_CONFIGURATION != jenkinsProxyConfiguration) { + synchronized (CLIENT) { + if (EFFECTIVE_PROXY_CONFIGURATION != jenkinsProxyConfiguration) { + org.eclipse.jetty.client.ProxyConfiguration proxyConfig = CLIENT.getProxyConfiguration(); + List proxies = proxyConfig.getProxies(); + proxies.clear(); + + String proxyHost = jenkinsProxyConfiguration.getName(); + int proxyPort = jenkinsProxyConfiguration.getPort(); + List noProxyHostPatterns = jenkinsProxyConfiguration.getNoProxyHostPatterns(); + proxies.add(new HttpProxy(proxyHost, proxyPort) { + @Override + public boolean matches(Origin origin) { + Origin.Address address = origin.getAddress(); + String host = address.getHost(); + for (Pattern noProxyHostPattern : noProxyHostPatterns) { + if (noProxyHostPattern.matcher(host).matches()) { + return false; + } + } + return true; + } + }); + + EFFECTIVE_PROXY_CONFIGURATION = jenkinsProxyConfiguration; + } + } + } + } + + private final long timeoutMillis; + private final HttpRetryPolicy.Factory retryPolicyFactory; + + public HttpClient(long timeoutMillis) { + this.timeoutMillis = timeoutMillis; + this.retryPolicyFactory = new HttpRetryPolicy.Factory( + getEnv(MAX_REQUEST_RETRIES_ENV_VAR, MAX_REQUEST_RETRIES_DEFAULT), + getEnv(INITIAL_RETRY_DELAY_MILLIS_ENV_VAR, INITIAL_RETRY_DELAY_MILLIS_DEFAULT), + getEnv(RETRY_DELAY_FACTOR_ENV_VAR, RETRY_DELAY_FACTOR_DEFAULT)); + } + + public T get(String url, Map headers, Function responseParser) throws ExecutionException, InterruptedException, TimeoutException { + ensureProxyConfiguration(); + + return executeSynchronously( + requestSupplier(url, HttpMethod.GET, headers, null, null), + retryPolicyFactory.create(), + responseParser); + } + + public T post(String url, Map headers, String contentType, byte[] body, Function responseParser) throws ExecutionException, InterruptedException, TimeoutException { + ensureProxyConfiguration(); + + return executeSynchronously( + requestSupplier(url, HttpMethod.POST, headers, contentType, body), + retryPolicyFactory.create(), + responseParser); + } + + public void postAsynchronously(String url, Map headers, String contentType, byte[] body) { + ensureProxyConfiguration(); + + executeAsynchronously( + requestSupplier( + url, + HttpMethod.POST, + headers, + contentType, + body), + retryPolicyFactory.create() + ); + } + + public void sendAsynchronously(HttpMessage message) { + ensureProxyConfiguration(); + + String url = message.getURL().toString(); + HttpMethod httpMethod = HttpMethod.fromString(message.getMethod().name()); + String contentType = message.getContentType(); + byte[] body = message.getPayload(); + executeAsynchronously( + requestSupplier( + url, + httpMethod, + Collections.emptyMap(), + contentType, + body), + retryPolicyFactory.create() + ); + } + + private Supplier requestSupplier(String url, HttpMethod method, Map headers, String contentType, byte[] body) { + return () -> { + Request request = CLIENT + .newRequest(url) + .method(method) + .timeout(timeoutMillis, TimeUnit.MILLISECONDS); + for (Map.Entry e : headers.entrySet()) { + request.header(e.getKey(), e.getValue()); + } + if (contentType != null) { + request.header(HttpHeader.CONTENT_TYPE, contentType); + } + if (body != null) { + request.content(new BytesContentProvider(contentType, body)); + } + return request; + }; + } + + private static T executeSynchronously(Supplier requestSupplier, HttpRetryPolicy retryPolicy, Function responseParser) throws InterruptedException, TimeoutException, ExecutionException { + while (true) { + ContentResponse response; + try { + Request request = requestSupplier.get(); + response = request.send(); + + } catch (TimeoutException | ExecutionException e) { + if (retryPolicy.shouldRetry(null)) { + Thread.sleep(retryPolicy.backoff()); + continue; + } else { + throw e; + } + } + + int status = response.getStatus(); + if (status >= 200 && status < 300) { + if (responseParser == null) { + return null; + } + + String content = response.getContentAsString(); + return responseParser.apply(content); + + } else { + if (retryPolicy.shouldRetry(response)) { + Thread.sleep(retryPolicy.backoff()); + continue; + } + + String additionalHint; + switch (status) { + case HttpStatus.FORBIDDEN_403: + additionalHint = "API key might be invalid, please check your config"; + break; + case HttpStatus.NOT_FOUND_404: + case HttpStatus.BAD_REQUEST_400: + additionalHint = "Request URL might be invalid, please check your config"; + break; + default: + additionalHint = ""; + break; + } + + throw new ResponseProcessingException("Received erroneous response " + response + ". " + additionalHint); + } + } + } + + private static void executeAsynchronously(Supplier requestSupplier, HttpRetryPolicy retryPolicy) { + Request request = requestSupplier.get(); + request.send(new ResponseListener(getEnv(MAX_RESPONSE_LENGTH_BYTES_ENV_VAR, MAX_RESPONSE_LENGTH_BYTES_DEFAULT), requestSupplier, retryPolicy)); + } + + private static final class ResponseListener extends BufferingResponseListener { + private final Supplier requestSupplier; + private final HttpRetryPolicy retryPolicy; + + public ResponseListener(int maxLength, Supplier requestSupplier, HttpRetryPolicy retryPolicy) { + super(maxLength); + this.requestSupplier = requestSupplier; + this.retryPolicy = retryPolicy; + } + + @Override + public void onComplete(Result result) { + try { + Response response = result.getResponse(); + int responseCode = response != null ? response.getStatus() : -1; + if (responseCode > 0 && responseCode < 400) { + // successful response + return; + } + + if (retryPolicy.shouldRetry(response)) { + Thread.sleep(retryPolicy.backoff()); + requestSupplier.get().send(this); + } else { + Throwable failure = result.getFailure(); + DatadogUtilities.severe(logger, failure, "HTTP request failed: " + result.getRequest() + ", response: " + response); + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + private static final class ResponseProcessingException extends ExecutionException { + public ResponseProcessingException(String message) { + super(message); + } + } + + private static int getEnv(String envVar, int defaultValue) { + String value = System.getenv(envVar); + if (value != null) { + try { + return Integer.parseInt(value); + } catch (Exception e) { + DatadogUtilities.severe(logger, null, "Invalid value " + value + " provided for env var " + envVar + ": integer number expected"); + } + } + return defaultValue; + } + + private static double getEnv(String envVar, double defaultValue) { + String value = System.getenv(envVar); + if (value != null) { + try { + return Double.parseDouble(value); + } catch (Exception e) { + DatadogUtilities.severe(logger, null, "Invalid value " + value + " provided for env var " + envVar + ": double number expected"); + } + } + return defaultValue; + } +} diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpRetryPolicy.java b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpRetryPolicy.java new file mode 100644 index 000000000..2644ff994 --- /dev/null +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpRetryPolicy.java @@ -0,0 +1,134 @@ +package org.datadog.jenkins.plugins.datadog.clients; + +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import javax.annotation.Nullable; +import javax.annotation.concurrent.NotThreadSafe; +import org.datadog.jenkins.plugins.datadog.DatadogUtilities; +import org.eclipse.jetty.client.api.Response; + +/** + * A policy which encapsulates retry rules for HTTP calls. Whether to retry and how long to wait + * before the next attempt is determined by received HTTP response. + * + *

Logic is the following: + * + *

    + *
  • if there was no response, or response code is 5XX, wait for specified delay time and + * retry (there is exponential backoff, so each consecutive retry takes longer than the + * previous one). Maximum number of retries and delay multiplication factor are provided + * during policy instantiation + *
  • if response code is 429 (Too Many Requests), try to get wait time from + * x-ratelimit-reset response header. Depending on the result: + *
      + *
    • if time is less than 10 seconds, wait for the specified amount + random(0, 0.4sec) + * (there will be only one retry in this case) + *
    • if time is more than 10 seconds, do not retry + *
    • if time was not provided or could not be parsed, do a regular retry (same logic as + * for 5XX responses) + *
    + *
  • in all other cases do not retry + *
+ * + *

Instances of this class are not thread-safe and not reusable: each HTTP call requires its own + * instance. + */ +@NotThreadSafe +public class HttpRetryPolicy { + + private static final Logger logger = Logger.getLogger(HttpRetryPolicy.class.getName()); + + private static final int NO_RESPONSE_RECEIVED = -1; + private static final int TOO_MANY_REQUESTS_HTTP_CODE = 429; + private static final String X_RATELIMIT_RESET_HTTP_HEADER = "x-ratelimit-reset"; + private static final int RATE_LIMIT_RESET_TIME_UNDEFINED = -1; + private static final int MAX_ALLOWED_WAIT_TIME_SECONDS = 10; + private static final int RATE_LIMIT_DELAY_RANDOM_COMPONENT_MAX_MILLIS = 401; + + private int retriesLeft; + private long delay; + private final double delayFactor; + + private HttpRetryPolicy(int retriesLeft, long delay, double delayFactor) { + this.retriesLeft = retriesLeft; + this.delay = delay; + this.delayFactor = delayFactor; + } + + public boolean shouldRetry(@Nullable Response response) { + if (retriesLeft == 0) { + return false; + } + + int responseCode = response != null ? response.getStatus() : NO_RESPONSE_RECEIVED; + if (responseCode >= 500 || responseCode == NO_RESPONSE_RECEIVED) { + retriesLeft--; + return true; + + } else if (responseCode == TOO_MANY_REQUESTS_HTTP_CODE) { + long waitTimeSeconds = getRateLimitResetTime(response); + if (waitTimeSeconds == RATE_LIMIT_RESET_TIME_UNDEFINED) { + retriesLeft--; // doing a regular retry if proper reset time was not provided + return true; + } + + if (waitTimeSeconds > MAX_ALLOWED_WAIT_TIME_SECONDS) { + return false; // too long to wait, will not retry + } + + retriesLeft = 0; + delay = + TimeUnit.SECONDS.toMillis(waitTimeSeconds) + + ThreadLocalRandom.current().nextInt(RATE_LIMIT_DELAY_RANDOM_COMPONENT_MAX_MILLIS); + return true; + + } else { + return false; + } + } + + private long getRateLimitResetTime(Response response) { + String rateLimitHeader = response.getHeaders().get(X_RATELIMIT_RESET_HTTP_HEADER); + if (rateLimitHeader == null) { + return RATE_LIMIT_RESET_TIME_UNDEFINED; + } + + try { + return Long.parseLong(rateLimitHeader); + } catch (NumberFormatException e) { + DatadogUtilities.severe(logger, e, + "Could not parse " + + X_RATELIMIT_RESET_HTTP_HEADER + + " header contents: " + + rateLimitHeader); + return RATE_LIMIT_RESET_TIME_UNDEFINED; + } + } + + public long backoff() { + long currentDelay = delay; + delay = (long) (delay * delayFactor * randomJitter(0.15)); + return currentDelay; + } + + private static double randomJitter(double percentage) { + return ThreadLocalRandom.current().nextDouble(1 - percentage, 1 + percentage); + } + + public static final class Factory { + private final int maxRetries; + private final long initialDelay; + private final double delayFactor; + + public Factory(int maxRetries, int initialDelay, double delayFactor) { + this.maxRetries = maxRetries; + this.initialDelay = initialDelay; + this.delayFactor = delayFactor; + } + + public HttpRetryPolicy create() { + return new HttpRetryPolicy(maxRetries, initialDelay, delayFactor); + } + } +} diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpMessage.java b/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpMessage.java index 49b5fbf87..886cd713b 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpMessage.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpMessage.java @@ -35,6 +35,15 @@ public byte[] getPayload() { return this.payload; } + @Override + public String toString() { + return "HttpMessage{" + + "url=" + url + + ", method=" + method + + ", contentType='" + contentType + '\'' + + '}'; + } + public enum HttpMethod { PUT } diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpSender.java b/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpSender.java index c9ee3b2fb..6d0a0da13 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpSender.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/transport/HttpSender.java @@ -1,16 +1,11 @@ package org.datadog.jenkins.plugins.datadog.transport; -import static org.datadog.jenkins.plugins.datadog.DatadogUtilities.getHttpURLConnection; - -import org.datadog.jenkins.plugins.datadog.util.SuppressFBWarnings; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.HttpURLConnection; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import org.datadog.jenkins.plugins.datadog.DatadogUtilities; +import org.datadog.jenkins.plugins.datadog.clients.HttpClient; public class HttpSender implements Runnable { @@ -18,27 +13,31 @@ public class HttpSender implements Runnable { private final BlockingQueue queue; private final HttpErrorHandler errorHandler; - private final int httpTimeoutMs; + private final HttpClient client; private volatile boolean shutdown; HttpSender(final int queueSize, final HttpErrorHandler errorHandler, final int httpTimeoutMs) { - this(new LinkedBlockingQueue(queueSize), errorHandler, httpTimeoutMs); + this(new ArrayBlockingQueue<>(queueSize), errorHandler, httpTimeoutMs); } HttpSender(final BlockingQueue queue, final HttpErrorHandler errorHandler, final int httpTimeoutMs) { this.queue = queue; this.errorHandler = errorHandler; - this.httpTimeoutMs = httpTimeoutMs; + this.client = new HttpClient(httpTimeoutMs); } - @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_BAD_PRACTICE") boolean send(final HttpMessage message){ - if(!shutdown){ - queue.offer(message); + if (shutdown) { + return false; + } + try { + queue.put(message); return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; } - return false; } @Override @@ -51,8 +50,8 @@ public void run() { // with 1 second of timeout to avoid blocking // the thread indefinitely. final HttpMessage message = queue.poll(1, TimeUnit.SECONDS); - if(null != message) { - blockingSend(message); + if (null != message) { + process(message); } } catch (final InterruptedException e) { if (shutdown) { @@ -64,43 +63,14 @@ public void run() { } } - protected void blockingSend(HttpMessage message) { - HttpURLConnection conn = null; + protected void process(HttpMessage message) { try { - conn = getHttpURLConnection(message.getURL(), httpTimeoutMs); - conn.setRequestMethod(message.getMethod().name()); - conn.setRequestProperty("Content-Type", message.getContentType()); - conn.setUseCaches(false); - conn.setDoOutput(true); - - final byte[] payload = message.getPayload(); - final OutputStream outputStream = conn.getOutputStream(); - outputStream.write(payload); - outputStream.close(); - - int httpStatus = conn.getResponseCode(); - logger.fine("HTTP/"+message.getMethod()+" " + message.getURL() + " ["+payload.length+" bytes] --> HTTP " + httpStatus); - if(httpStatus >= 400) { - logger.severe("Failed to send HTTP request: "+message.getMethod()+" "+ message.getURL()+ " - Status: HTTP "+httpStatus); - } - } catch (Exception ex) { - errorHandler.handle(ex); - try { - if(conn != null) { - logger.severe("Failed to send HTTP request: "+message.getMethod()+" "+ message.getURL()+ " - Status: HTTP "+conn.getResponseCode()); - } - } catch (IOException ioex) { - errorHandler.handle(ioex); - } - } finally { - if(conn != null) { - conn.disconnect(); - } + client.sendAsynchronously(message); + } catch (Exception e) { + DatadogUtilities.severe(logger, e, "Error while sending message: " + message); } } - - void shutdown() { shutdown = true; } diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/transport/NonBlockingHttpClient.java b/src/main/java/org/datadog/jenkins/plugins/datadog/transport/NonBlockingHttpClient.java index 96a16fe3a..960c44534 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/transport/NonBlockingHttpClient.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/transport/NonBlockingHttpClient.java @@ -14,6 +14,7 @@ public class NonBlockingHttpClient implements HttpClient { private static final int DEFAULT_TIMEOUT_MS = 10 * 1000; + private static final int DEFAULT_MAX_QUEUE_SIZE = 10_000; private static final int SIZE_SPANS_SEND_BUFFER = 100; private static final Logger logger = Logger.getLogger(NonBlockingHttpClient.class.getName()); @@ -37,7 +38,7 @@ public class NonBlockingHttpClient implements HttpClient { }); private NonBlockingHttpClient(final Builder builder) { - final int queueSize = builder.queueSize != null ? builder.queueSize : Integer.MAX_VALUE; + final int queueSize = builder.queueSize != null ? builder.queueSize : DEFAULT_MAX_QUEUE_SIZE; final int httpTimeoutMs = builder.httpTimeoutMs != null ? builder.httpTimeoutMs : DEFAULT_TIMEOUT_MS; this.errorHandler = builder.errorHandler != null ? builder.errorHandler : NO_OP_HANDLER; this.messageFactoryByType = builder.messageFactoryByType; diff --git a/src/test/java/org/datadog/jenkins/plugins/datadog/clients/DatadogClientTest.java b/src/test/java/org/datadog/jenkins/plugins/datadog/clients/DatadogClientTest.java index 8bc0460c1..1d44c7b43 100644 --- a/src/test/java/org/datadog/jenkins/plugins/datadog/clients/DatadogClientTest.java +++ b/src/test/java/org/datadog/jenkins/plugins/datadog/clients/DatadogClientTest.java @@ -49,7 +49,7 @@ public class DatadogClientTest { @Test public void testHttpClientGetInstanceApiKey() { - //validateCongiguration throws an error when given an invalid API key when the urls are valid + //validateConfiguration throws an error when given an invalid API key when the urls are valid Exception exception = Assert.assertThrows(IllegalArgumentException.class, () -> { DatadogHttpClient.enableValidations = false; DatadogHttpClient client = (DatadogHttpClient) DatadogHttpClient.getInstance("http", "test", "test", null); @@ -63,7 +63,7 @@ public void testHttpClientGetInstanceApiKey() { @Test public void testHttpClientGetInstanceApiUrl() { - // validateCongiguration throws an error when given an invalid url + // validateConfiguration throws an error when given an invalid url Exception exception = Assert.assertThrows(IllegalArgumentException.class, () -> { DatadogHttpClient.enableValidations = false; DatadogHttpClient client = (DatadogHttpClient) DatadogHttpClient.getInstance("", null, null, null); @@ -86,7 +86,7 @@ public void testHttpClientGetInstanceEnableValidations() { @Test public void testDogstatsDClientGetInstanceTargetPort() { - // validateCongiguration throws an error when given an invalid port + // validateConfiguration throws an error when given an invalid port Exception exception = Assert.assertThrows(IllegalArgumentException.class, () -> { DatadogAgentClient.enableValidations = false; DatadogAgentClient client = (DatadogAgentClient) DatadogAgentClient.getInstance("test", null, null, null); diff --git a/src/test/java/org/datadog/jenkins/plugins/datadog/transport/FakeHttpSender.java b/src/test/java/org/datadog/jenkins/plugins/datadog/transport/FakeHttpSender.java index fadc46773..a13c01903 100644 --- a/src/test/java/org/datadog/jenkins/plugins/datadog/transport/FakeHttpSender.java +++ b/src/test/java/org/datadog/jenkins/plugins/datadog/transport/FakeHttpSender.java @@ -31,7 +31,7 @@ public void handle(Exception exception) { } @Override - protected void blockingSend(HttpMessage message) { + protected void process(HttpMessage message) { this.messageCount.incrementAndGet(); synchronized (this.latches) { httpMessages.add(message);