Merge branch 'master' into nikita-tkachenko/tracer-autoconfig

jenkinsci · Aug 29, 2023 · 9344634 · 9344634
2 parents 9ff58d1 + 0c0fea3
commit 9344634
Show file tree

Hide file tree

Showing 10 changed files with 274 additions and 35 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,25 @@
 Changes
 =======
+
+## 5.5.0 / 2023-08-25
+### Details
+https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.2...datadog-5.5.0
+
+***Added***:
+
+* Add option to use AWS instance ID as hostname. See [#345](https://github.com/jenkinsci/datadog-plugin/pull/345).
+
+***Fixed***:
+
+* Fix error status propagation to take into account catch/catchError/warnError blocks. See [#343](https://github.com/jenkinsci/datadog-plugin/pull/343).
+* Look up hostname from controller environment. See [#340](https://github.com/jenkinsci/datadog-plugin/pull/340). Thanks [Vlatombe](https://github.com/Vlatombe).
+
+## 5.4.2 / 2023-07-12
+### Details
+https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.1...datadog-5.4.2
+
+* [Fixed] Fix [CVE-2023-37944](https://www.jenkins.io/security/advisory/2023-07-12/#SECURITY-3130) and require Overall/Administer permission to access the affected HTTP endpoint. See [#350](https://github.com/jenkinsci/datadog-plugin/pull/350).
+
 ## 5.4.1 / 2023-05-24
 ### Details
 https://github.com/jenkinsci/datadog-plugin/compare/datadog-5.4.0...datadog-5.4.1

diff --git a/pom.xml b/pom.xml
@@ -15,7 +15,7 @@
   <url>https://github.com/jenkinsci/datadog-plugin</url>
   <groupId>org.datadog.jenkins.plugins</groupId>
   <artifactId>datadog</artifactId>
-  <version>5.4.2-SNAPSHOT</version>
+  <version>5.5.1-SNAPSHOT</version>
   <packaging>hpi</packaging>
 
   <properties>

diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogGlobalConfiguration.java
@@ -110,6 +110,7 @@ public class DatadogGlobalConfiguration extends GlobalConfiguration {
     private static final String RETRY_LOGS_PROPERTY = "DATADOG_JENKINS_PLUGIN_RETRY_LOGS";
     private static final String REFRESH_DOGSTATSD_CLIENT_PROPERTY = "DATADOG_REFRESH_STATSD_CLIENT";
     private static final String CACHE_BUILD_RUNS_PROPERTY = "DATADOG_CACHE_BUILD_RUNS";
+    private static final String USE_AWS_INSTANCE_HOSTNAME_PROPERTY = "DATADOG_USE_AWS_INSTANCE_HOSTNAME";
 
     private static final String ENABLE_CI_VISIBILITY_PROPERTY = "DATADOG_JENKINS_PLUGIN_ENABLE_CI_VISIBILITY";
     private static final String CI_VISIBILITY_CI_INSTANCE_NAME_PROPERTY = "DATADOG_JENKINS_PLUGIN_CI_VISIBILITY_CI_INSTANCE_NAME";
@@ -131,6 +132,7 @@ public class DatadogGlobalConfiguration extends GlobalConfiguration {
     private static final boolean DEFAULT_RETRY_LOGS_VALUE = true;
     private static final boolean DEFAULT_REFRESH_DOGSTATSD_CLIENT_VALUE = false;
     private static final boolean DEFAULT_CACHE_BUILD_RUNS_VALUE = true;
+    private static final boolean DEFAULT_USE_AWS_INSTANCE_HOSTNAME_VALUE = false;
 
     private String reportWith = DEFAULT_REPORT_WITH_VALUE;
     private String targetApiURL = DEFAULT_TARGET_API_URL_VALUE;
@@ -161,6 +163,7 @@ public class DatadogGlobalConfiguration extends GlobalConfiguration {
     private boolean retryLogs = DEFAULT_RETRY_LOGS_VALUE;
     private boolean refreshDogstatsdClient = DEFAULT_REFRESH_DOGSTATSD_CLIENT_VALUE;
     private boolean cacheBuildRuns = DEFAULT_CACHE_BUILD_RUNS_VALUE;
+    private boolean useAwsInstanceHostname = DEFAULT_USE_AWS_INSTANCE_HOSTNAME_VALUE;
 
     @DataBoundConstructor
     public DatadogGlobalConfiguration() {
@@ -301,6 +304,11 @@ private void loadEnvVariables(){
             this.cacheBuildRuns = Boolean.valueOf(cacheBuildRunsEnvVar);
         }
 
+        String useAwsInstanceHostnameEnvVar = System.getenv(USE_AWS_INSTANCE_HOSTNAME_PROPERTY);
+        if(StringUtils.isNotBlank(useAwsInstanceHostnameEnvVar)){
+            this.useAwsInstanceHostname = Boolean.valueOf(useAwsInstanceHostnameEnvVar);
+        }
+
         String enableCiVisibilityVar = System.getenv(ENABLE_CI_VISIBILITY_PROPERTY);
         if(StringUtils.isNotBlank(enableCiVisibilityVar)) {
             this.collectBuildTraces = Boolean.valueOf(enableCiVisibilityVar);
@@ -415,7 +423,7 @@ public FormValidation doTestConnection(
             @QueryParameter("targetCredentialsApiKey") final String targetCredentialsApiKey, 
             @QueryParameter("targetApiURL") final String targetApiURL)
             throws IOException, ServletException {
-
+        Jenkins.getInstance().checkPermission(Jenkins.ADMINISTER);
         final Secret secret = findSecret(targetApiKey, targetCredentialsApiKey);
         if (DatadogHttpClient.validateDefaultIntakeConnection(new HttpClient(60_000), targetApiURL, secret)) {
             return FormValidation.ok("Great! Your API key is valid.");
@@ -828,6 +836,7 @@ public boolean configure(final StaplerRequest req, final JSONObject formData) th
             this.setRetryLogs(formData.getBoolean("retryLogs"));
             this.setRefreshDogstatsdClient(formData.getBoolean("refreshDogstatsdClient"));
             this.setCacheBuildRuns(formData.getBoolean("cacheBuildRuns"));
+            this.setUseAwsInstanceHostname(formData.getBoolean("useAwsInstanceHostname"));
             this.setEmitSystemEvents(formData.getBoolean("emitSystemEvents"));
             this.setEmitConfigChangeEvents(formData.getBoolean("emitConfigChangeEvents"));
 
@@ -1412,6 +1421,24 @@ public void setCacheBuildRuns(boolean cacheBuildRuns) {
         this.cacheBuildRuns = cacheBuildRuns;
     }
 
+    /**
+     * @return - A {@link Boolean} indicating if the user has configured Datadog to use AWS instance as hostname
+     */
+    public boolean isUseAwsInstanceHostname() {
+        return useAwsInstanceHostname;
+    }
+
+
+    /**
+     * Set the checkbox in the UI, used for Jenkins data binding
+     *
+     * @param useAwsInstanceHostname - The checkbox status (checked/unchecked)
+     */
+    @DataBoundSetter
+    public void setUseAwsInstanceHostname(boolean useAwsInstanceHostname) {
+        this.useAwsInstanceHostname = useAwsInstanceHostname;
+    }
+
     /**
      * @return - A {@link Boolean} indicating if the user has configured Datadog to emit System related events.
      */

diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java b/src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java
@@ -463,6 +463,40 @@ public static Map<String, Set<String>> computeTagListFromVarList(EnvVars envVars
         return result;
     }
 
+    public static String getAwsInstanceID() throws IOException {
+        String metadataUrl = "http://169.254.169.254/latest/meta-data/instance-id";
+        HttpURLConnection conn = null;
+        String instance_id = null;
+        // Make request
+        conn = getHttpURLConnection(new URL(metadataUrl), 300);
+        conn.setRequestMethod("GET");
+
+        // Get response
+        BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"));
+        StringBuilder result = new StringBuilder();
+        String line;
+        while ((line = rd.readLine()) != null) {
+            result.append(line);
+        }
+        rd.close();
+
+        // Validate
+        instance_id = result.toString();
+        try {
+            if (conn.getResponseCode() == 404) {
+                logger.fine("Could not retrieve AWS instance ID");
+            }
+            conn.disconnect();
+        } catch (IOException e) {
+            logger.info("Failed to inspect HTTP response when getting AWS Instance ID");
+        }
+
+        if (instance_id.equals("")) {
+            return null;
+        }
+        return instance_id;
+    }
+
     /**
      * Getter function to return either the saved hostname global configuration,
      * or the hostname that is set in the Jenkins host itself. Returns null if no
@@ -471,6 +505,8 @@ public static Map<String, Set<String>> computeTagListFromVarList(EnvVars envVars
      * Tries, in order:
      * Jenkins configuration
      * Jenkins hostname environment variable
+     * AWS instance ID, if enabled
+     * System hostname environment variable
      * Unix hostname via `/bin/hostname -f`
      * Localhost hostname
      *
@@ -500,6 +536,27 @@ public static String getHostname(EnvVars envVars) {
                 return hostname;
             }
         }
+
+        final DatadogGlobalConfiguration datadogGlobalConfig = getDatadogGlobalDescriptor();
+        if (datadogGlobalConfig != null){
+            if (datadogGlobalConfig.isUseAwsInstanceHostname()) {
+                try {
+                    hostname = getAwsInstanceID();
+                } catch (IOException e) {
+                    logger.fine("Error retrieving AWS hostname: " + e);
+                }
+                if (hostname != null) {
+                    logger.fine("Using AWS instance ID as hostname. Hostname: " + hostname);
+                    return hostname;
+                }
+            }
+        }
+
+        if (isValidHostname(hostname)) {
+            logger.fine("Using hostname found in $HOSTNAME controller environment variable. Hostname: " + hostname);
+            return hostname;
+        }
+
         hostname = System.getenv("HOSTNAME");
         if (isValidHostname(hostname)) {
             logger.fine("Using hostname found in $HOSTNAME controller environment variable. Hostname: " + hostname);

diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpClient.java b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpClient.java
@@ -39,6 +39,23 @@
 public class HttpClient {
 
     private static final org.eclipse.jetty.client.HttpClient CLIENT = buildHttpClient();
+
+    private static final String MAX_THREADS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_MAX_THREADS";
+    private static final String MIN_THREADS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_MIN_THREADS";
+    private static final String IDLE_THREAD_TIMEOUT_MILLIS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_IDLE_THREAD_TIMEOUT";
+    private static final String RESERVED_THREADS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_RESERVED_THREADS";
+    private static final String MAX_REQUEST_RETRIES_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_REQUEST_RETRIES";
+    private static final String INITIAL_RETRY_DELAY_MILLIS_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_INITIAL_RETRY_DELAY";
+    private static final String RETRY_DELAY_FACTOR_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_RETRY_DELAY_FACTOR";
+    private static final String MAX_RESPONSE_LENGTH_BYTES_ENV_VAR = "DD_JENKINS_HTTP_CLIENT_MAX_RESPONSE_LENGTH";
+    private static final int MAX_THREADS_DEFAULT = 64;
+    private static final int MIN_THREADS_DEFAULT = 1;
+    private static final int IDLE_THREAD_TIMEOUT_MILLIS = 60_000;
+    private static final int RESERVED_THREADS_DEFAULT = -1;
+    private static final int MAX_REQUEST_RETRIES_DEFAULT = 5;
+    private static final int INITIAL_RETRY_DELAY_MILLIS_DEFAULT = 100;
+    private static final double RETRY_DELAY_FACTOR_DEFAULT = 2.0;
+    private static final int MAX_RESPONSE_LENGTH_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB
     private static volatile hudson.ProxyConfiguration EFFECTIVE_PROXY_CONFIGURATION;
 
     private static final Logger logger = Logger.getLogger(HttpClient.class.getName());
@@ -58,10 +75,10 @@ public Thread newThread(final Runnable r) {
         };
 
         QueuedThreadPool threadPool = new QueuedThreadPool(
-                64,
-                1,
-                60_000,
-                -1,
+                getEnv(MAX_THREADS_ENV_VAR, MAX_THREADS_DEFAULT),
+                getEnv(MIN_THREADS_ENV_VAR, MIN_THREADS_DEFAULT),
+                getEnv(IDLE_THREAD_TIMEOUT_MILLIS_ENV_VAR, IDLE_THREAD_TIMEOUT_MILLIS),
+                getEnv(RESERVED_THREADS_ENV_VAR, RESERVED_THREADS_DEFAULT),
                 queue,
                 null,
                 threadFactory
@@ -109,7 +126,7 @@ public boolean matches(Origin origin) {
                                     return false;
                                 }
                             }
-                            return super.matches(origin);
+                            return true;
                         }
                     });
 
@@ -124,7 +141,10 @@ public boolean matches(Origin origin) {
 
     public HttpClient(long timeoutMillis) {
         this.timeoutMillis = timeoutMillis;
-        this.retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0);
+        this.retryPolicyFactory = new HttpRetryPolicy.Factory(
+                getEnv(MAX_REQUEST_RETRIES_ENV_VAR, MAX_REQUEST_RETRIES_DEFAULT),
+                getEnv(INITIAL_RETRY_DELAY_MILLIS_ENV_VAR, INITIAL_RETRY_DELAY_MILLIS_DEFAULT),
+                getEnv(RETRY_DELAY_FACTOR_ENV_VAR, RETRY_DELAY_FACTOR_DEFAULT));
     }
 
     public <T> T get(String url, Map<String, String> headers, Function<String, T> responseParser) throws ExecutionException, InterruptedException, TimeoutException {
@@ -220,6 +240,7 @@ private static <T> T executeSynchronously(Supplier<Request> requestSupplier, Htt
             try {
                 Request request = requestSupplier.get();
                 response = request.send();
+
             } catch (TimeoutException | ExecutionException e) {
                 if (retryPolicy.shouldRetry(null)) {
                     Thread.sleep(retryPolicy.backoff());
@@ -265,7 +286,7 @@ private static <T> T executeSynchronously(Supplier<Request> requestSupplier, Htt
 
     private static void executeAsynchronously(Supplier<Request> requestSupplier, HttpRetryPolicy retryPolicy) {
         Request request = requestSupplier.get();
-        request.send(new ResponseListener(2 * 1024 * 1024, requestSupplier, retryPolicy));
+        request.send(new ResponseListener(getEnv(MAX_RESPONSE_LENGTH_BYTES_ENV_VAR, MAX_RESPONSE_LENGTH_BYTES_DEFAULT), requestSupplier, retryPolicy));
     }
 
     private static final class ResponseListener extends BufferingResponseListener {
@@ -281,22 +302,19 @@ public ResponseListener(int maxLength, Supplier<Request> requestSupplier, HttpRe
         @Override
         public void onComplete(Result result) {
             try {
-                Throwable failure = result.getFailure();
-                if (failure != null) {
-                    if (retryPolicy.shouldRetry(null)) {
-                        Thread.sleep(retryPolicy.backoff());
-                        requestSupplier.get().send(this);
-                    } else {
-                        DatadogUtilities.severe(logger, failure, "HTTP request failed: " + result.getRequest());
-                    }
+                Response response = result.getResponse();
+                int responseCode = response != null ? response.getStatus() : -1;
+                if (responseCode > 0 && responseCode < 400) {
+                    // successful response
+                    return;
                 }
 
-                Response response = result.getResponse();
                 if (retryPolicy.shouldRetry(response)) {
                     Thread.sleep(retryPolicy.backoff());
                     requestSupplier.get().send(this);
                 } else {
-                    DatadogUtilities.severe(logger, null, "HTTP request failed: " + result.getRequest() + ", response: " + response);
+                    Throwable failure = result.getFailure();
+                    DatadogUtilities.severe(logger, failure, "HTTP request failed: " + result.getRequest() + ", response: " + response);
                 }
 
             } catch (InterruptedException e) {
@@ -310,4 +328,28 @@ public ResponseProcessingException(String message) {
             super(message);
         }
     }
+
+    private static int getEnv(String envVar, int defaultValue) {
+        String value = System.getenv(envVar);
+        if (value != null) {
+            try {
+                return Integer.parseInt(value);
+            } catch (Exception e) {
+                DatadogUtilities.severe(logger, null, "Invalid value " + value + " provided for env var " + envVar + ": integer number expected");
+            }
+        }
+        return defaultValue;
+    }
+
+    private static double getEnv(String envVar, double defaultValue) {
+        String value = System.getenv(envVar);
+        if (value != null) {
+            try {
+                return Double.parseDouble(value);
+            } catch (Exception e) {
+                DatadogUtilities.severe(logger, null, "Invalid value " + value + " provided for env var " + envVar + ": double number expected");
+            }
+        }
+        return defaultValue;
+    }
 }
diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpRetryPolicy.java b/src/main/java/org/datadog/jenkins/plugins/datadog/clients/HttpRetryPolicy.java
@@ -62,7 +62,11 @@ public boolean shouldRetry(@Nullable Response response) {
     }
 
     int responseCode = response != null ? response.getStatus() : NO_RESPONSE_RECEIVED;
-    if (responseCode == TOO_MANY_REQUESTS_HTTP_CODE) {
+    if (responseCode >= 500 || responseCode == NO_RESPONSE_RECEIVED) {
+      retriesLeft--;
+      return true;
+
+    } else if (responseCode == TOO_MANY_REQUESTS_HTTP_CODE) {
       long waitTimeSeconds = getRateLimitResetTime(response);
       if (waitTimeSeconds == RATE_LIMIT_RESET_TIME_UNDEFINED) {
         retriesLeft--; // doing a regular retry if proper reset time was not provided
@@ -79,10 +83,6 @@ public boolean shouldRetry(@Nullable Response response) {
               + ThreadLocalRandom.current().nextInt(RATE_LIMIT_DELAY_RANDOM_COMPONENT_MAX_MILLIS);
       return true;
 
-    } else if (responseCode >= 500 || responseCode == NO_RESPONSE_RECEIVED) {
-      retriesLeft--;
-      return true;
-
     } else {
       return false;
     }