Skip to content

Commit

Permalink
[fix](cloud) Fix aws sdk default retry policy not taking effect (apac…
Browse files Browse the repository at this point in the history
…he#43575)

* AWS SDK default retry policy check response xml `code` to determine
retry or not, but other object storage such as `OSS`, `COS` not
compatible with AWS S3, so adding a custome retry policy for solving the
problem
  • Loading branch information
SWJTU-ZhangLei authored Nov 12, 2024
1 parent d05487c commit 936ad96
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 5 deletions.
5 changes: 3 additions & 2 deletions be/src/util/s3_util.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
#include "cpp/obj_retry_strategy.h"
#include "cpp/sync_point.h"
#ifdef USE_AZURE
#include "io/fs/azure_obj_storage_client.h"
Expand Down Expand Up @@ -307,8 +308,8 @@ std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_s3_client(
aws_config.scheme = Aws::Http::Scheme::HTTP;
}

aws_config.retryStrategy =
std::make_shared<Aws::Client::DefaultRetryStrategy>(config::max_s3_client_retry);
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
config::max_s3_client_retry /*scaleFactor = 25*/);
std::shared_ptr<Aws::S3::S3Client> new_client;
if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
Expand Down
11 changes: 8 additions & 3 deletions common/cpp/obj_retry_strategy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "obj_retry_strategy.h"

#include <aws/core/http/HttpResponse.h>
#include <bvar/reducer.h>

namespace doris {
Expand All @@ -29,12 +30,16 @@ S3CustomRetryStrategy::~S3CustomRetryStrategy() = default;

bool S3CustomRetryStrategy::ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error,
long attemptedRetries) const {
if (attemptedRetries < m_maxRetries &&
error.GetResponseCode() == Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS) {
if (attemptedRetries >= m_maxRetries) {
return false;
}

if (Aws::Http::IsRetryableHttpResponseCode(error.GetResponseCode()) || error.ShouldRetry()) {
s3_too_many_request_retry_cnt << 1;
return true;
}
return Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);

return false;
}
#ifdef USE_AZURE
AzureRetryRecordPolicy::AzureRetryRecordPolicy(int retry_cnt) : retry_cnt(retry_cnt) {}
Expand Down

0 comments on commit 936ad96

Please sign in to comment.