From bb9d781823d7920292cc4f46e4f29567e203591c Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Fri, 19 Jan 2024 12:31:52 +0900 Subject: [PATCH] add global s3dialer --- aws/awss3/client.go | 57 +++++++++++++++-- aws/awss3/options/global/global_test.go | 63 +++++++++++++++++++ aws/awss3/options/global/s3dialer/s3dialer.go | 50 +++++++++++++++ 3 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 aws/awss3/options/global/global_test.go create mode 100644 aws/awss3/options/global/s3dialer/s3dialer.go diff --git a/aws/awss3/client.go b/aws/awss3/client.go index 894f7b3..aefec16 100644 --- a/aws/awss3/client.go +++ b/aws/awss3/client.go @@ -5,14 +5,26 @@ import ( "context" "encoding/gob" "fmt" + "net" + "sync" - "github.com/88labs/go-utils/aws/awsconfig" - - "github.com/88labs/go-utils/aws/ctxawslocal" "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/88labs/go-utils/aws/awsconfig" + "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" + "github.com/88labs/go-utils/aws/ctxawslocal" +) + +var ( + // GlobalDialer Global http dialer settings for awss3 library + GlobalDialer *s3dialer.ConfGlobalDialer + + customMu sync.Mutex + customEndpointClient *s3.Client ) // GetClient @@ -20,12 +32,34 @@ import ( // Using ctxawslocal.WithContext, you can make requests for local mocks func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) { if localProfile, ok := getLocalEndpoint(ctx); ok { - return getClientLocal(ctx, *localProfile) + customMu.Lock() + defer customMu.Unlock() + var err error + if customEndpointClient != nil { + return customEndpointClient, err + } + customEndpointClient, err = getClientLocal(ctx, *localProfile) + return customEndpointClient, err + } + awsHttpClient := awshttp.NewBuildableClient() + if GlobalDialer != nil { + awsHttpClient.WithDialerOptions(func(dialer *net.Dialer) { + if GlobalDialer.Timeout != 0 { + dialer.Timeout = GlobalDialer.Timeout + } + if GlobalDialer.Deadline != nil { + dialer.Deadline = *GlobalDialer.Deadline + } + if GlobalDialer.KeepAlive != 0 { + dialer.KeepAlive = GlobalDialer.KeepAlive + } + }) } // S3 Client awsCfg, err := awsConfig.LoadDefaultConfig( ctx, awsConfig.WithRegion(region.String()), + awsConfig.WithHTTPClient(awsHttpClient), ) if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) @@ -34,7 +68,22 @@ func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) } func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, error) { + awsHttpClient := awshttp.NewBuildableClient() + if GlobalDialer != nil { + awsHttpClient.WithDialerOptions(func(dialer *net.Dialer) { + if GlobalDialer.Timeout != 0 { + dialer.Timeout = GlobalDialer.Timeout + } + if GlobalDialer.Deadline != nil { + dialer.Deadline = *GlobalDialer.Deadline + } + if GlobalDialer.KeepAlive != 0 { + dialer.KeepAlive = GlobalDialer.KeepAlive + } + }) + } awsCfg, err := awsConfig.LoadDefaultConfig(ctx, + awsConfig.WithHTTPClient(awsHttpClient), awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: aws.Credentials{ AccessKeyID: localProfile.AccessKey, diff --git a/aws/awss3/options/global/global_test.go b/aws/awss3/options/global/global_test.go new file mode 100644 index 0000000..c6387aa --- /dev/null +++ b/aws/awss3/options/global/global_test.go @@ -0,0 +1,63 @@ +package global_test + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + "github.com/88labs/go-utils/ulid" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + + "github.com/88labs/go-utils/aws/awsconfig" + "github.com/88labs/go-utils/aws/awss3" + "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" + "github.com/88labs/go-utils/aws/ctxawslocal" +) + +const ( + TestBucket = "test" + TestRegion = awsconfig.RegionTokyo +) + +func TestGlobalOptionWithHeadObject(t *testing.T) { + ctx := ctxawslocal.WithContext( + context.Background(), + ctxawslocal.WithS3Endpoint("http://127.0.0.1:29000"), // use Minio + ctxawslocal.WithAccessKey("DUMMYACCESSKEYEXAMPLE"), + ctxawslocal.WithSecretAccessKey("DUMMYSECRETKEYEXAMPLE"), + ) + s3Client, err := awss3.GetClient(ctx, TestRegion) + assert.NoError(t, err) + + createFixture := func(fileSize int) awss3.Key { + key := fmt.Sprintf("awstest/%s.txt", ulid.MustNew()) + uploader := manager.NewUploader(s3Client) + input := s3.PutObjectInput{ + Body: bytes.NewReader(bytes.Repeat([]byte{1}, fileSize)), + Bucket: aws.String(TestBucket), + Key: aws.String(key), + Expires: aws.Time(time.Now().Add(10 * time.Minute)), + } + if _, err := uploader.Upload(ctx, &input); err != nil { + assert.NoError(t, err) + } + return awss3.Key(key) + } + + t.Run("If the option is specified", func(t *testing.T) { + key := createFixture(100) + dialer := s3dialer.NewConfGlobalDialer() + dialer.WithTimeout(time.Second) + dialer.WithKeepAlive(2 * time.Second) + dialer.WithDeadline(time.Now().Add(time.Second)) + awss3.GlobalDialer = dialer + res, err := awss3.HeadObject(ctx, TestRegion, TestBucket, key) + assert.NoError(t, err) + assert.Equal(t, aws.Int64(100), res.ContentLength) + }) +} diff --git a/aws/awss3/options/global/s3dialer/s3dialer.go b/aws/awss3/options/global/s3dialer/s3dialer.go new file mode 100644 index 0000000..1ae3c04 --- /dev/null +++ b/aws/awss3/options/global/s3dialer/s3dialer.go @@ -0,0 +1,50 @@ +package s3dialer + +import "time" + +type ConfGlobalDialer struct { + // Timeout is the maximum amount of time a dial will wait for + // a connect to complete. If Deadline is also set, it may fail + // earlier. + // + // The default is no timeout. + // + // When using TCP and dialing a host name with multiple IP + // addresses, the timeout may be divided between them. + // + // With or without a timeout, the operating system may impose + // its own earlier timeout. For instance, TCP timeouts are + // often around 3 minutes. + Timeout time.Duration + + // Deadline is the absolute point in time after which dials + // will fail. If Timeout is set, it may fail earlier. + // Zero means no deadline, or dependent on the operating system + // as with the Timeout option. + Deadline *time.Time + + // KeepAlive specifies the interval between keep-alive + // probes for an active network connection. + // If zero, keep-alive probes are sent with a default value + // (currently 15 seconds), if supported by the protocol and operating + // system. Network protocols or operating systems that do + // not support keep-alives ignore this field. + // If negative, keep-alive probes are disabled. + KeepAlive time.Duration +} + +func NewConfGlobalDialer() *ConfGlobalDialer { + return &ConfGlobalDialer{} +} + +func (c *ConfGlobalDialer) WithTimeout(timeout time.Duration) { + c.Timeout = timeout +} + +func (c *ConfGlobalDialer) WithDeadline(deadline time.Time) { + c.Deadline = &deadline +} + +func (c *ConfGlobalDialer) WithKeepAlive(keepAlive time.Duration) { + c.KeepAlive = keepAlive +}