diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index a52de8c..6b90f8c 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -33,12 +33,10 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install Go uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - - name: Cache Go modules id: cache-go uses: actions/cache@v4 @@ -52,7 +50,6 @@ jobs: shell: bash if: ${{ steps.cache-go.outputs.cache-hit != 'true' }} run: go mod download - - name: Setup Docker working-directory: ${{ env.testdir }} env: @@ -63,14 +60,13 @@ jobs: mkdir -p ./docker/dynamodb/data sudo chmod 777 ./docker/dynamodb/data docker compose up -d - - name: Test working-directory: ${{ env.testdir }} run: | + go install gotest.tools/gotestsum@latest # shellcheck disable=SC2046 - go test -p 4 -parallel 4 -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... - + gotestsum --junitfile unit-tests.xml -- -p 4 -parallel 4 -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ${{ env.testdir }}/coverage.txt + files: ./unit-tests.xml,./coverage.txt diff --git a/aws/awss3/client.go b/aws/awss3/client.go index ac322fc..894f7b3 100644 --- a/aws/awss3/client.go +++ b/aws/awss3/client.go @@ -5,29 +5,14 @@ import ( "context" "encoding/gob" "fmt" - "net" - "net/url" - "sync" + "github.com/88labs/go-utils/aws/awsconfig" + + "github.com/88labs/go-utils/aws/ctxawslocal" "github.com/aws/aws-sdk-go-v2/aws" - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" - smithyendpoints "github.com/aws/smithy-go/endpoints" - - "github.com/88labs/go-utils/aws/awsconfig" - "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" - "github.com/88labs/go-utils/aws/ctxawslocal" -) - -var ( - // GlobalDialer Global http dialer settings for awss3 library - GlobalDialer *s3dialer.ConfGlobalDialer - - customMu sync.Mutex - customEndpoint string - customEndpointClient *s3.Client ) // GetClient @@ -35,34 +20,12 @@ var ( // Using ctxawslocal.WithContext, you can make requests for local mocks func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) { if localProfile, ok := getLocalEndpoint(ctx); ok { - customMu.Lock() - defer customMu.Unlock() - var err error - if customEndpointClient != nil { - return customEndpointClient, err - } - customEndpointClient, err = getClientLocal(ctx, *localProfile) - return customEndpointClient, err - } - awsHttpClient := awshttp.NewBuildableClient() - if GlobalDialer != nil { - awsHttpClient.WithDialerOptions(func(dialer *net.Dialer) { - if GlobalDialer.Timeout != 0 { - dialer.Timeout = GlobalDialer.Timeout - } - if GlobalDialer.Deadline != nil { - dialer.Deadline = *GlobalDialer.Deadline - } - if GlobalDialer.KeepAlive != 0 { - dialer.KeepAlive = GlobalDialer.KeepAlive - } - }) + return getClientLocal(ctx, *localProfile) } // S3 Client awsCfg, err := awsConfig.LoadDefaultConfig( ctx, awsConfig.WithRegion(region.String()), - awsConfig.WithHTTPClient(awsHttpClient), ) if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) @@ -70,64 +33,8 @@ func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) return s3.NewFromConfig(awsCfg), nil } -type staticResolver struct{} - -// ResolveEndpoint -// Local test mocks endpoints to connect to minio -// -// FIXME: EndpointResolverWithOptionsFunc substitutes staticResolver for endpoint mock -func (*staticResolver) ResolveEndpoint(_ context.Context, p s3.EndpointParameters) ( - smithyendpoints.Endpoint, error, -) { - if customEndpoint != "" { - endpoint, err := url.Parse(customEndpoint) - if err != nil { - return smithyendpoints.Endpoint{}, fmt.Errorf("unable to parse endpoint, %w", err) - } - if p.Bucket != nil { - endpoint = endpoint.JoinPath(*p.Bucket) - } - // This value will be used as-is when making the request. - return smithyendpoints.Endpoint{ - URI: *endpoint, - }, nil - } - return smithyendpoints.Endpoint{}, &aws.EndpointNotFoundError{} -} - func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, error) { - // FIXME: EndpointResolverWithOptionsFunc substitutes staticResolver for endpoint mock - // because HostnameImmutable is not enabled. (github.com/aws/aws-sdk-go-v2/config v1.25.4) - // https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/endpoints/ - //customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { - // if service == s3.ServiceID { - // return aws.Endpoint{ - // PartitionID: "aws", - // URL: localProfile.Endpoint, - // SigningRegion: region, - // HostnameImmutable: true, - // }, nil - // } - // // returning EndpointNotFoundError will allow the service to fallback to it's default resolution - // return aws.Endpoint{}, &aws.EndpointNotFoundError{} - //}) - awsHttpClient := awshttp.NewBuildableClient() - if GlobalDialer != nil { - awsHttpClient.WithDialerOptions(func(dialer *net.Dialer) { - if GlobalDialer.Timeout != 0 { - dialer.Timeout = GlobalDialer.Timeout - } - if GlobalDialer.Deadline != nil { - dialer.Deadline = *GlobalDialer.Deadline - } - if GlobalDialer.KeepAlive != 0 { - dialer.KeepAlive = GlobalDialer.KeepAlive - } - }) - } awsCfg, err := awsConfig.LoadDefaultConfig(ctx, - awsConfig.WithHTTPClient(awsHttpClient), - //awsConfig.WithEndpointResolverWithOptions(customResolver), awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: aws.Credentials{ AccessKeyID: localProfile.AccessKey, @@ -139,9 +46,9 @@ func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) } - customEndpoint = localProfile.Endpoint return s3.NewFromConfig(awsCfg, func(o *s3.Options) { - o.EndpointResolverV2 = &staticResolver{} + o.BaseEndpoint = aws.String(localProfile.Endpoint) + o.UsePathStyle = true }), nil } diff --git a/aws/awss3/options/global/global_test.go b/aws/awss3/options/global/global_test.go deleted file mode 100644 index c6387aa..0000000 --- a/aws/awss3/options/global/global_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package global_test - -import ( - "bytes" - "context" - "fmt" - "testing" - "time" - - "github.com/88labs/go-utils/ulid" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/stretchr/testify/assert" - - "github.com/88labs/go-utils/aws/awsconfig" - "github.com/88labs/go-utils/aws/awss3" - "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" - "github.com/88labs/go-utils/aws/ctxawslocal" -) - -const ( - TestBucket = "test" - TestRegion = awsconfig.RegionTokyo -) - -func TestGlobalOptionWithHeadObject(t *testing.T) { - ctx := ctxawslocal.WithContext( - context.Background(), - ctxawslocal.WithS3Endpoint("http://127.0.0.1:29000"), // use Minio - ctxawslocal.WithAccessKey("DUMMYACCESSKEYEXAMPLE"), - ctxawslocal.WithSecretAccessKey("DUMMYSECRETKEYEXAMPLE"), - ) - s3Client, err := awss3.GetClient(ctx, TestRegion) - assert.NoError(t, err) - - createFixture := func(fileSize int) awss3.Key { - key := fmt.Sprintf("awstest/%s.txt", ulid.MustNew()) - uploader := manager.NewUploader(s3Client) - input := s3.PutObjectInput{ - Body: bytes.NewReader(bytes.Repeat([]byte{1}, fileSize)), - Bucket: aws.String(TestBucket), - Key: aws.String(key), - Expires: aws.Time(time.Now().Add(10 * time.Minute)), - } - if _, err := uploader.Upload(ctx, &input); err != nil { - assert.NoError(t, err) - } - return awss3.Key(key) - } - - t.Run("If the option is specified", func(t *testing.T) { - key := createFixture(100) - dialer := s3dialer.NewConfGlobalDialer() - dialer.WithTimeout(time.Second) - dialer.WithKeepAlive(2 * time.Second) - dialer.WithDeadline(time.Now().Add(time.Second)) - awss3.GlobalDialer = dialer - res, err := awss3.HeadObject(ctx, TestRegion, TestBucket, key) - assert.NoError(t, err) - assert.Equal(t, aws.Int64(100), res.ContentLength) - }) -} diff --git a/aws/awss3/options/global/s3dialer/s3dialer.go b/aws/awss3/options/global/s3dialer/s3dialer.go deleted file mode 100644 index 1ae3c04..0000000 --- a/aws/awss3/options/global/s3dialer/s3dialer.go +++ /dev/null @@ -1,50 +0,0 @@ -package s3dialer - -import "time" - -type ConfGlobalDialer struct { - // Timeout is the maximum amount of time a dial will wait for - // a connect to complete. If Deadline is also set, it may fail - // earlier. - // - // The default is no timeout. - // - // When using TCP and dialing a host name with multiple IP - // addresses, the timeout may be divided between them. - // - // With or without a timeout, the operating system may impose - // its own earlier timeout. For instance, TCP timeouts are - // often around 3 minutes. - Timeout time.Duration - - // Deadline is the absolute point in time after which dials - // will fail. If Timeout is set, it may fail earlier. - // Zero means no deadline, or dependent on the operating system - // as with the Timeout option. - Deadline *time.Time - - // KeepAlive specifies the interval between keep-alive - // probes for an active network connection. - // If zero, keep-alive probes are sent with a default value - // (currently 15 seconds), if supported by the protocol and operating - // system. Network protocols or operating systems that do - // not support keep-alives ignore this field. - // If negative, keep-alive probes are disabled. - KeepAlive time.Duration -} - -func NewConfGlobalDialer() *ConfGlobalDialer { - return &ConfGlobalDialer{} -} - -func (c *ConfGlobalDialer) WithTimeout(timeout time.Duration) { - c.Timeout = timeout -} - -func (c *ConfGlobalDialer) WithDeadline(deadline time.Time) { - c.Deadline = &deadline -} - -func (c *ConfGlobalDialer) WithKeepAlive(keepAlive time.Duration) { - c.KeepAlive = keepAlive -} diff --git a/aws/go.mod b/aws/go.mod index b54b08a..e938855 100644 --- a/aws/go.mod +++ b/aws/go.mod @@ -6,11 +6,11 @@ require ( github.com/88labs/go-utils/ulid v0.3.0 github.com/88labs/go-utils/utf8bom v0.4.0 github.com/aws/aws-sdk-go-v2 v1.24.1 - github.com/aws/aws-sdk-go-v2/config v1.26.4 - github.com/aws/aws-sdk-go-v2/credentials v1.16.15 + github.com/aws/aws-sdk-go-v2/config v1.26.5 + github.com/aws/aws-sdk-go-v2/credentials v1.16.16 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.15 github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.15 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.12 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13 github.com/aws/aws-sdk-go-v2/service/cognitoidentity v1.21.7 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.26.9 github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 @@ -37,7 +37,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/aws/go.sum b/aws/go.sum index 98775a4..067c604 100644 --- a/aws/go.sum +++ b/aws/go.sum @@ -8,8 +8,12 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= github.com/aws/aws-sdk-go-v2/config v1.26.4 h1:Juj7LhtxNudNUlfX22K5AnLafO+v4eq9PA3VWSCIQs4= github.com/aws/aws-sdk-go-v2/config v1.26.4/go.mod h1:tioqQ7wvxMYnTDpoTTLHhV3Zh+z261i/f2oz+ds8eNI= +github.com/aws/aws-sdk-go-v2/config v1.26.5 h1:lodGSevz7d+kkFJodfauThRxK9mdJbyutUxGq1NNhvw= +github.com/aws/aws-sdk-go-v2/config v1.26.5/go.mod h1:DxHrz6diQJOc9EwDslVRh84VjjrE17g+pVZXUeSxaDU= github.com/aws/aws-sdk-go-v2/credentials v1.16.15 h1:P0/m1LU08MF2kRzx4P//+7lNjiJod1z4xI2WpWhdpTQ= github.com/aws/aws-sdk-go-v2/credentials v1.16.15/go.mod h1:pgtMCf7Dx4GWw5EpHOTc2Sy17LIP0A0N2C9nQ83pQ/0= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.15 h1:34/YcQBavs4hTUqy9wq3k7f8b+eQqvKDHRithYOD4Gw= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.15/go.mod h1:DbKcs3L/AKDeVNZzKOttxLu/x9bm4VKercDGy42/AGo= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.15 h1:nNOuo7q+hjvf97f5dmnv84IwsySeIyzFdGb5Jyrspjw= @@ -18,6 +22,8 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tC github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.12 h1:0FMZy36RSYvcvVzEf1xbNdebLHZewW40QWP+P8jCMVk= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.12/go.mod h1:+chyahvarkb3HibkNei9IQEM9P5cWD5w2kgXCa3Hh0I= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13 h1:8Nt4LBUEKV0FxLBO2BmRzDKax3hp2LRMKySMBwL4vMc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13/go.mod h1:t5QEDu/FBJJM4kslbQlTSpYtnhoWDNmHSsgQojIxE0o= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= @@ -48,6 +54,8 @@ github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 h1:tRNrFDGRm81e6nTX5Q4CFblea99e github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7/go.mod h1:8GWUDux5Z2h6z2efAtr54RdHXtLm8sq7Rg85ZNY/CZM= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=