From 99e0929de974a630065dda2a552edca13716b2bb Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 18:26:36 +0900 Subject: [PATCH 01/16] fix: awss3 client mock use customResolver --- aws/awss3/client.go | 82 ++++++++++----------------------------------- 1 file changed, 18 insertions(+), 64 deletions(-) diff --git a/aws/awss3/client.go b/aws/awss3/client.go index ac322fc..eb8fe8c 100644 --- a/aws/awss3/client.go +++ b/aws/awss3/client.go @@ -6,28 +6,20 @@ import ( "encoding/gob" "fmt" "net" - "net/url" - "sync" + "github.com/88labs/go-utils/aws/awsconfig" + "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" + "github.com/88labs/go-utils/aws/ctxawslocal" "github.com/aws/aws-sdk-go-v2/aws" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" - smithyendpoints "github.com/aws/smithy-go/endpoints" - - "github.com/88labs/go-utils/aws/awsconfig" - "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" - "github.com/88labs/go-utils/aws/ctxawslocal" ) var ( // GlobalDialer Global http dialer settings for awss3 library GlobalDialer *s3dialer.ConfGlobalDialer - - customMu sync.Mutex - customEndpoint string - customEndpointClient *s3.Client ) // GetClient @@ -35,14 +27,7 @@ var ( // Using ctxawslocal.WithContext, you can make requests for local mocks func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) { if localProfile, ok := getLocalEndpoint(ctx); ok { - customMu.Lock() - defer customMu.Unlock() - var err error - if customEndpointClient != nil { - return customEndpointClient, err - } - customEndpointClient, err = getClientLocal(ctx, *localProfile) - return customEndpointClient, err + return getClientLocal(ctx, *localProfile) } awsHttpClient := awshttp.NewBuildableClient() if GlobalDialer != nil { @@ -70,47 +55,19 @@ func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) return s3.NewFromConfig(awsCfg), nil } -type staticResolver struct{} - -// ResolveEndpoint -// Local test mocks endpoints to connect to minio -// -// FIXME: EndpointResolverWithOptionsFunc substitutes staticResolver for endpoint mock -func (*staticResolver) ResolveEndpoint(_ context.Context, p s3.EndpointParameters) ( - smithyendpoints.Endpoint, error, -) { - if customEndpoint != "" { - endpoint, err := url.Parse(customEndpoint) - if err != nil { - return smithyendpoints.Endpoint{}, fmt.Errorf("unable to parse endpoint, %w", err) - } - if p.Bucket != nil { - endpoint = endpoint.JoinPath(*p.Bucket) - } - // This value will be used as-is when making the request. - return smithyendpoints.Endpoint{ - URI: *endpoint, - }, nil - } - return smithyendpoints.Endpoint{}, &aws.EndpointNotFoundError{} -} - func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, error) { - // FIXME: EndpointResolverWithOptionsFunc substitutes staticResolver for endpoint mock - // because HostnameImmutable is not enabled. (github.com/aws/aws-sdk-go-v2/config v1.25.4) - // https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/endpoints/ - //customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { - // if service == s3.ServiceID { - // return aws.Endpoint{ - // PartitionID: "aws", - // URL: localProfile.Endpoint, - // SigningRegion: region, - // HostnameImmutable: true, - // }, nil - // } - // // returning EndpointNotFoundError will allow the service to fallback to it's default resolution - // return aws.Endpoint{}, &aws.EndpointNotFoundError{} - //}) + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + if service == s3.ServiceID { + return aws.Endpoint{ + PartitionID: "aws", + URL: localProfile.Endpoint, + SigningRegion: region, + HostnameImmutable: true, + }, nil + } + // returning EndpointNotFoundError will allow the service to fallback to it's default resolution + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + }) awsHttpClient := awshttp.NewBuildableClient() if GlobalDialer != nil { awsHttpClient.WithDialerOptions(func(dialer *net.Dialer) { @@ -127,7 +84,7 @@ func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, } awsCfg, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithHTTPClient(awsHttpClient), - //awsConfig.WithEndpointResolverWithOptions(customResolver), + awsConfig.WithEndpointResolverWithOptions(customResolver), awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: aws.Credentials{ AccessKeyID: localProfile.AccessKey, @@ -139,10 +96,7 @@ func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) } - customEndpoint = localProfile.Endpoint - return s3.NewFromConfig(awsCfg, func(o *s3.Options) { - o.EndpointResolverV2 = &staticResolver{} - }), nil + return s3.NewFromConfig(awsCfg), nil } type LocalProfile struct { From dd7f39e9de573822f39167741837f5aab0c6c16e Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 18:46:59 +0900 Subject: [PATCH 02/16] remove test parallels --- .github/workflows/test-aws.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index a52de8c..32262bd 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -68,7 +68,7 @@ jobs: working-directory: ${{ env.testdir }} run: | # shellcheck disable=SC2046 - go test -p 4 -parallel 4 -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... + go test -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... - uses: codecov/codecov-action@v3 with: From d6cbff40bb6dea2dab937040c4a0cef293b3e2ad Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 18:56:16 +0900 Subject: [PATCH 03/16] remove global dialer --- .github/workflows/test-aws.yaml | 2 +- aws/awss3/client.go | 38 ----------- aws/awss3/options/global/global_test.go | 63 ------------------- aws/awss3/options/global/s3dialer/s3dialer.go | 50 --------------- 4 files changed, 1 insertion(+), 152 deletions(-) delete mode 100644 aws/awss3/options/global/global_test.go delete mode 100644 aws/awss3/options/global/s3dialer/s3dialer.go diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index 32262bd..a52de8c 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -68,7 +68,7 @@ jobs: working-directory: ${{ env.testdir }} run: | # shellcheck disable=SC2046 - go test -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... + go test -p 4 -parallel 4 -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... - uses: codecov/codecov-action@v3 with: diff --git a/aws/awss3/client.go b/aws/awss3/client.go index eb8fe8c..9893e0e 100644 --- a/aws/awss3/client.go +++ b/aws/awss3/client.go @@ -5,23 +5,15 @@ import ( "context" "encoding/gob" "fmt" - "net" "github.com/88labs/go-utils/aws/awsconfig" - "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" "github.com/88labs/go-utils/aws/ctxawslocal" "github.com/aws/aws-sdk-go-v2/aws" - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" ) -var ( - // GlobalDialer Global http dialer settings for awss3 library - GlobalDialer *s3dialer.ConfGlobalDialer -) - // GetClient // Get s3 client for aws-sdk-go v2. // Using ctxawslocal.WithContext, you can make requests for local mocks @@ -29,25 +21,10 @@ func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) if localProfile, ok := getLocalEndpoint(ctx); ok { return getClientLocal(ctx, *localProfile) } - awsHttpClient := awshttp.NewBuildableClient() - if GlobalDialer != nil { - awsHttpClient.WithDialerOptions(func(dialer *net.Dialer) { - if GlobalDialer.Timeout != 0 { - dialer.Timeout = GlobalDialer.Timeout - } - if GlobalDialer.Deadline != nil { - dialer.Deadline = *GlobalDialer.Deadline - } - if GlobalDialer.KeepAlive != 0 { - dialer.KeepAlive = GlobalDialer.KeepAlive - } - }) - } // S3 Client awsCfg, err := awsConfig.LoadDefaultConfig( ctx, awsConfig.WithRegion(region.String()), - awsConfig.WithHTTPClient(awsHttpClient), ) if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) @@ -68,22 +45,7 @@ func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, // returning EndpointNotFoundError will allow the service to fallback to it's default resolution return aws.Endpoint{}, &aws.EndpointNotFoundError{} }) - awsHttpClient := awshttp.NewBuildableClient() - if GlobalDialer != nil { - awsHttpClient.WithDialerOptions(func(dialer *net.Dialer) { - if GlobalDialer.Timeout != 0 { - dialer.Timeout = GlobalDialer.Timeout - } - if GlobalDialer.Deadline != nil { - dialer.Deadline = *GlobalDialer.Deadline - } - if GlobalDialer.KeepAlive != 0 { - dialer.KeepAlive = GlobalDialer.KeepAlive - } - }) - } awsCfg, err := awsConfig.LoadDefaultConfig(ctx, - awsConfig.WithHTTPClient(awsHttpClient), awsConfig.WithEndpointResolverWithOptions(customResolver), awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: aws.Credentials{ diff --git a/aws/awss3/options/global/global_test.go b/aws/awss3/options/global/global_test.go deleted file mode 100644 index c6387aa..0000000 --- a/aws/awss3/options/global/global_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package global_test - -import ( - "bytes" - "context" - "fmt" - "testing" - "time" - - "github.com/88labs/go-utils/ulid" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/stretchr/testify/assert" - - "github.com/88labs/go-utils/aws/awsconfig" - "github.com/88labs/go-utils/aws/awss3" - "github.com/88labs/go-utils/aws/awss3/options/global/s3dialer" - "github.com/88labs/go-utils/aws/ctxawslocal" -) - -const ( - TestBucket = "test" - TestRegion = awsconfig.RegionTokyo -) - -func TestGlobalOptionWithHeadObject(t *testing.T) { - ctx := ctxawslocal.WithContext( - context.Background(), - ctxawslocal.WithS3Endpoint("http://127.0.0.1:29000"), // use Minio - ctxawslocal.WithAccessKey("DUMMYACCESSKEYEXAMPLE"), - ctxawslocal.WithSecretAccessKey("DUMMYSECRETKEYEXAMPLE"), - ) - s3Client, err := awss3.GetClient(ctx, TestRegion) - assert.NoError(t, err) - - createFixture := func(fileSize int) awss3.Key { - key := fmt.Sprintf("awstest/%s.txt", ulid.MustNew()) - uploader := manager.NewUploader(s3Client) - input := s3.PutObjectInput{ - Body: bytes.NewReader(bytes.Repeat([]byte{1}, fileSize)), - Bucket: aws.String(TestBucket), - Key: aws.String(key), - Expires: aws.Time(time.Now().Add(10 * time.Minute)), - } - if _, err := uploader.Upload(ctx, &input); err != nil { - assert.NoError(t, err) - } - return awss3.Key(key) - } - - t.Run("If the option is specified", func(t *testing.T) { - key := createFixture(100) - dialer := s3dialer.NewConfGlobalDialer() - dialer.WithTimeout(time.Second) - dialer.WithKeepAlive(2 * time.Second) - dialer.WithDeadline(time.Now().Add(time.Second)) - awss3.GlobalDialer = dialer - res, err := awss3.HeadObject(ctx, TestRegion, TestBucket, key) - assert.NoError(t, err) - assert.Equal(t, aws.Int64(100), res.ContentLength) - }) -} diff --git a/aws/awss3/options/global/s3dialer/s3dialer.go b/aws/awss3/options/global/s3dialer/s3dialer.go deleted file mode 100644 index 1ae3c04..0000000 --- a/aws/awss3/options/global/s3dialer/s3dialer.go +++ /dev/null @@ -1,50 +0,0 @@ -package s3dialer - -import "time" - -type ConfGlobalDialer struct { - // Timeout is the maximum amount of time a dial will wait for - // a connect to complete. If Deadline is also set, it may fail - // earlier. - // - // The default is no timeout. - // - // When using TCP and dialing a host name with multiple IP - // addresses, the timeout may be divided between them. - // - // With or without a timeout, the operating system may impose - // its own earlier timeout. For instance, TCP timeouts are - // often around 3 minutes. - Timeout time.Duration - - // Deadline is the absolute point in time after which dials - // will fail. If Timeout is set, it may fail earlier. - // Zero means no deadline, or dependent on the operating system - // as with the Timeout option. - Deadline *time.Time - - // KeepAlive specifies the interval between keep-alive - // probes for an active network connection. - // If zero, keep-alive probes are sent with a default value - // (currently 15 seconds), if supported by the protocol and operating - // system. Network protocols or operating systems that do - // not support keep-alives ignore this field. - // If negative, keep-alive probes are disabled. - KeepAlive time.Duration -} - -func NewConfGlobalDialer() *ConfGlobalDialer { - return &ConfGlobalDialer{} -} - -func (c *ConfGlobalDialer) WithTimeout(timeout time.Duration) { - c.Timeout = timeout -} - -func (c *ConfGlobalDialer) WithDeadline(deadline time.Time) { - c.Deadline = &deadline -} - -func (c *ConfGlobalDialer) WithKeepAlive(keepAlive time.Duration) { - c.KeepAlive = keepAlive -} From 1ed2ebd90acfa35fc41a6cbb9d9e8f03e2e2fec1 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:05:43 +0900 Subject: [PATCH 04/16] change docker compose --- .github/workflows/test-aws.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index a52de8c..30097e2 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -54,15 +54,9 @@ jobs: run: go mod download - name: Setup Docker - working-directory: ${{ env.testdir }} - env: - DOCKER_BUILDKIT: 1 - run: | - # Create the directory for the volume of dynamodb in advance, otherwise permission error will occur. - # https://stackoverflow.com/questions/45850688/unable-to-open-local-dynamodb-database-file-after-power-outage - mkdir -p ./docker/dynamodb/data - sudo chmod 777 ./docker/dynamodb/data - docker compose up -d + uses: actions/setup-go@v5 + with: + compose-file: %{{ env.testdir }}/docker-compose.yml - name: Test working-directory: ${{ env.testdir }} From d74073be2b64f322dd846220c09d6d1913f51bb3 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:07:47 +0900 Subject: [PATCH 05/16] change docker compose --- .github/workflows/test-aws.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index 30097e2..9be17fd 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -54,7 +54,7 @@ jobs: run: go mod download - name: Setup Docker - uses: actions/setup-go@v5 + uses: isbang/compose-action@v1.5.1 with: compose-file: %{{ env.testdir }}/docker-compose.yml From 07e8140e02b9d838d9261085f3c9a53e8e892c04 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:10:23 +0900 Subject: [PATCH 06/16] change docker compose --- .github/workflows/test-aws.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index 9be17fd..5ab707b 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -56,7 +56,7 @@ jobs: - name: Setup Docker uses: isbang/compose-action@v1.5.1 with: - compose-file: %{{ env.testdir }}/docker-compose.yml + compose-file: ${{ env.testdir }}/docker-compose.yml - name: Test working-directory: ${{ env.testdir }} From 796a8f1f49f0ce884f27862a252b78b6d97adfcb Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:11:23 +0900 Subject: [PATCH 07/16] change docker compose --- .github/workflows/test-aws.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index 5ab707b..e744191 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -56,7 +56,7 @@ jobs: - name: Setup Docker uses: isbang/compose-action@v1.5.1 with: - compose-file: ${{ env.testdir }}/docker-compose.yml + compose-file: ${{ env.testdir }}/docker-compose.yaml - name: Test working-directory: ${{ env.testdir }} From 7a344f544507371a25dbf0ce5e53d40b960573b2 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:19:46 +0900 Subject: [PATCH 08/16] change docker compose --- .github/workflows/test-aws.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index e744191..d047a69 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -57,6 +57,7 @@ jobs: uses: isbang/compose-action@v1.5.1 with: compose-file: ${{ env.testdir }}/docker-compose.yaml + up-flags: --detach - name: Test working-directory: ${{ env.testdir }} From 4d417911ce42968cffbcddb35cde0427a60badd3 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:31:08 +0900 Subject: [PATCH 09/16] change docker compose --- .github/workflows/test-aws.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index d047a69..db0a753 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -33,12 +33,10 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install Go uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - - name: Cache Go modules id: cache-go uses: actions/cache@v4 @@ -52,20 +50,18 @@ jobs: shell: bash if: ${{ steps.cache-go.outputs.cache-hit != 'true' }} run: go mod download - - name: Setup Docker uses: isbang/compose-action@v1.5.1 with: compose-file: ${{ env.testdir }}/docker-compose.yaml up-flags: --detach - - name: Test working-directory: ${{ env.testdir }} run: | + go install gotest.tools/gotestsum@latest # shellcheck disable=SC2046 - go test -p 4 -parallel 4 -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... - + gotestsum --junitfile unit-tests.xml -- -p 4 -parallel 4 -v ./... -race -coverprofile="coverage.txt" -covermode=atomic -coverpkg=./... - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ${{ env.testdir }}/coverage.txt + files: ./unit-tests.xml,./coverage.txt From 45be9e907bf020eaa87e8f89e8dd27a39545ddd6 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:37:09 +0900 Subject: [PATCH 10/16] change docker compose --- .github/workflows/test-aws.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index db0a753..efa7458 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -54,7 +54,13 @@ jobs: uses: isbang/compose-action@v1.5.1 with: compose-file: ${{ env.testdir }}/docker-compose.yaml - up-flags: --detach + up-flags: -d + services: | + elasticmq + minio + miniocreatebuckets + dynamodb + dynamodbcreatetable - name: Test working-directory: ${{ env.testdir }} run: | From 5ca6d07945594537d6a3bb672fd11a06f035cc9a Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:37:24 +0900 Subject: [PATCH 11/16] change docker compose --- .github/workflows/test-aws.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index efa7458..160d1a3 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -54,7 +54,7 @@ jobs: uses: isbang/compose-action@v1.5.1 with: compose-file: ${{ env.testdir }}/docker-compose.yaml - up-flags: -d + up-flags: "-d" services: | elasticmq minio From 3f6734989ff6b4c69179196a78ec838e1ac33f18 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Thu, 18 Jan 2024 19:44:47 +0900 Subject: [PATCH 12/16] change docker compose --- .github/workflows/test-aws.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-aws.yaml b/.github/workflows/test-aws.yaml index 160d1a3..6b90f8c 100644 --- a/.github/workflows/test-aws.yaml +++ b/.github/workflows/test-aws.yaml @@ -51,16 +51,15 @@ jobs: if: ${{ steps.cache-go.outputs.cache-hit != 'true' }} run: go mod download - name: Setup Docker - uses: isbang/compose-action@v1.5.1 - with: - compose-file: ${{ env.testdir }}/docker-compose.yaml - up-flags: "-d" - services: | - elasticmq - minio - miniocreatebuckets - dynamodb - dynamodbcreatetable + working-directory: ${{ env.testdir }} + env: + DOCKER_BUILDKIT: 1 + run: | + # Create the directory for the volume of dynamodb in advance, otherwise permission error will occur. + # https://stackoverflow.com/questions/45850688/unable-to-open-local-dynamodb-database-file-after-power-outage + mkdir -p ./docker/dynamodb/data + sudo chmod 777 ./docker/dynamodb/data + docker compose up -d - name: Test working-directory: ${{ env.testdir }} run: | From 666d78b45208b43c47c719955cb75e446ee4b37a Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Fri, 19 Jan 2024 10:27:32 +0900 Subject: [PATCH 13/16] update gomod --- aws/go.mod | 8 ++++---- aws/go.sum | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/aws/go.mod b/aws/go.mod index b54b08a..e938855 100644 --- a/aws/go.mod +++ b/aws/go.mod @@ -6,11 +6,11 @@ require ( github.com/88labs/go-utils/ulid v0.3.0 github.com/88labs/go-utils/utf8bom v0.4.0 github.com/aws/aws-sdk-go-v2 v1.24.1 - github.com/aws/aws-sdk-go-v2/config v1.26.4 - github.com/aws/aws-sdk-go-v2/credentials v1.16.15 + github.com/aws/aws-sdk-go-v2/config v1.26.5 + github.com/aws/aws-sdk-go-v2/credentials v1.16.16 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.15 github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.15 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.12 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13 github.com/aws/aws-sdk-go-v2/service/cognitoidentity v1.21.7 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.26.9 github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 @@ -37,7 +37,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/aws/go.sum b/aws/go.sum index 98775a4..067c604 100644 --- a/aws/go.sum +++ b/aws/go.sum @@ -8,8 +8,12 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= github.com/aws/aws-sdk-go-v2/config v1.26.4 h1:Juj7LhtxNudNUlfX22K5AnLafO+v4eq9PA3VWSCIQs4= github.com/aws/aws-sdk-go-v2/config v1.26.4/go.mod h1:tioqQ7wvxMYnTDpoTTLHhV3Zh+z261i/f2oz+ds8eNI= +github.com/aws/aws-sdk-go-v2/config v1.26.5 h1:lodGSevz7d+kkFJodfauThRxK9mdJbyutUxGq1NNhvw= +github.com/aws/aws-sdk-go-v2/config v1.26.5/go.mod h1:DxHrz6diQJOc9EwDslVRh84VjjrE17g+pVZXUeSxaDU= github.com/aws/aws-sdk-go-v2/credentials v1.16.15 h1:P0/m1LU08MF2kRzx4P//+7lNjiJod1z4xI2WpWhdpTQ= github.com/aws/aws-sdk-go-v2/credentials v1.16.15/go.mod h1:pgtMCf7Dx4GWw5EpHOTc2Sy17LIP0A0N2C9nQ83pQ/0= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.15 h1:34/YcQBavs4hTUqy9wq3k7f8b+eQqvKDHRithYOD4Gw= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.15/go.mod h1:DbKcs3L/AKDeVNZzKOttxLu/x9bm4VKercDGy42/AGo= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.15 h1:nNOuo7q+hjvf97f5dmnv84IwsySeIyzFdGb5Jyrspjw= @@ -18,6 +22,8 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tC github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.12 h1:0FMZy36RSYvcvVzEf1xbNdebLHZewW40QWP+P8jCMVk= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.12/go.mod h1:+chyahvarkb3HibkNei9IQEM9P5cWD5w2kgXCa3Hh0I= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13 h1:8Nt4LBUEKV0FxLBO2BmRzDKax3hp2LRMKySMBwL4vMc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13/go.mod h1:t5QEDu/FBJJM4kslbQlTSpYtnhoWDNmHSsgQojIxE0o= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= @@ -48,6 +54,8 @@ github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 h1:tRNrFDGRm81e6nTX5Q4CFblea99e github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7/go.mod h1:8GWUDux5Z2h6z2efAtr54RdHXtLm8sq7Rg85ZNY/CZM= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= From e46c95ea08cbeed7c46d47774dae16781e1489a8 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Fri, 19 Jan 2024 10:31:07 +0900 Subject: [PATCH 14/16] change BaseEndpoint --- aws/awss3/client.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/aws/awss3/client.go b/aws/awss3/client.go index 9893e0e..0394f6f 100644 --- a/aws/awss3/client.go +++ b/aws/awss3/client.go @@ -33,20 +33,7 @@ func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) } func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, error) { - customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { - if service == s3.ServiceID { - return aws.Endpoint{ - PartitionID: "aws", - URL: localProfile.Endpoint, - SigningRegion: region, - HostnameImmutable: true, - }, nil - } - // returning EndpointNotFoundError will allow the service to fallback to it's default resolution - return aws.Endpoint{}, &aws.EndpointNotFoundError{} - }) awsCfg, err := awsConfig.LoadDefaultConfig(ctx, - awsConfig.WithEndpointResolverWithOptions(customResolver), awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: aws.Credentials{ AccessKeyID: localProfile.AccessKey, @@ -58,7 +45,9 @@ func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) } - return s3.NewFromConfig(awsCfg), nil + return s3.NewFromConfig(awsCfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(localProfile.Endpoint) + }), nil } type LocalProfile struct { From 367d7f48706082c16863aa29648a2897cda726ed Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Fri, 19 Jan 2024 12:08:51 +0900 Subject: [PATCH 15/16] change BaseEndpoint --- aws/awss3/client.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/aws/awss3/client.go b/aws/awss3/client.go index 0394f6f..d7c888d 100644 --- a/aws/awss3/client.go +++ b/aws/awss3/client.go @@ -5,6 +5,7 @@ import ( "context" "encoding/gob" "fmt" + "net/url" "github.com/88labs/go-utils/aws/awsconfig" "github.com/88labs/go-utils/aws/ctxawslocal" @@ -12,6 +13,7 @@ import ( awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + smithyendpoints "github.com/aws/smithy-go/endpoints" ) // GetClient @@ -32,6 +34,18 @@ func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) return s3.NewFromConfig(awsCfg), nil } +type staticResolver struct { + endpoint url.URL +} + +func (r *staticResolver) ResolveEndpoint(_ context.Context, _ s3.EndpointParameters) ( + smithyendpoints.Endpoint, error, +) { + return smithyendpoints.Endpoint{ + URI: r.endpoint, + }, nil +} + func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, error) { awsCfg, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ @@ -45,8 +59,23 @@ func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) } + //u, err := url.Parse(localProfile.Endpoint) + //if err != nil { + // return nil, fmt.Errorf("custom endpoint parse error, %w", err) + //} return s3.NewFromConfig(awsCfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(localProfile.Endpoint) + //o.EndpointResolverV2 = &staticResolver{ + // endpoint: *u, + //} + o.UsePathStyle = true + //o.Credentials = credentials.StaticCredentialsProvider{ + // Value: aws.Credentials{ + // AccessKeyID: localProfile.AccessKey, + // SecretAccessKey: localProfile.SecretAccessKey, + // SessionToken: localProfile.SessionToken, + // }, + //} }), nil } From 4f04567b5801436af43db1c5ebfad9c946c33319 Mon Sep 17 00:00:00 2001 From: tom twinkle Date: Fri, 19 Jan 2024 12:12:26 +0900 Subject: [PATCH 16/16] change BaseEndpoint --- aws/awss3/client.go | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/aws/awss3/client.go b/aws/awss3/client.go index d7c888d..894f7b3 100644 --- a/aws/awss3/client.go +++ b/aws/awss3/client.go @@ -5,15 +5,14 @@ import ( "context" "encoding/gob" "fmt" - "net/url" "github.com/88labs/go-utils/aws/awsconfig" + "github.com/88labs/go-utils/aws/ctxawslocal" "github.com/aws/aws-sdk-go-v2/aws" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" - smithyendpoints "github.com/aws/smithy-go/endpoints" ) // GetClient @@ -34,18 +33,6 @@ func GetClient(ctx context.Context, region awsconfig.Region) (*s3.Client, error) return s3.NewFromConfig(awsCfg), nil } -type staticResolver struct { - endpoint url.URL -} - -func (r *staticResolver) ResolveEndpoint(_ context.Context, _ s3.EndpointParameters) ( - smithyendpoints.Endpoint, error, -) { - return smithyendpoints.Endpoint{ - URI: r.endpoint, - }, nil -} - func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, error) { awsCfg, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ @@ -59,23 +46,9 @@ func getClientLocal(ctx context.Context, localProfile LocalProfile) (*s3.Client, if err != nil { return nil, fmt.Errorf("unable to load SDK config, %w", err) } - //u, err := url.Parse(localProfile.Endpoint) - //if err != nil { - // return nil, fmt.Errorf("custom endpoint parse error, %w", err) - //} return s3.NewFromConfig(awsCfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(localProfile.Endpoint) - //o.EndpointResolverV2 = &staticResolver{ - // endpoint: *u, - //} o.UsePathStyle = true - //o.Credentials = credentials.StaticCredentialsProvider{ - // Value: aws.Credentials{ - // AccessKeyID: localProfile.AccessKey, - // SecretAccessKey: localProfile.SecretAccessKey, - // SessionToken: localProfile.SessionToken, - // }, - //} }), nil }