From 295107e5ad675290160d234548c6191fb8052df8 Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 3 Jun 2024 12:12:52 +1000 Subject: [PATCH] feat: Implement an AWS Secrets Manager secrets provider + resolver #1545 (#1575) Please note that this is not integrated into the CLI and controller to prevent potential conflicts with https://github.com/TBD54566975/ftl/issues/1473 Related issue: #1545 --- common/configuration/asm.go | 162 +++++++++++++++++++++++++++++++ common/configuration/asm_test.go | 126 ++++++++++++++++++++++++ docker-compose.yml | 7 ++ go.mod | 14 +++ go.sum | 28 ++++++ 5 files changed, 337 insertions(+) create mode 100644 common/configuration/asm.go create mode 100644 common/configuration/asm_test.go diff --git a/common/configuration/asm.go b/common/configuration/asm.go new file mode 100644 index 0000000000..d24c4e9b51 --- /dev/null +++ b/common/configuration/asm.go @@ -0,0 +1,162 @@ +package configuration + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/TBD54566975/ftl/internal/slices" + + . "github.com/alecthomas/types/optional" //nolint:stylecheck + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" + "github.com/aws/smithy-go" +) + +// ASM implements Resolver and Provider for AWS Secrets Manager (ASM). +// +// The resolver does a direct/proxy map from a Ref to a URL, module.name <-> asm://module.name and does not access ASM at all. +type ASM struct { + Client secretsmanager.Client +} + +var _ Resolver[Secrets] = &ASM{} +var _ Provider[Secrets] = &ASM{} + +func asmURLForRef(ref Ref) *url.URL { + return &url.URL{ + Scheme: "asm", + Host: ref.String(), + } +} + +func (ASM) Role() Secrets { + return Secrets{} +} + +func (ASM) Key() string { + return "asm" +} + +func (ASM) Get(ctx context.Context, ref Ref) (*url.URL, error) { + return asmURLForRef(ref), nil +} + +func (ASM) Set(ctx context.Context, ref Ref, key *url.URL) error { + expectedKey := asmURLForRef(ref) + if key.String() != expectedKey.String() { + return fmt.Errorf("key does not match expected key for ref: %s", expectedKey) + } + + return nil +} + +// Unset does nothing because this resolver does not record any state. +func (ASM) Unset(ctx context.Context, ref Ref) error { + return nil +} + +// List all secrets in the account. This might require multiple calls to the AWS API if there are more than 100 secrets. +func (a ASM) List(ctx context.Context) ([]Entry, error) { + nextToken := None[string]() + entries := []Entry{} + for { + out, err := a.Client.ListSecrets(ctx, &secretsmanager.ListSecretsInput{ + MaxResults: aws.Int32(100), + NextToken: nextToken.Ptr(), + }) + if err != nil { + return nil, fmt.Errorf("unable to list secrets: %w", err) + } + + var activeSecrets = slices.Filter(out.SecretList, func(s types.SecretListEntry) bool { + return s.DeletedDate == nil + }) + page, err := slices.MapErr(activeSecrets, func(s types.SecretListEntry) (Entry, error) { + var ref Ref + ref, err = ParseRef(*s.Name) + if err != nil { + return Entry{}, fmt.Errorf("unable to parse ref: %w", err) + } + + return Entry{ + Ref: ref, + Accessor: asmURLForRef(ref), + }, nil + }) + if err != nil { + return nil, err + } + + entries = append(entries, page...) + + nextToken = Ptr[string](out.NextToken) + if !nextToken.Ok() { + break + } + } + + return entries, nil +} + +// Load only supports loading "string" secrets, not binary secrets. +func (a ASM) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { + expectedKey := asmURLForRef(ref) + if key.String() != expectedKey.String() { + return nil, fmt.Errorf("key does not match expected key for ref: %s", expectedKey) + } + + out, err := a.Client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(ref.String()), + }) + if err != nil { + return nil, fmt.Errorf("unable to retrieve secret: %w", err) + } + + // Secret is a string + if out.SecretBinary != nil { + return nil, fmt.Errorf("secret is not a string") + } + + return []byte(*out.SecretString), nil +} + +// Store and if the secret already exists, update it. +func (a ASM) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + _, err := a.Client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ + Name: aws.String(ref.String()), + SecretString: aws.String(string(value)), + }) + + // https://github.com/aws/aws-sdk-go-v2/issues/1110#issuecomment-1054643716 + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceExistsException" { + _, err = a.Client.UpdateSecret(ctx, &secretsmanager.UpdateSecretInput{ + SecretId: aws.String(ref.String()), + SecretString: aws.String(string(value)), + }) + if err != nil { + return nil, fmt.Errorf("unable to update secret: %w", err) + } + + } else if err != nil { + return nil, fmt.Errorf("unable to store secret: %w", err) + } + + return asmURLForRef(ref), nil +} + +func (a ASM) Delete(ctx context.Context, ref Ref) error { + var t = true + _, err := a.Client.DeleteSecret(ctx, &secretsmanager.DeleteSecretInput{ + SecretId: aws.String(ref.String()), + ForceDeleteWithoutRecovery: &t, + }) + if err != nil { + return fmt.Errorf("unable to delete secret: %w", err) + } + + return nil +} diff --git a/common/configuration/asm_test.go b/common/configuration/asm_test.go new file mode 100644 index 0000000000..9057281d1b --- /dev/null +++ b/common/configuration/asm_test.go @@ -0,0 +1,126 @@ +package configuration + +import ( + "context" + "fmt" + "sort" + "testing" + + "github.com/TBD54566975/ftl/internal/log" + + "github.com/alecthomas/assert/v2" + . "github.com/alecthomas/types/optional" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" +) + +func localstack(ctx context.Context, t *testing.T) ASM { + t.Helper() + cc := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("test", "test", "")) + cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(cc), config.WithRegion("us-west-2")) + if err != nil { + t.Fatal(err) + } + + sm := secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) { + o.BaseEndpoint = aws.String("http://localhost:4566") + }) + asm := ASM{Client: *sm} + return asm +} + +func TestASMWorkflow(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + asm := localstack(ctx, t) + ref := Ref{Module: Some("foo"), Name: "bar"} + var mySecret = []byte("my secret") + manager, err := New(ctx, asm, []Provider[Secrets]{asm}) + assert.NoError(t, err) + + var gotSecret []byte + err = manager.Get(ctx, ref, &gotSecret) + assert.Error(t, err) + + items, err := manager.List(ctx) + assert.NoError(t, err) + assert.Equal(t, items, []Entry{}) + + err = manager.Set(ctx, "asm", ref, mySecret) + assert.NoError(t, err) + + items, err = manager.List(ctx) + assert.NoError(t, err) + assert.Equal(t, items, []Entry{{Ref: ref, Accessor: URL("asm://foo.bar")}}) + + err = manager.Get(ctx, ref, &gotSecret) + assert.NoError(t, err) + + // Set again to make sure it updates. + mySecret = []byte("hunter1") + err = manager.Set(ctx, "asm", ref, mySecret) + assert.NoError(t, err) + + err = manager.Get(ctx, ref, &gotSecret) + assert.NoError(t, err) + assert.Equal(t, gotSecret, mySecret) + + err = manager.Unset(ctx, "asm", ref) + assert.NoError(t, err) + + items, err = manager.List(ctx) + assert.NoError(t, err) + assert.Equal(t, items, []Entry{}) + + err = manager.Get(ctx, ref, &gotSecret) + assert.Error(t, err) +} + +// Suggest not running this against a real AWS account (especially in CI) due to the cost. Maybe costs a few $. +func TestASMPagination(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + asm := localstack(ctx, t) + manager, err := New(ctx, asm, []Provider[Secrets]{asm}) + assert.NoError(t, err) + + // Create 210 secrets, so we paginate at least twice. + for i := range 210 { + ref := NewRef("foo", fmt.Sprintf("bar%03d", i)) + err := manager.Set(ctx, "asm", ref, []byte(fmt.Sprintf("hunter%03d", i))) + assert.NoError(t, err) + } + + items, err := manager.List(ctx) + assert.NoError(t, err) + assert.Equal(t, len(items), 210) + + // Check each secret. + sort.Slice(items, func(i, j int) bool { + return items[i].Ref.Name < items[j].Ref.Name + }) + for i, item := range items { + assert.Equal(t, item.Ref.Name, fmt.Sprintf("bar%03d", i)) + + // Just to save on requests, skip by 10 + if i%10 != 0 { + continue + } + var secret []byte + err := manager.Get(ctx, item.Ref, &secret) + assert.NoError(t, err) + assert.Equal(t, secret, []byte(fmt.Sprintf("hunter%03d", i))) + } + + // Delete them + for i := range 210 { + ref := NewRef("foo", fmt.Sprintf("bar%03d", i)) + err := manager.Unset(ctx, "asm", ref) + assert.NoError(t, err) + } + + // Make sure they are all gone + items, err = manager.List(ctx) + assert.NoError(t, err) + assert.Equal(t, len(items), 0) +} diff --git a/docker-compose.yml b/docker-compose.yml index ef62b98b9f..eec3fa77d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,3 +29,10 @@ services: ports: - 4317:4317 # OTLP gRPC receiver - 4318:4318 # OTLP http receiver + localstack: + image: localstack/localstack + ports: + - 4566:4566 + environment: + SERVICES: secretsmanager + DEBUG: 1 \ No newline at end of file diff --git a/go.mod b/go.mod index afe9a4ee6d..2f79cb73eb 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,11 @@ require ( github.com/alecthomas/participle/v2 v2.1.1 github.com/alecthomas/types v0.16.0 github.com/amacneil/dbmate/v2 v2.16.0 + github.com/aws/aws-sdk-go-v2 v1.27.0 + github.com/aws/aws-sdk-go-v2/config v1.27.4 + github.com/aws/aws-sdk-go-v2/credentials v1.17.4 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.29.1 + github.com/aws/smithy-go v1.20.2 github.com/beevik/etree v1.4.0 github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/deckarep/golang-set/v2 v2.6.0 @@ -63,6 +68,15 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/go.sum b/go.sum index cb5ec563b3..77cce42fdc 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,34 @@ github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4u github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/amacneil/dbmate/v2 v2.16.0 h1:uIah9qFOxA9cIyS1TuPMUgn0KVnwRRSx0MhKu7sFpcI= github.com/amacneil/dbmate/v2 v2.16.0/go.mod h1:TlxMqFDBxsLVertnF2cAxZxPyt9UjlCrt04iy1UxRXc= +github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo= +github.com/aws/aws-sdk-go-v2 v1.27.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/config v1.27.4 h1:AhfWb5ZwimdsYTgP7Od8E9L1u4sKmDW2ZVeLcf2O42M= +github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.4 h1:h5Vztbd8qLppiPwX+y0Q6WiwMZgpd9keKe2EAENgAuI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 h1:lf/8VTF2cM+N4SLzaYJERKEWAXq8MOMpZfU6wEPWsPk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7/go.mod h1:4SjkU7QiqK2M9oozyMzfZ/23LmUY+h3oFqhdeP5OMiI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 h1:4OYVp0705xu8yjdyoWix0r9wPIRXnIzzOoUpQVHIJ/g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7/go.mod h1:vd7ESTEvI76T2Na050gODNmNU7+OyKrIKroYTu4ABiI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 h1:5ffmXjPtwRExp1zc7gENLgCPyHFbhEPwVTkTiH9niSk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.29.1 h1:NSWsFzdHN41mJ5I/DOFzxgkKSYNHQADHn7Mu+lU/AKw= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.29.1/go.mod h1:5mMk0DgUgaHlcqtN65fNyZI0ZDX3i9Cw+nwq75HKB3U= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 h1:utEGkfdQ4L6YW/ietH7111ZYglLJvS+sLriHJ1NBJEQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 h1:9/GylMS45hGGFCcMrUZDVayQE1jYSIN6da9jo7RAYIw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 h1:3I2cBEYgKhrWlwyZgfpSO2BpaMY1LHPqXYk/QGlu2ew= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/beevik/etree v1.4.0 h1:oz1UedHRepuY3p4N5OjE0nK1WLCqtzHf25bxplKOHLs=