wip: tests parse

common/configuration/aws_secrets.go

@@ -2,115 +2,215 @@ package configuration
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"sync"
+
+	"github.com/TBD54566975/ftl/internal/slices"
+
+	"github.com/alecthomas/types/optional"
 	"github.com/aws/aws-sdk-go-v2/aws"
-	_ "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
-	_ "github.com/aws/aws-sdk-go-v2/service/secretsmanager"
-	"log"
-	"net/url"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
+	"github.com/aws/smithy-go"
 )
 
-type AWSSecrets[R Role] struct {
-	client *secretsmanager.Client
+const schemeKey = "asm"
 
+// AWSSecrets implements Resolver and Provider for AWS Secrets Manager (ASM).
+//
+// The resolver does a direct/proxy map from a Ref to a URL, module.name <-> asm://module.name and does not access ASM at all.
+type AWSSecrets[R Role] struct {
 	AccessKeyId     string
 	SecretAccessKey string
 	Region          string
-	Endpoint        string
+	Endpoint        optional.Option[string]
 }
 
 var _ Resolver[Secrets] = AWSSecrets[Secrets]{}
 var _ Provider[Secrets] = AWSSecrets[Secrets]{}
 var _ MutableProvider[Secrets] = AWSSecrets[Secrets]{}
 
-func (a AWSSecrets[R]) getClient(ctx context.Context) (*AWSSecrets[Secrets], error) {
-	cfg, err := config.LoadDefaultConfig(ctx,
-		config.WithRegion(a.region),
-	)
-	if err != nil {
-		return nil, err
+var (
+	asmClient *secretsmanager.Client
+	asmOnce   sync.Once
+	asmErr    error
+)
+
+func urlForRef(ref Ref) *url.URL {
+	return &url.URL{
+		Scheme: schemeKey,
+		Host:   ref.String(),
 	}
+}
+
+func (a AWSSecrets[R]) client(ctx context.Context) (*secretsmanager.Client, error) {
+	asmOnce.Do(func() {
+		var optFns []func(*config.LoadOptions) error
+
+		// Use a static credentials provider if access key and secret are provided.
+		// Otherwise, the SDK will use the default credential chain (env vars, iam, etc).
+		if a.AccessKeyId != "" {
+			credentials := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(a.AccessKeyId, a.SecretAccessKey, ""))
+			optFns = append(optFns, config.WithCredentialsProvider(credentials))
+		}
+
+		if a.Region != "" {
+			optFns = append(optFns, config.WithRegion(a.Region))
+		}
 
-	svc := secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) {
-		e, ok := endpoint.Get()
-		if ok {
-			o.BaseEndpoint = aws.String(e)
+		cfg, err := config.LoadDefaultConfig(ctx, optFns...)
+		if err != nil {
+			err = fmt.Errorf("unable to load aws config: %w", err)
+			return
 		}
+
+		asmClient = secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) {
+			e, ok := a.Endpoint.Get()
+			if ok {
+				o.BaseEndpoint = aws.String(e)
+			}
+		})
+
 	})
 
-	return &AWSSecrets[Secrets]{
-		AccessKeyId:     accessKeyId,
-		SecretAccessKey: secretAccessKey,
-		Region:          region,
-		client:          svc,
-	}, nil
+	return asmClient, asmErr
 }
 
 func (a AWSSecrets[R]) Role() R {
 	var r R
 	return r
 }
 
-func (a AWSSecrets[R]) Get(ctx context.Context, ref Ref) (*url.URL, error) {}
+func (a AWSSecrets[R]) Key() string {
+	return schemeKey
+}
+
+func (a AWSSecrets[R]) Get(ctx context.Context, ref Ref) (*url.URL, error) {
+	return urlForRef(ref), nil
+}
 
 func (a AWSSecrets[R]) Set(ctx context.Context, ref Ref, key *url.URL) error {
-	//this will have to do a list/check to see if the secret exists
+	expectedKey := urlForRef(ref)
+	if key.String() != expectedKey.String() {
+		return fmt.Errorf("key does not match expected key for ref: %s", expectedKey)
+	}
 
+	return nil
 }
 
-func (a AWSSecrets[R]) Unset(ctx context.Context, ref Ref) error {}
+// Unset does nothing because this resolver does not record any state.
+func (a AWSSecrets[R]) Unset(ctx context.Context, ref Ref) error {
+	return nil
+}
 
-func (a AWSSecrets[R]) List(ctx context.Context) ([]Entry, error) {}
+func (a AWSSecrets[R]) List(ctx context.Context) ([]Entry, error) {
+	c, err := a.client(ctx)
+	if err != nil {
+		return nil, err
+	}
 
-func (a AWSSecrets[R]) Key() string {
-	return "asm"
-}
+	out, err := c.ListSecrets(ctx, &secretsmanager.ListSecretsInput{})
+	if err != nil {
+		return nil, fmt.Errorf("unable to list secrets: %w", err)
+	}
 
-func (a AWSSecrets[R]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {}
+	var activeSecrets = slices.Filter(out.SecretList, func(s types.SecretListEntry) bool {
+		return s.DeletedDate == nil
+	})
 
-func (a AWSSecrets[R]) Writer() bool {
-	return true
+	return slices.MapErr(activeSecrets, func(s types.SecretListEntry) (Entry, error) {
+		var ref Ref
+		ref, err = ParseRef(*s.Name)
+		if err != nil {
+			return Entry{}, fmt.Errorf("unable to parse ref: %w", err)
+		}
+
+		return Entry{
+			Ref:      ref,
+			Accessor: urlForRef(ref),
+		}, nil
+	})
 }
 
-func (a AWSSecrets[R]) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) {}
+// Load only supports loading "string" secrets, not binary secrets.
+func (a AWSSecrets[R]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {
+	expectedKey := urlForRef(ref)
+	if key.String() != expectedKey.String() {
+		return nil, fmt.Errorf("key does not match expected key for ref: %s", expectedKey)
+	}
+
+	c, err := a.client(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := c.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
+		SecretId: aws.String(ref.String()),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to retrieve secret: %w", err)
+	}
 
-func (a AWSSecrets[R]) Delete(ctx context.Context, ref Ref) error {}
+	// Secret is a string
+	if out.SecretBinary != nil {
+		return nil, fmt.Errorf("secret is not a string")
+	}
 
-func brainstorm() {
-	endpoint := "http://localhost:4566"
+	return []byte(*out.SecretString), nil
+}
 
-	cfg, err := config.LoadDefaultConfig(context.TODO(),
-		config.WithRegion("us-west-2"),
-	)
+func (a AWSSecrets[R]) Writer() bool {
+	return true
+}
 
+// Store and if the secret already exists, update it.
+func (a AWSSecrets[R]) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) {
+	c, err := a.client(ctx)
 	if err != nil {
-		log.Fatalf("unable to load SDK config, %v", err)
+		return nil, err
 	}
 
-	svc := secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) {
-		o.BaseEndpoint = aws.String(endpoint)
+	_, err = c.CreateSecret(ctx, &secretsmanager.CreateSecretInput{
+		Name:         aws.String(ref.String()),
+		SecretString: aws.String(string(value)),
 	})
 
-	// create a secret
-	name := "test-secret3"
-	secret := "hunter1"
-	_, err = svc.CreateSecret(context.TODO(), &secretsmanager.CreateSecretInput{
-		Name:         &name,
-		SecretString: &secret,
-	})
+	// https://github.com/aws/aws-sdk-go-v2/issues/1110#issuecomment-1054643716
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceExistsException" {
+		_, err = c.UpdateSecret(ctx, &secretsmanager.UpdateSecretInput{
+			SecretId:     aws.String(ref.String()),
+			SecretString: aws.String(string(value)),
+		})
+		if err != nil {
+			return nil, fmt.Errorf("unable to update secret: %w", err)
+		}
+
+	} else if err != nil {
+		return nil, fmt.Errorf("unable to store secret: %w", err)
+	}
+
+	return urlForRef(ref), nil
+}
+
+func (a AWSSecrets[R]) Delete(ctx context.Context, ref Ref) error {
+	c, err := a.client(ctx)
 	if err != nil {
-		log.Fatalf("failed to create secret, %v", err)
+		return err
 	}
 
-	// get the secret
-	out, err := svc.GetSecretValue(context.TODO(), &secretsmanager.GetSecretValueInput{
-		SecretId: &name,
+	var t = true
+	_, err = c.DeleteSecret(ctx, &secretsmanager.DeleteSecretInput{
+		SecretId:                   aws.String(ref.String()),
+		ForceDeleteWithoutRecovery: &t,
 	})
 	if err != nil {
-		log.Fatalf("failed to retrieve secret, %v", err)
+		return fmt.Errorf("unable to delete secret: %w", err)
 	}
 
-	log.Printf("retrieved secret: %s", *out.SecretString)
-
+	return nil
 }

common/configuration/aws_secrets_test.go

@@ -1,9 +1,69 @@
 package configuration
 
-import "testing"
+import (
+	"context"
+	"testing"
 
-func TestMoo(t *testing.T) {
-	t.Log("moo")
+	"github.com/TBD54566975/ftl/internal/log"
 
-	brainstorm()
+	"github.com/alecthomas/assert/v2"
+	. "github.com/alecthomas/types/optional"
+)
+
+func TestAWSSecretsBasics(t *testing.T) {
+	ctx := log.ContextWithNewDefaultLogger(context.Background())
+
+	// Localstack!
+	asm := AWSSecrets[Secrets]{
+		AccessKeyId:     "test",
+		SecretAccessKey: "test",
+		Region:          "us-west-2",
+		Endpoint:        Some("http://localhost:4566"),
+	}
+	url := URL("asm://foo.bar")
+	ref := Ref{Module: Some("foo"), Name: "bar"}
+	var mySecret = []byte("my secret")
+
+	err := asm.Set(ctx, ref, url)
+	assert.NoError(t, err)
+
+	url1, err := asm.Get(ctx, ref)
+	assert.NoError(t, err)
+	assert.Equal(t, url, url1)
+
+	items, err := asm.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, items, []Entry{})
+
+	url2, err := asm.Store(ctx, ref, mySecret)
+	assert.NoError(t, err)
+	assert.Equal(t, url, url2)
+
+	items, err = asm.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, items, []Entry{{Ref: ref, Accessor: url}})
+
+	item, err := asm.Load(ctx, ref, url)
+	assert.NoError(t, err)
+	assert.Equal(t, item, mySecret)
+
+	// Store a second time to make sure it is updating
+	var mySecret2 = []byte("hunter1")
+	url3, err := asm.Store(ctx, ref, mySecret2)
+	assert.NoError(t, err)
+	assert.Equal(t, url, url3)
+
+	item, err = asm.Load(ctx, ref, url)
+	assert.NoError(t, err)
+	assert.Equal(t, item, mySecret2)
+
+	err = asm.Delete(ctx, ref)
+	assert.NoError(t, err)
+
+	items, err = asm.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, items, []Entry{})
+
+	_, err = asm.Load(ctx, ref, url)
+	assert.Error(t, err)
 }

go.mod

@@ -17,6 +17,9 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.1
 	github.com/alecthomas/types v0.15.0
 	github.com/amacneil/dbmate/v2 v2.16.0
+	github.com/aws/aws-sdk-go-v2 v1.27.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.4
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.29.1
 	github.com/beevik/etree v1.4.0
 	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/deckarep/golang-set/v2 v2.6.0
@@ -64,6 +67,17 @@ require (
 
 require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect