wip: kms part 1, core encryption functionality and deprecations.

backend/controller/controller.go

@@ -85,15 +85,15 @@ func (c *CommonConfig) Validate() error {
 	return nil
 }
 
-type EncryptionKeys struct {
+type DeprecatedEncryptionKeys struct {
 	Logs  string `name:"log-key" help:"Key for sensitive log data in internal FTL tables." env:"FTL_LOG_ENCRYPTION_KEY"`
 	Async string `name:"async-key" help:"Key for sensitive async call data in internal FTL tables." env:"FTL_ASYNC_ENCRYPTION_KEY"`
 }
 
-func (e EncryptionKeys) Encryptors(required bool) (*dal.Encryptors, error) {
+func (e DeprecatedEncryptionKeys) Encryptors(required bool) (*dal.Encryptors, error) {
 	encryptors := dal.Encryptors{}
 	if e.Logs != "" {
-		enc, err := encryption.NewForKeyOrURI(e.Logs)
+		enc, err := encryption.NewDeprecatedKeyKeyOrURI(e.Logs)
 		if err != nil {
 			return nil, fmt.Errorf("could not create log encryptor: %w", err)
 		}
@@ -105,7 +105,7 @@ func (e EncryptionKeys) Encryptors(required bool) (*dal.Encryptors, error) {
 	}
 
 	if e.Async != "" {
-		enc, err := encryption.NewForKeyOrURI(e.Async)
+		enc, err := encryption.NewDeprecatedKeyKeyOrURI(e.Async)
 		if err != nil {
 			return nil, fmt.Errorf("could not create async calls encryptor: %w", err)
 		}
@@ -133,7 +133,8 @@ type Config struct {
 	ModuleUpdateFrequency        time.Duration       `help:"Frequency to send module updates." default:"30s"`
 	EventLogRetention            *time.Duration      `help:"Delete call logs after this time period. 0 to disable" env:"FTL_EVENT_LOG_RETENTION" default:"24h"`
 	ArtefactChunkSize            int                 `help:"Size of each chunk streamed to the client." default:"1048576"`
-	EncryptionKeys
+	KMSURI                       *url.URL            `help:"URI for KMS key e.g. aws-kms://arn:aws:kms:ap-southeast-2:12345:key/0000-1111" env:"FTL_KMS_URI"`
+	DeprecatedEncryptionKeys
 	CommonConfig
 }
 

backend/controller/dal/dal.go

@@ -227,8 +227,8 @@ type DAL struct {
 }
 
 type Encryptors struct {
-	Logs  encryption.Encryptable
-	Async encryption.Encryptable
+	Logs  encryption.DeprecatedEncryptable
+	Async encryption.DeprecatedEncryptable
 }
 
 // NoOpEncryptors do not encrypt potentially sensitive data.

cmd/ftl-controller/main.go

@@ -43,7 +43,7 @@ func main() {
 	)
 	cli.ControllerConfig.SetDefaults()
 
-	encryptors, err := cli.ControllerConfig.EncryptionKeys.Encryptors(true)
+	encryptors, err := cli.ControllerConfig.DeprecatedEncryptionKeys.Encryptors(true)
 	kctx.FatalIfErrorf(err, "failed to create encryptors")
 
 	ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, cli.LogConfig))

cmd/ftl/cmd_box_run.go

@@ -62,7 +62,7 @@ func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) er
 	if err != nil {
 		return fmt.Errorf("failed to bring up DB connection: %w", err)
 	}
-	encryptors, err := config.EncryptionKeys.Encryptors(false)
+	encryptors, err := config.DeprecatedEncryptionKeys.Encryptors(false)
 	if err != nil {
 		return fmt.Errorf("failed to create encryptors: %w", err)
 	}

cmd/ftl/cmd_serve.go

@@ -152,7 +152,7 @@ func (s *serveCmd) run(ctx context.Context, projConfig projectconfig.Config, ini
 		if err != nil {
 			return fmt.Errorf("failed to bring up DB connection: %w", err)
 		}
-		encryptors, err := config.EncryptionKeys.Encryptors(false)
+		encryptors, err := config.DeprecatedEncryptionKeys.Encryptors(false)
 		if err != nil {
 			return fmt.Errorf("failed to create encryptors: %w", err)
 		}

common/configuration/asm_test.go

@@ -22,8 +22,6 @@ import (
 	"github.com/alecthomas/types/optional"
 	. "github.com/alecthomas/types/optional"
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
 )
@@ -38,10 +36,7 @@ func setUp(ctx context.Context, t *testing.T, router optional.Option[Router[Secr
 		router = optional.Some[Router[Secrets]](ProjectConfigResolver[Secrets]{Config: projectPath})
 	}
 
-	cc := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("test", "test", ""))
-	cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(cc), config.WithRegion("us-west-2"))
-	assert.NoError(t, err)
-
+	cfg := testutils.NewLocalstackConfig(t, ctx)
 	externalClient := secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) {
 		o.BaseEndpoint = aws.String("http://localhost:4566")
 	})

go.mod

@@ -18,9 +18,11 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.1
 	github.com/alecthomas/types v0.16.0
 	github.com/amacneil/dbmate/v2 v2.19.0
+	github.com/aws/aws-sdk-go v1.44.267
 	github.com/aws/aws-sdk-go-v2 v1.30.3
 	github.com/aws/aws-sdk-go-v2/config v1.27.27
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.27
+	github.com/aws/aws-sdk-go-v2/service/kms v1.35.3
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.4
 	github.com/aws/smithy-go v1.20.3
 	github.com/beevik/etree v1.4.1
@@ -44,6 +46,7 @@ require (
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/sqlc-dev/pqtype v0.3.0
 	github.com/swaggest/jsonschema-go v0.3.72
+	github.com/tink-crypto/tink-go-awskms v0.0.0-20230616072154-ba4f9f22c3e9
 	github.com/tink-crypto/tink-go/v2 v2.2.0
 	github.com/titanous/json5 v1.0.0
 	github.com/tliron/commonlog v0.2.17
@@ -87,6 +90,7 @@ require (
 	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/iancoleman/strcase v0.3.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect