From 7a9d9cf9dc50e3da06f130440e1ceb7cd3979484 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Sat, 14 Sep 2024 17:37:55 +1000 Subject: [PATCH] refactor: consolidate encryption packages into backend (#2675) This is just a mechanical move. I'll combine the two APIs into a single one in a followup. --- backend/controller/controller.go | 4 ++-- backend/controller/cronjobs/cronjobs.go | 4 ++-- backend/controller/cronjobs/cronjobs_test.go | 4 ++-- .../dal/internal/sql/async_queries.sql.go | 4 ++-- backend/controller/dal/async_calls.go | 4 ++-- backend/controller/dal/async_calls_test.go | 4 ++-- backend/controller/dal/dal.go | 10 ++++----- backend/controller/dal/dal_test.go | 8 +++---- backend/controller/dal/fsm.go | 6 ++--- backend/controller/dal/fsm_test.go | 4 ++-- .../dal/internal/sql/async_queries.sql.go | 4 ++-- backend/controller/dal/internal/sql/models.go | 12 +++++----- .../controller/dal/internal/sql/querier.go | 4 ++-- .../dal/internal/sql/queries.sql.go | 22 +++++++++---------- backend/controller/dal/pubsub.go | 4 ++-- .../controller/encryption/api}/database.go | 2 +- .../controller/encryption/api}/encryption.go | 2 +- .../encryption/api}/encryption_test.go | 2 +- .../encryption/api}/integration_test.go | 2 +- .../api}/testdata/go/encryption/encryption.go | 0 .../api}/testdata/go/encryption/ftl.toml | 0 .../api}/testdata/go/encryption/go.mod | 0 .../api}/testdata/go/encryption/go.sum | 0 backend/controller/encryption/dal/dal.go | 14 ++++++------ .../encryption/dal/internal/sql/querier.go | 4 ++-- .../dal/internal/sql/queries.sql.go | 8 +++---- .../encryption/{encryption.go => service.go} | 14 ++++++------ .../{encryption_test.go => service_test.go} | 10 ++++----- sqlc.yaml | 8 +++---- 29 files changed, 82 insertions(+), 82 deletions(-) rename {internal/encryption => backend/controller/encryption/api}/database.go (98%) rename {internal/encryption => backend/controller/encryption/api}/encryption.go (99%) rename {internal/encryption => backend/controller/encryption/api}/encryption_test.go (98%) rename {internal/encryption => backend/controller/encryption/api}/integration_test.go (99%) rename {internal/encryption => backend/controller/encryption/api}/testdata/go/encryption/encryption.go (100%) rename {internal/encryption => backend/controller/encryption/api}/testdata/go/encryption/ftl.toml (100%) rename {internal/encryption => backend/controller/encryption/api}/testdata/go/encryption/go.mod (100%) rename {internal/encryption => backend/controller/encryption/api}/testdata/go/encryption/go.sum (100%) rename backend/controller/encryption/{encryption.go => service.go} (78%) rename backend/controller/encryption/{encryption_test.go => service_test.go} (78%) diff --git a/backend/controller/controller.go b/backend/controller/controller.go index faf24eb8ce..2afd227d15 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -39,6 +39,7 @@ import ( "github.com/TBD54566975/ftl/backend/controller/cronjobs" "github.com/TBD54566975/ftl/backend/controller/dal" "github.com/TBD54566975/ftl/backend/controller/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/ingress" "github.com/TBD54566975/ftl/backend/controller/leases" leasesdal "github.com/TBD54566975/ftl/backend/controller/leases/dal" @@ -55,7 +56,6 @@ import ( frontend "github.com/TBD54566975/ftl/frontend/console" cf "github.com/TBD54566975/ftl/internal/configuration/manager" "github.com/TBD54566975/ftl/internal/cors" - ftlencryption "github.com/TBD54566975/ftl/internal/encryption" ftlhttp "github.com/TBD54566975/ftl/internal/http" "github.com/TBD54566975/ftl/internal/log" ftlmaps "github.com/TBD54566975/ftl/internal/maps" @@ -232,7 +232,7 @@ func New(ctx context.Context, conn *sql.DB, config Config, devel bool) (*Service config.ControllerTimeout = time.Second * 5 } - encryptionSrv, err := encryption.New(ctx, conn, ftlencryption.NewBuilder().WithKMSURI(optional.Ptr(config.KMSURI))) + encryptionSrv, err := encryption.New(ctx, conn, api.NewBuilder().WithKMSURI(optional.Ptr(config.KMSURI))) if err != nil { return nil, fmt.Errorf("failed to create encryption dal: %w", err) } diff --git a/backend/controller/cronjobs/cronjobs.go b/backend/controller/cronjobs/cronjobs.go index e586c144c9..df0dce0ec6 100644 --- a/backend/controller/cronjobs/cronjobs.go +++ b/backend/controller/cronjobs/cronjobs.go @@ -11,10 +11,10 @@ import ( "github.com/TBD54566975/ftl/backend/controller/cronjobs/dal" parentdal "github.com/TBD54566975/ftl/backend/controller/dal" encryptionsvc "github.com/TBD54566975/ftl/backend/controller/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/internal/cron" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" ) @@ -178,7 +178,7 @@ func (s *Service) scheduleCronJob(ctx context.Context, tx *dal.DAL, job model.Cr logger.Tracef("Scheduling cron job %q async_call execution at %s", job.Key, nextAttemptForJob) origin := &parentdal.AsyncOriginCron{CronJobKey: job.Key} - var request encryption.EncryptedColumn[encryption.AsyncSubKey] + var request api.EncryptedColumn[api.AsyncSubKey] err = s.encryption.Encrypt([]byte(`{}`), &request) if err != nil { return fmt.Errorf("failed to encrypt request for job %q: %w", job.Key, err) diff --git a/backend/controller/cronjobs/cronjobs_test.go b/backend/controller/cronjobs/cronjobs_test.go index af3642704d..a393866001 100644 --- a/backend/controller/cronjobs/cronjobs_test.go +++ b/backend/controller/cronjobs/cronjobs_test.go @@ -15,11 +15,11 @@ import ( "github.com/TBD54566975/ftl/backend/controller/cronjobs/dal" parentdal "github.com/TBD54566975/ftl/backend/controller/dal" "github.com/TBD54566975/ftl/backend/controller/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltest" "github.com/TBD54566975/ftl/backend/libdal" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/internal/cron" - ftlencryption "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" ) @@ -37,7 +37,7 @@ func TestNewCronJobsForModule(t *testing.T) { dal := dal.New(conn) uri := "fake-kms://CK6YwYkBElQKSAowdHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuQWVzR2NtS2V5EhIaEJy4TIQgfCuwxA3ZZgChp_wYARABGK6YwYkBIAE" - encryption, err := encryption.New(ctx, conn, ftlencryption.NewBuilder().WithKMSURI(optional.Some(uri))) + encryption, err := encryption.New(ctx, conn, api.NewBuilder().WithKMSURI(optional.Some(uri))) assert.NoError(t, err) parentDAL := parentdal.New(ctx, conn, encryption) diff --git a/backend/controller/cronjobs/dal/internal/sql/async_queries.sql.go b/backend/controller/cronjobs/dal/internal/sql/async_queries.sql.go index 181199435a..e6002dd786 100644 --- a/backend/controller/cronjobs/dal/internal/sql/async_queries.sql.go +++ b/backend/controller/cronjobs/dal/internal/sql/async_queries.sql.go @@ -10,9 +10,9 @@ import ( "encoding/json" "time" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/alecthomas/types/optional" ) @@ -61,7 +61,7 @@ type CreateAsyncCallParams struct { ScheduledAt time.Time Verb schema.RefKey Origin string - Request encryption.EncryptedAsyncColumn + Request api.EncryptedAsyncColumn RemainingAttempts int32 Backoff sqltypes.Duration MaxBackoff sqltypes.Duration diff --git a/backend/controller/dal/async_calls.go b/backend/controller/dal/async_calls.go index 7ba0f4dbe0..d53262d62d 100644 --- a/backend/controller/dal/async_calls.go +++ b/backend/controller/dal/async_calls.go @@ -13,11 +13,11 @@ import ( "github.com/alecthomas/types/optional" "github.com/TBD54566975/ftl/backend/controller/dal/internal/sql" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" leasedal "github.com/TBD54566975/ftl/backend/controller/leases/dal" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/libdal" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" ) @@ -191,7 +191,7 @@ func (d *DAL) CompleteAsyncCall(ctx context.Context, didScheduleAnotherCall = false switch result := result.(type) { case either.Left[[]byte, string]: // Successful response. - var encryptedResult encryption.EncryptedAsyncColumn + var encryptedResult api.EncryptedAsyncColumn err := tx.encryption.Encrypt(result.Get(), &encryptedResult) if err != nil { return false, fmt.Errorf("failed to encrypt async call result: %w", err) diff --git a/backend/controller/dal/async_calls_test.go b/backend/controller/dal/async_calls_test.go index 7cf5b43c63..5f3b988332 100644 --- a/backend/controller/dal/async_calls_test.go +++ b/backend/controller/dal/async_calls_test.go @@ -7,10 +7,10 @@ import ( "github.com/alecthomas/assert/v2" "github.com/TBD54566975/ftl/backend/controller/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltest" "github.com/TBD54566975/ftl/backend/libdal" "github.com/TBD54566975/ftl/backend/schema" - ftlencryption "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" ) @@ -18,7 +18,7 @@ import ( func TestNoCallToAcquire(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) conn := sqltest.OpenForTesting(ctx, t) - encryption, err := encryption.New(ctx, conn, ftlencryption.NewBuilder()) + encryption, err := encryption.New(ctx, conn, api.NewBuilder()) assert.NoError(t, err) dal := New(ctx, conn, encryption) diff --git a/backend/controller/dal/dal.go b/backend/controller/dal/dal.go index 7f790ccaaa..90c088212f 100644 --- a/backend/controller/dal/dal.go +++ b/backend/controller/dal/dal.go @@ -17,12 +17,12 @@ import ( dalsql "github.com/TBD54566975/ftl/backend/controller/dal/internal/sql" "github.com/TBD54566975/ftl/backend/controller/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" leasedal "github.com/TBD54566975/ftl/backend/controller/leases/dal" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/libdal" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/schema" - ftlencryption "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/maps" "github.com/TBD54566975/ftl/internal/model" @@ -605,7 +605,7 @@ func (d *DAL) SetDeploymentReplicas(ctx context.Context, key model.DeploymentKey return libdal.TranslatePGError(err) } } - var payload ftlencryption.EncryptedTimelineColumn + var payload api.EncryptedTimelineColumn err = d.encryption.EncryptJSON(map[string]interface{}{ "prev_min_replicas": deployment.MinReplicas, "min_replicas": minReplicas, @@ -679,7 +679,7 @@ func (d *DAL) ReplaceDeployment(ctx context.Context, newDeploymentKey model.Depl } } - var payload ftlencryption.EncryptedTimelineColumn + var payload api.EncryptedTimelineColumn err = d.encryption.EncryptJSON(map[string]any{ "min_replicas": int32(minReplicas), "replaced": replacedDeploymentKey, @@ -892,7 +892,7 @@ func (d *DAL) InsertLogEvent(ctx context.Context, log *LogEvent) error { "error": log.Error, "stack": log.Stack, } - var encryptedPayload ftlencryption.EncryptedTimelineColumn + var encryptedPayload api.EncryptedTimelineColumn err := d.encryption.EncryptJSON(payload, &encryptedPayload) if err != nil { return fmt.Errorf("failed to encrypt log payload: %w", err) @@ -973,7 +973,7 @@ func (d *DAL) InsertCallEvent(ctx context.Context, call *CallEvent) error { if pr, ok := call.ParentRequestKey.Get(); ok { parentRequestKey = optional.Some(pr.String()) } - var payload ftlencryption.EncryptedTimelineColumn + var payload api.EncryptedTimelineColumn err := d.encryption.EncryptJSON(map[string]any{ "duration_ms": call.Duration.Milliseconds(), "request": call.Request, diff --git a/backend/controller/dal/dal_test.go b/backend/controller/dal/dal_test.go index 56462c5ac1..d554b4e683 100644 --- a/backend/controller/dal/dal_test.go +++ b/backend/controller/dal/dal_test.go @@ -14,11 +14,11 @@ import ( "golang.org/x/sync/errgroup" "github.com/TBD54566975/ftl/backend/controller/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltest" "github.com/TBD54566975/ftl/backend/libdal" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/schema" - ftlencryption "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" "github.com/TBD54566975/ftl/internal/sha256" @@ -28,7 +28,7 @@ import ( func TestDAL(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) conn := sqltest.OpenForTesting(ctx, t) - encryption, err := encryption.New(ctx, conn, ftlencryption.NewBuilder()) + encryption, err := encryption.New(ctx, conn, api.NewBuilder()) assert.NoError(t, err) dal := New(ctx, conn, encryption) @@ -294,7 +294,7 @@ func TestDAL(t *testing.T) { func TestCreateArtefactConflict(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) conn := sqltest.OpenForTesting(ctx, t) - encryption, err := encryption.New(ctx, conn, ftlencryption.NewBuilder()) + encryption, err := encryption.New(ctx, conn, api.NewBuilder()) assert.NoError(t, err) dal := New(ctx, conn, encryption) @@ -373,7 +373,7 @@ func assertEventsEqual(t *testing.T, expected, actual []TimelineEvent) { func TestDeleteOldEvents(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) conn := sqltest.OpenForTesting(ctx, t) - encryption, err := encryption.New(ctx, conn, ftlencryption.NewBuilder()) + encryption, err := encryption.New(ctx, conn, api.NewBuilder()) assert.NoError(t, err) dal := New(ctx, conn, encryption) diff --git a/backend/controller/dal/fsm.go b/backend/controller/dal/fsm.go index 1007674556..8b0b8cc581 100644 --- a/backend/controller/dal/fsm.go +++ b/backend/controller/dal/fsm.go @@ -10,12 +10,12 @@ import ( "github.com/alecthomas/types/optional" sql2 "github.com/TBD54566975/ftl/backend/controller/dal/internal/sql" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/leases" "github.com/TBD54566975/ftl/backend/controller/observability" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/libdal" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" ) // StartFSMTransition sends an event to an executing instance of an FSM. @@ -32,7 +32,7 @@ import ( // // Note: no validation of the FSM is performed. func (d *DAL) StartFSMTransition(ctx context.Context, fsm schema.RefKey, instanceKey string, destinationState schema.RefKey, request []byte, encrypted bool, retryParams schema.RetryParams) (err error) { - var encryptedRequest encryption.EncryptedAsyncColumn + var encryptedRequest api.EncryptedAsyncColumn if encrypted { encryptedRequest.Set(request) } else { @@ -154,7 +154,7 @@ func (d *DAL) PopNextFSMEvent(ctx context.Context, fsm schema.RefKey, instanceKe } func (d *DAL) SetNextFSMEvent(ctx context.Context, fsm schema.RefKey, instanceKey string, nextState schema.RefKey, request json.RawMessage, requestType schema.Type) error { - var encryptedRequest encryption.EncryptedAsyncColumn + var encryptedRequest api.EncryptedAsyncColumn err := d.encryption.EncryptJSON(request, &encryptedRequest) if err != nil { return fmt.Errorf("failed to encrypt FSM request: %w", err) diff --git a/backend/controller/dal/fsm_test.go b/backend/controller/dal/fsm_test.go index 690658c738..1cd415ab58 100644 --- a/backend/controller/dal/fsm_test.go +++ b/backend/controller/dal/fsm_test.go @@ -9,18 +9,18 @@ import ( "github.com/alecthomas/types/either" "github.com/TBD54566975/ftl/backend/controller/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" leasedal "github.com/TBD54566975/ftl/backend/controller/leases/dal" "github.com/TBD54566975/ftl/backend/controller/sql/sqltest" "github.com/TBD54566975/ftl/backend/libdal" "github.com/TBD54566975/ftl/backend/schema" - ftlencryption "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" ) func TestSendFSMEvent(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) conn := sqltest.OpenForTesting(ctx, t) - encryption, err := encryption.New(ctx, conn, ftlencryption.NewBuilder()) + encryption, err := encryption.New(ctx, conn, api.NewBuilder()) assert.NoError(t, err) dal := New(ctx, conn, encryption) diff --git a/backend/controller/dal/internal/sql/async_queries.sql.go b/backend/controller/dal/internal/sql/async_queries.sql.go index 181199435a..e6002dd786 100644 --- a/backend/controller/dal/internal/sql/async_queries.sql.go +++ b/backend/controller/dal/internal/sql/async_queries.sql.go @@ -10,9 +10,9 @@ import ( "encoding/json" "time" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/alecthomas/types/optional" ) @@ -61,7 +61,7 @@ type CreateAsyncCallParams struct { ScheduledAt time.Time Verb schema.RefKey Origin string - Request encryption.EncryptedAsyncColumn + Request api.EncryptedAsyncColumn RemainingAttempts int32 Backoff sqltypes.Duration MaxBackoff sqltypes.Duration diff --git a/backend/controller/dal/internal/sql/models.go b/backend/controller/dal/internal/sql/models.go index d5e201eb46..a46fe7d077 100644 --- a/backend/controller/dal/internal/sql/models.go +++ b/backend/controller/dal/internal/sql/models.go @@ -10,9 +10,9 @@ import ( "fmt" "time" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" "github.com/sqlc-dev/pqtype" @@ -328,8 +328,8 @@ type AsyncCall struct { State AsyncCallState Origin string ScheduledAt time.Time - Request encryption.EncryptedAsyncColumn - Response encryption.OptionalEncryptedAsyncColumn + Request api.EncryptedAsyncColumn + Response api.OptionalEncryptedAsyncColumn Error optional.Option[string] RemainingAttempts int32 Backoff sqltypes.Duration @@ -389,7 +389,7 @@ type FsmNextEvent struct { CreatedAt time.Time FsmInstanceID int64 NextState schema.RefKey - Request encryption.EncryptedAsyncColumn + Request api.EncryptedAsyncColumn RequestType sqltypes.Type } @@ -409,7 +409,7 @@ type Timeline struct { CustomKey2 optional.Option[string] CustomKey3 optional.Option[string] CustomKey4 optional.Option[string] - Payload encryption.EncryptedTimelineColumn + Payload api.EncryptedTimelineColumn ParentRequestID optional.Option[string] } @@ -428,7 +428,7 @@ type TopicEvent struct { CreatedAt time.Time Key model.TopicEventKey TopicID int64 - Payload encryption.EncryptedAsyncColumn + Payload api.EncryptedAsyncColumn Caller optional.Option[string] RequestKey optional.Option[string] TraceContext pqtype.NullRawMessage diff --git a/backend/controller/dal/internal/sql/querier.go b/backend/controller/dal/internal/sql/querier.go index 382abea77d..690159a59d 100644 --- a/backend/controller/dal/internal/sql/querier.go +++ b/backend/controller/dal/internal/sql/querier.go @@ -8,9 +8,9 @@ import ( "context" "time" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" ) @@ -102,7 +102,7 @@ type Querier interface { // // "key" is the unique identifier for the FSM execution. StartFSMTransition(ctx context.Context, arg StartFSMTransitionParams) (FsmInstance, error) - SucceedAsyncCall(ctx context.Context, response encryption.OptionalEncryptedAsyncColumn, iD int64) (bool, error) + SucceedAsyncCall(ctx context.Context, response api.OptionalEncryptedAsyncColumn, iD int64) (bool, error) SucceedFSMInstance(ctx context.Context, fsm schema.RefKey, key string) (bool, error) UpdateCronJobExecution(ctx context.Context, arg UpdateCronJobExecutionParams) error UpsertController(ctx context.Context, key model.ControllerKey, endpoint string) (int64, error) diff --git a/backend/controller/dal/internal/sql/queries.sql.go b/backend/controller/dal/internal/sql/queries.sql.go index 5992ca86c8..f2a32178b8 100644 --- a/backend/controller/dal/internal/sql/queries.sql.go +++ b/backend/controller/dal/internal/sql/queries.sql.go @@ -10,10 +10,10 @@ import ( "encoding/json" "time" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/leases" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" "github.com/google/uuid" @@ -68,7 +68,7 @@ type AcquireAsyncCallRow struct { Origin string Verb schema.RefKey CatchVerb optional.Option[schema.RefKey] - Request encryption.EncryptedAsyncColumn + Request api.EncryptedAsyncColumn ScheduledAt time.Time RemainingAttempts int32 Error optional.Option[string] @@ -1183,7 +1183,7 @@ LIMIT 1 type GetNextEventForSubscriptionRow struct { Event optional.Option[model.TopicEventKey] - Payload encryption.OptionalEncryptedAsyncColumn + Payload api.OptionalEncryptedAsyncColumn CreatedAt optional.Option[time.Time] Caller optional.Option[string] RequestKey optional.Option[string] @@ -1841,7 +1841,7 @@ type InsertTimelineCallEventParams struct { SourceVerb optional.Option[string] DestModule string DestVerb string - Payload encryption.EncryptedTimelineColumn + Payload api.EncryptedTimelineColumn } func (q *Queries) InsertTimelineCallEvent(ctx context.Context, arg InsertTimelineCallEventParams) error { @@ -1884,7 +1884,7 @@ type InsertTimelineDeploymentCreatedEventParams struct { DeploymentKey model.DeploymentKey Language string ModuleName string - Payload encryption.EncryptedTimelineColumn + Payload api.EncryptedTimelineColumn } func (q *Queries) InsertTimelineDeploymentCreatedEvent(ctx context.Context, arg InsertTimelineDeploymentCreatedEventParams) error { @@ -1922,7 +1922,7 @@ type InsertTimelineDeploymentUpdatedEventParams struct { DeploymentKey model.DeploymentKey Language string ModuleName string - Payload encryption.EncryptedTimelineColumn + Payload api.EncryptedTimelineColumn } func (q *Queries) InsertTimelineDeploymentUpdatedEvent(ctx context.Context, arg InsertTimelineDeploymentUpdatedEventParams) error { @@ -1952,7 +1952,7 @@ type InsertTimelineEventParams struct { CustomKey2 optional.Option[string] CustomKey3 optional.Option[string] CustomKey4 optional.Option[string] - Payload encryption.EncryptedTimelineColumn + Payload api.EncryptedTimelineColumn } func (q *Queries) InsertTimelineEvent(ctx context.Context, arg InsertTimelineEventParams) error { @@ -1999,7 +1999,7 @@ type InsertTimelineLogEventParams struct { RequestKey optional.Option[string] TimeStamp time.Time Level int32 - Payload encryption.EncryptedTimelineColumn + Payload api.EncryptedTimelineColumn } func (q *Queries) InsertTimelineLogEvent(ctx context.Context, arg InsertTimelineLogEventParams) error { @@ -2151,7 +2151,7 @@ type PublishEventForTopicParams struct { Module string Topic string Caller string - Payload encryption.EncryptedAsyncColumn + Payload api.EncryptedAsyncColumn RequestKey string TraceContext json.RawMessage } @@ -2196,7 +2196,7 @@ type SetNextFSMEventParams struct { Fsm schema.RefKey InstanceKey string Event schema.RefKey - Request encryption.EncryptedAsyncColumn + Request api.EncryptedAsyncColumn RequestType sqltypes.Type } @@ -2294,7 +2294,7 @@ WHERE id = $2 RETURNING true ` -func (q *Queries) SucceedAsyncCall(ctx context.Context, response encryption.OptionalEncryptedAsyncColumn, iD int64) (bool, error) { +func (q *Queries) SucceedAsyncCall(ctx context.Context, response api.OptionalEncryptedAsyncColumn, iD int64) (bool, error) { row := q.db.QueryRowContext(ctx, succeedAsyncCall, response, iD) var column_1 bool err := row.Scan(&column_1) diff --git a/backend/controller/dal/pubsub.go b/backend/controller/dal/pubsub.go index d117605d16..559fab474c 100644 --- a/backend/controller/dal/pubsub.go +++ b/backend/controller/dal/pubsub.go @@ -9,11 +9,11 @@ import ( "github.com/alecthomas/types/optional" sql2 "github.com/TBD54566975/ftl/backend/controller/dal/internal/sql" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/observability" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/libdal" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" "github.com/TBD54566975/ftl/internal/rpc" @@ -21,7 +21,7 @@ import ( ) func (d *DAL) PublishEventForTopic(ctx context.Context, module, topic, caller string, payload []byte) error { - var encryptedPayload encryption.EncryptedAsyncColumn + var encryptedPayload api.EncryptedAsyncColumn err := d.encryption.Encrypt(payload, &encryptedPayload) if err != nil { return fmt.Errorf("failed to encrypt payload: %w", err) diff --git a/internal/encryption/database.go b/backend/controller/encryption/api/database.go similarity index 98% rename from internal/encryption/database.go rename to backend/controller/encryption/api/database.go index 3d19171d69..0d477e52c2 100644 --- a/internal/encryption/database.go +++ b/backend/controller/encryption/api/database.go @@ -1,4 +1,4 @@ -package encryption +package api import ( "database/sql" diff --git a/internal/encryption/encryption.go b/backend/controller/encryption/api/encryption.go similarity index 99% rename from internal/encryption/encryption.go rename to backend/controller/encryption/api/encryption.go index 93dff3060d..f0d3dd3272 100644 --- a/internal/encryption/encryption.go +++ b/backend/controller/encryption/api/encryption.go @@ -1,4 +1,4 @@ -package encryption +package api import ( "bytes" diff --git a/internal/encryption/encryption_test.go b/backend/controller/encryption/api/encryption_test.go similarity index 98% rename from internal/encryption/encryption_test.go rename to backend/controller/encryption/api/encryption_test.go index 23eb8b094b..5b20f5751b 100644 --- a/internal/encryption/encryption_test.go +++ b/backend/controller/encryption/api/encryption_test.go @@ -1,4 +1,4 @@ -package encryption +package api import ( "testing" diff --git a/internal/encryption/integration_test.go b/backend/controller/encryption/api/integration_test.go similarity index 99% rename from internal/encryption/integration_test.go rename to backend/controller/encryption/api/integration_test.go index 5c19af3665..4911ea4da7 100644 --- a/internal/encryption/integration_test.go +++ b/backend/controller/encryption/api/integration_test.go @@ -1,6 +1,6 @@ //go:build integration -package encryption +package api import ( "context" diff --git a/internal/encryption/testdata/go/encryption/encryption.go b/backend/controller/encryption/api/testdata/go/encryption/encryption.go similarity index 100% rename from internal/encryption/testdata/go/encryption/encryption.go rename to backend/controller/encryption/api/testdata/go/encryption/encryption.go diff --git a/internal/encryption/testdata/go/encryption/ftl.toml b/backend/controller/encryption/api/testdata/go/encryption/ftl.toml similarity index 100% rename from internal/encryption/testdata/go/encryption/ftl.toml rename to backend/controller/encryption/api/testdata/go/encryption/ftl.toml diff --git a/internal/encryption/testdata/go/encryption/go.mod b/backend/controller/encryption/api/testdata/go/encryption/go.mod similarity index 100% rename from internal/encryption/testdata/go/encryption/go.mod rename to backend/controller/encryption/api/testdata/go/encryption/go.mod diff --git a/internal/encryption/testdata/go/encryption/go.sum b/backend/controller/encryption/api/testdata/go/encryption/go.sum similarity index 100% rename from internal/encryption/testdata/go/encryption/go.sum rename to backend/controller/encryption/api/testdata/go/encryption/go.sum diff --git a/backend/controller/encryption/dal/dal.go b/backend/controller/encryption/dal/dal.go index 02fc564c1a..3717fe3b09 100644 --- a/backend/controller/encryption/dal/dal.go +++ b/backend/controller/encryption/dal/dal.go @@ -6,9 +6,9 @@ import ( "github.com/alecthomas/types/optional" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/encryption/dal/internal/sql" "github.com/TBD54566975/ftl/backend/libdal" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" ) @@ -62,7 +62,7 @@ func (d *DAL) EnsureKey(ctx context.Context, generateKey func() ([]byte, error)) const verification = "FTL - Towards a 𝝺-calculus for large-scale systems" -func (d *DAL) VerifyEncryptor(ctx context.Context, encryptor encryption.DataEncryptor) (err error) { +func (d *DAL) VerifyEncryptor(ctx context.Context, encryptor api.DataEncryptor) (err error) { tx, err := d.Begin(ctx) if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) @@ -115,11 +115,11 @@ func (d *DAL) VerifyEncryptor(ctx context.Context, encryptor encryption.DataEncr // verifySubkey checks if the subkey is set and if not, sets it to a verification string. // returns (nil, nil) if verified and not changed -func verifySubkey[SK encryption.SubKey]( - encryptor encryption.DataEncryptor, - encrypted optional.Option[encryption.EncryptedColumn[SK]], -) (optional.Option[encryption.EncryptedColumn[SK]], error) { - type EC = encryption.EncryptedColumn[SK] +func verifySubkey[SK api.SubKey]( + encryptor api.DataEncryptor, + encrypted optional.Option[api.EncryptedColumn[SK]], +) (optional.Option[api.EncryptedColumn[SK]], error) { + type EC = api.EncryptedColumn[SK] verifyField, ok := encrypted.Get() if !ok { diff --git a/backend/controller/encryption/dal/internal/sql/querier.go b/backend/controller/encryption/dal/internal/sql/querier.go index b11be04259..2fe670a002 100644 --- a/backend/controller/encryption/dal/internal/sql/querier.go +++ b/backend/controller/encryption/dal/internal/sql/querier.go @@ -7,13 +7,13 @@ package sql import ( "context" - "github.com/TBD54566975/ftl/internal/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" ) type Querier interface { CreateOnlyEncryptionKey(ctx context.Context, key []byte) error GetOnlyEncryptionKey(ctx context.Context) (GetOnlyEncryptionKeyRow, error) - UpdateEncryptionVerification(ctx context.Context, verifyTimeline encryption.OptionalEncryptedTimelineColumn, verifyAsync encryption.OptionalEncryptedAsyncColumn) error + UpdateEncryptionVerification(ctx context.Context, verifyTimeline api.OptionalEncryptedTimelineColumn, verifyAsync api.OptionalEncryptedAsyncColumn) error } var _ Querier = (*Queries)(nil) diff --git a/backend/controller/encryption/dal/internal/sql/queries.sql.go b/backend/controller/encryption/dal/internal/sql/queries.sql.go index 16502e2157..1f31884844 100644 --- a/backend/controller/encryption/dal/internal/sql/queries.sql.go +++ b/backend/controller/encryption/dal/internal/sql/queries.sql.go @@ -8,7 +8,7 @@ package sql import ( "context" - "github.com/TBD54566975/ftl/internal/encryption" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" ) const createOnlyEncryptionKey = `-- name: CreateOnlyEncryptionKey :exec @@ -29,8 +29,8 @@ WHERE id = 1 type GetOnlyEncryptionKeyRow struct { Key []byte - VerifyTimeline encryption.OptionalEncryptedTimelineColumn - VerifyAsync encryption.OptionalEncryptedAsyncColumn + VerifyTimeline api.OptionalEncryptedTimelineColumn + VerifyAsync api.OptionalEncryptedAsyncColumn } func (q *Queries) GetOnlyEncryptionKey(ctx context.Context) (GetOnlyEncryptionKeyRow, error) { @@ -47,7 +47,7 @@ SET verify_timeline = $1, WHERE id = 1 ` -func (q *Queries) UpdateEncryptionVerification(ctx context.Context, verifyTimeline encryption.OptionalEncryptedTimelineColumn, verifyAsync encryption.OptionalEncryptedAsyncColumn) error { +func (q *Queries) UpdateEncryptionVerification(ctx context.Context, verifyTimeline api.OptionalEncryptedTimelineColumn, verifyAsync api.OptionalEncryptedAsyncColumn) error { _, err := q.db.ExecContext(ctx, updateEncryptionVerification, verifyTimeline, verifyAsync) return err } diff --git a/backend/controller/encryption/encryption.go b/backend/controller/encryption/service.go similarity index 78% rename from backend/controller/encryption/encryption.go rename to backend/controller/encryption/service.go index 4631e6aeae..96a5c6c705 100644 --- a/backend/controller/encryption/encryption.go +++ b/backend/controller/encryption/service.go @@ -5,16 +5,16 @@ import ( "encoding/json" "fmt" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/encryption/dal" "github.com/TBD54566975/ftl/backend/libdal" - "github.com/TBD54566975/ftl/internal/encryption" ) type Service struct { - encryptor encryption.DataEncryptor + encryptor api.DataEncryptor } -func New(ctx context.Context, conn libdal.Connection, encryptionBuilder encryption.Builder) (*Service, error) { +func New(ctx context.Context, conn libdal.Connection, encryptionBuilder api.Builder) (*Service, error) { d := dal.New(ctx, conn) encryptor, err := encryptionBuilder.Build(ctx, d) @@ -30,7 +30,7 @@ func New(ctx context.Context, conn libdal.Connection, encryptionBuilder encrypti } // EncryptJSON encrypts the given JSON object and stores it in the provided destination. -func (s *Service) EncryptJSON(v any, dest encryption.Encrypted) error { +func (s *Service) EncryptJSON(v any, dest api.Encrypted) error { serialized, err := json.Marshal(v) if err != nil { return fmt.Errorf("failed to marshal JSON: %w", err) @@ -40,7 +40,7 @@ func (s *Service) EncryptJSON(v any, dest encryption.Encrypted) error { } // DecryptJSON decrypts the given encrypted object and stores it in the provided destination. -func (s *Service) DecryptJSON(encrypted encryption.Encrypted, v any) error { +func (s *Service) DecryptJSON(encrypted api.Encrypted, v any) error { decrypted, err := s.Decrypt(encrypted) if err != nil { return fmt.Errorf("failed to decrypt json with subkey %s: %w", encrypted.SubKey(), err) @@ -53,7 +53,7 @@ func (s *Service) DecryptJSON(encrypted encryption.Encrypted, v any) error { return nil } -func (s *Service) Encrypt(cleartext []byte, dest encryption.Encrypted) error { +func (s *Service) Encrypt(cleartext []byte, dest api.Encrypted) error { err := s.encryptor.Encrypt(cleartext, dest) if err != nil { return fmt.Errorf("failed to encrypt binary with subkey %s: %w", dest.SubKey(), err) @@ -62,7 +62,7 @@ func (s *Service) Encrypt(cleartext []byte, dest encryption.Encrypted) error { return nil } -func (s *Service) Decrypt(encrypted encryption.Encrypted) ([]byte, error) { +func (s *Service) Decrypt(encrypted api.Encrypted) ([]byte, error) { v, err := s.encryptor.Decrypt(encrypted) if err != nil { return nil, fmt.Errorf("failed to decrypt binary with subkey %s: %w", encrypted.SubKey(), err) diff --git a/backend/controller/encryption/encryption_test.go b/backend/controller/encryption/service_test.go similarity index 78% rename from backend/controller/encryption/encryption_test.go rename to backend/controller/encryption/service_test.go index 5dfc6610fb..b67ccc268e 100644 --- a/backend/controller/encryption/encryption_test.go +++ b/backend/controller/encryption/service_test.go @@ -8,8 +8,8 @@ import ( "github.com/alecthomas/assert/v2" "github.com/alecthomas/types/optional" + "github.com/TBD54566975/ftl/backend/controller/encryption/api" "github.com/TBD54566975/ftl/backend/controller/sql/sqltest" - ftlencryption "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" ) @@ -19,7 +19,7 @@ func TestEncryptionService(t *testing.T) { uri := "fake-kms://CK6YwYkBElQKSAowdHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuQWVzR2NtS2V5EhIaEJy4TIQgfCuwxA3ZZgChp_wYARABGK6YwYkBIAE" t.Run("EncryptDecryptJSON", func(t *testing.T) { - service, err := New(ctx, conn, ftlencryption.NewBuilder().WithKMSURI(optional.Some(uri))) + service, err := New(ctx, conn, api.NewBuilder().WithKMSURI(optional.Some(uri))) assert.NoError(t, err) type TestStruct struct { @@ -28,7 +28,7 @@ func TestEncryptionService(t *testing.T) { } original := TestStruct{Name: "John Doe", Age: 30} - var encrypted ftlencryption.EncryptedTimelineColumn + var encrypted api.EncryptedTimelineColumn err = service.EncryptJSON(original, &encrypted) assert.NoError(t, err) @@ -40,11 +40,11 @@ func TestEncryptionService(t *testing.T) { }) t.Run("EncryptDecryptBinary", func(t *testing.T) { - service, err := New(ctx, conn, ftlencryption.NewBuilder().WithKMSURI(optional.Some(uri))) + service, err := New(ctx, conn, api.NewBuilder().WithKMSURI(optional.Some(uri))) assert.NoError(t, err) original := []byte("Hello, World!") - var encrypted ftlencryption.EncryptedTimelineColumn + var encrypted api.EncryptedTimelineColumn err = service.Encrypt(original, &encrypted) assert.NoError(t, err) diff --git a/sqlc.yaml b/sqlc.yaml index 1b1792132e..0c46f8f1d2 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -66,15 +66,15 @@ sql: go_type: type: "optional.Option[model.CronJobKey]" - db_type: "encrypted_async" - go_type: "github.com/TBD54566975/ftl/internal/encryption.EncryptedAsyncColumn" + go_type: "github.com/TBD54566975/ftl/backend/controller/encryption/api.EncryptedAsyncColumn" - db_type: "encrypted_async" nullable: true - go_type: "github.com/TBD54566975/ftl/internal/encryption.OptionalEncryptedAsyncColumn" + go_type: "github.com/TBD54566975/ftl/backend/controller/encryption/api.OptionalEncryptedAsyncColumn" - db_type: "encrypted_timeline" - go_type: "github.com/TBD54566975/ftl/internal/encryption.EncryptedTimelineColumn" + go_type: "github.com/TBD54566975/ftl/backend/controller/encryption/api.EncryptedTimelineColumn" - db_type: "encrypted_timeline" nullable: true - go_type: "github.com/TBD54566975/ftl/internal/encryption.OptionalEncryptedTimelineColumn" + go_type: "github.com/TBD54566975/ftl/backend/controller/encryption/api.OptionalEncryptedTimelineColumn" - db_type: "lease_key" go_type: "github.com/TBD54566975/ftl/backend/controller/leases.Key" - db_type: "lease_key"