Merge branch 'main' into dli/split-dal-soup

.github/workflows/ci.yml

@@ -52,6 +52,18 @@ jobs:
         run: just init-db
       - name: Vet SQL
         run: sqlc vet
+  ensure-frozen-migrations:
+    name: Ensure Frozen Migrations
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Init Hermit
+        uses: cashapp/activate-hermit@v1
+      - name: Freeze Migrations
+        run: just ensure-frozen-migrations
   lint:
     name: Lint
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -174,9 +174,9 @@ jobs:
       - name: Publish
         run: |
           set -euo pipefail
-          version="$(git describe --tags --abbrev=0)"
+          version="$(git describe --tags --abbrev=0 | sed 's/^v//')"
           echo "Publishing version $version"
-          jq ".version = \"$version\"" extensions/vscode/package.json > extensions/vscode/package.json.tmp
+          jq --arg version "$version" '.version = $version' extensions/vscode/package.json > extensions/vscode/package.json.tmp
           mv extensions/vscode/package.json.tmp extensions/vscode/package.json
           just publish-extension
         env:

Dockerfile.runner

@@ -34,6 +34,8 @@ RUN just build ftl
 
 # Finally create the runtime image.
 FROM ubuntu:24.04
+RUN apt-get update
+RUN apt-get install -y ca-certificates
 
 WORKDIR /root/
 

Justfile

@@ -133,6 +133,10 @@ test-readme *args:
 tidy:
   find . -name go.mod -execdir go mod tidy \;
 
+# Check for changes in existing SQL migrations compared to main
+ensure-frozen-migrations:
+  scripts/ensure-frozen-migrations
+
 # Run backend tests
 test-backend:
   @gotestsum --hide-summary skipped --format-hide-empty-pkg -- -short -fullpath ./...

backend/controller/admin/admin.go

@@ -9,29 +9,20 @@ import (
 
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
-	"github.com/TBD54566975/ftl/backend/schema"
 	cf "github.com/TBD54566975/ftl/common/configuration"
-	"github.com/TBD54566975/ftl/go-runtime/encoding"
-	"github.com/TBD54566975/ftl/internal/log"
 )
 
 type AdminService struct {
-	schr SchemaRetriever
-	cm   *cf.Manager[cf.Configuration]
-	sm   *cf.Manager[cf.Secrets]
+	cm *cf.Manager[cf.Configuration]
+	sm *cf.Manager[cf.Secrets]
 }
 
 var _ ftlv1connect.AdminServiceHandler = (*AdminService)(nil)
 
-type SchemaRetriever interface {
-	GetActiveSchema(ctx context.Context) (*schema.Schema, error)
-}
-
-func NewAdminService(cm *cf.Manager[cf.Configuration], sm *cf.Manager[cf.Secrets], schr SchemaRetriever) *AdminService {
+func NewAdminService(cm *cf.Manager[cf.Configuration], sm *cf.Manager[cf.Secrets]) *AdminService {
 	return &AdminService{
-		schr: schr,
-		cm:   cm,
-		sm:   sm,
+		cm: cm,
+		sm: sm,
 	}
 }
 
@@ -43,13 +34,13 @@ func (s *AdminService) Ping(ctx context.Context, req *connect.Request[ftlv1.Ping
 func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv1.ListConfigRequest]) (*connect.Response[ftlv1.ListConfigResponse], error) {
 	listing, err := s.cm.List(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("failed to list configs: %w", err)
+		return nil, err
 	}
 
 	configs := []*ftlv1.ListConfigResponse_Config{}
 	for _, config := range listing {
 		module, ok := config.Module.Get()
-		if *req.Msg.Module != "" && module != *req.Msg.Module {
+		if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
 			continue
 		}
 
@@ -82,13 +73,13 @@ func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv
 // ConfigGet returns the configuration value for a given ref string.
 func (s *AdminService) ConfigGet(ctx context.Context, req *connect.Request[ftlv1.GetConfigRequest]) (*connect.Response[ftlv1.GetConfigResponse], error) {
 	var value any
-	err := s.cm.Get(ctx, refFromConfigRef(req.Msg.GetRef()), &value)
+	err := s.cm.Get(ctx, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), &value)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get from config manager: %w", err)
+		return nil, err
 	}
 	vb, err := json.MarshalIndent(value, "", "  ")
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal value: %w", err)
+		return nil, err
 	}
 	return connect.NewResponse(&ftlv1.GetConfigResponse{Value: vb}), nil
 }
@@ -110,25 +101,20 @@ func configProviderKey(p *ftlv1.ConfigProvider) string {
 
 // ConfigSet sets the configuration at the given ref to the provided value.
 func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1.SetConfigRequest]) (*connect.Response[ftlv1.SetConfigResponse], error) {
-	err := s.validateAgainstSchema(ctx, false, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
-	if err != nil {
-		return nil, err
-	}
-
 	pkey := configProviderKey(req.Msg.Provider)
-	err = s.cm.SetJSON(ctx, pkey, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
+	err := s.cm.SetJSON(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), req.Msg.Value)
 	if err != nil {
-		return nil, fmt.Errorf("failed to set config: %w", err)
+		return nil, err
 	}
 	return connect.NewResponse(&ftlv1.SetConfigResponse{}), nil
 }
 
 // ConfigUnset unsets the config value at the given ref.
 func (s *AdminService) ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error) {
 	pkey := configProviderKey(req.Msg.Provider)
-	err := s.cm.Unset(ctx, pkey, refFromConfigRef(req.Msg.GetRef()))
+	err := s.cm.Unset(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name))
 	if err != nil {
-		return nil, fmt.Errorf("failed to unset config: %w", err)
+		return nil, err
 	}
 	return connect.NewResponse(&ftlv1.UnsetConfigResponse{}), nil
 }
@@ -137,12 +123,16 @@ func (s *AdminService) ConfigUnset(ctx context.Context, req *connect.Request[ftl
 func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error) {
 	listing, err := s.sm.List(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("failed to list secrets: %w", err)
+		return nil, err
 	}
 	secrets := []*ftlv1.ListSecretsResponse_Secret{}
 	for _, secret := range listing {
+		if req.Msg.Provider != nil && cf.ProviderKeyForAccessor(secret.Accessor) != secretProviderKey(req.Msg.Provider) {
+			// Skip secrets that don't match the provider in the request
+			continue
+		}
 		module, ok := secret.Module.Get()
-		if *req.Msg.Module != "" && module != *req.Msg.Module {
+		if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
 			continue
 		}
 		ref := secret.Name
@@ -172,13 +162,13 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl
 // SecretGet returns the secret value for a given ref string.
 func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1.GetSecretRequest]) (*connect.Response[ftlv1.GetSecretResponse], error) {
 	var value any
-	err := s.sm.Get(ctx, refFromConfigRef(req.Msg.GetRef()), &value)
+	err := s.sm.Get(ctx, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), &value)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get from secret manager: %w", err)
+		return nil, err
 	}
 	vb, err := json.MarshalIndent(value, "", "  ")
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal value: %w", err)
+		return nil, err
 	}
 	return connect.NewResponse(&ftlv1.GetSecretResponse{Value: vb}), nil
 }
@@ -204,79 +194,20 @@ func secretProviderKey(p *ftlv1.SecretProvider) string {
 
 // SecretSet sets the secret at the given ref to the provided value.
 func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error) {
-	err := s.validateAgainstSchema(ctx, true, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
-	if err != nil {
-		return nil, err
-	}
-
 	pkey := secretProviderKey(req.Msg.Provider)
-	err = s.sm.SetJSON(ctx, pkey, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
+	err := s.sm.SetJSON(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), req.Msg.Value)
 	if err != nil {
-		return nil, fmt.Errorf("failed to set secret: %w", err)
+		return nil, err
 	}
 	return connect.NewResponse(&ftlv1.SetSecretResponse{}), nil
 }
 
 // SecretUnset unsets the secret value at the given ref.
 func (s *AdminService) SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error) {
 	pkey := secretProviderKey(req.Msg.Provider)
-	err := s.sm.Unset(ctx, pkey, refFromConfigRef(req.Msg.GetRef()))
+	err := s.sm.Unset(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name))
 	if err != nil {
-		return nil, fmt.Errorf("failed to unset secret: %w", err)
+		return nil, err
 	}
 	return connect.NewResponse(&ftlv1.UnsetSecretResponse{}), nil
 }
-
-func refFromConfigRef(cr *ftlv1.ConfigRef) cf.Ref {
-	return cf.NewRef(cr.GetModule(), cr.GetName())
-}
-
-func (s *AdminService) validateAgainstSchema(ctx context.Context, isSecret bool, ref cf.Ref, value json.RawMessage) error {
-	logger := log.FromContext(ctx)
-
-	// Globals aren't in the module schemas, so we have nothing to validate against.
-	if !ref.Module.Ok() {
-		return nil
-	}
-
-	// If we can't retrieve an active schema, skip validation.
-	sch, err := s.schr.GetActiveSchema(ctx)
-	if err != nil {
-		logger.Debugf("skipping validation; could not get the active schema: %v", err)
-		return nil
-	}
-
-	r := schema.RefKey{Module: ref.Module.Default(""), Name: ref.Name}.ToRef()
-	decl, ok := sch.Resolve(r).Get()
-	if !ok {
-		return fmt.Errorf("declaration %q not found", ref.Name)
-	}
-
-	var fieldType schema.Type
-	if isSecret {
-		decl, ok := decl.(*schema.Secret)
-		if !ok {
-			return fmt.Errorf("%q is not a secret declaration", ref.Name)
-		}
-		fieldType = decl.Type
-	} else {
-		decl, ok := decl.(*schema.Config)
-		if !ok {
-			return fmt.Errorf("%q is not a config declaration", ref.Name)
-		}
-		fieldType = decl.Type
-	}
-
-	var v any
-	err = encoding.Unmarshal(value, &v)
-	if err != nil {
-		return fmt.Errorf("could not unmarshal JSON value: %w", err)
-	}
-
-	err = schema.ValidateJSONValue(fieldType, []string{ref.Name}, v, sch)
-	if err != nil {
-		return fmt.Errorf("JSON validation failed: %w", err)
-	}
-
-	return nil
-}