From 01256a1721f04e3bd8e032340a530870c7d67539 Mon Sep 17 00:00:00 2001 From: Han Qiao Date: Tue, 24 Sep 2024 17:19:42 +0800 Subject: [PATCH 1/3] feat: support remote config overrides (#2704) * feat: setup basics for branch config override * fix: use pointers for falsy values to determine emptyness * chore: refactor turn Auth.EnableSignup into pointer * wip: attemps non pointer approach * fix: use direct toml parsing to distinguish undefined values * chore: restore gomod * chore: refactor to a single function leverage mergo * chore: fix lint * fix: add branch override logic to LoadConfigFs * fix: inverted logic * chore: add env branch override test * chore: add test for slices merging * chore: remote config overrides * chore: validate project id * Apply suggestions from code review Co-authored-by: Andrew Valleteau --------- Co-authored-by: avallete Co-authored-by: Andrew Valleteau --- pkg/config/config.go | 57 +++++++++++++++++++++++++++++---- pkg/config/config_test.go | 35 ++++++++++++++++++++ pkg/config/testdata/config.toml | 14 ++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index db441e9bc..18414546a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "io/fs" + "maps" "net" "net/http" "net/url" @@ -119,7 +120,8 @@ func (c CustomClaims) NewToken() *jwt.Token { // // Default values for internal configs should be added to `var Config` initializer. type ( - config struct { + // Common config fields between our "base" config and any "remote" branch specific + baseConfig struct { ProjectId string `toml:"project_id"` Hostname string `toml:"-"` Api api `toml:"api"` @@ -135,6 +137,12 @@ type ( Experimental experimental `toml:"experimental" mapstructure:"-"` } + config struct { + baseConfig + Overrides map[string]interface{} `toml:"remotes"` + Remotes map[string]baseConfig `toml:"-"` + } + api struct { Enabled bool `toml:"enabled"` Image string `toml:"-"` @@ -438,6 +446,16 @@ type ( } ) +func (c *baseConfig) Clone() baseConfig { + copy := *c + copy.Storage.Buckets = maps.Clone(c.Storage.Buckets) + copy.Functions = maps.Clone(c.Functions) + copy.Auth.External = maps.Clone(c.Auth.External) + copy.Auth.Email.Template = maps.Clone(c.Auth.Email.Template) + copy.Auth.Sms.TestOTP = maps.Clone(c.Auth.Sms.TestOTP) + return copy +} + type ConfigEditor func(*config) func WithHostname(hostname string) ConfigEditor { @@ -447,7 +465,7 @@ func WithHostname(hostname string) ConfigEditor { } func NewConfig(editors ...ConfigEditor) config { - initial := config{ + initial := config{baseConfig: baseConfig{ Hostname: "127.0.0.1", Api: api{ Image: postgrestImage, @@ -543,7 +561,7 @@ func NewConfig(editors ...ConfigEditor) config { EdgeRuntime: edgeRuntime{ Image: edgeRuntimeImage, }, - } + }} for _, apply := range editors { apply(&initial) } @@ -587,7 +605,6 @@ func (c *config) Load(path string, fsys fs.FS) error { if _, err := dec.Decode(c); err != nil { return errors.Errorf("failed to decode config template: %w", err) } - // Load user defined config if metadata, err := toml.DecodeFS(fsys, builder.ConfigPath, c); err != nil { cwd, osErr := os.Getwd() if osErr != nil { @@ -595,7 +612,11 @@ func (c *config) Load(path string, fsys fs.FS) error { } return errors.Errorf("cannot read config in %s: %w", cwd, err) } else if undecoded := metadata.Undecoded(); len(undecoded) > 0 { - fmt.Fprintf(os.Stderr, "Unknown config fields: %+v\n", undecoded) + for _, key := range undecoded { + if key[0] != "remotes" { + fmt.Fprintf(os.Stderr, "Unknown config field: [%s]\n", key) + } + } } // Load secrets from .env file if err := loadDefaultEnv(); err != nil { @@ -685,10 +706,32 @@ func (c *config) Load(path string, fsys fs.FS) error { } c.Functions[slug] = function } - return c.Validate() + if err := c.baseConfig.Validate(); err != nil { + return err + } + c.Remotes = make(map[string]baseConfig, len(c.Overrides)) + for name, remote := range c.Overrides { + base := c.baseConfig.Clone() + // Encode a toml file with only config overrides + var buf bytes.Buffer + if err := toml.NewEncoder(&buf).Encode(remote); err != nil { + return errors.Errorf("failed to encode map to TOML: %w", err) + } + // Decode overrides using base config as defaults + if metadata, err := toml.NewDecoder(&buf).Decode(&base); err != nil { + return errors.Errorf("failed to decode remote config: %w", err) + } else if undecoded := metadata.Undecoded(); len(undecoded) > 0 { + fmt.Fprintf(os.Stderr, "Unknown config fields: %+v\n", undecoded) + } + if err := base.Validate(); err != nil { + return err + } + c.Remotes[name] = base + } + return nil } -func (c *config) Validate() error { +func (c *baseConfig) Validate() error { if c.ProjectId == "" { return errors.New("Missing required field in config: project_id") } else if sanitized := sanitizeProjectId(c.ProjectId); sanitized != c.ProjectId { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 29a605059..733b3c7b7 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -55,6 +55,41 @@ func TestConfigParsing(t *testing.T) { // Run test assert.Error(t, config.Load("", fsys)) }) + + t.Run("config file with remotes", func(t *testing.T) { + config := NewConfig() + // Setup in-memory fs + fsys := fs.MapFS{ + "supabase/config.toml": &fs.MapFile{Data: testInitConfigEmbed}, + "supabase/templates/invite.html": &fs.MapFile{}, + } + // Run test + t.Setenv("TWILIO_AUTH_TOKEN", "token") + t.Setenv("AZURE_CLIENT_ID", "hello") + t.Setenv("AZURE_SECRET", "this is cool") + t.Setenv("AUTH_SEND_SMS_SECRETS", "v1,whsec_aWxpa2VzdXBhYmFzZXZlcnltdWNoYW5kaWhvcGV5b3Vkb3Rvbw==") + t.Setenv("SENDGRID_API_KEY", "sendgrid") + assert.NoError(t, config.Load("", fsys)) + // Check the default value in the config + assert.Equal(t, "http://127.0.0.1:3000", config.Auth.SiteUrl) + assert.Equal(t, true, config.Auth.EnableSignup) + assert.Equal(t, true, config.Auth.External["azure"].Enabled) + assert.Equal(t, []string{"image/png", "image/jpeg"}, config.Storage.Buckets["images"].AllowedMimeTypes) + // Check the values for remotes override + production, ok := config.Remotes["production"] + assert.True(t, ok) + staging, ok := config.Remotes["staging"] + assert.True(t, ok) + // Check the values for production override + assert.Equal(t, config.ProjectId, production.ProjectId) + assert.Equal(t, "http://feature-auth-branch.com/", production.Auth.SiteUrl) + assert.Equal(t, false, production.Auth.EnableSignup) + assert.Equal(t, false, production.Auth.External["azure"].Enabled) + assert.Equal(t, "nope", production.Auth.External["azure"].ClientId) + // Check the values for the staging override + assert.Equal(t, "staging-project", staging.ProjectId) + assert.Equal(t, []string{"image/png"}, staging.Storage.Buckets["images"].AllowedMimeTypes) + }) } func TestFileSizeLimitConfigParsing(t *testing.T) { diff --git a/pkg/config/testdata/config.toml b/pkg/config/testdata/config.toml index f7061c1e7..a7aa36544 100644 --- a/pkg/config/testdata/config.toml +++ b/pkg/config/testdata/config.toml @@ -217,3 +217,17 @@ s3_region = "ap-southeast-1" s3_access_key = "" # Configures AWS_SECRET_ACCESS_KEY for S3 bucket s3_secret_key = "" + +[remotes.production.auth] +site_url = "http://feature-auth-branch.com/" +enable_signup = false + +[remotes.production.auth.external.azure] +enabled = false +client_id = "nope" + +[remotes.staging] +project_id = "staging-project" + +[remotes.staging.storage.buckets.images] +allowed_mime_types = ["image/png"] From 11c5004fcbe963aae9e7a48e3884018d179a1a95 Mon Sep 17 00:00:00 2001 From: Andrew Valleteau Date: Tue, 24 Sep 2024 12:05:22 +0200 Subject: [PATCH 2/3] fix: disable security opts for db test on bitbucket runner (#2705) hotfix(cli): disable security opts for db test on bitbucket runner --- internal/utils/docker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/utils/docker.go b/internal/utils/docker.go index 80106583b..6264c4056 100644 --- a/internal/utils/docker.go +++ b/internal/utils/docker.go @@ -287,6 +287,9 @@ func DockerStart(ctx context.Context, config container.Config, hostConfig contai // Skip named volume for BitBucket pipeline if os.Getenv("BITBUCKET_CLONE_DIR") != "" { hostConfig.Binds = binds + // Bitbucket doesn't allow for --security-opt option to be set + // https://support.atlassian.com/bitbucket-cloud/docs/run-docker-commands-in-bitbucket-pipelines/#Full-list-of-restricted-commands + hostConfig.SecurityOpt = nil } else { // Create named volumes with labels for _, name := range sources { From 2ad2d330be6a0b8e74041f7bf725fdeff39a0358 Mon Sep 17 00:00:00 2001 From: Ivan Vasilov Date: Tue, 24 Sep 2024 14:49:29 +0200 Subject: [PATCH 3/3] fix: Bump studio to the latest image version 20240923 (#2706) Update studio version to 20240923 --- pkg/config/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/constants.go b/pkg/config/constants.go index 93331225a..d75ce61bc 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -10,7 +10,7 @@ const ( inbucketImage = "inbucket/inbucket:3.0.3" postgrestImage = "postgrest/postgrest:v12.2.0" pgmetaImage = "supabase/postgres-meta:v0.83.2" - studioImage = "supabase/studio:20240729-ce42139" + studioImage = "supabase/studio:20240923-2e3e90c" imageProxyImage = "darthsim/imgproxy:v3.8.0" edgeRuntimeImage = "supabase/edge-runtime:v1.58.3" vectorImage = "timberio/vector:0.28.1-alpine"