From 15ff758841e8a90c11205c99c97a23bdacff4edb Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 11:27:15 +0200 Subject: [PATCH 01/20] feat: add remote utils for api config --- internal/link/link.go | 16 +-- pkg/config/api.go | 72 ++++++++++++ pkg/config/api_test.go | 77 ++++++++++++ pkg/config/config.go | 27 ++--- pkg/config/diff.go | 261 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 422 insertions(+), 31 deletions(-) create mode 100644 pkg/config/api.go create mode 100644 pkg/config/api_test.go create mode 100644 pkg/config/diff.go diff --git a/internal/link/link.go b/internal/link/link.go index 0c2aa1de4..fd2d57b43 100644 --- a/internal/link/link.go +++ b/internal/link/link.go @@ -1,7 +1,6 @@ package link import ( - "bytes" "context" "fmt" "os" @@ -9,7 +8,6 @@ import ( "strings" "sync" - "github.com/BurntSushi/toml" "github.com/go-errors/errors" "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" @@ -25,7 +23,7 @@ import ( ) func Run(ctx context.Context, projectRef string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - original := toTomlBytes(map[string]interface{}{ + original := cliConfig.ToTomlBytes(map[string]interface{}{ "api": utils.Config.Api, "db": utils.Config.Db, }) @@ -60,7 +58,7 @@ func Run(ctx context.Context, projectRef string, fsys afero.Fs, options ...func( fmt.Fprintln(os.Stdout, "Finished "+utils.Aqua("supabase link")+".") // 4. Suggest config update - updated := toTomlBytes(map[string]interface{}{ + updated := cliConfig.ToTomlBytes(map[string]interface{}{ "api": utils.Config.Api, "db": utils.Config.Db, }) @@ -72,16 +70,6 @@ func Run(ctx context.Context, projectRef string, fsys afero.Fs, options ...func( return nil } -func toTomlBytes(config any) []byte { - var buf bytes.Buffer - enc := toml.NewEncoder(&buf) - enc.Indent = "" - if err := enc.Encode(config); err != nil { - fmt.Fprintln(utils.GetDebugLogger(), "failed to marshal toml config:", err) - } - return buf.Bytes() -} - func LinkServices(ctx context.Context, projectRef, anonKey string, fsys afero.Fs) { // Ignore non-fatal errors linking services var wg sync.WaitGroup diff --git a/pkg/config/api.go b/pkg/config/api.go new file mode 100644 index 000000000..f502b9281 --- /dev/null +++ b/pkg/config/api.go @@ -0,0 +1,72 @@ +package config + +import ( + "strings" + + v1API "github.com/supabase/cli/pkg/api" +) + +type ( + api struct { + Enabled bool `toml:"enabled"` + Image string `toml:"-"` + KongImage string `toml:"-"` + Port uint16 `toml:"port"` + Schemas []string `toml:"schemas"` + ExtraSearchPath []string `toml:"extra_search_path"` + MaxRows uint `toml:"max_rows"` + Tls tlsKong `toml:"tls"` + // TODO: replace [auth|studio].api_url + ExternalUrl string `toml:"external_url"` + } + + tlsKong struct { + Enabled bool `toml:"enabled"` + } +) + +func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { + body := v1API.UpdatePostgrestConfigBody{} + + // Convert Schemas to a comma-separated string + if len(a.Schemas) > 0 { + schemas := strings.Join(a.Schemas, ",") + body.DbSchema = &schemas + } + + // Convert ExtraSearchPath to a comma-separated string + if len(a.ExtraSearchPath) > 0 { + extraSearchPath := strings.Join(a.ExtraSearchPath, ",") + body.DbExtraSearchPath = &extraSearchPath + } + + // Convert MaxRows to int pointer + if a.MaxRows > 0 { + maxRows := int(a.MaxRows) + body.MaxRows = &maxRows + } + + // Note: DbPool is not present in the Api struct, so it's not set here + return body +} + +func (a *api) FromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) api { + result := *a + // Update Schemas if present in remoteConfig + result.Schemas = strings.Split(remoteConfig.DbSchema, ",") + + // Update ExtraSearchPath if present in remoteConfig + result.ExtraSearchPath = strings.Split(remoteConfig.DbExtraSearchPath, ",") + + // Update MaxRows if present in remoteConfig + result.MaxRows = uint(remoteConfig.MaxRows) + + return result +} + +func (a *api) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { + // Convert the config values into easily comparable remoteConfig values + currentValue := ToTomlBytes(a) + remoteCompare := ToTomlBytes(a.FromRemoteApiConfig(remoteConfig)) + return Diff("remote[api]", remoteCompare, "local[api]", currentValue) +} diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go new file mode 100644 index 000000000..881cd4893 --- /dev/null +++ b/pkg/config/api_test.go @@ -0,0 +1,77 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1API "github.com/supabase/cli/pkg/api" +) + +func TestApiToUpdatePostgrestConfigBody(t *testing.T) { + t.Run("converts all fields correctly", func(t *testing.T) { + api := &api{ + Schemas: []string{"public", "private"}, + ExtraSearchPath: []string{"extensions", "public"}, + MaxRows: 1000, + } + + body := api.ToUpdatePostgrestConfigBody() + + assert.Equal(t, "public,private", *body.DbSchema) + assert.Equal(t, "extensions,public", *body.DbExtraSearchPath) + assert.Equal(t, 1000, *body.MaxRows) + }) + + t.Run("handles empty fields", func(t *testing.T) { + api := &api{} + + body := api.ToUpdatePostgrestConfigBody() + + assert.Nil(t, body.DbSchema) + assert.Nil(t, body.DbExtraSearchPath) + assert.Nil(t, body.MaxRows) + }) +} + +func TestApiDiffWithRemote(t *testing.T) { + t.Run("detects differences", func(t *testing.T) { + api := &api{ + Schemas: []string{"public", "private"}, + ExtraSearchPath: []string{"extensions", "public"}, + MaxRows: 1000, + } + + remoteConfig := v1API.PostgrestConfigWithJWTSecretResponse{ + DbSchema: "public", + DbExtraSearchPath: "public", + MaxRows: 500, + } + + diff := api.DiffWithRemote(remoteConfig) + + assert.Contains(t, string(diff), "-schemas = [\"public\"]") + assert.Contains(t, string(diff), "+schemas = [\"public\", \"private\"]") + assert.Contains(t, string(diff), "-extra_search_path = [\"public\"]") + assert.Contains(t, string(diff), "+extra_search_path = [\"extensions\", \"public\"]") + assert.Contains(t, string(diff), "-max_rows = 500") + assert.Contains(t, string(diff), "+max_rows = 1000") + }) + + t.Run("handles no differences", func(t *testing.T) { + api := &api{ + Schemas: []string{"public"}, + ExtraSearchPath: []string{"public"}, + MaxRows: 500, + } + + remoteConfig := v1API.PostgrestConfigWithJWTSecretResponse{ + DbSchema: "public", + DbExtraSearchPath: "public", + MaxRows: 500, + } + + diff := api.DiffWithRemote(remoteConfig) + + assert.Empty(t, diff) + }) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 80270124c..e00fc5b8e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -145,23 +145,6 @@ type ( Remotes map[string]baseConfig `toml:"-"` } - api struct { - Enabled bool `toml:"enabled"` - Image string `toml:"-"` - KongImage string `toml:"-"` - Port uint16 `toml:"port"` - Schemas []string `toml:"schemas"` - ExtraSearchPath []string `toml:"extra_search_path"` - MaxRows uint `toml:"max_rows"` - Tls tlsKong `toml:"tls"` - // TODO: replace [auth|studio].api_url - ExternalUrl string `toml:"external_url"` - } - - tlsKong struct { - Enabled bool `toml:"enabled"` - } - db struct { Image string `toml:"-"` Port uint16 `toml:"port"` @@ -1322,3 +1305,13 @@ func (a *auth) ResolveJWKS(ctx context.Context) (string, error) { return string(jwksEncoded), nil } + +func ToTomlBytes(config any) []byte { + var buf bytes.Buffer + enc := toml.NewEncoder(&buf) + enc.Indent = "" + if err := enc.Encode(config); err != nil { + fmt.Fprintln(os.Stderr, "failed to marshal toml config:", err) + } + return buf.Bytes() +} diff --git a/pkg/config/diff.go b/pkg/config/diff.go new file mode 100644 index 000000000..04aa76ed4 --- /dev/null +++ b/pkg/config/diff.go @@ -0,0 +1,261 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package config + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +// A pair is a pair of values tracked for both the x and y side of a diff. +// It is typically a pair of line indexes. +type pair struct{ x, y int } + +// Diff returns an anchored diff of the two texts old and new +// in the “unified diff” format. If old and new are identical, +// Diff returns a nil slice (no output). +// +// Unix diff implementations typically look for a diff with +// the smallest number of lines inserted and removed, +// which can in the worst case take time quadratic in the +// number of lines in the texts. As a result, many implementations +// either can be made to run for a long time or cut off the search +// after a predetermined amount of work. +// +// In contrast, this implementation looks for a diff with the +// smallest number of “unique” lines inserted and removed, +// where unique means a line that appears just once in both old and new. +// We call this an “anchored diff” because the unique lines anchor +// the chosen matching regions. An anchored diff is usually clearer +// than a standard diff, because the algorithm does not try to +// reuse unrelated blank lines or closing braces. +// The algorithm also guarantees to run in O(n log n) time +// instead of the standard O(n²) time. +// +// Some systems call this approach a “patience diff,” named for +// the “patience sorting” algorithm, itself named for a solitaire card game. +// We avoid that name for two reasons. First, the name has been used +// for a few different variants of the algorithm, so it is imprecise. +// Second, the name is frequently interpreted as meaning that you have +// to wait longer (to be patient) for the diff, meaning that it is a slower algorithm, +// when in fact the algorithm is faster than the standard one. +func Diff(oldName string, old []byte, newName string, new []byte) []byte { + if bytes.Equal(old, new) { + return nil + } + x := lines(old) + y := lines(new) + + // Print diff header. + var out bytes.Buffer + fmt.Fprintf(&out, "diff %s %s\n", oldName, newName) + fmt.Fprintf(&out, "--- %s\n", oldName) + fmt.Fprintf(&out, "+++ %s\n", newName) + + // Loop over matches to consider, + // expanding each match to include surrounding lines, + // and then printing diff chunks. + // To avoid setup/teardown cases outside the loop, + // tgs returns a leading {0,0} and trailing {len(x), len(y)} pair + // in the sequence of matches. + var ( + done pair // printed up to x[:done.x] and y[:done.y] + chunk pair // start lines of current chunk + count pair // number of lines from each side in current chunk + ctext []string // lines for current chunk + ) + for _, m := range tgs(x, y) { + if m.x < done.x { + // Already handled scanning forward from earlier match. + continue + } + + // Expand matching lines as far as possible, + // establishing that x[start.x:end.x] == y[start.y:end.y]. + // Note that on the first (or last) iteration we may (or definitely do) + // have an empty match: start.x==end.x and start.y==end.y. + start := m + for start.x > done.x && start.y > done.y && x[start.x-1] == y[start.y-1] { + start.x-- + start.y-- + } + end := m + for end.x < len(x) && end.y < len(y) && x[end.x] == y[end.y] { + end.x++ + end.y++ + } + + // Emit the mismatched lines before start into this chunk. + // (No effect on first sentinel iteration, when start = {0,0}.) + for _, s := range x[done.x:start.x] { + ctext = append(ctext, "-"+s) + count.x++ + } + for _, s := range y[done.y:start.y] { + ctext = append(ctext, "+"+s) + count.y++ + } + + // If we're not at EOF and have too few common lines, + // the chunk includes all the common lines and continues. + const C = 3 // number of context lines + if (end.x < len(x) || end.y < len(y)) && + (end.x-start.x < C || (len(ctext) > 0 && end.x-start.x < 2*C)) { + for _, s := range x[start.x:end.x] { + ctext = append(ctext, " "+s) + count.x++ + count.y++ + } + done = end + continue + } + + // End chunk with common lines for context. + if len(ctext) > 0 { + n := end.x - start.x + if n > C { + n = C + } + for _, s := range x[start.x : start.x+n] { + ctext = append(ctext, " "+s) + count.x++ + count.y++ + } + done = pair{start.x + n, start.y + n} + + // Format and emit chunk. + // Convert line numbers to 1-indexed. + // Special case: empty file shows up as 0,0 not 1,0. + if count.x > 0 { + chunk.x++ + } + if count.y > 0 { + chunk.y++ + } + fmt.Fprintf(&out, "@@ -%d,%d +%d,%d @@\n", chunk.x, count.x, chunk.y, count.y) + for _, s := range ctext { + out.WriteString(s) + } + count.x = 0 + count.y = 0 + ctext = ctext[:0] + } + + // If we reached EOF, we're done. + if end.x >= len(x) && end.y >= len(y) { + break + } + + // Otherwise start a new chunk. + chunk = pair{end.x - C, end.y - C} + for _, s := range x[chunk.x:end.x] { + ctext = append(ctext, " "+s) + count.x++ + count.y++ + } + done = end + } + + return out.Bytes() +} + +// lines returns the lines in the file x, including newlines. +// If the file does not end in a newline, one is supplied +// along with a warning about the missing newline. +func lines(x []byte) []string { + l := strings.SplitAfter(string(x), "\n") + if l[len(l)-1] == "" { + l = l[:len(l)-1] + } else { + // Treat last line as having a message about the missing newline attached, + // using the same text as BSD/GNU diff (including the leading backslash). + l[len(l)-1] += "\n\\ No newline at end of file\n" + } + return l +} + +// tgs returns the pairs of indexes of the longest common subsequence +// of unique lines in x and y, where a unique line is one that appears +// once in x and once in y. +// +// The longest common subsequence algorithm is as described in +// Thomas G. Szymanski, “A Special Case of the Maximal Common +// Subsequence Problem,” Princeton TR #170 (January 1975), +// available at https://research.swtch.com/tgs170.pdf. +func tgs(x, y []string) []pair { + // Count the number of times each string appears in a and b. + // We only care about 0, 1, many, counted as 0, -1, -2 + // for the x side and 0, -4, -8 for the y side. + // Using negative numbers now lets us distinguish positive line numbers later. + m := make(map[string]int) + for _, s := range x { + if c := m[s]; c > -2 { + m[s] = c - 1 + } + } + for _, s := range y { + if c := m[s]; c > -8 { + m[s] = c - 4 + } + } + + // Now unique strings can be identified by m[s] = -1+-4. + // + // Gather the indexes of those strings in x and y, building: + // xi[i] = increasing indexes of unique strings in x. + // yi[i] = increasing indexes of unique strings in y. + // inv[i] = index j such that x[xi[i]] = y[yi[j]]. + var xi, yi, inv []int + for i, s := range y { + if m[s] == -1+-4 { + m[s] = len(yi) + yi = append(yi, i) + } + } + for i, s := range x { + if j, ok := m[s]; ok && j >= 0 { + xi = append(xi, i) + inv = append(inv, j) + } + } + + // Apply Algorithm A from Szymanski's paper. + // In those terms, A = J = inv and B = [0, n). + // We add sentinel pairs {0,0}, and {len(x),len(y)} + // to the returned sequence, to help the processing loop. + J := inv + n := len(xi) + T := make([]int, n) + L := make([]int, n) + for i := range T { + T[i] = n + 1 + } + for i := 0; i < n; i++ { + k := sort.Search(n, func(k int) bool { + return T[k] >= J[i] + }) + T[k] = J[i] + L[i] = k + 1 + } + k := 0 + for _, v := range L { + if k < v { + k = v + } + } + seq := make([]pair, 2+k) + seq[1+k] = pair{len(x), len(y)} // sentinel at end + lastj := n + for i := n - 1; i >= 0; i-- { + if L[i] == k && J[i] < lastj { + seq[k] = pair{xi[i], yi[J[i]]} + k-- + } + } + seq[0] = pair{0, 0} // sentinel at start + return seq +} From 23173ec9830d8ae97b43d0fa12c183b1d99f7069 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 14:24:27 +0200 Subject: [PATCH 02/20] chore: split api and remote api --- pkg/config/api.go | 23 +++++++++++++---------- pkg/config/api_test.go | 8 ++++---- pkg/config/config.go | 9 +++++++++ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index f502b9281..824acaf6e 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -7,15 +7,18 @@ import ( ) type ( - api struct { - Enabled bool `toml:"enabled"` - Image string `toml:"-"` - KongImage string `toml:"-"` - Port uint16 `toml:"port"` + RemoteApi struct { Schemas []string `toml:"schemas"` ExtraSearchPath []string `toml:"extra_search_path"` MaxRows uint `toml:"max_rows"` - Tls tlsKong `toml:"tls"` + } + api struct { + RemoteApi + Enabled bool `toml:"enabled"` + Image string `toml:"-"` + KongImage string `toml:"-"` + Port uint16 `toml:"port"` + Tls tlsKong `toml:"tls"` // TODO: replace [auth|studio].api_url ExternalUrl string `toml:"external_url"` } @@ -25,7 +28,7 @@ type ( } ) -func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { +func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { body := v1API.UpdatePostgrestConfigBody{} // Convert Schemas to a comma-separated string @@ -50,7 +53,7 @@ func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { return body } -func (a *api) FromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) api { +func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) RemoteApi { result := *a // Update Schemas if present in remoteConfig result.Schemas = strings.Split(remoteConfig.DbSchema, ",") @@ -64,9 +67,9 @@ func (a *api) FromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecre return result } -func (a *api) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { +func (a *RemoteApi) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { // Convert the config values into easily comparable remoteConfig values currentValue := ToTomlBytes(a) - remoteCompare := ToTomlBytes(a.FromRemoteApiConfig(remoteConfig)) + remoteCompare := ToTomlBytes(a.fromRemoteApiConfig(remoteConfig)) return Diff("remote[api]", remoteCompare, "local[api]", currentValue) } diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index 881cd4893..cbbb9da64 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -9,7 +9,7 @@ import ( func TestApiToUpdatePostgrestConfigBody(t *testing.T) { t.Run("converts all fields correctly", func(t *testing.T) { - api := &api{ + api := &RemoteApi{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -23,7 +23,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { }) t.Run("handles empty fields", func(t *testing.T) { - api := &api{} + api := &RemoteApi{} body := api.ToUpdatePostgrestConfigBody() @@ -35,7 +35,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { func TestApiDiffWithRemote(t *testing.T) { t.Run("detects differences", func(t *testing.T) { - api := &api{ + api := &RemoteApi{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -58,7 +58,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles no differences", func(t *testing.T) { - api := &api{ + api := &RemoteApi{ Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, MaxRows: 500, diff --git a/pkg/config/config.go b/pkg/config/config.go index e00fc5b8e..a59c91313 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1306,6 +1306,15 @@ func (a *auth) ResolveJWKS(ctx context.Context) (string, error) { return string(jwksEncoded), nil } +// Retrieve the final base config to use taking into account the remotes override +func (c *config) GetRemoteOverride(remotes_name string) (overrideConfig baseConfig, overrideExist bool) { + overrideConfig, exist := c.Remotes[remotes_name] + if exist { + return overrideConfig, true + } + return c.baseConfig, false +} + func ToTomlBytes(config any) []byte { var buf bytes.Buffer enc := toml.NewEncoder(&buf) From 65bc6d32004a3fb25fee19a246cf9231f726e45d Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 14:29:09 +0200 Subject: [PATCH 03/20] Revert "chore: split api and remote api" This reverts commit 23173ec9830d8ae97b43d0fa12c183b1d99f7069. --- pkg/config/api.go | 23 ++++++++++------------- pkg/config/api_test.go | 8 ++++---- pkg/config/config.go | 9 --------- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index 824acaf6e..f502b9281 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -7,18 +7,15 @@ import ( ) type ( - RemoteApi struct { + api struct { + Enabled bool `toml:"enabled"` + Image string `toml:"-"` + KongImage string `toml:"-"` + Port uint16 `toml:"port"` Schemas []string `toml:"schemas"` ExtraSearchPath []string `toml:"extra_search_path"` MaxRows uint `toml:"max_rows"` - } - api struct { - RemoteApi - Enabled bool `toml:"enabled"` - Image string `toml:"-"` - KongImage string `toml:"-"` - Port uint16 `toml:"port"` - Tls tlsKong `toml:"tls"` + Tls tlsKong `toml:"tls"` // TODO: replace [auth|studio].api_url ExternalUrl string `toml:"external_url"` } @@ -28,7 +25,7 @@ type ( } ) -func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { +func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { body := v1API.UpdatePostgrestConfigBody{} // Convert Schemas to a comma-separated string @@ -53,7 +50,7 @@ func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBod return body } -func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) RemoteApi { +func (a *api) FromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) api { result := *a // Update Schemas if present in remoteConfig result.Schemas = strings.Split(remoteConfig.DbSchema, ",") @@ -67,9 +64,9 @@ func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJW return result } -func (a *RemoteApi) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { +func (a *api) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { // Convert the config values into easily comparable remoteConfig values currentValue := ToTomlBytes(a) - remoteCompare := ToTomlBytes(a.fromRemoteApiConfig(remoteConfig)) + remoteCompare := ToTomlBytes(a.FromRemoteApiConfig(remoteConfig)) return Diff("remote[api]", remoteCompare, "local[api]", currentValue) } diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index cbbb9da64..881cd4893 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -9,7 +9,7 @@ import ( func TestApiToUpdatePostgrestConfigBody(t *testing.T) { t.Run("converts all fields correctly", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -23,7 +23,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { }) t.Run("handles empty fields", func(t *testing.T) { - api := &RemoteApi{} + api := &api{} body := api.ToUpdatePostgrestConfigBody() @@ -35,7 +35,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { func TestApiDiffWithRemote(t *testing.T) { t.Run("detects differences", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -58,7 +58,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles no differences", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, MaxRows: 500, diff --git a/pkg/config/config.go b/pkg/config/config.go index a59c91313..e00fc5b8e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1306,15 +1306,6 @@ func (a *auth) ResolveJWKS(ctx context.Context) (string, error) { return string(jwksEncoded), nil } -// Retrieve the final base config to use taking into account the remotes override -func (c *config) GetRemoteOverride(remotes_name string) (overrideConfig baseConfig, overrideExist bool) { - overrideConfig, exist := c.Remotes[remotes_name] - if exist { - return overrideConfig, true - } - return c.baseConfig, false -} - func ToTomlBytes(config any) []byte { var buf bytes.Buffer enc := toml.NewEncoder(&buf) From 21270cf95460b42e6c32cda6983786653b312efd Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 14:29:56 +0200 Subject: [PATCH 04/20] chore: make api public --- pkg/config/api.go | 10 +++++----- pkg/config/api_test.go | 8 ++++---- pkg/config/config.go | 13 +++++++++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index f502b9281..92802432a 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -7,7 +7,7 @@ import ( ) type ( - api struct { + Api struct { Enabled bool `toml:"enabled"` Image string `toml:"-"` KongImage string `toml:"-"` @@ -25,7 +25,7 @@ type ( } ) -func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { +func (a *Api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { body := v1API.UpdatePostgrestConfigBody{} // Convert Schemas to a comma-separated string @@ -50,7 +50,7 @@ func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { return body } -func (a *api) FromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) api { +func (a *Api) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) Api { result := *a // Update Schemas if present in remoteConfig result.Schemas = strings.Split(remoteConfig.DbSchema, ",") @@ -64,9 +64,9 @@ func (a *api) FromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecre return result } -func (a *api) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { +func (a *Api) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { // Convert the config values into easily comparable remoteConfig values currentValue := ToTomlBytes(a) - remoteCompare := ToTomlBytes(a.FromRemoteApiConfig(remoteConfig)) + remoteCompare := ToTomlBytes(a.fromRemoteApiConfig(remoteConfig)) return Diff("remote[api]", remoteCompare, "local[api]", currentValue) } diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index 881cd4893..b0fd6a3b3 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -9,7 +9,7 @@ import ( func TestApiToUpdatePostgrestConfigBody(t *testing.T) { t.Run("converts all fields correctly", func(t *testing.T) { - api := &api{ + api := &Api{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -23,7 +23,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { }) t.Run("handles empty fields", func(t *testing.T) { - api := &api{} + api := &Api{} body := api.ToUpdatePostgrestConfigBody() @@ -35,7 +35,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { func TestApiDiffWithRemote(t *testing.T) { t.Run("detects differences", func(t *testing.T) { - api := &api{ + api := &Api{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -58,7 +58,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles no differences", func(t *testing.T) { - api := &api{ + api := &Api{ Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, MaxRows: 500, diff --git a/pkg/config/config.go b/pkg/config/config.go index e00fc5b8e..89c690f59 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -126,7 +126,7 @@ type ( baseConfig struct { ProjectId string `toml:"project_id"` Hostname string `toml:"-"` - Api api `toml:"api"` + Api Api `toml:"api"` Db db `toml:"db" mapstructure:"db"` Realtime realtime `toml:"realtime"` Studio studio `toml:"studio"` @@ -454,7 +454,7 @@ func WithHostname(hostname string) ConfigEditor { func NewConfig(editors ...ConfigEditor) config { initial := config{baseConfig: baseConfig{ Hostname: "127.0.0.1", - Api: api{ + Api: Api{ Image: postgrestImage, KongImage: kongImage, }, @@ -1306,6 +1306,15 @@ func (a *auth) ResolveJWKS(ctx context.Context) (string, error) { return string(jwksEncoded), nil } +// Retrieve the final base config to use taking into account the remotes override +func (c *config) GetRemoteOverride(remotes_name string) (overrideConfig baseConfig, overrideExist bool) { + overrideConfig, exist := c.Remotes[remotes_name] + if exist { + return overrideConfig, true + } + return c.baseConfig, false +} + func ToTomlBytes(config any) []byte { var buf bytes.Buffer enc := toml.NewEncoder(&buf) From b7ad1e0f1d606fc497caab9a0a6c2c4493885c39 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 14:35:42 +0200 Subject: [PATCH 05/20] Revert "Revert "chore: split api and remote api"" This reverts commit 65bc6d32004a3fb25fee19a246cf9231f726e45d. --- pkg/config/api.go | 21 ++++++++++++--------- pkg/config/api_test.go | 8 ++++---- pkg/config/config.go | 4 ++-- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index 92802432a..824acaf6e 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -7,15 +7,18 @@ import ( ) type ( - Api struct { - Enabled bool `toml:"enabled"` - Image string `toml:"-"` - KongImage string `toml:"-"` - Port uint16 `toml:"port"` + RemoteApi struct { Schemas []string `toml:"schemas"` ExtraSearchPath []string `toml:"extra_search_path"` MaxRows uint `toml:"max_rows"` - Tls tlsKong `toml:"tls"` + } + api struct { + RemoteApi + Enabled bool `toml:"enabled"` + Image string `toml:"-"` + KongImage string `toml:"-"` + Port uint16 `toml:"port"` + Tls tlsKong `toml:"tls"` // TODO: replace [auth|studio].api_url ExternalUrl string `toml:"external_url"` } @@ -25,7 +28,7 @@ type ( } ) -func (a *Api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { +func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { body := v1API.UpdatePostgrestConfigBody{} // Convert Schemas to a comma-separated string @@ -50,7 +53,7 @@ func (a *Api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { return body } -func (a *Api) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) Api { +func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) RemoteApi { result := *a // Update Schemas if present in remoteConfig result.Schemas = strings.Split(remoteConfig.DbSchema, ",") @@ -64,7 +67,7 @@ func (a *Api) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecre return result } -func (a *Api) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { +func (a *RemoteApi) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { // Convert the config values into easily comparable remoteConfig values currentValue := ToTomlBytes(a) remoteCompare := ToTomlBytes(a.fromRemoteApiConfig(remoteConfig)) diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index b0fd6a3b3..cbbb9da64 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -9,7 +9,7 @@ import ( func TestApiToUpdatePostgrestConfigBody(t *testing.T) { t.Run("converts all fields correctly", func(t *testing.T) { - api := &Api{ + api := &RemoteApi{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -23,7 +23,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { }) t.Run("handles empty fields", func(t *testing.T) { - api := &Api{} + api := &RemoteApi{} body := api.ToUpdatePostgrestConfigBody() @@ -35,7 +35,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { func TestApiDiffWithRemote(t *testing.T) { t.Run("detects differences", func(t *testing.T) { - api := &Api{ + api := &RemoteApi{ Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -58,7 +58,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles no differences", func(t *testing.T) { - api := &Api{ + api := &RemoteApi{ Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, MaxRows: 500, diff --git a/pkg/config/config.go b/pkg/config/config.go index 89c690f59..a59c91313 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -126,7 +126,7 @@ type ( baseConfig struct { ProjectId string `toml:"project_id"` Hostname string `toml:"-"` - Api Api `toml:"api"` + Api api `toml:"api"` Db db `toml:"db" mapstructure:"db"` Realtime realtime `toml:"realtime"` Studio studio `toml:"studio"` @@ -454,7 +454,7 @@ func WithHostname(hostname string) ConfigEditor { func NewConfig(editors ...ConfigEditor) config { initial := config{baseConfig: baseConfig{ Hostname: "127.0.0.1", - Api: Api{ + Api: api{ Image: postgrestImage, KongImage: kongImage, }, From 327a09a4b333537388537e2111b06483a80b6fb6 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 16:31:55 +0200 Subject: [PATCH 06/20] chore: handle api enable --- pkg/config/api.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index 824acaf6e..8108393e5 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -8,13 +8,13 @@ import ( type ( RemoteApi struct { + Enabled bool `toml:"enabled"` Schemas []string `toml:"schemas"` ExtraSearchPath []string `toml:"extra_search_path"` MaxRows uint `toml:"max_rows"` } api struct { RemoteApi - Enabled bool `toml:"enabled"` Image string `toml:"-"` KongImage string `toml:"-"` Port uint16 `toml:"port"` @@ -49,6 +49,12 @@ func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBod body.MaxRows = &maxRows } + // When the api is disabled, remote side it just set the dbSchema to an empty value + if !a.Enabled { + emptyString := "" + body.DbSchema = &emptyString + } + // Note: DbPool is not present in the Api struct, so it's not set here return body } @@ -64,6 +70,9 @@ func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJW // Update MaxRows if present in remoteConfig result.MaxRows = uint(remoteConfig.MaxRows) + // If the remote schema is empty it means the api is disabled + result.Enabled = remoteConfig.DbSchema != "" + return result } From 018ab576af749ecc488c15f9cb8008ed1fddd9bd Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 16:59:51 +0200 Subject: [PATCH 07/20] chore: make convert whitespace resilient --- pkg/config/api.go | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index 8108393e5..bc18a7063 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -31,6 +31,13 @@ type ( func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { body := v1API.UpdatePostgrestConfigBody{} + // When the api is disabled, remote side it just set the dbSchema to an empty value + if !a.Enabled { + emptyString := "" + body.DbSchema = &emptyString + return body + } + // Convert Schemas to a comma-separated string if len(a.Schemas) > 0 { schemas := strings.Join(a.Schemas, ",") @@ -49,30 +56,34 @@ func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBod body.MaxRows = &maxRows } - // When the api is disabled, remote side it just set the dbSchema to an empty value - if !a.Enabled { - emptyString := "" - body.DbSchema = &emptyString - } - // Note: DbPool is not present in the Api struct, so it's not set here return body } func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) RemoteApi { result := *a + if remoteConfig.DbSchema == "" { + result.Enabled = false + return result + } // Update Schemas if present in remoteConfig - result.Schemas = strings.Split(remoteConfig.DbSchema, ",") + schemas := strings.Split(remoteConfig.DbSchema, ",") + result.Schemas = make([]string, len(schemas)) + // TODO: use slices.Map when upgrade go version + for i, schema := range schemas { + result.Schemas[i] = strings.TrimSpace(schema) + } // Update ExtraSearchPath if present in remoteConfig - result.ExtraSearchPath = strings.Split(remoteConfig.DbExtraSearchPath, ",") + extraSearchPath := strings.Split(remoteConfig.DbExtraSearchPath, ",") + result.ExtraSearchPath = make([]string, len(extraSearchPath)) + for i, path := range extraSearchPath { + result.ExtraSearchPath[i] = strings.TrimSpace(path) + } // Update MaxRows if present in remoteConfig result.MaxRows = uint(remoteConfig.MaxRows) - // If the remote schema is empty it means the api is disabled - result.Enabled = remoteConfig.DbSchema != "" - return result } From f0b9d57cfd297bcf19ce1f85fa25c2f33af5be55 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 11 Oct 2024 19:04:06 +0200 Subject: [PATCH 08/20] feat: add some errors handling for remotes config --- pkg/config/api_test.go | 17 +++++++++++++++++ pkg/config/config.go | 27 +++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index cbbb9da64..2ea6bd634 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -72,6 +72,23 @@ func TestApiDiffWithRemote(t *testing.T) { diff := api.DiffWithRemote(remoteConfig) + assert.Empty(t, diff) + }) + t.Run("handles multiple schemas and search paths with spaces", func(t *testing.T) { + api := &RemoteApi{ + Schemas: []string{"public", "private"}, + ExtraSearchPath: []string{"extensions", "public"}, + MaxRows: 500, + } + + remoteConfig := v1API.PostgrestConfigWithJWTSecretResponse{ + DbSchema: "public, private", + DbExtraSearchPath: "extensions, public", + MaxRows: 500, + } + + diff := api.DiffWithRemote(remoteConfig) + assert.Empty(t, diff) }) } diff --git a/pkg/config/config.go b/pkg/config/config.go index a59c91313..9abca621d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -702,6 +702,7 @@ func (c *config) Load(path string, fsys fs.FS) error { return err } c.Remotes = make(map[string]baseConfig, len(c.Overrides)) + duplicatedRemotesProjectsIds := make(map[string][]string) for name, remote := range c.Overrides { base := c.baseConfig.Clone() // Encode a toml file with only config overrides @@ -715,7 +716,19 @@ func (c *config) Load(path string, fsys fs.FS) error { } else if undecoded := metadata.Undecoded(); len(undecoded) > 0 { fmt.Fprintf(os.Stderr, "Unknown config fields: %+v\n", undecoded) } + if base.ProjectId == c.baseConfig.ProjectId { + fmt.Fprintf(os.Stderr, "WARN: project_id is missing for remote %s this config won't apply to any branch\n", name) + } else { + duplicatedRemotesProjectsIds[base.ProjectId] = append(duplicatedRemotesProjectsIds[base.ProjectId], name) + } + // Check for duplicate project IDs across remotes + for projectId, remotes := range duplicatedRemotesProjectsIds { + if len(remotes) > 1 { + fmt.Fprintf(os.Stderr, "WARN: Multiple remotes (%s) have the same project_id: %s. This may lead to unexpected config override.\n", strings.Join(remotes, ", "), projectId) + } + } if err := base.Validate(fsys); err != nil { + fmt.Fprintf(os.Stderr, "Error with remote config %s\n", name) return err } c.Remotes[name] = base @@ -1307,11 +1320,17 @@ func (a *auth) ResolveJWKS(ctx context.Context) (string, error) { } // Retrieve the final base config to use taking into account the remotes override -func (c *config) GetRemoteOverride(remotes_name string) (overrideConfig baseConfig, overrideExist bool) { - overrideConfig, exist := c.Remotes[remotes_name] - if exist { - return overrideConfig, true +func (c *config) GetRemoteOverride(project_ref string) (overrideConfig baseConfig, overrideExist bool) { + // Iterate over all the config.Remotes + for _, remoteConfig := range c.Remotes { + // Check if there is one matching project_id + if remoteConfig.ProjectId == project_ref { + // Return the matching remote config and true to indicate an override exists + return remoteConfig, true + } } + + // If no matching remote config is found, return the base config and false return c.baseConfig, false } From 935b0f7cd0e9d0ee24d3e431a748c8068514b0e0 Mon Sep 17 00:00:00 2001 From: avallete Date: Sat, 12 Oct 2024 12:33:13 +0200 Subject: [PATCH 09/20] chore: move diff into own package --- internal/link/link.go | 3 +- internal/{link => utils/diff}/diff.go | 2 +- pkg/config/api.go | 3 +- pkg/config/diff.go | 261 -------------------------- 4 files changed, 5 insertions(+), 264 deletions(-) rename internal/{link => utils/diff}/diff.go (99%) delete mode 100644 pkg/config/diff.go diff --git a/internal/link/link.go b/internal/link/link.go index fd2d57b43..75cb99513 100644 --- a/internal/link/link.go +++ b/internal/link/link.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/viper" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/credentials" + "github.com/supabase/cli/internal/utils/diff" "github.com/supabase/cli/internal/utils/flags" "github.com/supabase/cli/internal/utils/tenant" "github.com/supabase/cli/pkg/api" @@ -63,7 +64,7 @@ func Run(ctx context.Context, projectRef string, fsys afero.Fs, options ...func( "db": utils.Config.Db, }) // if lineDiff := cmp.Diff(original, updated); len(lineDiff) > 0 { - if lineDiff := Diff(utils.ConfigPath, original, projectRef, updated); len(lineDiff) > 0 { + if lineDiff := diff.Diff(utils.ConfigPath, original, projectRef, updated); len(lineDiff) > 0 { fmt.Fprintln(os.Stderr, utils.Yellow("WARNING:"), "Local config differs from linked project. Try updating", utils.Bold(utils.ConfigPath)) fmt.Println(string(lineDiff)) } diff --git a/internal/link/diff.go b/internal/utils/diff/diff.go similarity index 99% rename from internal/link/diff.go rename to internal/utils/diff/diff.go index 84f2e3494..6a40b23fc 100644 --- a/internal/link/diff.go +++ b/internal/utils/diff/diff.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package link +package diff import ( "bytes" diff --git a/pkg/config/api.go b/pkg/config/api.go index bc18a7063..2d676fade 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -3,6 +3,7 @@ package config import ( "strings" + "github.com/supabase/cli/internal/utils/diff" v1API "github.com/supabase/cli/pkg/api" ) @@ -91,5 +92,5 @@ func (a *RemoteApi) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecr // Convert the config values into easily comparable remoteConfig values currentValue := ToTomlBytes(a) remoteCompare := ToTomlBytes(a.fromRemoteApiConfig(remoteConfig)) - return Diff("remote[api]", remoteCompare, "local[api]", currentValue) + return diff.Diff("remote[api]", remoteCompare, "local[api]", currentValue) } diff --git a/pkg/config/diff.go b/pkg/config/diff.go deleted file mode 100644 index 04aa76ed4..000000000 --- a/pkg/config/diff.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package config - -import ( - "bytes" - "fmt" - "sort" - "strings" -) - -// A pair is a pair of values tracked for both the x and y side of a diff. -// It is typically a pair of line indexes. -type pair struct{ x, y int } - -// Diff returns an anchored diff of the two texts old and new -// in the “unified diff” format. If old and new are identical, -// Diff returns a nil slice (no output). -// -// Unix diff implementations typically look for a diff with -// the smallest number of lines inserted and removed, -// which can in the worst case take time quadratic in the -// number of lines in the texts. As a result, many implementations -// either can be made to run for a long time or cut off the search -// after a predetermined amount of work. -// -// In contrast, this implementation looks for a diff with the -// smallest number of “unique” lines inserted and removed, -// where unique means a line that appears just once in both old and new. -// We call this an “anchored diff” because the unique lines anchor -// the chosen matching regions. An anchored diff is usually clearer -// than a standard diff, because the algorithm does not try to -// reuse unrelated blank lines or closing braces. -// The algorithm also guarantees to run in O(n log n) time -// instead of the standard O(n²) time. -// -// Some systems call this approach a “patience diff,” named for -// the “patience sorting” algorithm, itself named for a solitaire card game. -// We avoid that name for two reasons. First, the name has been used -// for a few different variants of the algorithm, so it is imprecise. -// Second, the name is frequently interpreted as meaning that you have -// to wait longer (to be patient) for the diff, meaning that it is a slower algorithm, -// when in fact the algorithm is faster than the standard one. -func Diff(oldName string, old []byte, newName string, new []byte) []byte { - if bytes.Equal(old, new) { - return nil - } - x := lines(old) - y := lines(new) - - // Print diff header. - var out bytes.Buffer - fmt.Fprintf(&out, "diff %s %s\n", oldName, newName) - fmt.Fprintf(&out, "--- %s\n", oldName) - fmt.Fprintf(&out, "+++ %s\n", newName) - - // Loop over matches to consider, - // expanding each match to include surrounding lines, - // and then printing diff chunks. - // To avoid setup/teardown cases outside the loop, - // tgs returns a leading {0,0} and trailing {len(x), len(y)} pair - // in the sequence of matches. - var ( - done pair // printed up to x[:done.x] and y[:done.y] - chunk pair // start lines of current chunk - count pair // number of lines from each side in current chunk - ctext []string // lines for current chunk - ) - for _, m := range tgs(x, y) { - if m.x < done.x { - // Already handled scanning forward from earlier match. - continue - } - - // Expand matching lines as far as possible, - // establishing that x[start.x:end.x] == y[start.y:end.y]. - // Note that on the first (or last) iteration we may (or definitely do) - // have an empty match: start.x==end.x and start.y==end.y. - start := m - for start.x > done.x && start.y > done.y && x[start.x-1] == y[start.y-1] { - start.x-- - start.y-- - } - end := m - for end.x < len(x) && end.y < len(y) && x[end.x] == y[end.y] { - end.x++ - end.y++ - } - - // Emit the mismatched lines before start into this chunk. - // (No effect on first sentinel iteration, when start = {0,0}.) - for _, s := range x[done.x:start.x] { - ctext = append(ctext, "-"+s) - count.x++ - } - for _, s := range y[done.y:start.y] { - ctext = append(ctext, "+"+s) - count.y++ - } - - // If we're not at EOF and have too few common lines, - // the chunk includes all the common lines and continues. - const C = 3 // number of context lines - if (end.x < len(x) || end.y < len(y)) && - (end.x-start.x < C || (len(ctext) > 0 && end.x-start.x < 2*C)) { - for _, s := range x[start.x:end.x] { - ctext = append(ctext, " "+s) - count.x++ - count.y++ - } - done = end - continue - } - - // End chunk with common lines for context. - if len(ctext) > 0 { - n := end.x - start.x - if n > C { - n = C - } - for _, s := range x[start.x : start.x+n] { - ctext = append(ctext, " "+s) - count.x++ - count.y++ - } - done = pair{start.x + n, start.y + n} - - // Format and emit chunk. - // Convert line numbers to 1-indexed. - // Special case: empty file shows up as 0,0 not 1,0. - if count.x > 0 { - chunk.x++ - } - if count.y > 0 { - chunk.y++ - } - fmt.Fprintf(&out, "@@ -%d,%d +%d,%d @@\n", chunk.x, count.x, chunk.y, count.y) - for _, s := range ctext { - out.WriteString(s) - } - count.x = 0 - count.y = 0 - ctext = ctext[:0] - } - - // If we reached EOF, we're done. - if end.x >= len(x) && end.y >= len(y) { - break - } - - // Otherwise start a new chunk. - chunk = pair{end.x - C, end.y - C} - for _, s := range x[chunk.x:end.x] { - ctext = append(ctext, " "+s) - count.x++ - count.y++ - } - done = end - } - - return out.Bytes() -} - -// lines returns the lines in the file x, including newlines. -// If the file does not end in a newline, one is supplied -// along with a warning about the missing newline. -func lines(x []byte) []string { - l := strings.SplitAfter(string(x), "\n") - if l[len(l)-1] == "" { - l = l[:len(l)-1] - } else { - // Treat last line as having a message about the missing newline attached, - // using the same text as BSD/GNU diff (including the leading backslash). - l[len(l)-1] += "\n\\ No newline at end of file\n" - } - return l -} - -// tgs returns the pairs of indexes of the longest common subsequence -// of unique lines in x and y, where a unique line is one that appears -// once in x and once in y. -// -// The longest common subsequence algorithm is as described in -// Thomas G. Szymanski, “A Special Case of the Maximal Common -// Subsequence Problem,” Princeton TR #170 (January 1975), -// available at https://research.swtch.com/tgs170.pdf. -func tgs(x, y []string) []pair { - // Count the number of times each string appears in a and b. - // We only care about 0, 1, many, counted as 0, -1, -2 - // for the x side and 0, -4, -8 for the y side. - // Using negative numbers now lets us distinguish positive line numbers later. - m := make(map[string]int) - for _, s := range x { - if c := m[s]; c > -2 { - m[s] = c - 1 - } - } - for _, s := range y { - if c := m[s]; c > -8 { - m[s] = c - 4 - } - } - - // Now unique strings can be identified by m[s] = -1+-4. - // - // Gather the indexes of those strings in x and y, building: - // xi[i] = increasing indexes of unique strings in x. - // yi[i] = increasing indexes of unique strings in y. - // inv[i] = index j such that x[xi[i]] = y[yi[j]]. - var xi, yi, inv []int - for i, s := range y { - if m[s] == -1+-4 { - m[s] = len(yi) - yi = append(yi, i) - } - } - for i, s := range x { - if j, ok := m[s]; ok && j >= 0 { - xi = append(xi, i) - inv = append(inv, j) - } - } - - // Apply Algorithm A from Szymanski's paper. - // In those terms, A = J = inv and B = [0, n). - // We add sentinel pairs {0,0}, and {len(x),len(y)} - // to the returned sequence, to help the processing loop. - J := inv - n := len(xi) - T := make([]int, n) - L := make([]int, n) - for i := range T { - T[i] = n + 1 - } - for i := 0; i < n; i++ { - k := sort.Search(n, func(k int) bool { - return T[k] >= J[i] - }) - T[k] = J[i] - L[i] = k + 1 - } - k := 0 - for _, v := range L { - if k < v { - k = v - } - } - seq := make([]pair, 2+k) - seq[1+k] = pair{len(x), len(y)} // sentinel at end - lastj := n - for i := n - 1; i >= 0; i-- { - if L[i] == k && J[i] < lastj { - seq[k] = pair{xi[i], yi[J[i]]} - k-- - } - } - seq[0] = pair{0, 0} // sentinel at start - return seq -} From 6d84006b3d1a3eb329b6fc88bc041280d639ea30 Mon Sep 17 00:00:00 2001 From: avallete Date: Sat, 12 Oct 2024 12:52:38 +0200 Subject: [PATCH 10/20] chore: add some diff tests --- pkg/config/api.go | 1 + pkg/config/api_test.go | 52 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index 2d676fade..3711fd792 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -67,6 +67,7 @@ func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJW result.Enabled = false return result } + result.Enabled = true // Update Schemas if present in remoteConfig schemas := strings.Split(remoteConfig.DbSchema, ",") result.Schemas = make([]string, len(schemas)) diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index 2ea6bd634..ca5de36c8 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -10,6 +11,7 @@ import ( func TestApiToUpdatePostgrestConfigBody(t *testing.T) { t.Run("converts all fields correctly", func(t *testing.T) { api := &RemoteApi{ + Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -27,15 +29,15 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { body := api.ToUpdatePostgrestConfigBody() - assert.Nil(t, body.DbSchema) - assert.Nil(t, body.DbExtraSearchPath) - assert.Nil(t, body.MaxRows) + // remote api will be false by default, leading to an empty schema on api side + assert.Equal(t, "", *body.DbSchema) }) } func TestApiDiffWithRemote(t *testing.T) { t.Run("detects differences", func(t *testing.T) { api := &RemoteApi{ + Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -59,6 +61,7 @@ func TestApiDiffWithRemote(t *testing.T) { t.Run("handles no differences", func(t *testing.T) { api := &RemoteApi{ + Enabled: true, Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, MaxRows: 500, @@ -76,6 +79,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles multiple schemas and search paths with spaces", func(t *testing.T) { api := &RemoteApi{ + Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 500, @@ -91,4 +95,46 @@ func TestApiDiffWithRemote(t *testing.T) { assert.Empty(t, diff) }) + t.Run("handles api disabled on remote side", func(t *testing.T) { + api := &RemoteApi{ + Enabled: true, + Schemas: []string{"public", "private"}, + ExtraSearchPath: []string{"extensions", "public"}, + MaxRows: 500, + } + + remoteConfig := v1API.PostgrestConfigWithJWTSecretResponse{ + DbSchema: "", + DbExtraSearchPath: "", + MaxRows: 0, + } + + diff := api.DiffWithRemote(remoteConfig) + d := string(diff) + fmt.Println(d) + + assert.Contains(t, string(diff), "-enabled = false") + assert.Contains(t, string(diff), "+enabled = true") + }) + t.Run("handles api disabled on local side", func(t *testing.T) { + api := &RemoteApi{ + Enabled: false, + Schemas: []string{"public"}, + ExtraSearchPath: []string{"public"}, + MaxRows: 500, + } + + remoteConfig := v1API.PostgrestConfigWithJWTSecretResponse{ + DbSchema: "public", + DbExtraSearchPath: "public", + MaxRows: 500, + } + + diff := api.DiffWithRemote(remoteConfig) + d := string(diff) + fmt.Println(d) + + assert.Contains(t, string(diff), "-enabled = true") + assert.Contains(t, string(diff), "+enabled = false") + }) } From cfada5bb2ae358831fbf87b745089a3dab138cdc Mon Sep 17 00:00:00 2001 From: avallete Date: Mon, 14 Oct 2024 15:26:11 +0200 Subject: [PATCH 11/20] chore: fix golint casting lints --- internal/link/link.go | 3 ++- internal/utils/cast/cast.go | 21 +++++++++++++++++++++ pkg/config/api.go | 7 ++++--- pkg/config/api_test.go | 10 +++++----- pkg/parser/token.go | 3 ++- 5 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 internal/utils/cast/cast.go diff --git a/internal/link/link.go b/internal/link/link.go index 75cb99513..d204a044e 100644 --- a/internal/link/link.go +++ b/internal/link/link.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/viper" "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/internal/utils/cast" "github.com/supabase/cli/internal/utils/credentials" "github.com/supabase/cli/internal/utils/diff" "github.com/supabase/cli/internal/utils/flags" @@ -136,7 +137,7 @@ func linkPostgrestVersion(ctx context.Context, api tenant.TenantAPI, fsys afero. } func updateApiConfig(config api.PostgrestConfigWithJWTSecretResponse) { - utils.Config.Api.MaxRows = uint(config.MaxRows) + utils.Config.Api.MaxRows = cast.IntToUint(config.MaxRows) utils.Config.Api.ExtraSearchPath = readCsv(config.DbExtraSearchPath) utils.Config.Api.Schemas = readCsv(config.DbSchema) } diff --git a/internal/utils/cast/cast.go b/internal/utils/cast/cast.go new file mode 100644 index 000000000..cd4e220a0 --- /dev/null +++ b/internal/utils/cast/cast.go @@ -0,0 +1,21 @@ +package cast + +import "math" + +// UintToInt converts a uint to an *int, handling potential overflow +func UintToInt(value uint) int { + if value <= math.MaxInt { + result := int(value) + return result + } + maxInt := math.MaxInt + return maxInt +} + +// IntToUint converts an int to a uint, handling negative values +func IntToUint(value int) uint { + if value < 0 { + return 0 + } + return uint(value) +} diff --git a/pkg/config/api.go b/pkg/config/api.go index 3711fd792..44543db47 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -3,6 +3,7 @@ package config import ( "strings" + "github.com/supabase/cli/internal/utils/cast" "github.com/supabase/cli/internal/utils/diff" v1API "github.com/supabase/cli/pkg/api" ) @@ -53,8 +54,8 @@ func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBod // Convert MaxRows to int pointer if a.MaxRows > 0 { - maxRows := int(a.MaxRows) - body.MaxRows = &maxRows + intValue := cast.UintToInt(a.MaxRows) + body.MaxRows = &intValue } // Note: DbPool is not present in the Api struct, so it's not set here @@ -84,7 +85,7 @@ func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJW } // Update MaxRows if present in remoteConfig - result.MaxRows = uint(remoteConfig.MaxRows) + result.MaxRows = cast.IntToUint(remoteConfig.MaxRows) return result } diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index ca5de36c8..679de5052 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -11,7 +11,7 @@ import ( func TestApiToUpdatePostgrestConfigBody(t *testing.T) { t.Run("converts all fields correctly", func(t *testing.T) { api := &RemoteApi{ - Enabled: true, + Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -37,7 +37,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { func TestApiDiffWithRemote(t *testing.T) { t.Run("detects differences", func(t *testing.T) { api := &RemoteApi{ - Enabled: true, + Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 1000, @@ -61,7 +61,7 @@ func TestApiDiffWithRemote(t *testing.T) { t.Run("handles no differences", func(t *testing.T) { api := &RemoteApi{ - Enabled: true, + Enabled: true, Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, MaxRows: 500, @@ -79,7 +79,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles multiple schemas and search paths with spaces", func(t *testing.T) { api := &RemoteApi{ - Enabled: true, + Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 500, @@ -97,7 +97,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles api disabled on remote side", func(t *testing.T) { api := &RemoteApi{ - Enabled: true, + Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, MaxRows: 500, diff --git a/pkg/parser/token.go b/pkg/parser/token.go index 018a6f5d4..945ff1069 100644 --- a/pkg/parser/token.go +++ b/pkg/parser/token.go @@ -8,6 +8,7 @@ import ( "github.com/go-errors/errors" "github.com/spf13/viper" + "github.com/supabase/cli/internal/utils/cast" ) // Equal to `startBufSize` from `bufio/scan.go` @@ -83,7 +84,7 @@ func Split(sql io.Reader, transform ...func(string) string) (stats []string, err // Increase scanner capacity to support very long lines containing e.g. geodata buf := make([]byte, startBufSize) - maxbuf := int(viper.GetSizeInBytes("SCANNER_BUFFER_SIZE")) + maxbuf := cast.UintToInt(viper.GetSizeInBytes("SCANNER_BUFFER_SIZE")) if maxbuf == 0 { maxbuf = MaxScannerCapacity } From 274a43846448a99df2738b1caadfddec3350d51d Mon Sep 17 00:00:00 2001 From: Andrew Valleteau Date: Tue, 15 Oct 2024 09:51:11 +0200 Subject: [PATCH 12/20] Update internal/utils/cast/cast.go Co-authored-by: Han Qiao --- internal/utils/cast/cast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/utils/cast/cast.go b/internal/utils/cast/cast.go index cd4e220a0..7af9d9bd6 100644 --- a/internal/utils/cast/cast.go +++ b/internal/utils/cast/cast.go @@ -2,7 +2,7 @@ package cast import "math" -// UintToInt converts a uint to an *int, handling potential overflow +// UintToInt converts a uint to an int, handling potential overflow func UintToInt(value uint) int { if value <= math.MaxInt { result := int(value) From af51f758778a687d7fb7c6407eb0833120e2e03a Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 15 Oct 2024 10:31:53 +0200 Subject: [PATCH 13/20] chore: use Errorf remote config error --- pkg/config/config.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index ef79ba8ed..331e01ccf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -728,8 +728,7 @@ func (c *config) Load(path string, fsys fs.FS) error { } } if err := base.Validate(fsys); err != nil { - fmt.Fprintf(os.Stderr, "Error with remote config %s\n", name) - return err + return errors.Errorf("invalid remote config %s : %w", name, err) } c.Remotes[name] = base } From 70519e814747d391ac328b28b489909d170ab452 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 15 Oct 2024 10:34:43 +0200 Subject: [PATCH 14/20] chore: move diff and cast to pkg --- internal/link/link.go | 4 ++-- {internal/utils => pkg}/cast/cast.go | 0 pkg/config/api.go | 4 ++-- {internal/utils => pkg}/diff/diff.go | 0 pkg/parser/token.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename {internal/utils => pkg}/cast/cast.go (100%) rename {internal/utils => pkg}/diff/diff.go (100%) diff --git a/internal/link/link.go b/internal/link/link.go index d204a044e..02ed22114 100644 --- a/internal/link/link.go +++ b/internal/link/link.go @@ -14,13 +14,13 @@ import ( "github.com/spf13/afero" "github.com/spf13/viper" "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/internal/utils/cast" "github.com/supabase/cli/internal/utils/credentials" - "github.com/supabase/cli/internal/utils/diff" "github.com/supabase/cli/internal/utils/flags" "github.com/supabase/cli/internal/utils/tenant" "github.com/supabase/cli/pkg/api" + "github.com/supabase/cli/pkg/cast" cliConfig "github.com/supabase/cli/pkg/config" + "github.com/supabase/cli/pkg/diff" "github.com/supabase/cli/pkg/migration" ) diff --git a/internal/utils/cast/cast.go b/pkg/cast/cast.go similarity index 100% rename from internal/utils/cast/cast.go rename to pkg/cast/cast.go diff --git a/pkg/config/api.go b/pkg/config/api.go index 44543db47..89cef0952 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -3,9 +3,9 @@ package config import ( "strings" - "github.com/supabase/cli/internal/utils/cast" - "github.com/supabase/cli/internal/utils/diff" v1API "github.com/supabase/cli/pkg/api" + "github.com/supabase/cli/pkg/cast" + "github.com/supabase/cli/pkg/diff" ) type ( diff --git a/internal/utils/diff/diff.go b/pkg/diff/diff.go similarity index 100% rename from internal/utils/diff/diff.go rename to pkg/diff/diff.go diff --git a/pkg/parser/token.go b/pkg/parser/token.go index 945ff1069..db0084342 100644 --- a/pkg/parser/token.go +++ b/pkg/parser/token.go @@ -8,7 +8,7 @@ import ( "github.com/go-errors/errors" "github.com/spf13/viper" - "github.com/supabase/cli/internal/utils/cast" + "github.com/supabase/cli/pkg/cast" ) // Equal to `startBufSize` from `bufio/scan.go` From 3af42eb52c3fd698a954cadc9968a974014a8949 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 16 Oct 2024 11:28:20 +0800 Subject: [PATCH 15/20] chore: minor refactor --- internal/link/link.go | 12 +++++++++--- pkg/cast/cast.go | 4 ++++ pkg/config/api.go | 25 +++++++++++++++---------- pkg/config/api_test.go | 37 ++++++++++++++++++++----------------- pkg/config/config.go | 6 +++--- 5 files changed, 51 insertions(+), 33 deletions(-) diff --git a/internal/link/link.go b/internal/link/link.go index 02ed22114..8a75a8d3d 100644 --- a/internal/link/link.go +++ b/internal/link/link.go @@ -25,10 +25,13 @@ import ( ) func Run(ctx context.Context, projectRef string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - original := cliConfig.ToTomlBytes(map[string]interface{}{ + original, err := cliConfig.ToTomlBytes(map[string]interface{}{ "api": utils.Config.Api, "db": utils.Config.Db, }) + if err != nil { + fmt.Fprintln(utils.GetDebugLogger(), err) + } if err := checkRemoteProjectStatus(ctx, projectRef); err != nil { return err @@ -60,11 +63,14 @@ func Run(ctx context.Context, projectRef string, fsys afero.Fs, options ...func( fmt.Fprintln(os.Stdout, "Finished "+utils.Aqua("supabase link")+".") // 4. Suggest config update - updated := cliConfig.ToTomlBytes(map[string]interface{}{ + updated, err := cliConfig.ToTomlBytes(map[string]interface{}{ "api": utils.Config.Api, "db": utils.Config.Db, }) - // if lineDiff := cmp.Diff(original, updated); len(lineDiff) > 0 { + if err != nil { + fmt.Fprintln(utils.GetDebugLogger(), err) + } + if lineDiff := diff.Diff(utils.ConfigPath, original, projectRef, updated); len(lineDiff) > 0 { fmt.Fprintln(os.Stderr, utils.Yellow("WARNING:"), "Local config differs from linked project. Try updating", utils.Bold(utils.ConfigPath)) fmt.Println(string(lineDiff)) diff --git a/pkg/cast/cast.go b/pkg/cast/cast.go index 7af9d9bd6..b72cadbbf 100644 --- a/pkg/cast/cast.go +++ b/pkg/cast/cast.go @@ -19,3 +19,7 @@ func IntToUint(value int) uint { } return uint(value) } + +func Ptr[T any](v T) *T { + return &v +} diff --git a/pkg/config/api.go b/pkg/config/api.go index 89cef0952..62b71deb3 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -9,14 +9,12 @@ import ( ) type ( - RemoteApi struct { + api struct { Enabled bool `toml:"enabled"` Schemas []string `toml:"schemas"` ExtraSearchPath []string `toml:"extra_search_path"` MaxRows uint `toml:"max_rows"` - } - api struct { - RemoteApi + // Local only config Image string `toml:"-"` KongImage string `toml:"-"` Port uint16 `toml:"port"` @@ -30,7 +28,7 @@ type ( } ) -func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { +func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { body := v1API.UpdatePostgrestConfigBody{} // When the api is disabled, remote side it just set the dbSchema to an empty value @@ -62,12 +60,13 @@ func (a *RemoteApi) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBod return body } -func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) RemoteApi { +func (a *api) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) api { result := *a if remoteConfig.DbSchema == "" { result.Enabled = false return result } + result.Enabled = true // Update Schemas if present in remoteConfig schemas := strings.Split(remoteConfig.DbSchema, ",") @@ -90,9 +89,15 @@ func (a *RemoteApi) fromRemoteApiConfig(remoteConfig v1API.PostgrestConfigWithJW return result } -func (a *RemoteApi) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) []byte { +func (a *api) DiffWithRemote(remoteConfig v1API.PostgrestConfigWithJWTSecretResponse) ([]byte, error) { // Convert the config values into easily comparable remoteConfig values - currentValue := ToTomlBytes(a) - remoteCompare := ToTomlBytes(a.fromRemoteApiConfig(remoteConfig)) - return diff.Diff("remote[api]", remoteCompare, "local[api]", currentValue) + currentValue, err := ToTomlBytes(a) + if err != nil { + return nil, err + } + remoteCompare, err := ToTomlBytes(a.fromRemoteApiConfig(remoteConfig)) + if err != nil { + return nil, err + } + return diff.Diff("remote[api]", remoteCompare, "local[api]", currentValue), nil } diff --git a/pkg/config/api_test.go b/pkg/config/api_test.go index 679de5052..d48001244 100644 --- a/pkg/config/api_test.go +++ b/pkg/config/api_test.go @@ -1,7 +1,6 @@ package config import ( - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -10,7 +9,7 @@ import ( func TestApiToUpdatePostgrestConfigBody(t *testing.T) { t.Run("converts all fields correctly", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, @@ -25,7 +24,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { }) t.Run("handles empty fields", func(t *testing.T) { - api := &RemoteApi{} + api := &api{} body := api.ToUpdatePostgrestConfigBody() @@ -36,7 +35,7 @@ func TestApiToUpdatePostgrestConfigBody(t *testing.T) { func TestApiDiffWithRemote(t *testing.T) { t.Run("detects differences", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, @@ -49,7 +48,8 @@ func TestApiDiffWithRemote(t *testing.T) { MaxRows: 500, } - diff := api.DiffWithRemote(remoteConfig) + diff, err := api.DiffWithRemote(remoteConfig) + assert.NoError(t, err, string(diff)) assert.Contains(t, string(diff), "-schemas = [\"public\"]") assert.Contains(t, string(diff), "+schemas = [\"public\", \"private\"]") @@ -60,7 +60,7 @@ func TestApiDiffWithRemote(t *testing.T) { }) t.Run("handles no differences", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Enabled: true, Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, @@ -73,12 +73,14 @@ func TestApiDiffWithRemote(t *testing.T) { MaxRows: 500, } - diff := api.DiffWithRemote(remoteConfig) + diff, err := api.DiffWithRemote(remoteConfig) + assert.NoError(t, err) assert.Empty(t, diff) }) + t.Run("handles multiple schemas and search paths with spaces", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, @@ -91,12 +93,14 @@ func TestApiDiffWithRemote(t *testing.T) { MaxRows: 500, } - diff := api.DiffWithRemote(remoteConfig) + diff, err := api.DiffWithRemote(remoteConfig) + assert.NoError(t, err) assert.Empty(t, diff) }) + t.Run("handles api disabled on remote side", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Enabled: true, Schemas: []string{"public", "private"}, ExtraSearchPath: []string{"extensions", "public"}, @@ -109,15 +113,15 @@ func TestApiDiffWithRemote(t *testing.T) { MaxRows: 0, } - diff := api.DiffWithRemote(remoteConfig) - d := string(diff) - fmt.Println(d) + diff, err := api.DiffWithRemote(remoteConfig) + assert.NoError(t, err, string(diff)) assert.Contains(t, string(diff), "-enabled = false") assert.Contains(t, string(diff), "+enabled = true") }) + t.Run("handles api disabled on local side", func(t *testing.T) { - api := &RemoteApi{ + api := &api{ Enabled: false, Schemas: []string{"public"}, ExtraSearchPath: []string{"public"}, @@ -130,9 +134,8 @@ func TestApiDiffWithRemote(t *testing.T) { MaxRows: 500, } - diff := api.DiffWithRemote(remoteConfig) - d := string(diff) - fmt.Println(d) + diff, err := api.DiffWithRemote(remoteConfig) + assert.NoError(t, err, string(diff)) assert.Contains(t, string(diff), "-enabled = true") assert.Contains(t, string(diff), "+enabled = false") diff --git a/pkg/config/config.go b/pkg/config/config.go index 331e01ccf..e5ae9ea0a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1338,12 +1338,12 @@ func (c *config) GetRemoteOverride(project_ref string) (overrideConfig baseConfi return c.baseConfig, false } -func ToTomlBytes(config any) []byte { +func ToTomlBytes(config any) ([]byte, error) { var buf bytes.Buffer enc := toml.NewEncoder(&buf) enc.Indent = "" if err := enc.Encode(config); err != nil { - fmt.Fprintln(os.Stderr, "failed to marshal toml config:", err) + return nil, errors.Errorf("failed to marshal toml config: %w", err) } - return buf.Bytes() + return buf.Bytes(), nil } From f5f960ef8feef5af39ad16113d86df0e4c8723bf Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 16 Oct 2024 11:28:42 +0800 Subject: [PATCH 16/20] feat: implement remote config updater --- pkg/config/updater.go | 49 ++++++++++++++++++++++++++++ pkg/config/updater_test.go | 65 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 pkg/config/updater.go create mode 100644 pkg/config/updater_test.go diff --git a/pkg/config/updater.go b/pkg/config/updater.go new file mode 100644 index 000000000..467b7bb63 --- /dev/null +++ b/pkg/config/updater.go @@ -0,0 +1,49 @@ +package config + +import ( + "context" + "fmt" + "os" + + "github.com/go-errors/errors" + v1API "github.com/supabase/cli/pkg/api" +) + +type ConfigUpdater struct { + client v1API.ClientWithResponses +} + +func NewConfigUpdater(client v1API.ClientWithResponses) ConfigUpdater { + return ConfigUpdater{client: client} +} + +func (u *ConfigUpdater) UpdateRemoteConfig(ctx context.Context, remote baseConfig) error { + if err := u.UpdateApiConfig(ctx, remote.ProjectId, remote.Api); err != nil { + return err + } + // TODO: implement other service configs, ie. auth + return nil +} + +func (u *ConfigUpdater) UpdateApiConfig(ctx context.Context, projectRef string, c api) error { + apiConfig, err := u.client.V1GetPostgrestServiceConfigWithResponse(ctx, projectRef) + if err != nil { + return errors.Errorf("failed to read API config: %w", err) + } else if apiConfig.JSON200 == nil { + return errors.Errorf("unexpected status %d: %s", apiConfig.StatusCode(), string(apiConfig.Body)) + } + apiDiff, err := c.DiffWithRemote(*apiConfig.JSON200) + if err != nil { + return err + } else if len(apiDiff) == 0 { + fmt.Fprintln(os.Stderr, "Remote API config is up to date.") + return nil + } + fmt.Fprintln(os.Stderr, "Updating API service with config:", string(apiDiff)) + if resp, err := u.client.V1UpdatePostgrestServiceConfigWithResponse(ctx, projectRef, c.ToUpdatePostgrestConfigBody()); err != nil { + return errors.Errorf("failed to update API config: %w", err) + } else if resp.JSON200 == nil { + return errors.Errorf("unexpected status %d: %s", resp.StatusCode(), string(resp.Body)) + } + return nil +} diff --git a/pkg/config/updater_test.go b/pkg/config/updater_test.go new file mode 100644 index 000000000..241d612e9 --- /dev/null +++ b/pkg/config/updater_test.go @@ -0,0 +1,65 @@ +package config + +import ( + "context" + "net/http" + "testing" + + "github.com/h2non/gock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1API "github.com/supabase/cli/pkg/api" +) + +func TestUpdateApi(t *testing.T) { + server := "http://localhost" + client, err := v1API.NewClientWithResponses(server) + require.NoError(t, err) + + t.Run("updates remote config", func(t *testing.T) { + updater := NewConfigUpdater(*client) + // Setup mock server + defer gock.Off() + gock.New(server). + Get("/v1/projects/test-project/postgrest"). + Reply(http.StatusOK). + JSON(v1API.PostgrestConfigWithJWTSecretResponse{}) + gock.New(server). + Patch("/v1/projects/test-project/postgrest"). + Reply(http.StatusOK). + JSON(v1API.PostgrestConfigWithJWTSecretResponse{ + DbSchema: "public,graphql_public", + DbExtraSearchPath: "public,extensions", + MaxRows: 1000, + }) + // Run test + err := updater.UpdateApiConfig(context.Background(), "test-project", api{ + Enabled: true, + Schemas: []string{"public", "graphql_public"}, + ExtraSearchPath: []string{"public", "extensions"}, + MaxRows: 1000, + }) + // Check result + assert.NoError(t, err) + assert.True(t, gock.IsDone()) + }) + + t.Run("skips update if no diff", func(t *testing.T) { + updater := NewConfigUpdater(*client) + // Setup mock server + defer gock.Off() + gock.New(server). + Get("/v1/projects/test-project/postgrest"). + Reply(http.StatusOK). + JSON(v1API.PostgrestConfigWithJWTSecretResponse{ + DbSchema: "", + DbExtraSearchPath: "public,extensions", + MaxRows: 1000, + }) + // Run test + err := updater.UpdateApiConfig(context.Background(), "test-project", api{}) + // Check result + assert.NoError(t, err) + assert.True(t, gock.IsDone()) + }) +} From bd4b42dd9a8a2bf774d507271e7babb40a659559 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 16 Oct 2024 13:46:59 +0800 Subject: [PATCH 17/20] chore: minor style changes --- pkg/config/api.go | 6 ++---- pkg/config/config.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/config/api.go b/pkg/config/api.go index 62b71deb3..403e290e5 100644 --- a/pkg/config/api.go +++ b/pkg/config/api.go @@ -33,8 +33,7 @@ func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { // When the api is disabled, remote side it just set the dbSchema to an empty value if !a.Enabled { - emptyString := "" - body.DbSchema = &emptyString + body.DbSchema = cast.Ptr("") return body } @@ -52,8 +51,7 @@ func (a *api) ToUpdatePostgrestConfigBody() v1API.UpdatePostgrestConfigBody { // Convert MaxRows to int pointer if a.MaxRows > 0 { - intValue := cast.UintToInt(a.MaxRows) - body.MaxRows = &intValue + body.MaxRows = cast.Ptr(cast.UintToInt(a.MaxRows)) } // Note: DbPool is not present in the Api struct, so it's not set here diff --git a/pkg/config/config.go b/pkg/config/config.go index e5ae9ea0a..db662b7ca 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -728,7 +728,7 @@ func (c *config) Load(path string, fsys fs.FS) error { } } if err := base.Validate(fsys); err != nil { - return errors.Errorf("invalid remote config %s : %w", name, err) + return errors.Errorf("invalid remote config %s: %w", name, err) } c.Remotes[name] = base } From 1183979f2a1a513d3d939a7b58589ee3168f3a2c Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 16 Oct 2024 14:25:50 +0800 Subject: [PATCH 18/20] chore: refactor duplicate project ref check to getter --- pkg/config/config.go | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index db662b7ca..58d5d7f8e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -702,7 +702,6 @@ func (c *config) Load(path string, fsys fs.FS) error { return err } c.Remotes = make(map[string]baseConfig, len(c.Overrides)) - duplicatedRemotesProjectsIds := make(map[string][]string) for name, remote := range c.Overrides { base := c.baseConfig.Clone() // Encode a toml file with only config overrides @@ -714,18 +713,10 @@ func (c *config) Load(path string, fsys fs.FS) error { if metadata, err := toml.NewDecoder(&buf).Decode(&base); err != nil { return errors.Errorf("failed to decode remote config: %w", err) } else if undecoded := metadata.Undecoded(); len(undecoded) > 0 { - fmt.Fprintf(os.Stderr, "Unknown config fields: %+v\n", undecoded) + fmt.Fprintf(os.Stderr, "WARN: unknown config fields: %+v\n", undecoded) } if base.ProjectId == c.baseConfig.ProjectId { - fmt.Fprintf(os.Stderr, "WARN: project_id is missing for remote %s this config won't apply to any branch\n", name) - } else { - duplicatedRemotesProjectsIds[base.ProjectId] = append(duplicatedRemotesProjectsIds[base.ProjectId], name) - } - // Check for duplicate project IDs across remotes - for projectId, remotes := range duplicatedRemotesProjectsIds { - if len(remotes) > 1 { - fmt.Fprintf(os.Stderr, "WARN: Multiple remotes (%s) have the same project_id: %s. This may lead to unexpected config override.\n", strings.Join(remotes, ", "), projectId) - } + fmt.Fprintf(os.Stderr, "WARN: project_id is missing for [remotes.%s]\n", name) } if err := base.Validate(fsys); err != nil { return errors.Errorf("invalid remote config %s: %w", name, err) @@ -1324,18 +1315,23 @@ func (a *auth) ResolveJWKS(ctx context.Context) (string, error) { } // Retrieve the final base config to use taking into account the remotes override -func (c *config) GetRemoteOverride(project_ref string) (overrideConfig baseConfig, overrideExist bool) { +func (c *config) GetRemoteByProjectRef(projectRef string) (baseConfig, error) { + var result []string // Iterate over all the config.Remotes - for _, remoteConfig := range c.Remotes { + for name, remoteConfig := range c.Remotes { // Check if there is one matching project_id - if remoteConfig.ProjectId == project_ref { - // Return the matching remote config and true to indicate an override exists - return remoteConfig, true + if remoteConfig.ProjectId == projectRef { + // Check for duplicate project IDs across remotes + result = append(result, name) } } - + if len(result) == 0 { + return baseConfig{}, errors.Errorf("no remote found for project_id: %s", projectRef) + } else if len(result) > 1 { + return baseConfig{}, errors.Errorf("multiple remotes %v have the same project_id: %s", result, projectRef) + } // If no matching remote config is found, return the base config and false - return c.baseConfig, false + return c.Remotes[result[0]], nil } func ToTomlBytes(config any) ([]byte, error) { From 1a8b72ddbf707f40ef37c6102d56b7295972fc02 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 16 Oct 2024 14:37:04 +0800 Subject: [PATCH 19/20] chore: update error message for consistency --- pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 58d5d7f8e..4cc1b7d2e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -719,7 +719,7 @@ func (c *config) Load(path string, fsys fs.FS) error { fmt.Fprintf(os.Stderr, "WARN: project_id is missing for [remotes.%s]\n", name) } if err := base.Validate(fsys); err != nil { - return errors.Errorf("invalid remote config %s: %w", name, err) + return errors.Errorf("invalid config for [remotes.%s]: %w", name, err) } c.Remotes[name] = base } From 6a45ff3670eaf5dd03e9d625c015c92ab0460a30 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Wed, 16 Oct 2024 15:36:28 +0800 Subject: [PATCH 20/20] chore: validate duplicate remote early --- pkg/config/config.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 4cc1b7d2e..1682193bf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -701,6 +701,7 @@ func (c *config) Load(path string, fsys fs.FS) error { if err := c.baseConfig.Validate(fsys); err != nil { return err } + idToName := map[string]string{} c.Remotes = make(map[string]baseConfig, len(c.Overrides)) for name, remote := range c.Overrides { base := c.baseConfig.Clone() @@ -715,8 +716,13 @@ func (c *config) Load(path string, fsys fs.FS) error { } else if undecoded := metadata.Undecoded(); len(undecoded) > 0 { fmt.Fprintf(os.Stderr, "WARN: unknown config fields: %+v\n", undecoded) } + // Cross validate remote project id if base.ProjectId == c.baseConfig.ProjectId { fmt.Fprintf(os.Stderr, "WARN: project_id is missing for [remotes.%s]\n", name) + } else if other, exists := idToName[base.ProjectId]; exists { + return errors.Errorf("duplicate project_id for [remotes.%s] and [remotes.%s]", other, name) + } else { + idToName[base.ProjectId] = name } if err := base.Validate(fsys); err != nil { return errors.Errorf("invalid config for [remotes.%s]: %w", name, err) @@ -1325,13 +1331,15 @@ func (c *config) GetRemoteByProjectRef(projectRef string) (baseConfig, error) { result = append(result, name) } } + // If no matching remote config is found, return the base config if len(result) == 0 { - return baseConfig{}, errors.Errorf("no remote found for project_id: %s", projectRef) - } else if len(result) > 1 { - return baseConfig{}, errors.Errorf("multiple remotes %v have the same project_id: %s", result, projectRef) + return c.baseConfig, errors.Errorf("no remote found for project_id: %s", projectRef) + } + remote := c.Remotes[result[0]] + if len(result) > 1 { + return remote, errors.Errorf("multiple remotes %v have the same project_id: %s", result, projectRef) } - // If no matching remote config is found, return the base config and false - return c.Remotes[result[0]], nil + return remote, nil } func ToTomlBytes(config any) ([]byte, error) {