From fd04d07c60c8634326975b27b962bd729eaff182 Mon Sep 17 00:00:00 2001 From: Andrew Valleteau Date: Wed, 25 Sep 2024 08:28:05 +0200 Subject: [PATCH] feat: add custom seed path to config (#2702) * chore: replace SeedDataPath by DefaultSeedDataPath * wip: add path seed matching logic * chore: add test for utils.GetSeedFiles * chore: wip tests mock * chore: fix lint * chore: show seed path * chore: change comment * chore: apply pr suggestions * chore: fix lint * chore: keep default value assignation * chore: remove DefaultSeedPath * chore: keep consistent WARNING message * chore: inline get seed file path * chore: address review comments --------- Co-authored-by: Qiao Han --- cmd/db.go | 2 +- cmd/start.go | 2 +- internal/db/push/push.go | 6 ++- internal/db/push/push_test.go | 4 +- internal/db/reset/reset_test.go | 2 +- internal/db/start/start_test.go | 3 +- internal/init/init.go | 18 +------ internal/init/init_test.go | 13 ----- internal/migration/apply/apply.go | 8 +-- internal/migration/apply/apply_test.go | 10 ++-- internal/utils/misc.go | 22 +++++++- internal/utils/misc_test.go | 73 ++++++++++++++++++++++++++ pkg/config/config.go | 6 ++- pkg/config/templates/config.toml | 8 +++ pkg/config/testdata/config.toml | 8 +++ pkg/config/utils.go | 2 - pkg/migration/seed.go | 3 +- 17 files changed, 140 insertions(+), 50 deletions(-) diff --git a/cmd/db.go b/cmd/db.go index 2ddb4072f..2d64edd74 100644 --- a/cmd/db.go +++ b/cmd/db.go @@ -276,7 +276,7 @@ func init() { pushFlags := dbPushCmd.Flags() pushFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.") pushFlags.BoolVar(&includeRoles, "include-roles", false, "Include custom roles from "+utils.CustomRolesPath+".") - pushFlags.BoolVar(&includeSeed, "include-seed", false, "Include seed data from "+utils.SeedDataPath+".") + pushFlags.BoolVar(&includeSeed, "include-seed", false, "Include seed data from your config.") pushFlags.BoolVar(&dryRun, "dry-run", false, "Print the migrations that would be applied, but don't actually apply them.") pushFlags.String("db-url", "", "Pushes to the database specified by the connection string (must be percent-encoded).") pushFlags.Bool("linked", true, "Pushes to the linked project.") diff --git a/cmd/start.go b/cmd/start.go index ae3faf01e..a7af80e0c 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -27,7 +27,7 @@ func validateExcludedContainers(excludedContainers []string) { // Sort the names list so it's easier to visually spot the one you looking for sort.Strings(validContainers) warning := fmt.Sprintf("%s The following container names are not valid to exclude: %s\nValid containers to exclude are: %s\n", - utils.Yellow("Warning:"), + utils.Yellow("WARNING:"), utils.Aqua(strings.Join(invalidContainers, ", ")), utils.Aqua(strings.Join(validContainers, ", "))) fmt.Fprint(os.Stderr, warning) diff --git a/internal/db/push/push.go b/internal/db/push/push.go index 68e3be9ac..0bff15163 100644 --- a/internal/db/push/push.go +++ b/internal/db/push/push.go @@ -41,7 +41,11 @@ func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles, fmt.Fprintln(os.Stderr, "Would push these migrations:") fmt.Fprint(os.Stderr, utils.Bold(confirmPushAll(pending))) if includeSeed { - fmt.Fprintln(os.Stderr, "Would seed data "+utils.Bold(utils.SeedDataPath)+"...") + seedPaths, err := utils.GetSeedFiles(fsys) + if err != nil { + return err + } + fmt.Fprintf(os.Stderr, "Would seed data %v...\n", seedPaths) } } else { msg := fmt.Sprintf("Do you want to push these migrations to the remote database?\n%s\n", confirmPushAll(pending)) diff --git a/internal/db/push/push_test.go b/internal/db/push/push_test.go index e4f6353ec..72df15ef9 100644 --- a/internal/db/push/push_test.go +++ b/internal/db/push/push_test.go @@ -162,7 +162,9 @@ func TestPushAll(t *testing.T) { t.Run("throws error on seed failure", func(t *testing.T) { // Setup in-memory fs - fsys := &fstest.OpenErrorFs{DenyPath: utils.SeedDataPath} + seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql") + fsys := &fstest.OpenErrorFs{DenyPath: seedPath} + _, _ = fsys.Create(seedPath) path := filepath.Join(utils.MigrationsDir, "0_test.sql") require.NoError(t, afero.WriteFile(fsys, path, []byte{}, 0644)) // Setup mock postgres diff --git a/internal/db/reset/reset_test.go b/internal/db/reset/reset_test.go index 03da968c5..0be21f3b2 100644 --- a/internal/db/reset/reset_test.go +++ b/internal/db/reset/reset_test.go @@ -362,7 +362,7 @@ func TestResetRemote(t *testing.T) { fsys := afero.NewMemMapFs() path := filepath.Join(utils.MigrationsDir, "0_schema.sql") require.NoError(t, afero.WriteFile(fsys, path, nil, 0644)) - seedPath := filepath.Join(utils.SeedDataPath) + seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql") // Will raise an error when seeding require.NoError(t, afero.WriteFile(fsys, seedPath, []byte("INSERT INTO test_table;"), 0644)) // Setup mock postgres diff --git a/internal/db/start/start_test.go b/internal/db/start/start_test.go index dcc93b4c8..e4072df98 100644 --- a/internal/db/start/start_test.go +++ b/internal/db/start/start_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "path/filepath" "testing" "github.com/docker/docker/api/types" @@ -60,7 +61,7 @@ func TestStartDatabase(t *testing.T) { roles := "create role test" require.NoError(t, afero.WriteFile(fsys, utils.CustomRolesPath, []byte(roles), 0644)) seed := "INSERT INTO employees(name) VALUES ('Alice')" - require.NoError(t, afero.WriteFile(fsys, utils.SeedDataPath, []byte(seed), 0644)) + require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(seed), 0644)) // Setup mock docker require.NoError(t, apitest.MockDocker(utils.Docker)) defer gock.OffAll() diff --git a/internal/init/init.go b/internal/init/init.go index 033638dd9..f4e470b02 100644 --- a/internal/init/init.go +++ b/internal/init/init.go @@ -40,19 +40,14 @@ func Run(ctx context.Context, fsys afero.Fs, createVscodeSettings, createIntelli return err } - // 2. Create `seed.sql`. - if err := initSeed(fsys); err != nil { - return err - } - - // 3. Append to `.gitignore`. + // 2. Append to `.gitignore`. if utils.IsGitRepo() { if err := updateGitIgnore(utils.GitIgnorePath, fsys); err != nil { return err } } - // 4. Generate VS Code settings. + // 3. Generate VS Code settings. if createVscodeSettings != nil { if *createVscodeSettings { return writeVscodeConfig(fsys) @@ -77,15 +72,6 @@ func Run(ctx context.Context, fsys afero.Fs, createVscodeSettings, createIntelli return nil } -func initSeed(fsys afero.Fs) error { - f, err := fsys.OpenFile(utils.SeedDataPath, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return errors.Errorf("failed to create seed file: %w", err) - } - defer f.Close() - return nil -} - func updateGitIgnore(ignorePath string, fsys afero.Fs) error { var contents []byte diff --git a/internal/init/init_test.go b/internal/init/init_test.go index 09bf6ab0f..47e35b89e 100644 --- a/internal/init/init_test.go +++ b/internal/init/init_test.go @@ -28,10 +28,6 @@ func TestInitCommand(t *testing.T) { exists, err = afero.Exists(fsys, utils.GitIgnorePath) assert.NoError(t, err) assert.True(t, exists) - // Validate generated seed.sql - exists, err = afero.Exists(fsys, utils.SeedDataPath) - assert.NoError(t, err) - assert.True(t, exists) // Validate vscode settings file isn't generated exists, err = afero.Exists(fsys, settingsPath) assert.NoError(t, err) @@ -70,15 +66,6 @@ func TestInitCommand(t *testing.T) { assert.Error(t, Run(context.Background(), fsys, nil, nil, utils.InitParams{})) }) - t.Run("throws error on seed failure", func(t *testing.T) { - // Setup in-memory fs - fsys := &fstest.OpenErrorFs{DenyPath: utils.SeedDataPath} - // Run test - err := Run(context.Background(), fsys, nil, nil, utils.InitParams{}) - // Check error - assert.ErrorIs(t, err, os.ErrPermission) - }) - t.Run("creates vscode settings file", func(t *testing.T) { // Setup in-memory fs fsys := &afero.MemMapFs{} diff --git a/internal/migration/apply/apply.go b/internal/migration/apply/apply.go index 44195297f..6796930d2 100644 --- a/internal/migration/apply/apply.go +++ b/internal/migration/apply/apply.go @@ -27,11 +27,11 @@ func MigrateAndSeed(ctx context.Context, version string, conn *pgx.Conn, fsys af } func SeedDatabase(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error { - err := migration.SeedData(ctx, []string{utils.SeedDataPath}, conn, afero.NewIOFS(fsys)) - if errors.Is(err, os.ErrNotExist) { - return nil + seedPaths, err := utils.GetSeedFiles(fsys) + if err != nil { + return err } - return err + return migration.SeedData(ctx, seedPaths, conn, afero.NewIOFS(fsys)) } func CreateCustomRoles(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error { diff --git a/internal/migration/apply/apply_test.go b/internal/migration/apply/apply_test.go index b8c041749..6c5b915fd 100644 --- a/internal/migration/apply/apply_test.go +++ b/internal/migration/apply/apply_test.go @@ -44,7 +44,7 @@ func TestMigrateDatabase(t *testing.T) { path := filepath.Join(utils.MigrationsDir, "0_test.sql") sql := "create schema public" require.NoError(t, afero.WriteFile(fsys, path, []byte(sql), 0644)) - seedPath := filepath.Join(utils.SeedDataPath) + seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql") // This will raise an error when seeding require.NoError(t, afero.WriteFile(fsys, seedPath, []byte("INSERT INTO test_table;"), 0644)) // Setup mock postgres @@ -82,7 +82,7 @@ func TestSeedDatabase(t *testing.T) { fsys := afero.NewMemMapFs() // Setup seed file sql := "INSERT INTO employees(name) VALUES ('Alice')" - require.NoError(t, afero.WriteFile(fsys, utils.SeedDataPath, []byte(sql), 0644)) + require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(sql), 0644)) // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) @@ -100,7 +100,9 @@ func TestSeedDatabase(t *testing.T) { t.Run("throws error on read failure", func(t *testing.T) { // Setup in-memory fs - fsys := &fstest.OpenErrorFs{DenyPath: utils.SeedDataPath} + seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql") + fsys := &fstest.OpenErrorFs{DenyPath: seedPath} + _, _ = fsys.Create(seedPath) // Run test err := SeedDatabase(context.Background(), nil, fsys) // Check error @@ -112,7 +114,7 @@ func TestSeedDatabase(t *testing.T) { fsys := afero.NewMemMapFs() // Setup seed file sql := "INSERT INTO employees(name) VALUES ('Alice')" - require.NoError(t, afero.WriteFile(fsys, utils.SeedDataPath, []byte(sql), 0644)) + require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(sql), 0644)) // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) diff --git a/internal/utils/misc.go b/internal/utils/misc.go index 8118b8213..d4e49f046 100644 --- a/internal/utils/misc.go +++ b/internal/utils/misc.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "time" "github.com/docker/docker/client" @@ -148,7 +149,6 @@ var ( FallbackImportMapPath = filepath.Join(FunctionsDir, "import_map.json") FallbackEnvFilePath = filepath.Join(FunctionsDir, ".env") DbTestsDir = filepath.Join(SupabaseDirPath, "tests") - SeedDataPath = filepath.Join(SupabaseDirPath, "seed.sql") CustomRolesPath = filepath.Join(SupabaseDirPath, "roles.sql") ErrNotLinked = errors.Errorf("Cannot find project ref. Have you run %s?", Aqua("supabase link")) @@ -157,6 +157,26 @@ var ( ErrNotRunning = errors.Errorf("%s is not running.", Aqua("supabase start")) ) +// Match the glob patterns from the config to get a deduplicated +// array of all migrations files to apply in the declared order. +func GetSeedFiles(fsys afero.Fs) ([]string, error) { + seedPaths := Config.Db.Seed.SqlPaths + var files []string + for _, pattern := range seedPaths { + fullPattern := filepath.Join(SupabaseDirPath, pattern) + matches, err := afero.Glob(fsys, fullPattern) + if err != nil { + return nil, errors.Errorf("failed to apply glob pattern for %w", err) + } + if len(matches) == 0 { + fmt.Fprintf(os.Stderr, "%s Your pattern %s matched 0 seed files.\n", Yellow("WARNING:"), pattern) + } + sort.Strings(matches) + files = append(files, matches...) + } + return RemoveDuplicates(files), nil +} + func GetCurrentTimestamp() string { // Magic number: https://stackoverflow.com/q/45160822. return time.Now().UTC().Format("20060102150405") diff --git a/internal/utils/misc_test.go b/internal/utils/misc_test.go index 6472c2145..c16abdf00 100644 --- a/internal/utils/misc_test.go +++ b/internal/utils/misc_test.go @@ -75,3 +75,76 @@ func TestProjectRoot(t *testing.T) { assert.Equal(t, cwd, path) }) } + +func TestGetSeedFiles(t *testing.T) { + t.Run("returns seed files matching patterns", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Create seed files + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed1.sql", []byte("INSERT INTO table1 VALUES (1);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed2.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed3.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/another.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/ignore.sql", []byte("INSERT INTO table3 VALUES (3);"), 0644)) + // Mock config patterns + Config.Db.Seed.SqlPaths = []string{"seeds/seed[12].sql", "seeds/ano*.sql"} + + // Run test + files, err := GetSeedFiles(fsys) + + // Check error + assert.NoError(t, err) + // Validate files + assert.ElementsMatch(t, []string{"supabase/seeds/seed1.sql", "supabase/seeds/seed2.sql", "supabase/seeds/another.sql"}, files) + }) + t.Run("returns seed files matching patterns skip duplicates", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Create seed files + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed1.sql", []byte("INSERT INTO table1 VALUES (1);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed2.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed3.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/another.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644)) + require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/ignore.sql", []byte("INSERT INTO table3 VALUES (3);"), 0644)) + // Mock config patterns + Config.Db.Seed.SqlPaths = []string{"seeds/seed[12].sql", "seeds/ano*.sql", "seeds/seed*.sql"} + + // Run test + files, err := GetSeedFiles(fsys) + + // Check error + assert.NoError(t, err) + // Validate files + assert.ElementsMatch(t, []string{"supabase/seeds/seed1.sql", "supabase/seeds/seed2.sql", "supabase/seeds/another.sql", "supabase/seeds/seed3.sql"}, files) + }) + + t.Run("returns error on invalid pattern", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Mock config patterns + Config.Db.Seed.SqlPaths = []string{"[*!#@D#"} + + // Run test + files, err := GetSeedFiles(fsys) + + // Check error + assert.Nil(t, err) + // The resuling seed list should be empty + assert.ElementsMatch(t, []string{}, files) + }) + + t.Run("returns empty list if no files match", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Mock config patterns + Config.Db.Seed.SqlPaths = []string{"seeds/*.sql"} + + // Run test + files, err := GetSeedFiles(fsys) + + // Check error + assert.NoError(t, err) + // Validate files + assert.Empty(t, files) + }) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 18414546a..3636e7e9c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -172,7 +172,8 @@ type ( } seed struct { - Enabled bool `toml:"enabled"` + Enabled bool `toml:"enabled"` + SqlPaths []string `toml:"sql_paths"` } pooler struct { @@ -482,7 +483,8 @@ func NewConfig(editors ...ConfigEditor) config { SecretKeyBase: "EAx3IQ/wRG1v47ZD4NE4/9RzBI8Jmil3x0yhcW4V2NHBP6c2iPIzwjofi2Ep4HIG", }, Seed: seed{ - Enabled: true, + Enabled: true, + SqlPaths: []string{"./seed.sql"}, }, }, Realtime: realtime{ diff --git a/pkg/config/templates/config.toml b/pkg/config/templates/config.toml index 37646aa63..112237748 100644 --- a/pkg/config/templates/config.toml +++ b/pkg/config/templates/config.toml @@ -39,6 +39,14 @@ default_pool_size = 20 # Maximum number of client connections allowed. max_client_conn = 100 +[db.seed] +# If enabled, seeds the database after migrations during a db reset. +enabled = true +# Specifies an ordered list of seed files to load during db reset. +# Supports glob patterns relative to supabase directory. For example: +# sql_paths = ['./seeds/*.sql', '../project-src/seeds/*-load-testing.sql'] +sql_paths = ['./seed.sql'] + [realtime] enabled = true # Bind realtime via either IPv4 or IPv6. (default: IPv4) diff --git a/pkg/config/testdata/config.toml b/pkg/config/testdata/config.toml index a7aa36544..76e070c60 100644 --- a/pkg/config/testdata/config.toml +++ b/pkg/config/testdata/config.toml @@ -39,6 +39,14 @@ default_pool_size = 20 # Maximum number of client connections allowed. max_client_conn = 100 +[db.seed] +# If enabled, seeds the database after migrations during a db reset. +enabled = true +# Specifies an ordered list of seed files to load during db reset. +# Supports glob patterns relative to supabase directory. For example: +# sql_paths = ['./seeds/*.sql', '../project-src/seeds/*-load-testing.sql'] +sql_paths = ['./seed.sql'] + [realtime] enabled = true # Bind realtime via either IPv4 or IPv6. (default: IPv6) diff --git a/pkg/config/utils.go b/pkg/config/utils.go index b2318b5a9..ac26a38d2 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -29,7 +29,6 @@ type pathBuilder struct { FallbackImportMapPath string FallbackEnvFilePath string DbTestsDir string - SeedDataPath string CustomRolesPath string } @@ -63,7 +62,6 @@ func NewPathBuilder(configPath string) pathBuilder { FallbackImportMapPath: filepath.Join(base, "functions", "import_map.json"), FallbackEnvFilePath: filepath.Join(base, "functions", ".env"), DbTestsDir: filepath.Join(base, "tests"), - SeedDataPath: filepath.Join(base, "seed.sql"), CustomRolesPath: filepath.Join(base, "roles.sql"), } } diff --git a/pkg/migration/seed.go b/pkg/migration/seed.go index f299e9f05..62a2c875b 100644 --- a/pkg/migration/seed.go +++ b/pkg/migration/seed.go @@ -12,8 +12,7 @@ import ( func SeedData(ctx context.Context, pending []string, conn *pgx.Conn, fsys fs.FS) error { for _, path := range pending { - filename := filepath.Base(path) - fmt.Fprintf(os.Stderr, "Seeding data from %s...\n", filename) + fmt.Fprintf(os.Stderr, "Seeding data from %s...\n", path) // Batch seed commands, safe to use statement cache if seed, err := NewMigrationFromFile(path, fsys); err != nil { return err