Filter migrations

.gitignore

@@ -10,3 +10,14 @@
 dist
 docker-compose.override.yml
 node_modules
+
+# Devenv
+.devenv*
+devenv.local.nix
+
+# direnv
+.direnv
+
+# pre-commit
+.pre-commit-config.yaml
+

README.md

@@ -140,6 +140,7 @@ The following options are available with all commands. You must use command line
 - `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `DATABASE_URL`)_
 - `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from.
 - `--env-file ".env"` - specify an alternate environment variables file(s) to load.
+- `--migrations, -m "VERSION"` - operate on only the migrations specified. _(env: `DBMATE_MIGRATIONS`)_
 - `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `DBMATE_MIGRATIONS_DIR`)_
 - `--migrations-table "schema_migrations"` - database table to record migrations in. _(env: `DBMATE_MIGRATIONS_TABLE`)_
 - `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `DBMATE_SCHEMA_FILE`)_

main.go

@@ -59,6 +59,13 @@ func NewApp() *cli.App {
 			Value: cli.NewStringSlice(".env"),
 			Usage: "specify a file to load environment variables from",
 		},
+		&cli.StringSliceFlag{
+			Name:    "migrations",
+			Aliases: []string{"m"},
+			EnvVars: []string{"DBMATE_MIGRATIONS"},
+			Value:   cli.NewStringSlice(),
+			Usage:   "specify one or more specific migration to use",
+		},
 		&cli.StringSliceFlag{
 			Name:    "migrations-dir",
 			Aliases: []string{"d"},
@@ -292,6 +299,7 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc {
 		}
 		db := dbmate.New(u)
 		db.AutoDumpSchema = !c.Bool("no-dump-schema")
+		db.Migrations = c.StringSlice("migrations")
 		db.MigrationsDir = c.StringSlice("migrations-dir")
 		db.MigrationsTableName = c.String("migrations-table")
 		db.SchemaFile = c.String("schema-file")

pkg/dbmate/db.go

@@ -26,7 +26,7 @@ var (
 	ErrNoMigrationName       = errors.New("please specify a name for the new migration")
 	ErrMigrationAlreadyExist = errors.New("file already exists")
 	ErrMigrationDirNotFound  = errors.New("could not find migrations directory")
-	ErrMigrationNotFound     = errors.New("can't find migration file")
+	ErrMigrationNotFound     = errors.New("can't find migrations for")
 	ErrCreateDirectory       = errors.New("unable to create directory")
 )
 
@@ -43,6 +43,8 @@ type DB struct {
 	FS fs.FS
 	// Log is the interface to write stdout
 	Log io.Writer
+	// Migration specifies one or more specific migrations to look at
+	Migrations []string
 	// MigrationsDir specifies the directory or directories to find migration files
 	MigrationsDir []string
 	// MigrationsTableName specifies the database table to record migrations in
@@ -74,6 +76,7 @@ func New(databaseURL *url.URL) *DB {
 		DatabaseURL:         databaseURL,
 		FS:                  nil,
 		Log:                 os.Stdout,
+		Migrations:          []string{},
 		MigrationsDir:       []string{"./db/migrations"},
 		MigrationsTableName: "schema_migrations",
 		SchemaFile:          "./db/schema.sql",
@@ -462,7 +465,7 @@ func (db *DB) FindMigrations() ([]Migration, error) {
 		}
 	}
 
-	migrations := []Migration{}
+	foundMigrations := []Migration{}
 	for _, dir := range db.MigrationsDir {
 		// find filesystem migrations
 		files, err := db.readMigrationsDir(dir)
@@ -487,19 +490,20 @@ func (db *DB) FindMigrations() ([]Migration, error) {
 				FS:       db.FS,
 				Version:  matches[1],
 			}
+
 			if ok := appliedMigrations[migration.Version]; ok {
 				migration.Applied = true
 			}
 
-			migrations = append(migrations, migration)
+			foundMigrations = append(foundMigrations, migration)
 		}
 	}
 
-	sort.Slice(migrations, func(i, j int) bool {
-		return migrations[i].FileName < migrations[j].FileName
+	sort.Slice(foundMigrations, func(i, j int) bool {
+		return foundMigrations[i].FileName < foundMigrations[j].FileName
 	})
 
-	return migrations, nil
+	return filterMigrations(db.Migrations, foundMigrations)
 }
 
 // Rollback rolls back the most recent migration

pkg/dbmate/migration.go

@@ -2,9 +2,11 @@ package dbmate
 
 import (
 	"errors"
+	"fmt"
 	"io/fs"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 )
 
@@ -206,3 +208,176 @@ func substring(s string, begin, end int) string {
 	}
 	return s[begin:end]
 }
+
+type migrationRange struct {
+	start *string
+	end   *string
+}
+
+type migrationTracking struct {
+	lengthOfMigrations          int
+	currentMigration            int
+	singleMigrationsLookup      map[string]migrationRange
+	activeMigrationsLookup      map[string]migrationRange
+	inactiveMigrationsLookup    map[string]migrationRange
+	inactiveEndMigrationsLookup map[string]migrationRange
+}
+
+func filterMigrations(givenMigrations []string, migrations []Migration) ([]Migration, error) {
+	tracking := newMigrationTracking(givenMigrations, len(migrations))
+	if tracking == nil {
+		return migrations, nil
+	}
+	filteredMigrations := make([]Migration, 0, len(migrations))
+	for _, migration := range migrations {
+		if !tracking.addNext(migration) {
+			continue
+		}
+		filteredMigrations = append(filteredMigrations, migration)
+	}
+	notFound, err := tracking.givenMigrationsNotFound()
+	if err != nil {
+		return nil, err
+	}
+	if len(notFound) > 0 {
+		return nil, fmt.Errorf("%w `%v` %d", ErrMigrationNotFound, notFound, len(notFound))
+	}
+	return filteredMigrations, nil
+}
+
+// given migrations is a list of migrations
+// a migration can be either the migration version or the migration file name
+// Ranges are supported as well.
+//
+// dbmate -m ...version # everything before and including version
+// dbmate -m version... # everything starting at version and after
+// dbmate -m version...version2 # everything starting at version and ending at version2
+func newMigrationTracking(givenMigrations []string, lengthOfMigrations int) *migrationTracking {
+	if len(givenMigrations) == 0 {
+		return nil
+	}
+
+	singleMigrationsLookup := make(map[string]migrationRange)
+	inactiveMigrationsLookup := make(map[string]migrationRange)
+	inactiveEndMigrationsLookup := make(map[string]migrationRange)
+	activeMigrationsLookup := make(map[string]migrationRange)
+
+	for _, given := range givenMigrations {
+		if !strings.Contains(given, "...") {
+			singleMigrationsLookup[given] = migrationRange{}
+			continue
+		}
+
+		mr := migrationRange{}
+		split := strings.Split(given, "...")
+		start := split[0]
+		if start == "" {
+			mr.start = nil
+		} else {
+			mr.start = &start
+		}
+		if len(split) > 1 {
+			mr.end = &split[1]
+		}
+		// Empty range means all migrations
+		if mr.start == nil && mr.end == nil {
+			return nil
+		}
+		if mr.start == nil {
+			activeMigrationsLookup[*mr.end] = mr
+		} else {
+			inactiveMigrationsLookup[*mr.start] = mr
+			if mr.end != nil {
+				inactiveEndMigrationsLookup[*mr.end] = mr
+			}
+		}
+	}
+
+	return &migrationTracking{
+		currentMigration:            0,
+		lengthOfMigrations:          lengthOfMigrations,
+		singleMigrationsLookup:      singleMigrationsLookup,
+		activeMigrationsLookup:      activeMigrationsLookup,
+		inactiveMigrationsLookup:    inactiveMigrationsLookup,
+		inactiveEndMigrationsLookup: inactiveEndMigrationsLookup,
+	}
+}
+
+func (ar *migrationTracking) givenMigrationsNotFound() ([]string, error) {
+	notFound := make([]string, 0)
+	for m := range ar.singleMigrationsLookup {
+		notFound = append(notFound, m)
+	}
+	for _, mr := range ar.inactiveMigrationsLookup {
+		notFound = append(notFound, *mr.start)
+	}
+	for key, mr := range ar.activeMigrationsLookup {
+		if key != "" && mr.end != nil {
+			if _, ok := ar.inactiveEndMigrationsLookup[*mr.end]; !ok {
+				return notFound, fmt.Errorf("%w %v...%v because end comes before start- their order should be reversed", ErrMigrationNotFound, *mr.start, *mr.end)
+			}
+			notFound = append(notFound, *mr.end)
+		}
+	}
+	return notFound, nil
+}
+
+func (ar *migrationTracking) retrieveMigration(mapping map[string]migrationRange, migration Migration) (migrationRange, bool) {
+	if m, ok := mapping[migration.Version]; ok {
+		delete(mapping, migration.Version)
+		return m, true
+	}
+	if m, ok := mapping[migration.FileName]; ok {
+		delete(mapping, migration.FileName)
+		return m, true
+	}
+
+	// support using a numeric index with a leading plus
+	positiveStr := "+" + strconv.FormatInt(int64(ar.currentMigration), 10)
+	if m, ok := mapping[positiveStr]; ok {
+		delete(mapping, positiveStr)
+		return m, true
+	}
+	// the numeric index can be negative (this is more useful than positive)
+	negative := ar.currentMigration - ar.lengthOfMigrations
+	negativeStr := strconv.FormatInt(int64(negative), 10)
+	if m, ok := mapping[negativeStr]; ok {
+		delete(mapping, negativeStr)
+		return m, true
+	}
+
+	return migrationRange{}, false
+}
+
+// addNext returns true if the migration is selected
+func (ar *migrationTracking) addNext(migration Migration) bool {
+	ar.currentMigration++
+	selected := false
+	if _, ok := ar.retrieveMigration(ar.singleMigrationsLookup, migration); ok {
+		selected = true
+	}
+	if _, ok := ar.retrieveMigration(ar.activeMigrationsLookup, migration); ok {
+		selected = true
+	}
+	// Tracking just for error handling purposes
+	_, _ = ar.retrieveMigration(ar.inactiveEndMigrationsLookup, migration)
+
+	// transfer from inactive to active
+	if mr, ok := ar.retrieveMigration(ar.inactiveMigrationsLookup, migration); ok {
+		if mr.end == nil {
+			// active for all the rest of the migrations
+			ar.activeMigrationsLookup[""] = mr
+		} else {
+			// check to see if start and end are the same
+			if *mr.end != migration.Version && *mr.end != migration.FileName {
+				ar.activeMigrationsLookup[*mr.end] = mr
+			}
+		}
+	}
+
+	if len(ar.activeMigrationsLookup) > 0 {
+		return true
+	}
+
+	return selected
+}

pkg/dbmate/migration_test.go

@@ -200,3 +200,73 @@ DROP COLUMN status;
 		})
 	})
 }
+
+func TestMigrationRange(t *testing.T) {
+	abc := Migration{
+		Version:  "abc",
+		FileName: "abc.sql",
+	}
+	xyz := Migration{
+		Version:  "xyz",
+		FileName: "xyz.sql",
+	}
+	two := Migration{
+		Version:  "234",
+		FileName: "234.sql",
+	}
+	one := Migration{
+		Version:  "123",
+		FileName: "123.sql",
+	}
+	migrations := []Migration{abc, xyz, one, two}
+	t.Run("when no ranges given passes through", func(t *testing.T) {
+		ms, err := filterMigrations([]string{}, migrations)
+		require.NoError(t, err)
+		require.Equal(t, ms, migrations)
+	})
+
+	t.Run("when filters don't match", func(t *testing.T) {
+		_, err := filterMigrations([]string{"nobody"}, migrations)
+		require.Error(t, err)
+	})
+
+	t.Run("select single migrations by version", func(t *testing.T) {
+		ms, err := filterMigrations([]string{"xyz", "234"}, migrations)
+		require.NoError(t, err)
+		require.Equal(t, ms, []Migration{xyz, two})
+	})
+
+	t.Run("select single migrations by filename", func(t *testing.T) {
+		ms, err := filterMigrations([]string{"xyz.sql", "234.sql"}, migrations)
+		require.NoError(t, err)
+		require.Equal(t, []Migration{xyz, two}, ms)
+	})
+
+	t.Run("select single migrations by index position", func(t *testing.T) {
+		ms, err := filterMigrations([]string{"+2", "-1"}, migrations)
+		require.NoError(t, err)
+		require.Equal(t, []Migration{migrations[1], migrations[len(migrations)-2]}, ms)
+	})
+	t.Run("filter range", func(t *testing.T) {
+		ms, err := filterMigrations([]string{"abc...123"}, migrations)
+		require.NoError(t, err)
+		require.Equal(t, []Migration{abc, xyz, one}, ms)
+	})
+
+	t.Run("filter range given backwards", func(t *testing.T) {
+		_, err := filterMigrations([]string{"123...abc"}, migrations)
+		require.Error(t, err)
+	})
+
+	t.Run("filter range no end", func(t *testing.T) {
+		ms, err := filterMigrations([]string{"123..."}, migrations)
+		require.NoError(t, err)
+		require.Equal(t, []Migration{one, two}, ms)
+	})
+
+	t.Run("filter range no begin", func(t *testing.T) {
+		ms, err := filterMigrations([]string{"...123"}, migrations)
+		require.NoError(t, err)
+		require.Equal(t, []Migration{abc, xyz, one}, ms)
+	})
+}