meta: fix upgrade to v3 (#3048)

CHANGELOG.md

@@ -6,6 +6,7 @@ Changelog for NeoFS Node
 ### Added
 
 ### Fixed
+- Incomplete metabase migration to version 3 leading to node start failure (#3048)
 
 ### Changed
 

pkg/local_object_storage/metabase/control.go

@@ -52,27 +52,7 @@ func (db *DB) openBolt() error {
 
 	db.log.Debug("opened boltDB instance for Metabase")
 
-	db.log.Debug("checking metabase version")
-	return db.boltDB.View(func(tx *bbolt.Tx) error {
-		// The safest way to check if the metabase is fresh is to check if it has no buckets.
-		// However, shard info can be present. So here we check that the number of buckets is
-		// at most 1.
-		// Another thing to consider is that tests do not persist shard ID, we want to support
-		// this case too.
-		var n int
-		err := tx.ForEach(func([]byte, *bbolt.Bucket) error {
-			if n++; n >= 2 { // do not iterate a lot
-				return errBreakBucketForEach
-			}
-			return nil
-		})
-
-		if errors.Is(err, errBreakBucketForEach) {
-			db.initialized = true
-			err = nil
-		}
-		return err
-	})
+	return nil
 }
 
 // Init initializes metabase. It creates static (CID-independent) buckets in underlying BoltDB instance.
@@ -153,7 +133,7 @@ func (db *DB) init(reset bool) error {
 			if err != nil {
 				return err
 			}
-			err = updateVersion(tx, version)
+			err = updateVersion(tx, currentMetaVersion)
 			if err != nil {
 				return err
 			}

pkg/local_object_storage/metabase/db.go

@@ -40,8 +40,6 @@ type DB struct {
 	matchers map[object.SearchMatchType]matcher
 
 	boltDB *bbolt.DB
-
-	initialized bool
 }
 
 // Option is an option of DB constructor.

pkg/local_object_storage/metabase/status.go

@@ -47,7 +47,7 @@ func (db *DB) ObjectStatus(address oid.Address) (ObjectStatus, error) {
 	}
 
 	err = db.boltDB.View(func(tx *bbolt.Tx) error {
-		res.Version = getVersion(tx)
+		res.Version, _ = getVersion(tx)
 
 		oID := address.Object()
 		cID := address.Container()

pkg/local_object_storage/metabase/version.go

@@ -10,8 +10,8 @@ import (
 	"go.etcd.io/bbolt"
 )
 
-// version contains current metabase version.
-const version = 3
+// currentMetaVersion contains current metabase version.
+const currentMetaVersion = 3
 
 var versionKey = []byte("version")
 
@@ -21,39 +21,29 @@ var versionKey = []byte("version")
 var ErrOutdatedVersion = logicerr.New("invalid version, resynchronization is required")
 
 func (db *DB) checkVersion(tx *bbolt.Tx) error {
-	var knownVersion bool
+	stored, knownVersion := getVersion(tx)
 
-	b := tx.Bucket(shardInfoBucket)
-	if b != nil {
-		data := b.Get(versionKey)
-		if len(data) == 8 {
-			knownVersion = true
-
-			stored := binary.LittleEndian.Uint64(data)
-			if stored != version {
-				migrate, ok := migrateFrom[stored]
-				if !ok {
-					return fmt.Errorf("%w: expected=%d, stored=%d", ErrOutdatedVersion, version, stored)
-				}
-
-				err := migrate(db, tx)
-				if err != nil {
-					return fmt.Errorf("migrating from %d to %d version failed, consider database resync: %w", stored, version, err)
-				}
-			}
-		}
+	switch {
+	case !knownVersion:
+		// new database, write version
+		return updateVersion(tx, currentMetaVersion)
+	case stored == currentMetaVersion:
+		return nil
+	case stored > currentMetaVersion:
+		return fmt.Errorf("%w: expected=%d, stored=%d", ErrOutdatedVersion, currentMetaVersion, stored)
 	}
 
-	if !db.initialized {
-		// new database, write version
-		return updateVersion(tx, version)
-	} else if !knownVersion {
-		// db is initialized but no version
-		// has been found; that could happen
-		// if the db is corrupted or the version
-		// is <2 (is outdated and requires resync
-		// anyway)
-		return ErrOutdatedVersion
+	// Outdated, but can be migrated.
+	for i := stored; i < currentMetaVersion; i++ {
+		migrate, ok := migrateFrom[i]
+		if !ok {
+			return fmt.Errorf("%w: expected=%d, stored=%d", ErrOutdatedVersion, currentMetaVersion, stored)
+		}
+
+		err := migrate(db, tx)
+		if err != nil {
+			return fmt.Errorf("migrating from meta version %d failed, consider database resync: %w", i, err)
+		}
 	}
 
 	return nil
@@ -70,16 +60,16 @@ func updateVersion(tx *bbolt.Tx, version uint64) error {
 	return b.Put(versionKey, data)
 }
 
-func getVersion(tx *bbolt.Tx) uint64 {
+func getVersion(tx *bbolt.Tx) (uint64, bool) {
 	b := tx.Bucket(shardInfoBucket)
 	if b != nil {
 		data := b.Get(versionKey)
 		if len(data) == 8 {
-			return binary.LittleEndian.Uint64(data)
+			return binary.LittleEndian.Uint64(data), true
 		}
 	}
 
-	return 0
+	return 0, false
 }
 
 var migrateFrom = map[uint64]func(*DB, *bbolt.Tx) error{
@@ -95,7 +85,11 @@ func migrateFrom2Version(db *DB, tx *bbolt.Tx) error {
 	c := bkt.Cursor()
 
 	for k, v := c.First(); k != nil; k, v = c.Next() {
-		if l := len(v); l != addressKeySize {
+		l := len(v)
+		if l == addressKeySize+8 { // Because of a 0.44.0 bug we can have a migrated DB with version 2.
+			continue
+		}
+		if l != addressKeySize {
 			return fmt.Errorf("graveyard value with unexpected %d length", l)
 		}
 
@@ -109,5 +103,5 @@ func migrateFrom2Version(db *DB, tx *bbolt.Tx) error {
 		}
 	}
 
-	return nil
+	return updateVersion(tx, 3)
 }

pkg/local_object_storage/metabase/version_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	objectconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/object"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
@@ -42,8 +43,8 @@ func TestVersion(t *testing.T) {
 			if len(data) != 8 {
 				return errors.New("invalid version data")
 			}
-			if stored := binary.LittleEndian.Uint64(data); stored != version {
-				return fmt.Errorf("invalid version: %d != %d", stored, version)
+			if stored := binary.LittleEndian.Uint64(data); stored != currentMetaVersion {
+				return fmt.Errorf("invalid version: %d != %d", stored, currentMetaVersion)
 			}
 			return nil
 		}))
@@ -77,7 +78,7 @@ func TestVersion(t *testing.T) {
 		db := newDB(t)
 		require.NoError(t, db.Open(false))
 		require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error {
-			return updateVersion(tx, version+1)
+			return updateVersion(tx, currentMetaVersion+1)
 		}))
 		require.NoError(t, db.Close())
 
@@ -307,16 +308,20 @@ func TestMigrate2to3(t *testing.T) {
 	})
 	require.NoError(t, err)
 
+	// inhumeV2 stores data in the old format, but new DB has current version by default, force old version.
 	err = db.boltDB.Update(func(tx *bbolt.Tx) error {
-		err = updateVersion(tx, 2)
-		if err != nil {
-			return err
-		}
-
-		return migrateFrom2Version(db, tx)
+		return updateVersion(tx, 2)
 	})
 	require.NoError(t, err)
 
+	db.mode = mode.DegradedReadOnly // Force reload.
+	ok, err := db.Reload(WithPath(db.info.Path), WithEpochState(epochState{}))
+	require.NoError(t, err)
+	require.True(t, ok)
+
+	err = db.Init() // Migration happens here.
+	require.NoError(t, err)
+
 	err = db.boltDB.View(func(tx *bbolt.Tx) error {
 		return tx.Bucket(graveyardBucketName).ForEach(func(k, v []byte) error {
 			require.Len(t, v, addressKeySize+8)
@@ -327,4 +332,15 @@ func TestMigrate2to3(t *testing.T) {
 		})
 	})
 	require.NoError(t, err)
+	err = db.boltDB.View(func(tx *bbolt.Tx) error {
+		gotV, ok := getVersion(tx)
+		if !ok {
+			return errors.New("missing version")
+		}
+		if gotV != currentMetaVersion {
+			return errors.New("version was not updated")
+		}
+		return nil
+	})
+	require.NoError(t, err)
 }