Problem: app hash mismatch occurs when upgrade to iavl v1.2.0 (#1618)

* Problem: app hash mismatch occurs when upgrade to iavl v1.2.0

* try to reproduce in unit test

* reproduced

* make initial version compatible with iavl 1.2

* changelog

* remove debug log

* align submodule

* fix unit test

* fix lint

* better make test

* fix initial version

* cleanup

* cleanup

* cleanup

* lint

---------

Co-authored-by: huangyi <[email protected]>

.github/workflows/build.yml

@@ -74,10 +74,7 @@ jobs:
           signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
       - name: test & coverage report creation
         run: |
-          nix develop -c make test
-          nix develop -c make test-memiavl
-          nix develop -c make test-store
-          nix develop .#rocksdb -c make test-versiondb
+          nix develop .#rocksdb -c make test test-versiondb
         if: steps.changed-files.outputs.any_changed == 'true'
       - name: filter out proto files
         run: |

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## UNRELEASED
 
+### State Machine Breaking
+
+* (memiavl)[#1618](https://github.com/crypto-org-chain/cronos/pull/1618) memiavl change initial version logic to be
+  compatible with iavl 1.2.0.
+
 ### Improvements
 
 * [#1592](https://github.com/crypto-org-chain/cronos/pull/1592) Change the default parallelism of the block-stm to minimum between GOMAXPROCS and NumCPU

Makefile

@@ -98,7 +98,7 @@ build: check-network print-ledger go.sum
 install: check-network print-ledger go.sum
 	@go install -mod=readonly $(BUILD_FLAGS) ./cmd/cronosd
 
-test:
+test: test-memiavl test-store
 	@go test -tags=objstore -v -mod=readonly $(PACKAGES) -coverprofile=$(COVERAGE) -covermode=atomic
 
 test-memiavl:

go.mod

@@ -268,8 +268,6 @@ replace (
 replace (
 	// Use cosmos keyring
 	github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0
-	// FIXME: address the replacement when bump v1.2.0 to that triggers app hash mismatch in upgrade test
-	github.com/cosmos/iavl => github.com/cosmos/iavl v1.1.2
 	// dgrijalva/jwt-go is deprecated and doesn't receive security updates.
 	// TODO: remove it: https://github.com/cosmos/cosmos-sdk/issues/13134
 	github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2

go.sum

@@ -393,8 +393,8 @@ github.com/cosmos/gogogateway v1.2.0/go.mod h1:iQpLkGWxYcnCdz5iAdLcRBSw3h7NXeOkZ
 github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU=
 github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro=
 github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0=
-github.com/cosmos/iavl v1.1.2 h1:zL9FK7C4L/P4IF1Dm5fIwz0WXCnn7Bp1M2FxH0ayM7Y=
-github.com/cosmos/iavl v1.1.2/go.mod h1:jLeUvm6bGT1YutCaL2fIar/8vGUE8cPZvh/gXEWDaDM=
+github.com/cosmos/iavl v1.2.0 h1:kVxTmjTh4k0Dh1VNL046v6BXqKziqMDzxo93oh3kOfM=
+github.com/cosmos/iavl v1.2.0/go.mod h1:HidWWLVAtODJqFD6Hbne2Y0q3SdxByJepHUOeoH4LiI=
 github.com/cosmos/ibc-go/modules/apps/callbacks v0.0.0-20240913130017-6b2554360c0e h1:jMqihcJRBdpRrKGOMS1bDyyoo2JoQxv4QmMCwK3HSvI=
 github.com/cosmos/ibc-go/modules/apps/callbacks v0.0.0-20240913130017-6b2554360c0e/go.mod h1:akR14gsU5YD5S1G5I6lOI7z51OjR1vJko06Rs/3/Ym0=
 github.com/cosmos/ibc-go/modules/capability v1.0.1 h1:ibwhrpJ3SftEEZRxCRkH0fQZ9svjthrX2+oXdZvzgGI=

gomod2nix.toml

@@ -186,9 +186,8 @@ schema = 3
     version = "v1.7.0"
     hash = "sha256-ZkEUImxBBo8Q/6c7tVR0rybpLbtlplzvgfLl5xvtV00="
   [mod."github.com/cosmos/iavl"]
-    version = "v1.1.2"
-    hash = "sha256-fhh5fN1BMDxbF4PobERMQdIb9vIrxaSl0tRXas0WKmc="
-    replaced = "github.com/cosmos/iavl"
+    version = "v1.2.0"
+    hash = "sha256-NYSt6LOGyspP6eZXo9e5+2MFwyrWxD/rp2dRTtlWg2E="
   [mod."github.com/cosmos/ibc-go/modules/apps/callbacks"]
     version = "v0.0.0-20240913130017-6b2554360c0e"
     hash = "sha256-mL+g8WB+fLUlTKMnrdn8rTgw2gqXsHlky3/BTsyfek0="

integration_tests/utils.py

@@ -113,10 +113,9 @@ def wait_for_block(cli, height, timeout=240):
             print(f"get sync status failed: {e}", file=sys.stderr)
         else:
             current_height = int(get_sync_info(status)["latest_block_height"])
-            print("debug current height", current_height)
+            print("current block height", current_height)
             if current_height >= height:
                 break
-            print("current block height", current_height)
         time.sleep(0.5)
     else:
         raise TimeoutError(f"wait for block {height} timeout")

memiavl/db.go

@@ -699,7 +699,7 @@ func (db *DB) reloadMultiTree(mtree *MultiTree) error {
 
 	db.MultiTree = *mtree
 	// catch-up the pending changes
-	return db.MultiTree.applyWALEntry(db.pendingLog)
+	return db.applyWALEntry(db.pendingLog)
 }
 
 // rewriteIfApplicable execute the snapshot rewrite strategy according to current height

memiavl/db_test.go

@@ -180,6 +180,7 @@ func TestInitialVersion(t *testing.T) {
 	name2 := "new2"
 	key := "hello"
 	value := "world"
+	value1 := "world1"
 	for _, initialVersion := range []int64{0, 1, 100} {
 		dir := t.TempDir()
 		db, err := Load(dir, Options{CreateIfMissing: true, InitialStores: []string{name}})
@@ -188,66 +189,56 @@ func TestInitialVersion(t *testing.T) {
 		require.NoError(t, db.ApplyChangeSets(mockNameChangeSet(name, key, value)))
 		v, err := db.Commit()
 		require.NoError(t, err)
-		if initialVersion <= 1 {
-			require.Equal(t, int64(1), v)
-		} else {
-			require.Equal(t, initialVersion, v)
-		}
-		hash := db.LastCommitInfo().StoreInfos[0].CommitId.Hash
-		require.Equal(t, "6032661ab0d201132db7a8fa1da6a0afe427e6278bd122c301197680ab79ca02", hex.EncodeToString(hash))
-		require.NoError(t, db.ApplyChangeSets(mockNameChangeSet(name, key, "world1")))
+
+		realInitialVersion := max(initialVersion, 1)
+		require.Equal(t, realInitialVersion, v)
+
+		// the nodes are created with initial version to be compatible with iavl v1 behavior.
+		// with iavl v0, the nodes are created with version 1.
+		commitId := db.LastCommitInfo().StoreInfos[0].CommitId
+		require.Equal(t, commitId.Hash, HashNode(newLeafNode([]byte(key), []byte(value), uint32(commitId.Version))))
+
+		require.NoError(t, db.ApplyChangeSets(mockNameChangeSet(name, key, value1)))
 		v, err = db.Commit()
 		require.NoError(t, err)
-		hash = db.LastCommitInfo().StoreInfos[0].CommitId.Hash
-		if initialVersion <= 1 {
-			require.Equal(t, int64(2), v)
-			require.Equal(t, "ef0530f9bf1af56c19a3bac32a3ec4f76a6fefaacb2efd4027a2cf37240f60bb", hex.EncodeToString(hash))
-		} else {
-			require.Equal(t, initialVersion+1, v)
-			require.Equal(t, "a719e7d699d42ea8e5637ec84675a2c28f14a00a71fb518f20aa2c395673a3b8", hex.EncodeToString(hash))
-		}
+		commitId = db.LastCommitInfo().StoreInfos[0].CommitId
+		require.Equal(t, realInitialVersion+1, v)
+		require.Equal(t, commitId.Hash, HashNode(newLeafNode([]byte(key), []byte(value1), uint32(commitId.Version))))
 		require.NoError(t, db.Close())
 
+		// reload the db, check the contents are the same
 		db, err = Load(dir, Options{})
 		require.NoError(t, err)
 		require.Equal(t, uint32(initialVersion), db.initialVersion)
 		require.Equal(t, v, db.Version())
-		require.Equal(t, hex.EncodeToString(hash), hex.EncodeToString(db.LastCommitInfo().StoreInfos[0].CommitId.Hash))
+		require.Equal(t, hex.EncodeToString(commitId.Hash), hex.EncodeToString(db.LastCommitInfo().StoreInfos[0].CommitId.Hash))
 
+		// add a new store to a reloaded db
 		db.ApplyUpgrades([]*TreeNameUpgrade{{Name: name1}})
 		require.NoError(t, db.ApplyChangeSets(mockNameChangeSet(name1, key, value)))
 		v, err = db.Commit()
 		require.NoError(t, err)
-		if initialVersion <= 1 {
-			require.Equal(t, int64(3), v)
-		} else {
-			require.Equal(t, initialVersion+2, v)
-		}
+		require.Equal(t, realInitialVersion+2, v)
 		require.Equal(t, 2, len(db.lastCommitInfo.StoreInfos))
 		info := db.lastCommitInfo.StoreInfos[0]
 		require.Equal(t, name1, info.Name)
 		require.Equal(t, v, info.CommitId.Version)
-		require.Equal(t, "6032661ab0d201132db7a8fa1da6a0afe427e6278bd122c301197680ab79ca02", hex.EncodeToString(info.CommitId.Hash))
-		// the nodes are created with version 1, which is compatible with iavl behavior: https://github.com/cosmos/iavl/pull/660
-		require.Equal(t, info.CommitId.Hash, HashNode(newLeafNode([]byte(key), []byte(value), 1)))
+		require.Equal(t, info.CommitId.Hash, HashNode(newLeafNode([]byte(key), []byte(value), uint32(info.CommitId.Version))))
 
+		// test snapshot rewriting and reload
 		require.NoError(t, db.RewriteSnapshot())
 		require.NoError(t, db.Reload())
-
+		// add new store after snapshot rewriting
 		db.ApplyUpgrades([]*TreeNameUpgrade{{Name: name2}})
 		require.NoError(t, db.ApplyChangeSets(mockNameChangeSet(name2, key, value)))
 		v, err = db.Commit()
 		require.NoError(t, err)
-		if initialVersion <= 1 {
-			require.Equal(t, int64(4), v)
-		} else {
-			require.Equal(t, initialVersion+3, v)
-		}
+		require.Equal(t, realInitialVersion+3, v)
 		require.Equal(t, 3, len(db.lastCommitInfo.StoreInfos))
 		info2 := db.lastCommitInfo.StoreInfos[1]
 		require.Equal(t, name2, info2.Name)
 		require.Equal(t, v, info2.CommitId.Version)
-		require.Equal(t, hex.EncodeToString(info.CommitId.Hash), hex.EncodeToString(info2.CommitId.Hash))
+		require.Equal(t, info2.CommitId.Hash, HashNode(newLeafNode([]byte(key), []byte(value), uint32(info2.CommitId.Version))))
 	}
 }
 

memiavl/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/alitto/pond v1.8.3
 	github.com/cosmos/cosmos-db v1.0.2
 	github.com/cosmos/gogoproto v1.4.11
-	github.com/cosmos/iavl v1.1.2
+	github.com/cosmos/iavl v1.2.0
 	github.com/cosmos/ics23/go v0.10.0
 	github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263
 	github.com/stretchr/testify v1.8.4

memiavl/go.sum

@@ -30,8 +30,8 @@ github.com/cosmos/cosmos-db v1.0.2 h1:hwMjozuY1OlJs/uh6vddqnk9j7VamLv+0DBlbEXbAK
 github.com/cosmos/cosmos-db v1.0.2/go.mod h1:Z8IXcFJ9PqKK6BIsVOB3QXtkKoqUOp1vRvPT39kOXEA=
 github.com/cosmos/gogoproto v1.4.11 h1:LZcMHrx4FjUgrqQSWeaGC1v/TeuVFqSLa43CC6aWR2g=
 github.com/cosmos/gogoproto v1.4.11/go.mod h1:/g39Mh8m17X8Q/GDEs5zYTSNaNnInBSohtaxzQnYq1Y=
-github.com/cosmos/iavl v1.1.2 h1:zL9FK7C4L/P4IF1Dm5fIwz0WXCnn7Bp1M2FxH0ayM7Y=
-github.com/cosmos/iavl v1.1.2/go.mod h1:jLeUvm6bGT1YutCaL2fIar/8vGUE8cPZvh/gXEWDaDM=
+github.com/cosmos/iavl v1.2.0 h1:kVxTmjTh4k0Dh1VNL046v6BXqKziqMDzxo93oh3kOfM=
+github.com/cosmos/iavl v1.2.0/go.mod h1:HidWWLVAtODJqFD6Hbne2Y0q3SdxByJepHUOeoH4LiI=
 github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM=
 github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

memiavl/import.go

@@ -131,7 +131,7 @@ func (ai *TreeImporter) Close() error {
 // doImport a stream of `ExportNode`s into a new snapshot.
 func doImport(dir string, version int64, nodes <-chan *ExportNode) (returnErr error) {
 	if version > int64(math.MaxUint32) {
-		return errors.New("version overflows uint32")
+		return fmt.Errorf("version overflows uint32: %d", version)
 	}
 
 	return writeSnapshot(context.Background(), dir, uint32(version), func(w *snapshotWriter) (uint32, error) {
@@ -167,7 +167,7 @@ type importer struct {
 
 func (i *importer) Add(n *ExportNode) error {
 	if n.Version > int64(math.MaxUint32) {
-		return errors.New("version overflows uint32")
+		return fmt.Errorf("version overflows uint32: %d", n.Version)
 	}
 
 	if n.Height == 0 {

memiavl/multitree.go

@@ -105,8 +105,9 @@ func LoadMultiTree(dir string, zeroCopy bool, cacheSize int) (*MultiTree, error)
 		zeroCopy:       zeroCopy,
 		cacheSize:      cacheSize,
 	}
-	// initial version is nesserary for wal index conversion
-	mtree.setInitialVersion(metadata.InitialVersion)
+	// initial version is nesserary for wal index conversion,
+	// overflow checked in `readMetadata`.
+	mtree.setInitialVersion(uint32(metadata.InitialVersion))
 	return mtree, nil
 }
 
@@ -133,26 +134,28 @@ func (t *MultiTree) SetInitialVersion(initialVersion int64) error {
 	}
 
 	for _, entry := range t.trees {
-		if !entry.Tree.IsEmpty() {
+		if !entry.IsEmpty() {
 			return fmt.Errorf("tree is not empty: %s", entry.Name)
 		}
 	}
 
-	t.setInitialVersion(initialVersion)
+	t.setInitialVersion(uint32(initialVersion))
 	return nil
 }
 
-func (t *MultiTree) setInitialVersion(initialVersion int64) {
-	t.initialVersion = uint32(initialVersion)
-	for _, entry := range t.trees {
-		entry.Tree.initialVersion = t.initialVersion
+func (t *MultiTree) setInitialVersion(initialVersion uint32) {
+	t.initialVersion = initialVersion
+	if t.initialVersion > 1 {
+		for _, entry := range t.trees {
+			entry.setInitialVersion(t.initialVersion)
+		}
 	}
 }
 
 func (t *MultiTree) SetZeroCopy(zeroCopy bool) {
 	t.zeroCopy = zeroCopy
 	for _, entry := range t.trees {
-		entry.Tree.SetZeroCopy(zeroCopy)
+		entry.SetZeroCopy(zeroCopy)
 	}
 }
 
@@ -161,7 +164,7 @@ func (t *MultiTree) Copy(cacheSize int) *MultiTree {
 	trees := make([]NamedTree, len(t.trees))
 	treesByName := make(map[string]int, len(t.trees))
 	for i, entry := range t.trees {
-		tree := entry.Tree.Copy(cacheSize)
+		tree := entry.Copy(cacheSize)
 		trees[i] = NamedTree{Tree: tree, Name: entry.Name}
 		treesByName[entry.Name] = i
 	}
@@ -247,7 +250,7 @@ func (t *MultiTree) ApplyChangeSet(name string, changeSet ChangeSet) error {
 	if !found {
 		return fmt.Errorf("unknown tree name %s", name)
 	}
-	t.trees[i].Tree.ApplyChangeSet(changeSet)
+	t.trees[i].ApplyChangeSet(changeSet)
 	return nil
 }
 
@@ -271,7 +274,7 @@ func (t *MultiTree) WorkingCommitInfo() *CommitInfo {
 func (t *MultiTree) SaveVersion(updateCommitInfo bool) (int64, error) {
 	t.lastCommitInfo.Version = nextVersion(t.lastCommitInfo.Version, t.initialVersion)
 	for _, entry := range t.trees {
-		if _, _, err := entry.Tree.SaveVersion(updateCommitInfo); err != nil {
+		if _, _, err := entry.SaveVersion(updateCommitInfo); err != nil {
 			return 0, err
 		}
 	}
@@ -292,8 +295,8 @@ func (t *MultiTree) buildCommitInfo(version int64) *CommitInfo {
 		infos = append(infos, StoreInfo{
 			Name: entry.Name,
 			CommitId: CommitID{
-				Version: entry.Tree.Version(),
-				Hash:    entry.Tree.RootHash(),
+				Version: entry.Version(),
+				Hash:    entry.RootHash(),
 			},
 		})
 	}
@@ -410,7 +413,7 @@ func WriteFileSync(name string, data []byte) error {
 func (t *MultiTree) Close() error {
 	errs := make([]error, 0, len(t.trees))
 	for _, entry := range t.trees {
-		errs = append(errs, entry.Tree.Close())
+		errs = append(errs, entry.Close())
 	}
 	t.trees = nil
 	t.treesByName = nil