Problem: upgrade with versiondb and memiavl is not tested (#1205)

* Problem: upgrade with versiondb and memiavl is not tested Solution: - enable memiavl and versiondb in upgrade test. * test upgrade with v1.0.12 * fix versiondb migration * fix memiavl upgrade * rollback rocksdb to 8.1.1 * try fix lint * fix rest lint * fix lint * fix store go.mod --------- Signed-off-by: yihuang <[email protected]> Co-authored-by: mmsqe <[email protected]>
crypto-org-chain · Oct 16, 2023 · 03edb77 · 03edb77
1 parent c47cc56
commit 03edb77
Show file tree

Hide file tree

Showing 16 changed files with 46 additions and 60 deletions.
diff --git a/go.mod b/go.mod
@@ -239,6 +239,7 @@ replace (
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.0
 	github.com/jhump/protoreflect => github.com/jhump/protoreflect v1.9.0
+	github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.8.0
 	github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20230825054824-75403cd90c6e
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 )
diff --git a/go.sum b/go.sum
@@ -1057,8 +1057,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo=
-github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY=
+github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE=
+github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg=
 github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=

diff --git a/gomod2nix.toml b/gomod2nix.toml
@@ -388,8 +388,9 @@ schema = 3
     version = "v0.1.0"
     hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ="
   [mod."github.com/linxGnu/grocksdb"]
-    version = "v1.8.4"
-    hash = "sha256-AGQ0YEuwUaev/mxOkiTNgAOLB9c7reHyTmNxXuaRah8="
+    version = "v1.8.0"
+    hash = "sha256-o6zj18at4oN6pqSioFqd1AXduR/cx0xDgtL1rPPw+1M="
+    replaced = "github.com/linxGnu/grocksdb"
   [mod."github.com/magiconair/properties"]
     version = "v1.8.7"
     hash = "sha256-XQ2bnc2s7/IH3WxEO4GishZurMyKwEclZy1DXg+2xXc="

diff --git a/integration_tests/configs/cosmovisor.jsonnet b/integration_tests/configs/cosmovisor.jsonnet
@@ -5,27 +5,10 @@ config {
     'app-config'+: {
       'app-db-backend': 'rocksdb',
       'minimum-gas-prices': '100000000000basetcro',
-      memiavl:: super.memiavl,
-      store:: super.store,
-      streamers:: super.streamers,
       'iavl-lazy-loading':: super['iavl-lazy-loading'],
     },
-    validators: [super.validators[0] {
-      'app-config':: super['app-config'],
-    }] + super.validators[1:],
     genesis+: {
       app_state+: {
-        evm+: {
-          params+: {
-            // emulate the environment on production network
-            extra_eips: [
-              '2929',
-              '2200',
-              '1884',
-              '1344',
-            ],
-          },
-        },
         bank+: {
           params: {
             send_enabled: [

diff --git a/integration_tests/configs/upgrade-test-package.nix b/integration_tests/configs/upgrade-test-package.nix
@@ -7,7 +7,8 @@ let
       shortRev = builtins.substring 0 7 rev;
     };
   }).defaultNix;
-  released = (fetchFlake "crypto-org-chain/cronos" "b3da22dcff6176ec5120c84e446a79fb54e3d9de").default;
+  # v1.0.12
+  released = (fetchFlake "crypto-org-chain/cronos" "a54d8a9659a98b917d3fc1ae39489fbc0699df60").default;
   current = pkgs.callPackage ../../. { };
 in
 pkgs.linkFarm "upgrade-test-package" [

diff --git a/memiavl/db.go b/memiavl/db.go
@@ -1008,7 +1008,7 @@ func isSnapshotName(name string) bool {
 // it's needed for upgrade module to check store upgrades,
 // it returns 0 if db don't exists or is empty.
 func GetLatestVersion(dir string) (int64, error) {
-	metadata, err := readMetadata(dir)
+	metadata, err := readMetadata(currentPath(dir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return 0, nil

diff --git a/memiavl/go.mod b/memiavl/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/klauspost/compress v1.16.3 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
-	github.com/linxGnu/grocksdb v1.8.4 // indirect
+	github.com/linxGnu/grocksdb v1.8.0 // indirect
 	github.com/onsi/gomega v1.20.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

diff --git a/memiavl/go.sum b/memiavl/go.sum
@@ -95,8 +95,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 h1:LGEzZvf33Y1NhuP5+jI/ni9l1TFS6oYPDilgy74NusM=
 github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263/go.mod h1:OXgMDuUo2lZ3NpH29ZvMYbk+LxFd5ffDl2Z2mGMuY/I=
-github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo=
-github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY=
+github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE=
+github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=

diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix
@@ -21,13 +21,13 @@
 
 stdenv.mkDerivation rec {
   pname = "rocksdb";
-  version = "8.5.3";
+  version = "8.1.1";
 
   src = fetchFromGitHub {
     owner = "facebook";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-Qa4bAprXptA79ilNE5KSfggEDvNFHdrvDQ6SvzWMQus=";
+    sha256 = "sha256-79hRtc5QSWLLyjRGCmuYZSoIc9IcIsnl8UCinz2sVw4=";
   };
 
   nativeBuildInputs = [ cmake ninja ];

diff --git a/store/go.mod b/store/go.mod
@@ -72,7 +72,7 @@ require (
 	github.com/klauspost/compress v1.16.3 // indirect
 	github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
-	github.com/linxGnu/grocksdb v1.8.4 // indirect
+	github.com/linxGnu/grocksdb v1.8.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect

diff --git a/store/go.sum b/store/go.sum
@@ -336,8 +336,8 @@ github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263/go.mod h1:O
 github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
 github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
-github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo=
-github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY=
+github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE=
+github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=

diff --git a/store/rootmulti/import.go b/store/rootmulti/import.go
@@ -6,7 +6,7 @@ import (
 	"math"
 
 	"cosmossdk.io/errors"
-	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
+	"github.com/cosmos/cosmos-sdk/snapshots/types"
 	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 	protoio "github.com/cosmos/gogoproto/io"
 
@@ -16,50 +16,50 @@ import (
 // Implements interface Snapshotter
 func (rs *Store) Restore(
 	height uint64, format uint32, protoReader protoio.Reader,
-) (snapshottypes.SnapshotItem, error) {
+) (types.SnapshotItem, error) {
 	if rs.db != nil {
 		if err := rs.db.Close(); err != nil {
-			return snapshottypes.SnapshotItem{}, fmt.Errorf("failed to close db: %w", err)
+			return types.SnapshotItem{}, fmt.Errorf("failed to close db: %w", err)
 		}
 		rs.db = nil
 	}
 
 	item, err := rs.restore(height, format, protoReader)
 	if err != nil {
-		return snapshottypes.SnapshotItem{}, err
+		return types.SnapshotItem{}, err
 	}
 
 	return item, rs.LoadLatestVersion()
 }
 
 func (rs *Store) restore(
 	height uint64, format uint32, protoReader protoio.Reader,
-) (snapshottypes.SnapshotItem, error) {
+) (types.SnapshotItem, error) {
 	importer, err := memiavl.NewMultiTreeImporter(rs.dir, height)
 	if err != nil {
-		return snapshottypes.SnapshotItem{}, err
+		return types.SnapshotItem{}, err
 	}
 	defer importer.Close()
 
-	var snapshotItem snapshottypes.SnapshotItem
+	var snapshotItem types.SnapshotItem
 loop:
 	for {
-		snapshotItem = snapshottypes.SnapshotItem{}
+		snapshotItem = types.SnapshotItem{}
 		err := protoReader.ReadMsg(&snapshotItem)
 		if err == io.EOF {
 			break
 		} else if err != nil {
-			return snapshottypes.SnapshotItem{}, errors.Wrap(err, "invalid protobuf message")
+			return types.SnapshotItem{}, errors.Wrap(err, "invalid protobuf message")
 		}
 
 		switch item := snapshotItem.Item.(type) {
-		case *snapshottypes.SnapshotItem_Store:
+		case *types.SnapshotItem_Store:
 			if err := importer.AddTree(item.Store.Name); err != nil {
-				return snapshottypes.SnapshotItem{}, err
+				return types.SnapshotItem{}, err
 			}
-		case *snapshottypes.SnapshotItem_IAVL:
+		case *types.SnapshotItem_IAVL:
 			if item.IAVL.Height > math.MaxInt8 {
-				return snapshottypes.SnapshotItem{}, errors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v",
+				return types.SnapshotItem{}, errors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v",
 					item.IAVL.Height, math.MaxInt8)
 			}
 			node := &memiavl.ExportNode{
@@ -84,7 +84,7 @@ loop:
 	}
 
 	if err := importer.Finalize(); err != nil {
-		return snapshottypes.SnapshotItem{}, err
+		return types.SnapshotItem{}, err
 	}
 
 	return snapshotItem, nil

diff --git a/store/rootmulti/snapshot.go b/store/rootmulti/snapshot.go
@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"math"
 
-	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
+	"github.com/cosmos/cosmos-sdk/snapshots/types"
 	protoio "github.com/cosmos/gogoproto/io"
 
 	"github.com/crypto-org-chain/cronos/memiavl"
@@ -39,9 +39,9 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) (returnErr
 
 		switch item := item.(type) {
 		case *memiavl.ExportNode:
-			if err := protoWriter.WriteMsg(&snapshottypes.SnapshotItem{
-				Item: &snapshottypes.SnapshotItem_IAVL{
-					IAVL: &snapshottypes.SnapshotIAVLItem{
+			if err := protoWriter.WriteMsg(&types.SnapshotItem{
+				Item: &types.SnapshotItem_IAVL{
+					IAVL: &types.SnapshotIAVLItem{
 						Key:     item.Key,
 						Value:   item.Value,
 						Height:  int32(item.Height),
@@ -52,9 +52,9 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) (returnErr
 				return err
 			}
 		case string:
-			if err := protoWriter.WriteMsg(&snapshottypes.SnapshotItem{
-				Item: &snapshottypes.SnapshotItem_Store{
-					Store: &snapshottypes.SnapshotStoreItem{
+			if err := protoWriter.WriteMsg(&types.SnapshotItem{
+				Item: &types.SnapshotItem_Store{
+					Store: &types.SnapshotStoreItem{
 						Name: item,
 					},
 				},

diff --git a/versiondb/client/restore.go b/versiondb/client/restore.go
@@ -13,7 +13,7 @@ import (
 
 	"github.com/cosmos/cosmos-sdk/server"
 	"github.com/cosmos/cosmos-sdk/snapshots"
-	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
+	"github.com/cosmos/cosmos-sdk/snapshots/types"
 
 	"github.com/crypto-org-chain/cronos/versiondb"
 	"github.com/crypto-org-chain/cronos/versiondb/tsrocksdb"
@@ -82,13 +82,13 @@ func RestoreVersionDBCmd() *cobra.Command {
 // readSnapshotEntries reads key-value entries from protobuf reader and feed to the channel
 func readSnapshotEntries(protoReader protoio.Reader, ch chan<- versiondb.ImportEntry) error {
 	var (
-		snapshotItem snapshottypes.SnapshotItem
+		snapshotItem types.SnapshotItem
 		storeKey     string
 	)
 
 loop:
 	for {
-		snapshotItem = snapshottypes.SnapshotItem{}
+		snapshotItem = types.SnapshotItem{}
 		err := protoReader.ReadMsg(&snapshotItem)
 		if err == io.EOF {
 			break
@@ -97,9 +97,9 @@ loop:
 		}
 
 		switch item := snapshotItem.Item.(type) {
-		case *snapshottypes.SnapshotItem_Store:
+		case *types.SnapshotItem_Store:
 			storeKey = item.Store.Name
-		case *snapshottypes.SnapshotItem_IAVL:
+		case *types.SnapshotItem_IAVL:
 			if storeKey == "" {
 				return errors.Wrap(err, "invalid protobuf message, store name is empty")
 			}

diff --git a/versiondb/go.mod b/versiondb/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583
 	github.com/crypto-org-chain/cronos/memiavl v0.0.3
 	github.com/golang/snappy v0.0.4
-	github.com/linxGnu/grocksdb v1.8.4
+	github.com/linxGnu/grocksdb v1.8.0
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.4

diff --git a/versiondb/go.sum b/versiondb/go.sum
@@ -489,8 +489,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo=
-github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY=
+github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE=
+github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=