From 03edb775285c4fe8d17eeb12497fd21d484cefaa Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 16 Oct 2023 17:16:20 +0800 Subject: [PATCH] Problem: upgrade with versiondb and memiavl is not tested (#1205) * Problem: upgrade with versiondb and memiavl is not tested Solution: - enable memiavl and versiondb in upgrade test. * test upgrade with v1.0.12 * fix versiondb migration * fix memiavl upgrade * rollback rocksdb to 8.1.1 * try fix lint * fix rest lint * fix lint * fix store go.mod --------- Signed-off-by: yihuang Co-authored-by: mmsqe --- go.mod | 1 + go.sum | 4 +-- gomod2nix.toml | 5 ++-- integration_tests/configs/cosmovisor.jsonnet | 17 ----------- .../configs/upgrade-test-package.nix | 3 +- memiavl/db.go | 2 +- memiavl/go.mod | 2 +- memiavl/go.sum | 4 +-- nix/rocksdb.nix | 4 +-- store/go.mod | 2 +- store/go.sum | 4 +-- store/rootmulti/import.go | 28 +++++++++---------- store/rootmulti/snapshot.go | 14 +++++----- versiondb/client/restore.go | 10 +++---- versiondb/go.mod | 2 +- versiondb/go.sum | 4 +-- 16 files changed, 46 insertions(+), 60 deletions(-) diff --git a/go.mod b/go.mod index d02b93cd01..b7d53079ff 100644 --- a/go.mod +++ b/go.mod @@ -239,6 +239,7 @@ replace ( // TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409 github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.0 github.com/jhump/protoreflect => github.com/jhump/protoreflect v1.9.0 + github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.8.0 github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20230825054824-75403cd90c6e github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 ) diff --git a/go.sum b/go.sum index 2961b810f2..ab5c78d14d 100644 --- a/go.sum +++ b/go.sum @@ -1057,8 +1057,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo= -github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= +github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= diff --git a/gomod2nix.toml b/gomod2nix.toml index 950c6e06b8..c05a0e01f9 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -388,8 +388,9 @@ schema = 3 version = "v0.1.0" hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ=" [mod."github.com/linxGnu/grocksdb"] - version = "v1.8.4" - hash = "sha256-AGQ0YEuwUaev/mxOkiTNgAOLB9c7reHyTmNxXuaRah8=" + version = "v1.8.0" + hash = "sha256-o6zj18at4oN6pqSioFqd1AXduR/cx0xDgtL1rPPw+1M=" + replaced = "github.com/linxGnu/grocksdb" [mod."github.com/magiconair/properties"] version = "v1.8.7" hash = "sha256-XQ2bnc2s7/IH3WxEO4GishZurMyKwEclZy1DXg+2xXc=" diff --git a/integration_tests/configs/cosmovisor.jsonnet b/integration_tests/configs/cosmovisor.jsonnet index f5d77ba65b..51c9c7c3bd 100644 --- a/integration_tests/configs/cosmovisor.jsonnet +++ b/integration_tests/configs/cosmovisor.jsonnet @@ -5,27 +5,10 @@ config { 'app-config'+: { 'app-db-backend': 'rocksdb', 'minimum-gas-prices': '100000000000basetcro', - memiavl:: super.memiavl, - store:: super.store, - streamers:: super.streamers, 'iavl-lazy-loading':: super['iavl-lazy-loading'], }, - validators: [super.validators[0] { - 'app-config':: super['app-config'], - }] + super.validators[1:], genesis+: { app_state+: { - evm+: { - params+: { - // emulate the environment on production network - extra_eips: [ - '2929', - '2200', - '1884', - '1344', - ], - }, - }, bank+: { params: { send_enabled: [ diff --git a/integration_tests/configs/upgrade-test-package.nix b/integration_tests/configs/upgrade-test-package.nix index eef5c6476a..47e2b3f8bb 100644 --- a/integration_tests/configs/upgrade-test-package.nix +++ b/integration_tests/configs/upgrade-test-package.nix @@ -7,7 +7,8 @@ let shortRev = builtins.substring 0 7 rev; }; }).defaultNix; - released = (fetchFlake "crypto-org-chain/cronos" "b3da22dcff6176ec5120c84e446a79fb54e3d9de").default; + # v1.0.12 + released = (fetchFlake "crypto-org-chain/cronos" "a54d8a9659a98b917d3fc1ae39489fbc0699df60").default; current = pkgs.callPackage ../../. { }; in pkgs.linkFarm "upgrade-test-package" [ diff --git a/memiavl/db.go b/memiavl/db.go index 06e9dee111..457b357c00 100644 --- a/memiavl/db.go +++ b/memiavl/db.go @@ -1008,7 +1008,7 @@ func isSnapshotName(name string) bool { // it's needed for upgrade module to check store upgrades, // it returns 0 if db don't exists or is empty. func GetLatestVersion(dir string) (int64, error) { - metadata, err := readMetadata(dir) + metadata, err := readMetadata(currentPath(dir)) if err != nil { if os.IsNotExist(err) { return 0, nil diff --git a/memiavl/go.mod b/memiavl/go.mod index 0d065844fa..c3e28af119 100644 --- a/memiavl/go.mod +++ b/memiavl/go.mod @@ -34,7 +34,7 @@ require ( github.com/jmhodges/levigo v1.0.0 // indirect github.com/klauspost/compress v1.16.3 // indirect github.com/kr/pretty v0.3.1 // indirect - github.com/linxGnu/grocksdb v1.8.4 // indirect + github.com/linxGnu/grocksdb v1.8.0 // indirect github.com/onsi/gomega v1.20.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/memiavl/go.sum b/memiavl/go.sum index 1f756874c9..49d0c6d5bd 100644 --- a/memiavl/go.sum +++ b/memiavl/go.sum @@ -95,8 +95,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 h1:LGEzZvf33Y1NhuP5+jI/ni9l1TFS6oYPDilgy74NusM= github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263/go.mod h1:OXgMDuUo2lZ3NpH29ZvMYbk+LxFd5ffDl2Z2mGMuY/I= -github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo= -github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= +github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix index d40ac32b38..9b3333d173 100644 --- a/nix/rocksdb.nix +++ b/nix/rocksdb.nix @@ -21,13 +21,13 @@ stdenv.mkDerivation rec { pname = "rocksdb"; - version = "8.5.3"; + version = "8.1.1"; src = fetchFromGitHub { owner = "facebook"; repo = pname; rev = "v${version}"; - sha256 = "sha256-Qa4bAprXptA79ilNE5KSfggEDvNFHdrvDQ6SvzWMQus="; + sha256 = "sha256-79hRtc5QSWLLyjRGCmuYZSoIc9IcIsnl8UCinz2sVw4="; }; nativeBuildInputs = [ cmake ninja ]; diff --git a/store/go.mod b/store/go.mod index b04079b9f4..e59a0d5935 100644 --- a/store/go.mod +++ b/store/go.mod @@ -72,7 +72,7 @@ require ( github.com/klauspost/compress v1.16.3 // indirect github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect - github.com/linxGnu/grocksdb v1.8.4 // indirect + github.com/linxGnu/grocksdb v1.8.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect diff --git a/store/go.sum b/store/go.sum index 17f98efdb6..0b4e999997 100644 --- a/store/go.sum +++ b/store/go.sum @@ -336,8 +336,8 @@ github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263/go.mod h1:O github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= -github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo= -github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= +github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= diff --git a/store/rootmulti/import.go b/store/rootmulti/import.go index 8c79c7a481..f8a1ec48a4 100644 --- a/store/rootmulti/import.go +++ b/store/rootmulti/import.go @@ -6,7 +6,7 @@ import ( "math" "cosmossdk.io/errors" - snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types" + "github.com/cosmos/cosmos-sdk/snapshots/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" protoio "github.com/cosmos/gogoproto/io" @@ -16,17 +16,17 @@ import ( // Implements interface Snapshotter func (rs *Store) Restore( height uint64, format uint32, protoReader protoio.Reader, -) (snapshottypes.SnapshotItem, error) { +) (types.SnapshotItem, error) { if rs.db != nil { if err := rs.db.Close(); err != nil { - return snapshottypes.SnapshotItem{}, fmt.Errorf("failed to close db: %w", err) + return types.SnapshotItem{}, fmt.Errorf("failed to close db: %w", err) } rs.db = nil } item, err := rs.restore(height, format, protoReader) if err != nil { - return snapshottypes.SnapshotItem{}, err + return types.SnapshotItem{}, err } return item, rs.LoadLatestVersion() @@ -34,32 +34,32 @@ func (rs *Store) Restore( func (rs *Store) restore( height uint64, format uint32, protoReader protoio.Reader, -) (snapshottypes.SnapshotItem, error) { +) (types.SnapshotItem, error) { importer, err := memiavl.NewMultiTreeImporter(rs.dir, height) if err != nil { - return snapshottypes.SnapshotItem{}, err + return types.SnapshotItem{}, err } defer importer.Close() - var snapshotItem snapshottypes.SnapshotItem + var snapshotItem types.SnapshotItem loop: for { - snapshotItem = snapshottypes.SnapshotItem{} + snapshotItem = types.SnapshotItem{} err := protoReader.ReadMsg(&snapshotItem) if err == io.EOF { break } else if err != nil { - return snapshottypes.SnapshotItem{}, errors.Wrap(err, "invalid protobuf message") + return types.SnapshotItem{}, errors.Wrap(err, "invalid protobuf message") } switch item := snapshotItem.Item.(type) { - case *snapshottypes.SnapshotItem_Store: + case *types.SnapshotItem_Store: if err := importer.AddTree(item.Store.Name); err != nil { - return snapshottypes.SnapshotItem{}, err + return types.SnapshotItem{}, err } - case *snapshottypes.SnapshotItem_IAVL: + case *types.SnapshotItem_IAVL: if item.IAVL.Height > math.MaxInt8 { - return snapshottypes.SnapshotItem{}, errors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v", + return types.SnapshotItem{}, errors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v", item.IAVL.Height, math.MaxInt8) } node := &memiavl.ExportNode{ @@ -84,7 +84,7 @@ loop: } if err := importer.Finalize(); err != nil { - return snapshottypes.SnapshotItem{}, err + return types.SnapshotItem{}, err } return snapshotItem, nil diff --git a/store/rootmulti/snapshot.go b/store/rootmulti/snapshot.go index b601d1648e..e05f82fd2d 100644 --- a/store/rootmulti/snapshot.go +++ b/store/rootmulti/snapshot.go @@ -5,7 +5,7 @@ import ( "fmt" "math" - snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types" + "github.com/cosmos/cosmos-sdk/snapshots/types" protoio "github.com/cosmos/gogoproto/io" "github.com/crypto-org-chain/cronos/memiavl" @@ -39,9 +39,9 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) (returnErr switch item := item.(type) { case *memiavl.ExportNode: - if err := protoWriter.WriteMsg(&snapshottypes.SnapshotItem{ - Item: &snapshottypes.SnapshotItem_IAVL{ - IAVL: &snapshottypes.SnapshotIAVLItem{ + if err := protoWriter.WriteMsg(&types.SnapshotItem{ + Item: &types.SnapshotItem_IAVL{ + IAVL: &types.SnapshotIAVLItem{ Key: item.Key, Value: item.Value, Height: int32(item.Height), @@ -52,9 +52,9 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) (returnErr return err } case string: - if err := protoWriter.WriteMsg(&snapshottypes.SnapshotItem{ - Item: &snapshottypes.SnapshotItem_Store{ - Store: &snapshottypes.SnapshotStoreItem{ + if err := protoWriter.WriteMsg(&types.SnapshotItem{ + Item: &types.SnapshotItem_Store{ + Store: &types.SnapshotStoreItem{ Name: item, }, }, diff --git a/versiondb/client/restore.go b/versiondb/client/restore.go index a75afbf15f..a2e7d3f6f6 100644 --- a/versiondb/client/restore.go +++ b/versiondb/client/restore.go @@ -13,7 +13,7 @@ import ( "github.com/cosmos/cosmos-sdk/server" "github.com/cosmos/cosmos-sdk/snapshots" - snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types" + "github.com/cosmos/cosmos-sdk/snapshots/types" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/versiondb/tsrocksdb" @@ -82,13 +82,13 @@ func RestoreVersionDBCmd() *cobra.Command { // readSnapshotEntries reads key-value entries from protobuf reader and feed to the channel func readSnapshotEntries(protoReader protoio.Reader, ch chan<- versiondb.ImportEntry) error { var ( - snapshotItem snapshottypes.SnapshotItem + snapshotItem types.SnapshotItem storeKey string ) loop: for { - snapshotItem = snapshottypes.SnapshotItem{} + snapshotItem = types.SnapshotItem{} err := protoReader.ReadMsg(&snapshotItem) if err == io.EOF { break @@ -97,9 +97,9 @@ loop: } switch item := snapshotItem.Item.(type) { - case *snapshottypes.SnapshotItem_Store: + case *types.SnapshotItem_Store: storeKey = item.Store.Name - case *snapshottypes.SnapshotItem_IAVL: + case *types.SnapshotItem_IAVL: if storeKey == "" { return errors.Wrap(err, "invalid protobuf message, store name is empty") } diff --git a/versiondb/go.mod b/versiondb/go.mod index a0bca69e6f..0db4f595cb 100644 --- a/versiondb/go.mod +++ b/versiondb/go.mod @@ -12,7 +12,7 @@ require ( github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583 github.com/crypto-org-chain/cronos/memiavl v0.0.3 github.com/golang/snappy v0.0.4 - github.com/linxGnu/grocksdb v1.8.4 + github.com/linxGnu/grocksdb v1.8.0 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 diff --git a/versiondb/go.sum b/versiondb/go.sum index 7f3f4909ed..121350fdf6 100644 --- a/versiondb/go.sum +++ b/versiondb/go.sum @@ -489,8 +489,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.8.4 h1:ZMsBpPpJNtRLHiKKp0mI7gW+NT4s7UgfD5xHxx1jVRo= -github.com/linxGnu/grocksdb v1.8.4/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= +github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=