From de34f9e4416ec9df002686218a948097389e2158 Mon Sep 17 00:00:00 2001 From: yihuang Date: Wed, 22 May 2024 00:11:58 +0800 Subject: [PATCH] Problem: rocksdb is out of date (#1448) * Problem: rocksdb is out of date Solution: - upgrade to v9.1.1 * tag * changelog * Apply suggestions from code review Signed-off-by: yihuang * versiondb * fix versiondb integration test * add rocksdb.tools to shell for debugging --------- Signed-off-by: yihuang --- CHANGELOG.md | 1 + flake.nix | 12 ++++-------- go.mod | 2 +- go.sum | 4 ++-- gomod2nix.toml | 4 ++-- integration_tests/test_versiondb.py | 4 +++- nix/rocksdb.nix | 4 ++-- versiondb/go.mod | 2 +- versiondb/go.sum | 4 ++-- 9 files changed, 18 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c64cadaea6..77c018fcd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * (e2ee)[#1413](https://github.com/crypto-org-chain/cronos/pull/1413) Add custom keyring implementation for e2ee module. * (e2ee)[#1415](https://github.com/crypto-org-chain/cronos/pull/1415) Add batch keys query for e2ee module. * (e2ee)[#1421](https://github.com/crypto-org-chain/cronos/pull/1421) Validate e2ee key when register. +* (store) [#1448](https://github.com/crypto-org-chain/cronos/pull/1448) Upgrade rocksdb to `v9.1.1`. ### Bug Fixes diff --git a/flake.nix b/flake.nix index b27e354e0c..36fc0bfe97 100644 --- a/flake.nix +++ b/flake.nix @@ -50,7 +50,7 @@ }; defaultPackage = packages.cronosd; defaultApp = apps.cronosd; - devShells = { + devShells = rec { default = pkgs.mkShell { buildInputs = [ defaultPackage.go @@ -58,17 +58,13 @@ ]; }; rocksdb = pkgs.mkShell { - buildInputs = [ - defaultPackage.go - pkgs.gomod2nix + buildInputs = default.buildInputs ++ [ pkgs.rocksdb + pkgs.rocksdb.tools ]; }; full = pkgs.mkShell { - buildInputs = [ - defaultPackage.go - pkgs.gomod2nix - pkgs.rocksdb + buildInputs = rocksdb.buildInputs ++ [ pkgs.test-env ]; }; diff --git a/go.mod b/go.mod index 2bc0fd2360..28690775c9 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/go-metrics v0.5.3 - github.com/linxGnu/grocksdb v1.8.14 + github.com/linxGnu/grocksdb v1.9.1 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 28fe331fe2..d4df6e3a27 100644 --- a/go.sum +++ b/go.sum @@ -858,8 +858,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.8.14 h1:HTgyYalNwBSG/1qCQUIott44wU5b2Y9Kr3z7SK5OfGQ= -github.com/linxGnu/grocksdb v1.8.14/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= +github.com/linxGnu/grocksdb v1.9.1 h1:LmwuHzsWglxJrIES9jvS2O1xTPD2nnKYhAQDx5dIyRo= +github.com/linxGnu/grocksdb v1.9.1/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= diff --git a/gomod2nix.toml b/gomod2nix.toml index 0594db6b4f..11b79f9ba6 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -455,8 +455,8 @@ schema = 3 version = "v0.1.0" hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ=" [mod."github.com/linxGnu/grocksdb"] - version = "v1.8.14" - hash = "sha256-dT647UzB25Ye1Gv6b9JEtJZjuWKdJo4D8kw9cB0W8gA=" + version = "v1.9.1" + hash = "sha256-yRSGvZ5YeDqGOEmGPRSKxPjNoCQ/Xk/DMtNtutkbPGg=" [mod."github.com/magiconair/properties"] version = "v1.8.7" hash = "sha256-XQ2bnc2s7/IH3WxEO4GishZurMyKwEclZy1DXg+2xXc=" diff --git a/integration_tests/test_versiondb.py b/integration_tests/test_versiondb.py index 6e4d5f452f..d79dfa5ac3 100644 --- a/integration_tests/test_versiondb.py +++ b/integration_tests/test_versiondb.py @@ -17,7 +17,7 @@ def test_versiondb_migration(cronos: Cronos): - verify change set and save snapshot - restore pruned application.db from the snapshot - replace node1's application.db with the restored one - - build versiondb for node0 + - rebuild versiondb for node0 - start the nodes, now check: - the network can grow - node0 do support historical queries @@ -60,6 +60,8 @@ def test_versiondb_migration(cronos: Cronos): print("restore versiondb for node0") sst_dir = tempfile.mkdtemp(dir=cronos.base_dir) print(cli0.changeset_build_versiondb_sst(changeset_dir, sst_dir)) + # ingest-versiondb-sst expects an empty database + shutil.rmtree(cli0.data_dir / "data/versiondb") print( cli0.changeset_ingest_versiondb_sst( cli0.data_dir / "data/versiondb", sst_dir, maximum_version=latest_version diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix index 7d7528c824..916a0a0eb2 100644 --- a/nix/rocksdb.nix +++ b/nix/rocksdb.nix @@ -20,13 +20,13 @@ stdenv.mkDerivation (finalAttrs: { pname = "rocksdb"; - version = "8.11.3"; + version = "9.1.1"; src = fetchFromGitHub { owner = "facebook"; repo = finalAttrs.pname; rev = "v${finalAttrs.version}"; - hash = "sha256-OpEiMwGxZuxb9o3RQuSrwZMQGLhe9xLT1aa3HpI4KPs="; + hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc="; }; nativeBuildInputs = [ cmake ninja ]; diff --git a/versiondb/go.mod b/versiondb/go.mod index c1a4a88ae9..8716e80d99 100644 --- a/versiondb/go.mod +++ b/versiondb/go.mod @@ -17,7 +17,7 @@ require ( github.com/cosmos/ibc-go/modules/capability v1.0.0 github.com/crypto-org-chain/cronos/memiavl v0.0.3 github.com/golang/snappy v0.0.4 - github.com/linxGnu/grocksdb v1.8.14 + github.com/linxGnu/grocksdb v1.9.1 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 diff --git a/versiondb/go.sum b/versiondb/go.sum index 119dd47b84..8798584ffb 100644 --- a/versiondb/go.sum +++ b/versiondb/go.sum @@ -466,8 +466,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.8.14 h1:HTgyYalNwBSG/1qCQUIott44wU5b2Y9Kr3z7SK5OfGQ= -github.com/linxGnu/grocksdb v1.8.14/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= +github.com/linxGnu/grocksdb v1.9.1 h1:LmwuHzsWglxJrIES9jvS2O1xTPD2nnKYhAQDx5dIyRo= +github.com/linxGnu/grocksdb v1.9.1/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=