From 2cfdca4f1f83a8feeff11cf0400cdb40d7bd792b Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 13:27:09 +0800 Subject: [PATCH 01/10] Problem: nixpkgs-fmt is deprecated (backport: #1677) Solution: - switch to nixfmt-rfc-style --- .github/workflows/lint.yml | 2 +- Makefile | 2 +- contracts/shell.nix | 5 +- default.nix | 76 ++++--- flake.nix | 93 ++++---- integration_tests/configs/broken-cronosd.nix | 11 +- .../configs/upgrade-test-package.nix | 53 +++-- integration_tests/shell.nix | 5 +- nix/build_overlay.nix | 4 +- nix/bundle-win-exe.nix | 66 +++--- nix/cosmovisor.nix | 4 +- nix/cronos-matrix.nix | 90 ++++---- nix/dapptools-overlay.nix | 17 +- nix/default.nix | 40 +++- nix/gen-binding-shell.nix | 5 +- nix/go-ethereum.nix | 32 ++- nix/hermes.nix | 22 +- nix/rocksdb.nix | 141 ++++++------ nix/scripts.nix | 15 +- nix/sources.nix | 204 +++++++++++------- nix/testenv.nix | 21 +- nix/testground-image.nix | 26 ++- testground/benchmark/flake.nix | 61 +++--- testground/benchmark/overlay.nix | 72 ++++--- 24 files changed, 641 insertions(+), 426 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 068371cf8a..f4ac54f4b9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -102,5 +102,5 @@ jobs: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} if: steps.changed-files.outputs.any_changed == 'true' - - run: nix-shell -I nixpkgs=./nix -p nixpkgs-fmt --run "make lint-nix" + - run: nix develop -c make lint-nix if: steps.changed-files.outputs.any_changed == 'true' diff --git a/Makefile b/Makefile index 82246b56d7..03131ec304 100644 --- a/Makefile +++ b/Makefile @@ -131,7 +131,7 @@ lint-py: --format="::error file=%(path)s,line=%(row)d,col=%(col)d::%(path)s:%(row)d:%(col)d: %(code)s %(text)s" \ lint-nix: - find . -name "*.nix" ! -path './integration_tests/contracts/*' ! -path "./contracts/*" | xargs nixpkgs-fmt --check + find . -name "*.nix" ! -path './integration_tests/contracts/*' ! -path "./contracts/*" | xargs nixfmt -c .PHONY: lint lint-fix lint-py diff --git a/contracts/shell.nix b/contracts/shell.nix index a668db8fbc..41da5e4d6a 100644 --- a/contracts/shell.nix +++ b/contracts/shell.nix @@ -1,4 +1,7 @@ -{ system ? builtins.currentSystem, pkgs ? import ../nix { inherit system; } }: +{ + system ? builtins.currentSystem, + pkgs ? import ../nix { inherit system; }, +}: pkgs.mkShell { buildInputs = [ pkgs.which diff --git a/default.nix b/default.nix index d6c03837e1..6b822c297f 100644 --- a/default.nix +++ b/default.nix @@ -1,19 +1,26 @@ -{ lib -, stdenv -, buildGoApplication -, nix-gitignore -, buildPackages -, coverage ? false # https://tip.golang.org/doc/go1.20#cover -, rocksdb -, network ? "mainnet" # mainnet|testnet -, rev ? "dirty" -, static ? stdenv.hostPlatform.isStatic -, nativeByteOrder ? true # nativeByteOrder mode will panic on big endian machines +{ + lib, + stdenv, + buildGoApplication, + nix-gitignore, + buildPackages, + coverage ? false, # https://tip.golang.org/doc/go1.20#cover + rocksdb, + network ? "mainnet", # mainnet|testnet + rev ? "dirty", + static ? stdenv.hostPlatform.isStatic, + nativeByteOrder ? true, # nativeByteOrder mode will panic on big endian machines }: let version = "v1.3.3"; pname = "cronosd"; - tags = [ "ledger" "netgo" network "rocksdb" "grocksdb_no_link" ] ++ lib.optionals nativeByteOrder [ "nativebyteorder" ]; + tags = [ + "ledger" + "netgo" + network + "rocksdb" + "grocksdb_no_link" + ] ++ lib.optionals nativeByteOrder [ "nativebyteorder" ]; ldflags = lib.concatStringsSep "\n" ([ "-X github.com/cosmos/cosmos-sdk/version.Name=cronos" "-X github.com/cosmos/cosmos-sdk/version.AppName=${pname}" @@ -24,29 +31,40 @@ let buildInputs = [ rocksdb ]; in buildGoApplication rec { - inherit pname version buildInputs tags ldflags; - src = (nix-gitignore.gitignoreSourcePure [ - "/*" # ignore all, then add whitelists - "!/x/" - "!/app/" - "!/cmd/" - "!/client/" - "!/versiondb/" - "!/memiavl/" - "!/store/" - "!go.mod" - "!go.sum" - "!gomod2nix.toml" - ] ./.); + inherit + pname + version + buildInputs + tags + ldflags + ; + src = ( + nix-gitignore.gitignoreSourcePure [ + "/*" # ignore all, then add whitelists + "!/x/" + "!/app/" + "!/cmd/" + "!/client/" + "!/versiondb/" + "!/memiavl/" + "!/store/" + "!go.mod" + "!go.sum" + "!gomod2nix.toml" + ] ./. + ); modules = ./gomod2nix.toml; pwd = src; # needed to support replace subPackages = [ "cmd/cronosd" ]; buildFlags = lib.optionalString coverage "-cover"; CGO_ENABLED = "1"; CGO_LDFLAGS = lib.optionalString (rocksdb != null) ( - if static then "-lrocksdb -pthread -lstdc++ -ldl -lzstd -lsnappy -llz4 -lbz2 -lz" - else if stdenv.hostPlatform.isWindows then "-lrocksdb-shared" - else "-lrocksdb -pthread -lstdc++ -ldl" + if static then + "-lrocksdb -pthread -lstdc++ -ldl -lzstd -lsnappy -llz4 -lbz2 -lz" + else if stdenv.hostPlatform.isWindows then + "-lrocksdb-shared" + else + "-lrocksdb -pthread -lstdc++ -ldl" ); postFixup = lib.optionalString (stdenv.isDarwin && rocksdb != null) '' diff --git a/flake.nix b/flake.nix index b5d6d9a164..6cb3232b2d 100644 --- a/flake.nix +++ b/flake.nix @@ -18,7 +18,15 @@ }; }; - outputs = { self, nixpkgs, nix-bundle-exe, gomod2nix, flake-utils, poetry2nix }: + outputs = + { + self, + nixpkgs, + nix-bundle-exe, + gomod2nix, + flake-utils, + poetry2nix, + }: let rev = self.shortRev or "dirty"; mkApp = drv: { @@ -26,52 +34,49 @@ program = "${drv}/bin/${drv.meta.mainProgram}"; }; in - (flake-utils.lib.eachDefaultSystem - (system: - let - pkgs = import nixpkgs { - inherit system; - overlays = self.overlays.default; - config = { }; + (flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { + inherit system; + overlays = self.overlays.default; + config = { }; + }; + in + rec { + packages = pkgs.cronos-matrix // { + inherit (pkgs) rocksdb testground-image; + }; + apps = { + cronosd = mkApp packages.cronosd; + cronosd-testnet = mkApp packages.cronosd-testnet; + stateless-testcase = { + type = "app"; + program = "${pkgs.benchmark-testcase}/bin/stateless-testcase"; }; - in - rec { - packages = pkgs.cronos-matrix // { - inherit (pkgs) rocksdb testground-image; + }; + defaultPackage = packages.cronosd; + defaultApp = apps.cronosd; + devShells = rec { + default = pkgs.mkShell { + buildInputs = [ + defaultPackage.go + pkgs.gomod2nix + pkgs.nixfmt-rfc-style + ]; }; - apps = { - cronosd = mkApp packages.cronosd; - cronosd-testnet = mkApp packages.cronosd-testnet; - stateless-testcase = { - type = "app"; - program = "${pkgs.benchmark-testcase}/bin/stateless-testcase"; - }; + rocksdb = pkgs.mkShell { + buildInputs = default.buildInputs ++ [ + pkgs.rocksdb + pkgs.rocksdb.tools + ]; }; - defaultPackage = packages.cronosd; - defaultApp = apps.cronosd; - devShells = rec { - default = pkgs.mkShell { - buildInputs = [ - defaultPackage.go - pkgs.gomod2nix - ]; - }; - rocksdb = pkgs.mkShell { - buildInputs = default.buildInputs ++ [ - pkgs.rocksdb - pkgs.rocksdb.tools - ]; - }; - full = pkgs.mkShell { - buildInputs = rocksdb.buildInputs ++ [ - pkgs.test-env - ]; - }; - }; - legacyPackages = pkgs; - } - ) - ) // { + full = pkgs.mkShell { buildInputs = rocksdb.buildInputs ++ [ pkgs.test-env ]; }; + }; + legacyPackages = pkgs; + } + )) + // { overlays.default = [ (import ./nix/build_overlay.nix) poetry2nix.overlays.default diff --git a/integration_tests/configs/broken-cronosd.nix b/integration_tests/configs/broken-cronosd.nix index 9242c004ad..508761270d 100644 --- a/integration_tests/configs/broken-cronosd.nix +++ b/integration_tests/configs/broken-cronosd.nix @@ -1,8 +1,9 @@ -{ pkgs ? import ../../nix { } }: -let cronosd = (pkgs.callPackage ../../. { }); +{ + pkgs ? import ../../nix { }, +}: +let + cronosd = (pkgs.callPackage ../../. { }); in cronosd.overrideAttrs (oldAttrs: { - patches = oldAttrs.patches or [ ] ++ [ - ./broken-cronosd.patch - ]; + patches = oldAttrs.patches or [ ] ++ [ ./broken-cronosd.patch ]; }) diff --git a/integration_tests/configs/upgrade-test-package.nix b/integration_tests/configs/upgrade-test-package.nix index e60cebd3c0..988ac16af6 100644 --- a/integration_tests/configs/upgrade-test-package.nix +++ b/integration_tests/configs/upgrade-test-package.nix @@ -1,26 +1,47 @@ let pkgs = import ../../nix { }; - fetchFlake = repo: rev: (pkgs.flake-compat { - src = { - outPath = builtins.fetchTarball "https://github.com/${repo}/archive/${rev}.tar.gz"; - inherit rev; - shortRev = builtins.substring 0 7 rev; - }; - }).defaultNix; + fetchFlake = + repo: rev: + (pkgs.flake-compat { + src = { + outPath = builtins.fetchTarball "https://github.com/${repo}/archive/${rev}.tar.gz"; + inherit rev; + shortRev = builtins.substring 0 7 rev; + }; + }).defaultNix; # v0.7.0 - released_0 = (fetchFlake "crypto-org-chain/cronos" "799ac47e293403bd57580d2ff96bb8d9851c3cde").default; + released_0 = + (fetchFlake "crypto-org-chain/cronos" "799ac47e293403bd57580d2ff96bb8d9851c3cde").default; # v0.8.0 - released_1 = (fetchFlake "crypto-org-chain/cronos" "2f2cc88b501b47149690fdef05afbbbe5bc116c9").default; + released_1 = + (fetchFlake "crypto-org-chain/cronos" "2f2cc88b501b47149690fdef05afbbbe5bc116c9").default; # v1.0.15 - released_2 = (fetchFlake "crypto-org-chain/cronos" "1f5e2618362303d91f621b47cbc1115cf4fa0195").default; + released_2 = + (fetchFlake "crypto-org-chain/cronos" "1f5e2618362303d91f621b47cbc1115cf4fa0195").default; # v1.1.1 - released_3 = (fetchFlake "crypto-org-chain/cronos" "10b8eeb9052e3c52aa59dec15f5d3aca781d1271").default; + released_3 = + (fetchFlake "crypto-org-chain/cronos" "10b8eeb9052e3c52aa59dec15f5d3aca781d1271").default; current = pkgs.callPackage ../../. { }; in pkgs.linkFarm "upgrade-test-package" [ - { name = "genesis"; path = released_0; } - { name = "v0.7.0-hotfix"; path = released_1; } - { name = "v1.0.0"; path = released_2; } - { name = "v1.1.0"; path = released_3; } - { name = "v1.3"; path = current; } + { + name = "genesis"; + path = released_0; + } + { + name = "v0.7.0-hotfix"; + path = released_1; + } + { + name = "v1.0.0"; + path = released_2; + } + { + name = "v1.1.0"; + path = released_3; + } + { + name = "v1.3"; + path = current; + } ] diff --git a/integration_tests/shell.nix b/integration_tests/shell.nix index fe275138a5..78e8ccc79a 100644 --- a/integration_tests/shell.nix +++ b/integration_tests/shell.nix @@ -1,4 +1,7 @@ -{ system ? builtins.currentSystem, pkgs ? import ../nix { inherit system; } }: +{ + system ? builtins.currentSystem, + pkgs ? import ../nix { inherit system; }, +}: let renameExe = pkgs.callPackage ../nix/rename-exe.nix { }; in diff --git a/nix/build_overlay.nix b/nix/build_overlay.nix index 476654e669..31627bad22 100644 --- a/nix/build_overlay.nix +++ b/nix/build_overlay.nix @@ -1,4 +1,2 @@ # some basic overlays nessesary for the build -final: super: { - rocksdb = final.callPackage ./rocksdb.nix { }; -} +final: super: { rocksdb = final.callPackage ./rocksdb.nix { }; } diff --git a/nix/bundle-win-exe.nix b/nix/bundle-win-exe.nix index 44bcf323d7..5817b15823 100644 --- a/nix/bundle-win-exe.nix +++ b/nix/bundle-win-exe.nix @@ -1,35 +1,37 @@ -{ runCommand -, windows -, stdenv -, rocksdb -, bzip2 -, lz4 -, snappy -, zstd -, zlib -, cronosd +{ + runCommand, + windows, + stdenv, + rocksdb, + bzip2, + lz4, + snappy, + zstd, + zlib, + cronosd, }: runCommand "tarball-${cronosd.name}" -{ - # manually enumerate the runtime dependencies of cronosd on mingwW64 - deps = [ - "${rocksdb}/bin/librocksdb-shared.dll" - "${snappy}/bin/libsnappy.dll" - "${lz4.out}/lib/liblz4.dll" - "${bzip2.bin}/bin/libbz2-1.dll" - "${zlib}/bin/zlib1.dll" - "${zstd.bin}/bin/libzstd.dll" - "${windows.mingw_w64_pthreads}/bin/libwinpthread-1.dll" - "${windows.mcfgthreads}/bin/libmcfgthread-1.dll" - "${stdenv.cc.cc.lib}/x86_64-w64-mingw32/lib/libgcc_s_seh-1.dll" - "${stdenv.cc.cc.lib}/x86_64-w64-mingw32/lib/libstdc++-6.dll" - ]; -} '' - mkdir -p $out - for so in $deps - do - cp $so $out/ - done + { + # manually enumerate the runtime dependencies of cronosd on mingwW64 + deps = [ + "${rocksdb}/bin/librocksdb-shared.dll" + "${snappy}/bin/libsnappy.dll" + "${lz4.out}/lib/liblz4.dll" + "${bzip2.bin}/bin/libbz2-1.dll" + "${zlib}/bin/zlib1.dll" + "${zstd.bin}/bin/libzstd.dll" + "${windows.mingw_w64_pthreads}/bin/libwinpthread-1.dll" + "${windows.mcfgthreads}/bin/libmcfgthread-1.dll" + "${stdenv.cc.cc.lib}/x86_64-w64-mingw32/lib/libgcc_s_seh-1.dll" + "${stdenv.cc.cc.lib}/x86_64-w64-mingw32/lib/libstdc++-6.dll" + ]; + } + '' + mkdir -p $out + for so in $deps + do + cp $so $out/ + done - cp ${cronosd}/bin/${cronosd.meta.mainProgram} $out/ -'' + cp ${cronosd}/bin/${cronosd.meta.mainProgram} $out/ + '' diff --git a/nix/cosmovisor.nix b/nix/cosmovisor.nix index 16ca179e65..598cd908ad 100644 --- a/nix/cosmovisor.nix +++ b/nix/cosmovisor.nix @@ -1,6 +1,4 @@ -{ buildGoModule -, fetchFromGitHub -}: +{ buildGoModule, fetchFromGitHub }: let version = "1.5.0"; diff --git a/nix/cronos-matrix.nix b/nix/cronos-matrix.nix index e6016121bb..65f45e1261 100644 --- a/nix/cronos-matrix.nix +++ b/nix/cronos-matrix.nix @@ -1,25 +1,35 @@ -{ lib -, stdenv -, callPackage -, buildPackages -, runCommand -, bundle-exe -, rev ? "dirty" +{ + lib, + stdenv, + callPackage, + buildPackages, + runCommand, + bundle-exe, + rev ? "dirty", }: let # make-tarball don't follow symbolic links to avoid duplicate file, the bundle should have no external references. # reset the ownership and permissions to make the extract result more normal. - make-tarball = drv: runCommand "tarball-${drv.name}" - { - nativeBuildInputs = with buildPackages; [ gnutar gzip ]; - } '' - tar cfv - -C "${drv}" \ - --owner=0 --group=0 --mode=u+rw,uga+r --hard-dereference . \ - | gzip -9 > $out - ''; + make-tarball = + drv: + runCommand "tarball-${drv.name}" + { + nativeBuildInputs = with buildPackages; [ + gnutar + gzip + ]; + } + '' + tar cfv - -C "${drv}" \ + --owner=0 --group=0 --mode=u+rw,uga+r --hard-dereference . \ + | gzip -9 > $out + ''; bundle-win-exe = drv: callPackage ./bundle-win-exe.nix { cronosd = drv; }; matrix = lib.cartesianProductOfSets { - network = [ "mainnet" "testnet" ]; + network = [ + "mainnet" + "testnet" + ]; pkgtype = [ "nix" # normal nix package "bundle" # relocatable bundled package @@ -27,30 +37,26 @@ let ]; }; in -builtins.listToAttrs (builtins.map - ({ network, pkgtype }: { - name = builtins.concatStringsSep "-" ( - [ "cronosd" ] ++ - lib.optional (network != "mainnet") network ++ - lib.optional (pkgtype != "nix") pkgtype - ); - value = - let - cronosd = callPackage ../. { - inherit rev network; - }; - bundle = - if stdenv.hostPlatform.isWindows then - bundle-win-exe cronosd - else - bundle-exe cronosd; - in - if pkgtype == "bundle" then - bundle - else if pkgtype == "tarball" then - make-tarball bundle - else - cronosd; - }) - matrix +builtins.listToAttrs ( + builtins.map ( + { network, pkgtype }: + { + name = builtins.concatStringsSep "-" ( + [ "cronosd" ] + ++ lib.optional (network != "mainnet") network + ++ lib.optional (pkgtype != "nix") pkgtype + ); + value = + let + cronosd = callPackage ../. { inherit rev network; }; + bundle = if stdenv.hostPlatform.isWindows then bundle-win-exe cronosd else bundle-exe cronosd; + in + if pkgtype == "bundle" then + bundle + else if pkgtype == "tarball" then + make-tarball bundle + else + cronosd; + } + ) matrix ) diff --git a/nix/dapptools-overlay.nix b/nix/dapptools-overlay.nix index ace97cca95..1cb68e20a2 100644 --- a/nix/dapptools-overlay.nix +++ b/nix/dapptools-overlay.nix @@ -1,19 +1,16 @@ { dapptools-release, dapptools-master }: self: super: let - dapptools = { - x86_64-linux = - (import (dapptools-release + "/release.nix") { }).dapphub.linux.stable; - x86_64-darwin = - (import (dapptools-release + "/release.nix") { }).dapphub.darwin.stable; - }.${self.system} or (throw - "Unsupported system: ${self.system}"); + dapptools = + { + x86_64-linux = (import (dapptools-release + "/release.nix") { }).dapphub.linux.stable; + x86_64-darwin = (import (dapptools-release + "/release.nix") { }).dapphub.darwin.stable; + } + .${self.system} or (throw "Unsupported system: ${self.system}"); dapptools-patched = self.srcOnly { name = "dapptools-patched"; src = dapptools-master; - patches = [ - ./dapptools.patch - ]; + patches = [ ./dapptools.patch ]; }; in { diff --git a/nix/default.nix b/nix/default.nix index e6c1364dc5..3b417e2125 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,8 +1,15 @@ -{ sources ? import ./sources.nix, system ? builtins.currentSystem, ... }: +{ + sources ? import ./sources.nix, + system ? builtins.currentSystem, + ... +}: import sources.nixpkgs { overlays = [ (import ./build_overlay.nix) - (import ./dapptools-overlay.nix { dapptools-release = sources.dapptools; dapptools-master = sources.dapptools-master; }) + (import ./dapptools-overlay.nix { + dapptools-release = sources.dapptools; + dapptools-master = sources.dapptools-master; + }) (_: pkgs: { go = pkgs.go_1_22; go-ethereum = pkgs.callPackage ./go-ethereum.nix { @@ -15,15 +22,20 @@ import sources.nixpkgs { }) # update to a version that supports eip-1559 (import "${sources.poetry2nix}/overlay.nix") (import "${sources.gomod2nix}/overlay.nix") - (pkgs: _: + ( + pkgs: _: import ./scripts.nix { inherit pkgs; config = { cronos-config = ../scripts/cronos-devnet.yaml; geth-genesis = ../scripts/geth-genesis.json; - dotenv = builtins.path { name = "dotenv"; path = ../scripts/.env; }; + dotenv = builtins.path { + name = "dotenv"; + path = ../scripts/.env; + }; }; - }) + } + ) (_: pkgs: { gorc = pkgs.rustPlatform.buildRustPackage rec { name = "gorc"; @@ -31,21 +43,27 @@ import sources.nixpkgs { sourceRoot = "gravity-bridge-src/orchestrator"; cargoSha256 = "sha256-FQ43PFGbagIi+KZ6KUtjF7OClIkCqKd4pGzHaYr2Q+A="; cargoBuildFlags = "-p ${name} --features ethermint"; - buildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin - (with pkgs.darwin.apple_sdk.frameworks; [ CoreFoundation Security ]); + buildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin ( + with pkgs.darwin.apple_sdk.frameworks; + [ + CoreFoundation + Security + ] + ); doCheck = false; OPENSSL_NO_VENDOR = "1"; OPENSSL_DIR = pkgs.symlinkJoin { name = "openssl"; - paths = with pkgs.openssl; [ out dev ]; + paths = with pkgs.openssl; [ + out + dev + ]; }; }; hermes = pkgs.callPackage ./hermes.nix { src = sources.ibc-rs; }; }) (_: pkgs: { test-env = pkgs.callPackage ./testenv.nix { }; }) - (_: pkgs: { - cosmovisor = pkgs.callPackage ./cosmovisor.nix { }; - }) + (_: pkgs: { cosmovisor = pkgs.callPackage ./cosmovisor.nix { }; }) (_: pkgs: { rly = pkgs.buildGo120Module rec { name = "rly"; diff --git a/nix/gen-binding-shell.nix b/nix/gen-binding-shell.nix index 2ac5711a47..8409c332ff 100644 --- a/nix/gen-binding-shell.nix +++ b/nix/gen-binding-shell.nix @@ -1,4 +1,7 @@ -{ system ? builtins.currentSystem, pkgs ? import ./. { inherit system; } }: +{ + system ? builtins.currentSystem, + pkgs ? import ./. { inherit system; }, +}: let renameExe = pkgs.callPackage ./rename-exe.nix { }; in diff --git a/nix/go-ethereum.nix b/nix/go-ethereum.nix index b8651acaf1..7ef881495a 100644 --- a/nix/go-ethereum.nix +++ b/nix/go-ethereum.nix @@ -1,4 +1,11 @@ -{ lib, stdenv, buildGoModule, fetchFromGitHub, libobjc, IOKit }: +{ + lib, + stdenv, + buildGoModule, + fetchFromGitHub, + libobjc, + IOKit, +}: let # A list of binaries to put into separate outputs @@ -6,7 +13,6 @@ let "geth" "clef" ]; - in buildGoModule rec { pname = "go-ethereum"; @@ -27,7 +33,10 @@ buildGoModule rec { # Move binaries to separate outputs and symlink them back to $out postInstall = lib.concatStringsSep "\n" ( - builtins.map (bin: "mkdir -p \$${bin}/bin && mv $out/bin/${bin} \$${bin}/bin/ && ln -s \$${bin}/bin/${bin} $out/bin/") bins + builtins.map ( + bin: + "mkdir -p \$${bin}/bin && mv $out/bin/${bin} \$${bin}/bin/ && ln -s \$${bin}/bin/${bin} $out/bin/" + ) bins ); subPackages = [ @@ -47,13 +56,22 @@ buildGoModule rec { ]; # Fix for usb-related segmentation faults on darwin - propagatedBuildInputs = - lib.optionals stdenv.isDarwin [ libobjc IOKit ]; + propagatedBuildInputs = lib.optionals stdenv.isDarwin [ + libobjc + IOKit + ]; meta = with lib; { homepage = "https://geth.ethereum.org/"; description = "Official golang implementation of the Ethereum protocol"; - license = with licenses; [ lgpl3Plus gpl3Plus ]; - maintainers = with maintainers; [ adisbladis lionello RaghavSood ]; + license = with licenses; [ + lgpl3Plus + gpl3Plus + ]; + maintainers = with maintainers; [ + adisbladis + lionello + RaghavSood + ]; }; } diff --git a/nix/hermes.nix b/nix/hermes.nix index 241285515d..37804de5e6 100644 --- a/nix/hermes.nix +++ b/nix/hermes.nix @@ -1,11 +1,12 @@ -{ src -, lib -, stdenv -, darwin -, rustPlatform -, symlinkJoin -, openssl -, pkg-config +{ + src, + lib, + stdenv, + darwin, + rustPlatform, + symlinkJoin, + openssl, + pkg-config, }: rustPlatform.buildRustPackage rec { @@ -30,6 +31,9 @@ rustPlatform.buildRustPackage rec { OPENSSL_NO_VENDOR = "1"; OPENSSL_DIR = symlinkJoin { name = "openssl"; - paths = with openssl; [ out dev ]; + paths = with openssl; [ + out + dev + ]; }; } diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix index 6ddb033a98..66e37dd642 100644 --- a/nix/rocksdb.nix +++ b/nix/rocksdb.nix @@ -1,21 +1,22 @@ -{ lib -, stdenv -, fetchFromGitHub -, fetchpatch -, cmake -, ninja -, bzip2 -, lz4 -, snappy -, zlib -, zstd -, windows +{ + lib, + stdenv, + fetchFromGitHub, + fetchpatch, + cmake, + ninja, + bzip2, + lz4, + snappy, + zlib, + zstd, + windows, # only enable jemalloc for non-windows platforms # see: https://github.com/NixOS/nixpkgs/issues/216479 -, enableJemalloc ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isStatic -, jemalloc -, enableShared ? !stdenv.hostPlatform.isStatic -, sse42Support ? stdenv.hostPlatform.sse4_2Support + enableJemalloc ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isStatic, + jemalloc, + enableShared ? !stdenv.hostPlatform.isStatic, + sse42Support ? stdenv.hostPlatform.sse4_2Support, }: stdenv.mkDerivation (finalAttrs: { @@ -29,11 +30,21 @@ stdenv.mkDerivation (finalAttrs: { hash = "sha256-Zifn5Gu/4h6TaEqSaWQ2mFdryeAarqbHWW3fKUGGFac="; }; - nativeBuildInputs = [ cmake ninja ]; + nativeBuildInputs = [ + cmake + ninja + ]; - propagatedBuildInputs = [ bzip2 lz4 snappy zlib zstd ]; + propagatedBuildInputs = [ + bzip2 + lz4 + snappy + zlib + zstd + ]; - buildInputs = lib.optional enableJemalloc jemalloc + buildInputs = + lib.optional enableJemalloc jemalloc ++ lib.optional stdenv.hostPlatform.isMinGW windows.mingw_w64_pthreads; outputs = [ @@ -41,9 +52,7 @@ stdenv.mkDerivation (finalAttrs: { "tools" ]; - env.NIX_CFLAGS_COMPILE = toString (lib.optionals stdenv.cc.isClang [ - "-faligned-allocation" - ]); + env.NIX_CFLAGS_COMPILE = toString (lib.optionals stdenv.cc.isClang [ "-faligned-allocation" ]); cmakeFlags = [ "-DPORTABLE=1" @@ -68,46 +77,55 @@ stdenv.mkDerivation (finalAttrs: { # otherwise "cc1: error: -Wformat-security ignored without -Wformat [-Werror=format-security]" hardeningDisable = lib.optional stdenv.hostPlatform.isWindows "format"; - postPatch = lib.optionalString (lib.versionOlder finalAttrs.version "9") '' - # Fix gcc-13 build failures due to missing and - # includes, fixed upstream since 9.x - sed -e '1i #include ' -i options/offpeak_time_info.h - '' + lib.optionalString (lib.versionOlder finalAttrs.version "8") '' - # Fix gcc-13 build failures due to missing and - # includes, fixed upstream since 8.x - sed -e '1i #include ' -i db/compaction/compaction_iteration_stats.h - sed -e '1i #include ' -i table/block_based/data_block_hash_index.h - sed -e '1i #include ' -i util/string_util.h - sed -e '1i #include ' -i include/rocksdb/utilities/checkpoint.h - '' + lib.optionalString (lib.versionOlder finalAttrs.version "7") '' - # Fix gcc-13 build failures due to missing and - # includes, fixed upstream since 7.x - sed -e '1i #include ' -i third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h - '' + '' - # fixed in https://github.com/facebook/rocksdb/pull/12309 - sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i cmake/modules/Findzstd.cmake - sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i CMakeLists.txt - ''; + postPatch = + lib.optionalString (lib.versionOlder finalAttrs.version "9") '' + # Fix gcc-13 build failures due to missing and + # includes, fixed upstream since 9.x + sed -e '1i #include ' -i options/offpeak_time_info.h + '' + + lib.optionalString (lib.versionOlder finalAttrs.version "8") '' + # Fix gcc-13 build failures due to missing and + # includes, fixed upstream since 8.x + sed -e '1i #include ' -i db/compaction/compaction_iteration_stats.h + sed -e '1i #include ' -i table/block_based/data_block_hash_index.h + sed -e '1i #include ' -i util/string_util.h + sed -e '1i #include ' -i include/rocksdb/utilities/checkpoint.h + '' + + lib.optionalString (lib.versionOlder finalAttrs.version "7") '' + # Fix gcc-13 build failures due to missing and + # includes, fixed upstream since 7.x + sed -e '1i #include ' -i third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h + '' + + '' + # fixed in https://github.com/facebook/rocksdb/pull/12309 + sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i cmake/modules/Findzstd.cmake + sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i CMakeLists.txt + ''; - preInstall = '' - mkdir -p $tools/bin - cp tools/{ldb,sst_dump}${stdenv.hostPlatform.extensions.executable} $tools/bin/ - '' + lib.optionalString stdenv.isDarwin '' - ls -1 $tools/bin/* | xargs -I{} ${stdenv.cc.bintools.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib {} - '' + lib.optionalString (stdenv.isLinux && enableShared) '' - ls -1 $tools/bin/* | xargs -I{} patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib {} - ''; + preInstall = + '' + mkdir -p $tools/bin + cp tools/{ldb,sst_dump}${stdenv.hostPlatform.extensions.executable} $tools/bin/ + '' + + lib.optionalString stdenv.isDarwin '' + ls -1 $tools/bin/* | xargs -I{} ${stdenv.cc.bintools.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib {} + '' + + lib.optionalString (stdenv.isLinux && enableShared) '' + ls -1 $tools/bin/* | xargs -I{} patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib {} + ''; # Old version doesn't ship the .pc file, new version puts wrong paths in there. - postFixup = '' - if [ -f "$out"/lib/pkgconfig/rocksdb.pc ]; then - substituteInPlace "$out"/lib/pkgconfig/rocksdb.pc \ - --replace '="''${prefix}//' '="/' - fi - '' + lib.optionalString stdenv.isDarwin '' - ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/libsnappy.1.dylib" "${snappy}/lib/libsnappy.1.dylib" $out/lib/librocksdb.dylib - ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" "$out/lib/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib - ''; + postFixup = + '' + if [ -f "$out"/lib/pkgconfig/rocksdb.pc ]; then + substituteInPlace "$out"/lib/pkgconfig/rocksdb.pc \ + --replace '="''${prefix}//' '="/' + fi + '' + + lib.optionalString stdenv.isDarwin '' + ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/libsnappy.1.dylib" "${snappy}/lib/libsnappy.1.dylib" $out/lib/librocksdb.dylib + ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" "$out/lib/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib + ''; meta = with lib; { homepage = "https://rocksdb.org"; @@ -115,6 +133,9 @@ stdenv.mkDerivation (finalAttrs: { changelog = "https://github.com/facebook/rocksdb/raw/v${finalAttrs.version}/HISTORY.md"; license = licenses.asl20; platforms = platforms.all; - maintainers = with maintainers; [ adev magenbluten ]; + maintainers = with maintainers; [ + adev + magenbluten + ]; }; }) diff --git a/nix/scripts.nix b/nix/scripts.nix index 45aa631b3e..64a262acf8 100644 --- a/nix/scripts.nix +++ b/nix/scripts.nix @@ -1,7 +1,9 @@ -{ pkgs -, config -, cronos ? (import ../. { inherit pkgs; }) -}: rec { +{ + pkgs, + config, + cronos ? (import ../. { inherit pkgs; }), +}: +rec { start-cronos = pkgs.writeShellScriptBin "start-cronos" '' # rely on environment to provide cronosd export PATH=${pkgs.test-env}/bin:$PATH @@ -14,6 +16,9 @@ ''; start-scripts = pkgs.symlinkJoin { name = "start-scripts"; - paths = [ start-cronos start-geth ]; + paths = [ + start-cronos + start-geth + ]; }; } diff --git a/nix/sources.nix b/nix/sources.nix index 3f841046a8..cd40006431 100644 --- a/nix/sources.nix +++ b/nix/sources.nix @@ -6,113 +6,159 @@ let # The fetchers. fetch_ fetches specs of type . # - fetch_file = pkgs: name: spec: + fetch_file = + pkgs: name: spec: let name' = sanitizeName name + "-src"; in if spec.builtin or true then - builtins_fetchurl { inherit (spec) url sha256; name = name'; } + builtins_fetchurl { + inherit (spec) url sha256; + name = name'; + } else - pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + pkgs.fetchurl { + inherit (spec) url sha256; + name = name'; + }; - fetch_tarball = pkgs: name: spec: + fetch_tarball = + pkgs: name: spec: let name' = sanitizeName name + "-src"; in if spec.builtin or true then - builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + builtins_fetchTarball { + name = name'; + inherit (spec) url sha256; + } else - pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + pkgs.fetchzip { + name = name'; + inherit (spec) url sha256; + }; - fetch_git = name: spec: + fetch_git = + name: spec: let ref = - if spec ? ref then spec.ref else - if spec ? branch then "refs/heads/${spec.branch}" else - if spec ? tag then "refs/tags/${spec.tag}" else - abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + if spec ? ref then + spec.ref + else if spec ? branch then + "refs/heads/${spec.branch}" + else if spec ? tag then + "refs/tags/${spec.tag}" + else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; in - builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; + builtins.fetchGit { + url = spec.repo; + inherit (spec) rev; + inherit ref; + }; fetch_local = spec: spec.path; - fetch_builtin-tarball = name: throw - ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=tarball -a builtin=true''; + fetch_builtin-tarball = + name: + throw '' + [${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; - fetch_builtin-url = name: throw - ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=file -a builtin=true''; + fetch_builtin-url = + name: + throw '' + [${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; # # Various helpers # # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 - sanitizeName = name: - ( - concatMapStrings (s: if builtins.isList s then "-" else s) - ( - builtins.split "[^[:alnum:]+._?=-]+" - ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) - ) - ); + sanitizeName = + name: + (concatMapStrings (s: if builtins.isList s then "-" else s) ( + builtins.split "[^[:alnum:]+._?=-]+" ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + )); # The set of packages used when specs are fetched using non-builtins. - mkPkgs = sources: system: + mkPkgs = + sources: system: let - sourcesNixpkgs = - import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + sourcesNixpkgs = import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { + inherit system; + }; hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; hasThisAsNixpkgsPath = == ./.; in - if builtins.hasAttr "nixpkgs" sources - then sourcesNixpkgs - else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + if builtins.hasAttr "nixpkgs" sources then + sourcesNixpkgs + else if hasNixpkgsPath && !hasThisAsNixpkgsPath then import { } else - abort - '' - Please specify either (through -I or NIX_PATH=nixpkgs=...) or - add a package called "nixpkgs" to your sources.json. - ''; + abort '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; # The actual fetching function. - fetch = pkgs: name: spec: + fetch = + pkgs: name: spec: - if ! builtins.hasAttr "type" spec then + if !builtins.hasAttr "type" spec then abort "ERROR: niv spec ${name} does not have a 'type' attribute" - else if spec.type == "file" then fetch_file pkgs name spec - else if spec.type == "tarball" then fetch_tarball pkgs name spec - else if spec.type == "git" then fetch_git name spec - else if spec.type == "local" then fetch_local spec - else if spec.type == "builtin-tarball" then fetch_builtin-tarball name - else if spec.type == "builtin-url" then fetch_builtin-url name + else if spec.type == "file" then + fetch_file pkgs name spec + else if spec.type == "tarball" then + fetch_tarball pkgs name spec + else if spec.type == "git" then + fetch_git name spec + else if spec.type == "local" then + fetch_local spec + else if spec.type == "builtin-tarball" then + fetch_builtin-tarball name + else if spec.type == "builtin-url" then + fetch_builtin-url name else abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; # If the environment variable NIV_OVERRIDE_${name} is set, then use # the path directly as opposed to the fetched source. - replace = name: drv: + replace = + name: drv: let saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; in - if ersatz == "" then drv else - # this turns the string into an actual Nix path (for both absolute and - # relative paths) - if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + if ersatz == "" then + drv + else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then + /. + ersatz + else + /. + builtins.getEnv "PWD" + "/${ersatz}"; # Ports of functions for older nix versions # a Nix version of mapAttrs if the built-in doesn't exist - mapAttrs = builtins.mapAttrs or ( - f: set: with builtins; - listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) - ); + mapAttrs = + builtins.mapAttrs or ( + f: set: + with builtins; + listToAttrs ( + map (attr: { + name = attr; + value = f attr set.${attr}; + }) (attrNames set) + ) + ); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 - range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); + range = + first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); @@ -126,7 +172,12 @@ let optionalAttrs = cond: as: if cond then as else { }; # fetchTarball version that is compatible between all the versions of Nix - builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + builtins_fetchTarball = + { + url, + name ? null, + sha256, + }@attrs: let inherit (builtins) lessThan nixVersion fetchTarball; in @@ -136,7 +187,12 @@ let fetchTarball attrs; # fetchurl version that is compatible between all the versions of Nix - builtins_fetchurl = { url, name ? null, sha256 }@attrs: + builtins_fetchurl = + { + url, + name ? null, + sha256, + }@attrs: let inherit (builtins) lessThan nixVersion fetchurl; in @@ -146,32 +202,30 @@ let fetchurl attrs; # Create the final "sources" from the config - mkSources = config: - mapAttrs - ( - name: spec: - if builtins.hasAttr "outPath" spec - then - abort - "The values in sources.json should not have an 'outPath' attribute" - else - spec // { outPath = replace name (fetch config.pkgs name spec); } - ) - config.sources; + mkSources = + config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec then + abort "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) config.sources; # The "config" used by the fetchers mkConfig = - { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null - , sources ? if isNull sourcesFile then { } else builtins.fromJSON (builtins.readFile sourcesFile) - , system ? builtins.currentSystem - , pkgs ? mkPkgs sources system - }: rec { + { + sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null, + sources ? if isNull sourcesFile then { } else builtins.fromJSON (builtins.readFile sourcesFile), + system ? builtins.currentSystem, + pkgs ? mkPkgs sources system, + }: + rec { # The sources, i.e. the attribute set of spec name to spec inherit sources; # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers inherit pkgs; }; - in mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/nix/testenv.nix b/nix/testenv.nix index b73aa9ad3c..66952a071d 100644 --- a/nix/testenv.nix +++ b/nix/testenv.nix @@ -1,8 +1,13 @@ -{ poetry2nix, lib, python311 }: +{ + poetry2nix, + lib, + python311, +}: poetry2nix.mkPoetryEnv { projectDir = ../integration_tests; python = python311; - overrides = poetry2nix.overrides.withDefaults (self: super: + overrides = poetry2nix.overrides.withDefaults ( + self: super: let buildSystems = { pystarport = [ "poetry-core" ]; @@ -16,11 +21,11 @@ poetry2nix.mkPoetryEnv { eth-bloom = [ "setuptools" ]; }; in - lib.mapAttrs - (attr: systems: super.${attr}.overridePythonAttrs - (old: { - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; - })) - buildSystems + lib.mapAttrs ( + attr: systems: + super.${attr}.overridePythonAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; + }) + ) buildSystems ); } diff --git a/nix/testground-image.nix b/nix/testground-image.nix index 629ae3ae8c..86c345a442 100644 --- a/nix/testground-image.nix +++ b/nix/testground-image.nix @@ -1,9 +1,12 @@ -{ dockerTools, runCommandLocal, cronos-matrix, benchmark-testcase }: +{ + dockerTools, + runCommandLocal, + cronos-matrix, + benchmark-testcase, +}: let patched-cronosd = cronos-matrix.cronosd.overrideAttrs (oldAttrs: { - patches = oldAttrs.patches or [ ] ++ [ - ./testground-cronosd.patch - ]; + patches = oldAttrs.patches or [ ] ++ [ ./testground-cronosd.patch ]; }); in let @@ -20,10 +23,17 @@ dockerTools.buildLayeredImage { tmpDir ]; config = { - Expose = [ 9090 26657 26656 1317 26658 26660 26659 30000 ]; - Cmd = [ "/bin/stateless-testcase" ]; - Env = [ - "PYTHONUNBUFFERED=1" + Expose = [ + 9090 + 26657 + 26656 + 1317 + 26658 + 26660 + 26659 + 30000 ]; + Cmd = [ "/bin/stateless-testcase" ]; + Env = [ "PYTHONUNBUFFERED=1" ]; }; } diff --git a/testground/benchmark/flake.nix b/testground/benchmark/flake.nix index f29572553a..14097fcf80 100644 --- a/testground/benchmark/flake.nix +++ b/testground/benchmark/flake.nix @@ -9,34 +9,39 @@ }; }; - outputs = { self, nixpkgs, flake-utils, poetry2nix }: - flake-utils.lib.eachDefaultSystem - (system: - let - pkgs = import nixpkgs { - inherit system; - overlays = [ - poetry2nix.overlays.default - (import ./overlay.nix) - ]; - config = { }; + outputs = + { + self, + nixpkgs, + flake-utils, + poetry2nix, + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ + poetry2nix.overlays.default + (import ./overlay.nix) + ]; + config = { }; + }; + in + rec { + packages.default = pkgs.benchmark-testcase; + apps = { + default = { + type = "app"; + program = "${pkgs.benchmark-testcase}/bin/stateless-testcase"; }; - in - rec { - packages.default = pkgs.benchmark-testcase; - apps = { - default = { - type = "app"; - program = "${pkgs.benchmark-testcase}/bin/stateless-testcase"; - }; - stateless-testcase = { - type = "app"; - program = "${pkgs.benchmark-testcase}/bin/stateless-testcase"; - }; + stateless-testcase = { + type = "app"; + program = "${pkgs.benchmark-testcase}/bin/stateless-testcase"; }; - devShells.default = pkgs.mkShell { - buildInputs = [ pkgs.benchmark-testcase-env ]; - }; - legacyPackages = pkgs; - }); + }; + devShells.default = pkgs.mkShell { buildInputs = [ pkgs.benchmark-testcase-env ]; }; + legacyPackages = pkgs; + } + ); } diff --git a/testground/benchmark/overlay.nix b/testground/benchmark/overlay.nix index 280f3b46cb..29f77a8578 100644 --- a/testground/benchmark/overlay.nix +++ b/testground/benchmark/overlay.nix @@ -1,44 +1,64 @@ final: _: let - overrides = { lib, poetry2nix }: poetry2nix.overrides.withDefaults - (self: super: + overrides = + { lib, poetry2nix }: + poetry2nix.overrides.withDefaults ( + self: super: let buildSystems = { pystarport = [ "poetry-core" ]; durations = [ "setuptools" ]; multitail2 = [ "setuptools" ]; - docker = [ "hatchling" "hatch-vcs" ]; + docker = [ + "hatchling" + "hatch-vcs" + ]; pyunormalize = [ "setuptools" ]; pytest-github-actions-annotate-failures = [ "setuptools" ]; }; in - lib.mapAttrs - (attr: systems: super.${attr}.overridePythonAttrs - (old: { - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; - })) - buildSystems + lib.mapAttrs ( + attr: systems: + super.${attr}.overridePythonAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; + }) + ) buildSystems ); - src = nix-gitignore: nix-gitignore.gitignoreSourcePure [ - "/*" # ignore all, then add whitelists - "!/benchmark/" - "!poetry.lock" - "!pyproject.toml" - ] ./.; + src = + nix-gitignore: + nix-gitignore.gitignoreSourcePure [ + "/*" # ignore all, then add whitelists + "!/benchmark/" + "!poetry.lock" + "!pyproject.toml" + ] ./.; - benchmark = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryApplication { - projectDir = src nix-gitignore; - python = python311; - overrides = overrides { inherit lib poetry2nix; }; - }; - - benchmark-env = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryEnv { - projectDir = src nix-gitignore; - python = python311; - overrides = overrides { inherit lib poetry2nix; }; - }; + benchmark = + { + lib, + poetry2nix, + python311, + nix-gitignore, + }: + poetry2nix.mkPoetryApplication { + projectDir = src nix-gitignore; + python = python311; + overrides = overrides { inherit lib poetry2nix; }; + }; + benchmark-env = + { + lib, + poetry2nix, + python311, + nix-gitignore, + }: + poetry2nix.mkPoetryEnv { + projectDir = src nix-gitignore; + python = python311; + overrides = overrides { inherit lib poetry2nix; }; + }; in { benchmark-testcase = final.callPackage benchmark { }; From 19701820f7a0ac3857caa450b4d3b36e93f43f0b Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 11 Nov 2024 10:27:12 +0800 Subject: [PATCH 02/10] Problem: no command to fix corrupted data in versiondb (backport: #1685) Closes: #1683 Solution: - add fix command to fix corrupted data in versiondb rename support SkipVersionZero support SkipVersionZero cleanup cleanup cleanup fix test cleanup destroy log store name fix data manually cli Update versiondb/client/fixdata.go Signed-off-by: yihuang Update versiondb/client/fixdata.go Signed-off-by: yihuang rnemae Update CHANGELOG.md Signed-off-by: yihuang fix test don't return nil as empty slice add dryrun mode separete read from iteration add stores flag debug add timestamp api --- CHANGELOG.md | 1 + app/versiondb.go | 4 + versiondb/client/cmd.go | 1 + versiondb/client/fixdata.go | 59 ++++++++++++++ versiondb/tsrocksdb/iterator.go | 43 +++++++--- versiondb/tsrocksdb/opts.go | 14 ++++ versiondb/tsrocksdb/store.go | 131 ++++++++++++++++++++++++++---- versiondb/tsrocksdb/store_test.go | 87 ++++++++++++++++++++ versiondb/types.go | 10 ++- 9 files changed, 324 insertions(+), 26 deletions(-) create mode 100644 versiondb/client/fixdata.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d36d777951..d030066718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * [#1670](https://github.com/crypto-org-chain/cronos/pull/1670) Fix state overwrite in debug trace APIs. * [#1679](https://github.com/crypto-org-chain/cronos/pull/1679) Include no trace detail on insufficient balance fix. +* [#1685](https://github.com/crypto-org-chain/cronos/pull/1685) Add command to fix versiondb corrupted data. *Oct 14, 2024* diff --git a/app/versiondb.go b/app/versiondb.go index 10b0ad76ad..381db1027b 100644 --- a/app/versiondb.go +++ b/app/versiondb.go @@ -23,6 +23,7 @@ func (app *App) setupVersionDB( if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { return nil, err } + versionDB, err := tsrocksdb.NewStore(dataDir) if err != nil { return nil, err @@ -34,6 +35,9 @@ func (app *App) setupVersionDB( exposeStoreKeys = append(exposeStoreKeys, storeKey) } + // see: https://github.com/crypto-org-chain/cronos/issues/1683 + versionDB.SetSkipVersionZero(true) + service := versiondb.NewStreamingService(versionDB, exposeStoreKeys) app.SetStreamingService(service) diff --git a/versiondb/client/cmd.go b/versiondb/client/cmd.go index 4bf36343e8..5955037f71 100644 --- a/versiondb/client/cmd.go +++ b/versiondb/client/cmd.go @@ -28,6 +28,7 @@ func ChangeSetGroupCmd(opts Options) *cobra.Command { ChangeSetToVersionDBCmd(), RestoreAppDBCmd(opts), RestoreVersionDBCmd(), + FixDataCmd(opts.DefaultStores), ) return cmd } diff --git a/versiondb/client/fixdata.go b/versiondb/client/fixdata.go new file mode 100644 index 0000000000..e0b438988e --- /dev/null +++ b/versiondb/client/fixdata.go @@ -0,0 +1,59 @@ +package client + +import ( + "github.com/crypto-org-chain/cronos/versiondb/tsrocksdb" + "github.com/linxGnu/grocksdb" + "github.com/spf13/cobra" +) + +const ( + FlagDryRun = "dry-run" + FlagStore = "store" +) + +func FixDataCmd(defaultStores []string) *cobra.Command { + cmd := &cobra.Command{ + Use: "fixdata ", + Args: cobra.ExactArgs(1), + Short: "Fix wrong data in versiondb, see: https://github.com/crypto-org-chain/cronos/issues/1683", + RunE: func(cmd *cobra.Command, args []string) error { + dir := args[0] + dryRun, err := cmd.Flags().GetBool(FlagDryRun) + if err != nil { + return err + } + stores, err := cmd.Flags().GetStringArray(FlagStore) + if err != nil { + return err + } + if len(stores) == 0 { + stores = defaultStores + } + + var ( + db *grocksdb.DB + cfHandle *grocksdb.ColumnFamilyHandle + ) + + if dryRun { + db, cfHandle, err = tsrocksdb.OpenVersionDBForReadOnly(dir, false) + } else { + db, cfHandle, err = tsrocksdb.OpenVersionDB(dir) + } + if err != nil { + return err + } + + versionDB := tsrocksdb.NewStoreWithDB(db, cfHandle) + if err := versionDB.FixData(stores, dryRun); err != nil { + return err + } + + return nil + }, + } + + cmd.Flags().Bool(FlagDryRun, false, "Dry run, do not write to the database, open the database in read-only mode.") + cmd.Flags().StringArray(FlagStore, nil, "Store names to fix, if not specified, all stores will be fixed.") + return cmd +} diff --git a/versiondb/tsrocksdb/iterator.go b/versiondb/tsrocksdb/iterator.go index f4c4b3c777..9cb15e8a62 100644 --- a/versiondb/tsrocksdb/iterator.go +++ b/versiondb/tsrocksdb/iterator.go @@ -2,8 +2,9 @@ package tsrocksdb import ( "bytes" + "encoding/binary" - "github.com/cosmos/cosmos-sdk/store/types" + "github.com/crypto-org-chain/cronos/versiondb" "github.com/linxGnu/grocksdb" ) @@ -12,11 +13,14 @@ type rocksDBIterator struct { prefix, start, end []byte isReverse bool isInvalid bool + + // see: https://github.com/crypto-org-chain/cronos/issues/1683 + skipVersionZero bool } -var _ types.Iterator = (*rocksDBIterator)(nil) +var _ versiondb.Iterator = (*rocksDBIterator)(nil) -func newRocksDBIterator(source *grocksdb.Iterator, prefix, start, end []byte, isReverse bool) *rocksDBIterator { +func newRocksDBIterator(source *grocksdb.Iterator, prefix, start, end []byte, isReverse bool, skipVersionZero bool) *rocksDBIterator { if isReverse { if end == nil { source.SeekToLast() @@ -39,14 +43,18 @@ func newRocksDBIterator(source *grocksdb.Iterator, prefix, start, end []byte, is source.Seek(start) } } - return &rocksDBIterator{ - source: source, - prefix: prefix, - start: start, - end: end, - isReverse: isReverse, - isInvalid: false, + it := &rocksDBIterator{ + source: source, + prefix: prefix, + start: start, + end: end, + isReverse: isReverse, + isInvalid: false, + skipVersionZero: skipVersionZero, } + + it.trySkipZeroVersion() + return it } // Domain implements Iterator. @@ -114,6 +122,21 @@ func (itr rocksDBIterator) Next() { } else { itr.source.Next() } + + itr.trySkipZeroVersion() +} + +func (itr rocksDBIterator) Timestamp() []byte { + itr.assertIsValid() + return moveSliceToBytes(itr.source.Timestamp()) +} + +func (itr rocksDBIterator) trySkipZeroVersion() { + if itr.skipVersionZero { + for itr.Valid() && binary.LittleEndian.Uint64(itr.Timestamp()) == 0 { + itr.Next() + } + } } // Error implements Iterator. diff --git a/versiondb/tsrocksdb/opts.go b/versiondb/tsrocksdb/opts.go index 604c203b72..8218cd9692 100644 --- a/versiondb/tsrocksdb/opts.go +++ b/versiondb/tsrocksdb/opts.go @@ -64,6 +64,20 @@ func OpenVersionDB(dir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, erro return db, cfHandles[1], nil } +// OpenVersionDBForReadOnly open versiondb in readonly mode +func OpenVersionDBForReadOnly(dir string, errorIfWalFileExists bool) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) { + opts := grocksdb.NewDefaultOptions() + db, cfHandles, err := grocksdb.OpenDbForReadOnlyColumnFamilies( + opts, dir, []string{"default", VersionDBCFName}, + []*grocksdb.Options{opts, NewVersionDBOpts(false)}, + errorIfWalFileExists, + ) + if err != nil { + return nil, nil, err + } + return db, cfHandles[1], nil +} + // OpenVersionDBAndTrimHistory opens versiondb similar to `OpenVersionDB`, // but it also trim the versions newer than target one, can be used for rollback. func OpenVersionDBAndTrimHistory(dir string, version int64) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) { diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go index 7d4176c5d0..86a3f33e4f 100644 --- a/versiondb/tsrocksdb/store.go +++ b/versiondb/tsrocksdb/store.go @@ -1,6 +1,7 @@ package tsrocksdb import ( + "bytes" "encoding/binary" "errors" "fmt" @@ -38,6 +39,9 @@ func init() { type Store struct { db *grocksdb.DB cfHandle *grocksdb.ColumnFamilyHandle + + // see: https://github.com/crypto-org-chain/cronos/issues/1683 + skipVersionZero bool } func NewStore(dir string) (Store, error) { @@ -58,6 +62,10 @@ func NewStoreWithDB(db *grocksdb.DB, cfHandle *grocksdb.ColumnFamilyHandle) Stor } } +func (s *Store) SetSkipVersionZero(skip bool) { + s.skipVersionZero = skip +} + func (s Store) SetLatestVersion(version int64) error { var ts [TimestampSize]byte binary.LittleEndian.PutUint64(ts[:], uint64(version)) @@ -86,11 +94,23 @@ func (s Store) PutAtVersion(version int64, changeSet []types.StoreKVPair) error } func (s Store) GetAtVersionSlice(storeKey string, key []byte, version *int64) (*grocksdb.Slice, error) { - return s.db.GetCF( + value, ts, err := s.db.GetCFWithTS( newTSReadOptions(version), s.cfHandle, prependStoreKey(storeKey, key), ) + if err != nil { + return nil, err + } + defer ts.Free() + + if value.Exists() && s.skipVersionZero { + if binary.LittleEndian.Uint64(ts.Data()) == 0 { + return grocksdb.NewSlice(nil, 0), nil + } + } + + return value, err } // GetAtVersion implements VersionStore interface @@ -127,29 +147,25 @@ func (s Store) GetLatestVersion() (int64, error) { } // IteratorAtVersion implements VersionStore interface -func (s Store) IteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, errKeyEmpty - } - - prefix := storePrefix(storeKey) - start, end = iterateWithPrefix(prefix, start, end) - - itr := s.db.NewIteratorCF(newTSReadOptions(version), s.cfHandle) - return newRocksDBIterator(itr, prefix, start, end, false), nil +func (s Store) IteratorAtVersion(storeKey string, start, end []byte, version *int64) (versiondb.Iterator, error) { + return s.iteratorAtVersion(storeKey, start, end, version, false) } // ReverseIteratorAtVersion implements VersionStore interface -func (s Store) ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) { +func (s Store) ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (versiondb.Iterator, error) { + return s.iteratorAtVersion(storeKey, start, end, version, true) +} + +func (s Store) iteratorAtVersion(storeKey string, start, end []byte, version *int64, reverse bool) (versiondb.Iterator, error) { if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { return nil, errKeyEmpty } prefix := storePrefix(storeKey) - start, end = iterateWithPrefix(storePrefix(storeKey), start, end) + start, end = iterateWithPrefix(prefix, start, end) itr := s.db.NewIteratorCF(newTSReadOptions(version), s.cfHandle) - return newRocksDBIterator(itr, prefix, start, end, true), nil + return newRocksDBIterator(itr, prefix, start, end, reverse, s.skipVersionZero), nil } // FeedChangeSet is used to migrate legacy change sets into versiondb @@ -216,6 +232,93 @@ func (s Store) Flush() error { ) } +// FixData fixes wrong data written in versiondb due to rocksdb upgrade, the operation is idempotent. +// see: https://github.com/crypto-org-chain/cronos/issues/1683 +// call this before `SetSkipVersionZero(true)`. +func (s Store) FixData(storeNames []string, dryRun bool) error { + for _, storeName := range storeNames { + if err := s.fixDataStore(storeName, dryRun); err != nil { + return err + } + } + + return nil +} + +// fixDataStore iterate the wrong data at version 0, parse the timestamp from the key and write it again. +func (s Store) fixDataStore(storeName string, dryRun bool) error { + pairs, err := s.loadWrongData(storeName) + if err != nil { + return err + } + + batch := grocksdb.NewWriteBatch() + defer batch.Destroy() + + prefix := storePrefix(storeName) + readOpts := grocksdb.NewDefaultReadOptions() + defer readOpts.Destroy() + for _, pair := range pairs { + realKey := cloneAppend(prefix, pair.Key) + + readOpts.SetTimestamp(pair.Timestamp) + oldValue, err := s.db.GetCF(readOpts, s.cfHandle, realKey) + if err != nil { + return err + } + + clean := bytes.Equal(oldValue.Data(), pair.Value) + oldValue.Free() + + if clean { + continue + } + + if dryRun { + fmt.Printf("fix data: %s, key: %X, ts: %X\n", storeName, pair.Key, pair.Timestamp) + } else { + batch.PutCFWithTS(s.cfHandle, realKey, pair.Timestamp, pair.Value) + } + } + + if !dryRun { + return s.db.Write(defaultSyncWriteOpts, batch) + } + + return nil +} + +type KVPairWithTS struct { + Key []byte + Value []byte + Timestamp []byte +} + +func (s Store) loadWrongData(storeName string) ([]KVPairWithTS, error) { + var version int64 + iter, err := s.IteratorAtVersion(storeName, nil, nil, &version) + if err != nil { + return nil, err + } + defer iter.Close() + + var pairs []KVPairWithTS + for ; iter.Valid(); iter.Next() { + key := iter.Key() + if len(key) < TimestampSize { + return nil, fmt.Errorf("invalid key length: %X, store: %s", key, storeName) + } + + pairs = append(pairs, KVPairWithTS{ + Key: key[:len(key)-TimestampSize], + Timestamp: key[len(key)-TimestampSize:], + Value: iter.Value(), + }) + } + + return pairs, nil +} + func newTSReadOptions(version *int64) *grocksdb.ReadOptions { var ver uint64 if version == nil { diff --git a/versiondb/tsrocksdb/store_test.go b/versiondb/tsrocksdb/store_test.go index a5328977a6..f3ec3617fb 100644 --- a/versiondb/tsrocksdb/store_test.go +++ b/versiondb/tsrocksdb/store_test.go @@ -4,6 +4,8 @@ import ( "encoding/binary" "testing" + dbm "github.com/cometbft/cometbft-db" + "github.com/cosmos/cosmos-sdk/store/types" "github.com/crypto-org-chain/cronos/versiondb" "github.com/linxGnu/grocksdb" "github.com/stretchr/testify/require" @@ -153,3 +155,88 @@ func TestUserTimestampPruning(t *testing.T) { require.Equal(t, []byte{100}, bz.Data()) bz.Free() } + +func TestSkipVersionZero(t *testing.T) { + storeKey := "test" + + var wrongTz [8]byte + binary.LittleEndian.PutUint64(wrongTz[:], 100) + + key1 := []byte("hello1") + key2 := []byte("hello2") + key2Wrong := cloneAppend(key2, wrongTz[:]) + key3 := []byte("hello3") + + store, err := NewStore(t.TempDir()) + require.NoError(t, err) + + err = store.PutAtVersion(0, []types.StoreKVPair{ + {StoreKey: storeKey, Key: key2Wrong, Value: []byte{2}}, + }) + require.NoError(t, err) + err = store.PutAtVersion(100, []types.StoreKVPair{ + {StoreKey: storeKey, Key: key1, Value: []byte{1}}, + }) + require.NoError(t, err) + err = store.PutAtVersion(100, []types.StoreKVPair{ + {StoreKey: storeKey, Key: key3, Value: []byte{3}}, + }) + require.NoError(t, err) + + i := int64(999) + bz, err := store.GetAtVersion(storeKey, key2Wrong, &i) + require.NoError(t, err) + require.Equal(t, []byte{2}, bz) + + it, err := store.IteratorAtVersion(storeKey, nil, nil, &i) + require.NoError(t, err) + require.Equal(t, + []kvPair{ + {Key: key1, Value: []byte{1}}, + {Key: key2Wrong, Value: []byte{2}}, + {Key: key3, Value: []byte{3}}, + }, + consumeIterator(it), + ) + + store.SetSkipVersionZero(true) + + bz, err = store.GetAtVersion(storeKey, key2Wrong, &i) + require.NoError(t, err) + require.Empty(t, bz) + bz, err = store.GetAtVersion(storeKey, key1, &i) + require.NoError(t, err) + require.Equal(t, []byte{1}, bz) + + it, err = store.IteratorAtVersion(storeKey, nil, nil, &i) + require.NoError(t, err) + require.Equal(t, + []kvPair{ + {Key: key1, Value: []byte{1}}, + {Key: key3, Value: []byte{3}}, + }, + consumeIterator(it), + ) + + store.SetSkipVersionZero(false) + err = store.FixData([]string{storeKey}, false) + require.NoError(t, err) + + bz, err = store.GetAtVersion(storeKey, key2, &i) + require.NoError(t, err) + require.Equal(t, []byte{2}, bz) +} + +type kvPair struct { + Key []byte + Value []byte +} + +func consumeIterator(it dbm.Iterator) []kvPair { + var result []kvPair + for ; it.Valid(); it.Next() { + result = append(result, kvPair{it.Key(), it.Value()}) + } + it.Close() + return result +} diff --git a/versiondb/types.go b/versiondb/types.go index 6c69fbbf94..9f5eab3637 100644 --- a/versiondb/types.go +++ b/versiondb/types.go @@ -4,14 +4,20 @@ import ( "github.com/cosmos/cosmos-sdk/store/types" ) +type Iterator interface { + types.Iterator + + Timestamp() []byte +} + // VersionStore is a versioned storage of a flat key-value pairs. // it don't need to support merkle proof, so could be implemented in a much more efficient way. // `nil` version means the latest version. type VersionStore interface { GetAtVersion(storeKey string, key []byte, version *int64) ([]byte, error) HasAtVersion(storeKey string, key []byte, version *int64) (bool, error) - IteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) - ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) + IteratorAtVersion(storeKey string, start, end []byte, version *int64) (Iterator, error) + ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (Iterator, error) GetLatestVersion() (int64, error) // Persist the change set of a block, From ec7e78e43d4c9bd6a4d2cb5126ffb0b91127ed53 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 13:30:44 +0800 Subject: [PATCH 03/10] update rocksdb --- go.mod | 2 +- go.sum | 4 ++-- gomod2nix.toml | 4 ++-- nix/rocksdb.nix | 4 ++-- versiondb/go.mod | 2 +- versiondb/go.sum | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 386ce5014e..cc60d20c95 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 - github.com/linxGnu/grocksdb v1.9.2 + github.com/linxGnu/grocksdb v1.9.7 github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index 2231f62066..d48b77ff32 100644 --- a/go.sum +++ b/go.sum @@ -1085,8 +1085,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.9.2 h1:O3mzvO0wuzQ9mtlHbDrShixyVjVbmuqTjFrzlf43wZ8= -github.com/linxGnu/grocksdb v1.9.2/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= +github.com/linxGnu/grocksdb v1.9.7 h1:Bp2r1Yti/IXxEobZZnDooXAui/Q+5gVqgQMenLWyDUw= +github.com/linxGnu/grocksdb v1.9.7/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= diff --git a/gomod2nix.toml b/gomod2nix.toml index 345984dccd..1cff51efb6 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -403,8 +403,8 @@ schema = 3 version = "v0.1.0" hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ=" [mod."github.com/linxGnu/grocksdb"] - version = "v1.9.2" - hash = "sha256-ThXtaXx6LvRIFW4xLHsMrVWdsN2qobLPA0InLmlADOM=" + version = "v1.9.7" + hash = "sha256-ZSomnYZRo7gHB9/FW55MebNkNzn0DuR96RfsVpAwjIQ=" [mod."github.com/magiconair/properties"] version = "v1.8.7" hash = "sha256-XQ2bnc2s7/IH3WxEO4GishZurMyKwEclZy1DXg+2xXc=" diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix index 66e37dd642..32e1e0720e 100644 --- a/nix/rocksdb.nix +++ b/nix/rocksdb.nix @@ -21,13 +21,13 @@ stdenv.mkDerivation (finalAttrs: { pname = "rocksdb"; - version = "9.2.1"; + version = "9.7.4"; src = fetchFromGitHub { owner = "facebook"; repo = finalAttrs.pname; rev = "v${finalAttrs.version}"; - hash = "sha256-Zifn5Gu/4h6TaEqSaWQ2mFdryeAarqbHWW3fKUGGFac="; + hash = "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM="; }; nativeBuildInputs = [ diff --git a/versiondb/go.mod b/versiondb/go.mod index 9b1f0454f6..9fb00f47f5 100644 --- a/versiondb/go.mod +++ b/versiondb/go.mod @@ -14,7 +14,7 @@ require ( github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583 github.com/crypto-org-chain/cronos/memiavl v0.0.3 github.com/golang/snappy v0.0.4 - github.com/linxGnu/grocksdb v1.9.2 + github.com/linxGnu/grocksdb v1.9.7 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.9.0 diff --git a/versiondb/go.sum b/versiondb/go.sum index 5f08098e34..cb73e9e5a9 100644 --- a/versiondb/go.sum +++ b/versiondb/go.sum @@ -515,8 +515,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.9.2 h1:O3mzvO0wuzQ9mtlHbDrShixyVjVbmuqTjFrzlf43wZ8= -github.com/linxGnu/grocksdb v1.9.2/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= +github.com/linxGnu/grocksdb v1.9.7 h1:Bp2r1Yti/IXxEobZZnDooXAui/Q+5gVqgQMenLWyDUw= +github.com/linxGnu/grocksdb v1.9.7/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= From 2ea817ebc81363f91242307fea69525eac68d67e Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 14:10:51 +0800 Subject: [PATCH 04/10] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d030066718..71400f7b5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [#1670](https://github.com/crypto-org-chain/cronos/pull/1670) Fix state overwrite in debug trace APIs. * [#1679](https://github.com/crypto-org-chain/cronos/pull/1679) Include no trace detail on insufficient balance fix. * [#1685](https://github.com/crypto-org-chain/cronos/pull/1685) Add command to fix versiondb corrupted data. +* [#1688](https://github.com/crypto-org-chain/cronos/pull/1688) Add timestamp api to versiondb iterator. *Oct 14, 2024* From d831cf162b7154c59c8ce34278a7b33d96403dab Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 14:20:47 +0800 Subject: [PATCH 05/10] fix flag name conflict --- versiondb/client/fixdata.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versiondb/client/fixdata.go b/versiondb/client/fixdata.go index e0b438988e..efd95fbff4 100644 --- a/versiondb/client/fixdata.go +++ b/versiondb/client/fixdata.go @@ -8,7 +8,7 @@ import ( const ( FlagDryRun = "dry-run" - FlagStore = "store" + FlagStore = "store-name" ) func FixDataCmd(defaultStores []string) *cobra.Command { @@ -54,6 +54,6 @@ func FixDataCmd(defaultStores []string) *cobra.Command { } cmd.Flags().Bool(FlagDryRun, false, "Dry run, do not write to the database, open the database in read-only mode.") - cmd.Flags().StringArray(FlagStore, nil, "Store names to fix, if not specified, all stores will be fixed.") + cmd.Flags().StringArray(FlagStore, []string{}, "Store names to fix, if not specified, all stores will be fixed.") return cmd } From f7ae9ad7861c6fb3442b9daebb10f0ac73b23608 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 14:33:38 +0800 Subject: [PATCH 06/10] validate timestamp --- versiondb/tsrocksdb/store.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go index 86a3f33e4f..109a452f4e 100644 --- a/versiondb/tsrocksdb/store.go +++ b/versiondb/tsrocksdb/store.go @@ -304,7 +304,12 @@ func (s Store) loadWrongData(storeName string) ([]KVPairWithTS, error) { var pairs []KVPairWithTS for ; iter.Valid(); iter.Next() { + ts := iter.Timestamp() key := iter.Key() + if binary.LittleEndian.Uint64(ts) != 0 { + return nil, fmt.Errorf("invalid timestamp: %X, store: %s, key: %X", ts, storeName, key) + } + if len(key) < TimestampSize { return nil, fmt.Errorf("invalid key length: %X, store: %s", key, storeName) } From 0fa82128dc7b727f5c58a5242831aa17a21731d3 Mon Sep 17 00:00:00 2001 From: yihuang Date: Tue, 12 Nov 2024 14:41:18 +0800 Subject: [PATCH 07/10] Update CHANGELOG.md Signed-off-by: yihuang --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71400f7b5f..0b5d406386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * [#1679](https://github.com/crypto-org-chain/cronos/pull/1679) Include no trace detail on insufficient balance fix. * [#1685](https://github.com/crypto-org-chain/cronos/pull/1685) Add command to fix versiondb corrupted data. * [#1688](https://github.com/crypto-org-chain/cronos/pull/1688) Add timestamp api to versiondb iterator. +* [#1686](https://github.com/crypto-org-chain/cronos/pull/1686) Update rocksdb to 9.7.4. *Oct 14, 2024* From 4dd16459bd1c54861a7d4a919c9ce8355083f8a4 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 14:47:15 +0800 Subject: [PATCH 08/10] skip non-zero version --- versiondb/tsrocksdb/store.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go index 109a452f4e..9148928681 100644 --- a/versiondb/tsrocksdb/store.go +++ b/versiondb/tsrocksdb/store.go @@ -305,11 +305,12 @@ func (s Store) loadWrongData(storeName string) ([]KVPairWithTS, error) { var pairs []KVPairWithTS for ; iter.Valid(); iter.Next() { ts := iter.Timestamp() - key := iter.Key() if binary.LittleEndian.Uint64(ts) != 0 { - return nil, fmt.Errorf("invalid timestamp: %X, store: %s, key: %X", ts, storeName, key) + // FIXME: https://github.com/crypto-org-chain/cronos/issues/1689 + continue } + key := iter.Key() if len(key) < TimestampSize { return nil, fmt.Errorf("invalid key length: %X, store: %s", key, storeName) } From 74813d55f501afbd22850ec438a2fbbf454543de Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 16:44:46 +0800 Subject: [PATCH 09/10] flush after fix --- versiondb/tsrocksdb/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go index 9148928681..ef7ea45ca0 100644 --- a/versiondb/tsrocksdb/store.go +++ b/versiondb/tsrocksdb/store.go @@ -242,7 +242,7 @@ func (s Store) FixData(storeNames []string, dryRun bool) error { } } - return nil + return s.Flush() } // fixDataStore iterate the wrong data at version 0, parse the timestamp from the key and write it again. From d406df9ec4eb18c70797274b6c10f2724302fba6 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 12 Nov 2024 17:46:19 +0800 Subject: [PATCH 10/10] only flush if not dry-run --- versiondb/tsrocksdb/store.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go index ef7ea45ca0..706c357b98 100644 --- a/versiondb/tsrocksdb/store.go +++ b/versiondb/tsrocksdb/store.go @@ -242,7 +242,11 @@ func (s Store) FixData(storeNames []string, dryRun bool) error { } } - return s.Flush() + if !dryRun { + return s.Flush() + } + + return nil } // fixDataStore iterate the wrong data at version 0, parse the timestamp from the key and write it again.