State expiry mvp0.1: fix some bugs and opt performance;

cmd/geth/chaincmd.go

@@ -56,8 +56,7 @@ var (
 		Flags: flags.Merge([]cli.Flag{
 			utils.CachePreimagesFlag,
 			utils.StateSchemeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
@@ -87,8 +86,7 @@ It expects the genesis file as argument.`,
 		Name:      "dumpgenesis",
 		Usage:     "Dumps genesis block JSON configuration to stdout",
 		ArgsUsage: "",
-		Flags: append([]cli.Flag{utils.DataDirFlag,
-			utils.StateExpiryEnableFlag}, utils.NetworkFlags...),
+		Flags:     flags.Merge([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The dumpgenesis command prints the genesis configuration of the network preset
 if one is set.  Otherwise it prints the genesis from the datadir.`,
@@ -123,8 +121,7 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.TransactionHistoryFlag,
 			utils.StateSchemeFlag,
 			utils.StateHistoryFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The import command imports blocks from an RLP-encoded form. The form can be one file
 with several RLP-encoded blocks, or several files can be used.
@@ -141,8 +138,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 			utils.StateSchemeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
@@ -158,8 +154,7 @@ be gzipped.`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The import-preimages command imports hash preimages from an RLP encoded stream.
 It's deprecated, please use "geth db import" instead.
@@ -173,8 +168,7 @@ It's deprecated, please use "geth db import" instead.
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The export-preimages command exports hash preimages to an RLP encoded stream.
 It's deprecated, please use "geth db export" instead.
@@ -194,8 +188,7 @@ It's deprecated, please use "geth db export" instead.
 			utils.StartKeyFlag,
 			utils.DumpLimitFlag,
 			utils.StateSchemeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 This command dumps out the state for a given block (or latest, if none provided).
 `,

cmd/geth/config.go

@@ -49,7 +49,7 @@ var (
 		Name:        "dumpconfig",
 		Usage:       "Export configuration values in a TOML format",
 		ArgsUsage:   "<dumpfile (optional)>",
-		Flags:       flags.Merge(nodeFlags, rpcFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+		Flags:       flags.Merge(nodeFlags, rpcFlags, utils.StateExpiryBaseFlags),
 		Description: `Export configuration values in TOML format (to stdout by default).`,
 	}
 

cmd/geth/consolecmd.go

@@ -33,7 +33,7 @@ var (
 		Action: localConsole,
 		Name:   "console",
 		Usage:  "Start an interactive JavaScript environment",
-		Flags:  flags.Merge(nodeFlags, rpcFlags, consoleFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+		Flags:  flags.Merge(nodeFlags, rpcFlags, consoleFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.

cmd/geth/dbcmd.go

@@ -50,7 +50,7 @@ var (
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: "",
-		Flags:     flags.Merge(utils.DatabasePathFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+		Flags:     flags.Merge(utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 Remove blockchain and state databases`,
 	}
@@ -86,8 +86,7 @@ Remove blockchain and state databases`,
 		ArgsUsage: "<prefix> <start>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
@@ -97,16 +96,15 @@ Remove blockchain and state databases`,
 		ArgsUsage: "<blocknum> <jobnum>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Usage:       "Inspect the MPT tree of the account and contract.",
 		Description: `This commands iterates the entrie WorldState.`,
 	}
 	dbCheckStateContentCmd = &cli.Command{
 		Action:    checkStateContent,
 		Name:      "check-state-content",
 		ArgsUsage: "<start (optional)>",
-		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Usage:     "Verify that state data is cryptographically correct",
 		Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes.
 For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates
@@ -157,8 +155,7 @@ a data corruption.`,
 		Usage:  "Print leveldb statistics",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 	}
 	dbCompactCmd = &cli.Command{
 		Action: dbCompact,
@@ -168,8 +165,7 @@ a data corruption.`,
 			utils.SyncModeFlag,
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `This command performs a database compaction. 
 WARNING: This operation may take a very long time to finish, and may cause database
 corruption if it is aborted during execution'!`,
@@ -181,8 +177,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = &cli.Command{
@@ -192,8 +187,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `This command deletes the specified database key from the database. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -204,8 +198,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `This command sets a given database key to the given value. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -217,8 +210,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
 			utils.StateSchemeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDumpFreezerIndex = &cli.Command{
@@ -228,8 +220,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "This command displays information about the freezer index.",
 	}
 	dbImportCmd = &cli.Command{
@@ -239,8 +230,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<dumpfile> <start (optional)",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
 	dbExportCmd = &cli.Command{
@@ -250,8 +240,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<type> <dumpfile>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
 	dbMetadataCmd = &cli.Command{
@@ -260,17 +249,16 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Usage:  "Shows metadata about the chain status.",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateExpiryEnableFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
 	ancientInspectCmd = &cli.Command{
 		Action: ancientInspect,
 		Name:   "inspect-reserved-oldest-blocks",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-			utils.StateExpiryEnableFlag,
-		},
+		Flags: flags.Merge(
+			[]cli.Flag{utils.DataDirFlag},
+			utils.StateExpiryBaseFlags,
+		),
 		Usage: "Inspect the ancientStore information",
 		Description: `This commands will read current offset from kvdb, which is the current offset and starting BlockNumber
 of ancientStore, will also displays the reserved number of blocks in ancientStore `,
@@ -441,6 +429,9 @@ func inspectTrie(ctx *cli.Context) error {
 			return err
 		}
 		theInspect, err := trie.NewInspector(trieDB, theTrie, blockNumber, jobnum)
+		if err != nil {
+			return err
+		}
 		theInspect.Run()
 		theInspect.DisplayResult()
 	}

cmd/geth/main.go

@@ -223,6 +223,7 @@ var (
 		utils.StateExpiryStateEpoch2BlockFlag,
 		utils.StateExpiryStateEpochPeriodFlag,
 		utils.StateExpiryEnableLocalReviveFlag,
+		utils.StateExpiryEnableRemoteModeFlag,
 	}
 )
 

cmd/geth/snapshot.go

@@ -21,11 +21,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/ethereum/go-ethereum/core"
 	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/ethereum/go-ethereum/core"
+
 	"github.com/prometheus/tsdb/fileutil"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -61,9 +62,9 @@ var (
 				Flags: flags.Merge([]cli.Flag{
 					utils.BloomFilterSizeFlag,
 					utils.TriesInMemoryFlag,
-					utils.StateExpiryEnableFlag,
+					utils.StateExpiryMaxThreadFlag,
 					configFileFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth snapshot prune-state <state-root>
 will prune historical state data with the help of the state snapshot.
@@ -81,14 +82,13 @@ WARNING: it's only supported in hash mode(--state.scheme=hash)".
 				Usage:    "Prune block data offline",
 				Action:   pruneBlock,
 				Category: "MISCELLANEOUS COMMANDS",
-				Flags: []cli.Flag{
+				Flags: flags.Merge([]cli.Flag{
 					utils.DataDirFlag,
 					utils.AncientFlag,
 					utils.BlockAmountReserved,
 					utils.TriesInMemoryFlag,
 					utils.CheckSnapshotWithMPT,
-					utils.StateExpiryEnableFlag,
-				},
+				}, utils.StateExpiryBaseFlags),
 				Description: `
 geth offline prune-block for block data in ancientdb.
 The amount of blocks expected for remaining after prune can be specified via block-amount-reserved in this command,
@@ -108,8 +108,7 @@ so it's very necessary to do block data prune, this feature will handle it.
 				Action:    verifyState,
 				Flags: flags.Merge([]cli.Flag{
 					utils.StateSchemeFlag,
-					utils.StateExpiryEnableFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
@@ -124,11 +123,10 @@ In other words, this command does the snapshot to trie conversion.
 				ArgsUsage: "<genesisPath>",
 				Action:    pruneAllState,
 				Category:  "MISCELLANEOUS COMMANDS",
-				Flags: []cli.Flag{
+				Flags: flags.Merge([]cli.Flag{
 					utils.DataDirFlag,
 					utils.AncientFlag,
-					utils.StateExpiryEnableFlag,
-				},
+				}, utils.StateExpiryBaseFlags),
 				Description: `
 will prune all historical trie state data except genesis block.
 All trie nodes will be deleted from the database. 
@@ -146,7 +144,7 @@ the trie clean cache with default directory will be deleted.
 				Usage:     "Check that there is no 'dangling' snap storage",
 				ArgsUsage: "<root>",
 				Action:    checkDanglingStorage,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth snapshot check-dangling-storage <state-root> traverses the snap storage 
 data, and verifies that all snapshot storage data has a corresponding account. 
@@ -157,7 +155,7 @@ data, and verifies that all snapshot storage data has a corresponding account.
 				Usage:     "Check all snapshot layers for the a specific account",
 				ArgsUsage: "<address | hash>",
 				Action:    checkAccount,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth snapshot inspect-account <address | hash> checks all snapshot layers and prints out
 information about the specified address. 
@@ -170,8 +168,7 @@ information about the specified address.
 				Action:    traverseState,
 				Flags: flags.Merge([]cli.Flag{
 					utils.StateSchemeFlag,
-					utils.StateExpiryEnableFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth snapshot traverse-state <state-root>
 will traverse the whole state from the given state root and will abort if any
@@ -188,8 +185,7 @@ It's also usable without snapshot enabled.
 				Action:    traverseRawState,
 				Flags: flags.Merge([]cli.Flag{
 					utils.StateSchemeFlag,
-					utils.StateExpiryEnableFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth snapshot traverse-rawstate <state-root>
 will traverse the whole state from the given root and will abort if any referenced
@@ -212,8 +208,7 @@ It's also usable without snapshot enabled.
 					utils.DumpLimitFlag,
 					utils.TriesInMemoryFlag,
 					utils.StateSchemeFlag,
-					utils.StateExpiryEnableFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 This command is semantically equivalent to 'geth dump', but uses the snapshots
 as the backend data source, making this command a lot faster. 
@@ -450,6 +445,7 @@ func pruneState(ctx *cli.Context) error {
 		EnableStateExpiry: cfg.Eth.StateExpiryCfg.EnableExpiry(),
 		ChainConfig:       chainConfig,
 		CacheConfig:       cacheConfig,
+		MaxExpireThreads:  ctx.Uint64(utils.StateExpiryMaxThreadFlag.Name),
 	}
 	pruner, err := pruner.NewPruner(chaindb, prunerconfig, ctx.Uint64(utils.TriesInMemoryFlag.Name))
 	if err != nil {

cmd/geth/verkle.go

@@ -45,7 +45,7 @@ var (
 				Usage:     "verify the conversion of a MPT into a verkle tree",
 				ArgsUsage: "<root>",
 				Action:    verifyVerkle,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth verkle verify <state-root>
 This command takes a root commitment and attempts to rebuild the tree.
@@ -56,7 +56,7 @@ This command takes a root commitment and attempts to rebuild the tree.
 				Usage:     "Dump a verkle tree to a DOT file",
 				ArgsUsage: "<root> <key1> [<key 2> ...]",
 				Action:    expandVerkle,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, []cli.Flag{utils.StateExpiryEnableFlag}),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 				Description: `
 geth verkle dump <state-root> <key 1> [<key 2> ...]
 This command will produce a dot file representing the tree, rooted at <root>.