proof, trie: refactor MPTProofCache and add revive tests

.github/workflows/commit-lint.yml

@@ -5,11 +5,13 @@ on:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
   pull_request:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
 jobs:
   commitlint:

.github/workflows/integration-test.yml

@@ -5,11 +5,13 @@ on:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
   pull_request:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
 jobs:
   truffle-test:

.github/workflows/lint.yml

@@ -5,11 +5,13 @@ on:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
   pull_request:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
 jobs:
   golang-lint:

.github/workflows/unit-test.yml

@@ -5,11 +5,13 @@ on:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
   pull_request:
     branches: 
       - master
       - develop
+      - state_expiry_mvp0.1_dev
 
 jobs:
   unit-test:

accounts/abi/bind/backends/simulated.go

@@ -188,7 +188,7 @@ func (b *SimulatedBackend) stateByBlockNumber(ctx context.Context, blockNumber *
 	if err != nil {
 		return nil, err
 	}
-	return b.blockchain.StateAt(block.Root())
+	return b.blockchain.StateAt(block.Root(), block.Hash(), block.Number())
 }
 
 // CodeAt returns the code associated with a certain account in the blockchain.

cmd/geth/chaincmd.go

@@ -56,7 +56,7 @@ var (
 		Flags: flags.Merge([]cli.Flag{
 			utils.CachePreimagesFlag,
 			utils.StateSchemeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
@@ -86,7 +86,7 @@ It expects the genesis file as argument.`,
 		Name:      "dumpgenesis",
 		Usage:     "Dumps genesis block JSON configuration to stdout",
 		ArgsUsage: "",
-		Flags:     append([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags...),
+		Flags:     flags.Merge([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The dumpgenesis command prints the genesis configuration of the network preset
 if one is set.  Otherwise it prints the genesis from the datadir.`,
@@ -121,7 +121,7 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.TransactionHistoryFlag,
 			utils.StateSchemeFlag,
 			utils.StateHistoryFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The import command imports blocks from an RLP-encoded form. The form can be one file
 with several RLP-encoded blocks, or several files can be used.
@@ -138,7 +138,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 			utils.StateSchemeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
@@ -154,7 +154,7 @@ be gzipped.`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The import-preimages command imports hash preimages from an RLP encoded stream.
 It's deprecated, please use "geth db import" instead.
@@ -168,7 +168,7 @@ It's deprecated, please use "geth db import" instead.
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The export-preimages command exports hash preimages to an RLP encoded stream.
 It's deprecated, please use "geth db export" instead.
@@ -188,7 +188,7 @@ It's deprecated, please use "geth db export" instead.
 			utils.StartKeyFlag,
 			utils.DumpLimitFlag,
 			utils.StateSchemeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 This command dumps out the state for a given block (or latest, if none provided).
 `,

cmd/geth/config.go

@@ -49,7 +49,7 @@ var (
 		Name:        "dumpconfig",
 		Usage:       "Export configuration values in a TOML format",
 		ArgsUsage:   "<dumpfile (optional)>",
-		Flags:       flags.Merge(nodeFlags, rpcFlags),
+		Flags:       flags.Merge(nodeFlags, rpcFlags, utils.StateExpiryBaseFlags),
 		Description: `Export configuration values in TOML format (to stdout by default).`,
 	}
 

cmd/geth/consolecmd.go

@@ -33,7 +33,7 @@ var (
 		Action: localConsole,
 		Name:   "console",
 		Usage:  "Start an interactive JavaScript environment",
-		Flags:  flags.Merge(nodeFlags, rpcFlags, consoleFlags),
+		Flags:  flags.Merge(nodeFlags, rpcFlags, consoleFlags, utils.StateExpiryBaseFlags),
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.

cmd/geth/dbcmd.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -49,7 +50,7 @@ var (
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: "",
-		Flags:     utils.DatabasePathFlags,
+		Flags:     flags.Merge(utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `
 Remove blockchain and state databases`,
 	}
@@ -59,6 +60,7 @@ Remove blockchain and state databases`,
 		ArgsUsage: "",
 		Subcommands: []*cli.Command{
 			dbInspectCmd,
+			dbInspectTrieCmd,
 			dbStatCmd,
 			dbCompactCmd,
 			dbGetCmd,
@@ -84,15 +86,25 @@ Remove blockchain and state databases`,
 		ArgsUsage: "<prefix> <start>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
+	dbInspectTrieCmd = &cli.Command{
+		Action:    inspectTrie,
+		Name:      "inspect-trie",
+		ArgsUsage: "<blocknum> <jobnum>",
+		Flags: flags.Merge([]cli.Flag{
+			utils.SyncModeFlag,
+		}, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
+		Usage:       "Inspect the MPT tree of the account and contract.",
+		Description: `This commands iterates the entrie WorldState.`,
+	}
 	dbCheckStateContentCmd = &cli.Command{
 		Action:    checkStateContent,
 		Name:      "check-state-content",
 		ArgsUsage: "<start (optional)>",
-		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Usage:     "Verify that state data is cryptographically correct",
 		Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes.
 For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates
@@ -143,7 +155,7 @@ a data corruption.`,
 		Usage:  "Print leveldb statistics",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 	}
 	dbCompactCmd = &cli.Command{
 		Action: dbCompact,
@@ -153,7 +165,7 @@ a data corruption.`,
 			utils.SyncModeFlag,
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `This command performs a database compaction. 
 WARNING: This operation may take a very long time to finish, and may cause database
 corruption if it is aborted during execution'!`,
@@ -165,7 +177,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = &cli.Command{
@@ -175,7 +187,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `This command deletes the specified database key from the database. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -186,7 +198,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: `This command sets a given database key to the given value. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -198,7 +210,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
 			utils.StateSchemeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDumpFreezerIndex = &cli.Command{
@@ -208,7 +220,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "This command displays information about the freezer index.",
 	}
 	dbImportCmd = &cli.Command{
@@ -218,7 +230,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<dumpfile> <start (optional)",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
 	dbExportCmd = &cli.Command{
@@ -228,7 +240,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<type> <dumpfile>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
 	dbMetadataCmd = &cli.Command{
@@ -237,15 +249,16 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Usage:  "Shows metadata about the chain status.",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabasePathFlags, utils.StateExpiryBaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
 	ancientInspectCmd = &cli.Command{
 		Action: ancientInspect,
 		Name:   "inspect-reserved-oldest-blocks",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-		},
+		Flags: flags.Merge(
+			[]cli.Flag{utils.DataDirFlag},
+			utils.StateExpiryBaseFlags,
+		),
 		Usage: "Inspect the ancientStore information",
 		Description: `This commands will read current offset from kvdb, which is the current offset and starting BlockNumber
 of ancientStore, will also displays the reserved number of blocks in ancientStore `,
@@ -352,6 +365,79 @@ func ancientInspect(ctx *cli.Context) error {
 	return rawdb.AncientInspect(db)
 }
 
+func inspectTrie(ctx *cli.Context) error {
+	if ctx.NArg() < 1 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+
+	if ctx.NArg() > 3 {
+		return fmt.Errorf("Max 3 arguments: %v", ctx.Command.ArgsUsage)
+	}
+
+	var (
+		blockNumber uint64
+		blockRoot   common.Hash
+		jobnum      uint64
+	)
+
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, true, true)
+	defer db.Close()
+
+	if ctx.NArg() >= 1 {
+		if ctx.Args().Get(0) == "latest" {
+			headBlock := rawdb.ReadHeadBlock(db)
+			if headBlock == nil {
+				return errors.New("failed to load head block")
+			}
+			blockNumber = headBlock.NumberU64()
+			blockRoot = headBlock.Root()
+		} else if ctx.Args().Get(0) == "snapshot" {
+			blockRoot = rawdb.ReadSnapshotRoot(db)
+			blockNumber = math.MaxUint64
+		} else {
+			var err error
+			blockNumber, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64)
+			if err != nil {
+				return fmt.Errorf("failed to Parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err)
+			}
+			blockHash := rawdb.ReadCanonicalHash(db, blockNumber)
+			block := rawdb.ReadBlock(db, blockHash, blockNumber)
+			blockRoot = block.Root()
+		}
+
+		if ctx.NArg() == 1 {
+			jobnum = 1000
+		} else {
+			var err error
+			jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64)
+			if err != nil {
+				return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
+			}
+		}
+
+		if (blockRoot == common.Hash{}) {
+			log.Error("Empty root hash")
+		}
+		fmt.Printf("ReadBlockHeader, root: %v, blocknum: %v\n", blockRoot, blockNumber)
+		trieDB := trie.NewDatabase(db, nil)
+		theTrie, err := trie.New(trie.TrieID(blockRoot), trieDB)
+		if err != nil {
+			fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, blockRoot.String())
+			return err
+		}
+		theInspect, err := trie.NewInspector(trieDB, theTrie, blockNumber, jobnum)
+		if err != nil {
+			return err
+		}
+		theInspect.Run()
+		theInspect.DisplayResult()
+	}
+	return nil
+}
+
 func checkStateContent(ctx *cli.Context) error {
 	var (
 		prefix []byte