From 3f266bf0e382d4c03d85ec4927fc398d82312281 Mon Sep 17 00:00:00 2001 From: RealUncle Date: Thu, 2 Dec 2021 17:40:47 +0800 Subject: [PATCH 01/51] implement the framework of fast node --- cmd/geth/main.go | 1 + cmd/geth/snapshot.go | 76 ++++++++++++++++++++++- cmd/utils/flags.go | 7 +++ core/block_validator.go | 6 -- core/blockchain.go | 13 +++- core/state/database.go | 24 +++++++- core/state/pruner/pruner.go | 104 +++++++++++++++++++++++++++++++- core/state/snapshot/journal.go | 9 ++- core/state/snapshot/snapshot.go | 4 +- core/state/statedb.go | 96 +++++++++++++++++++---------- core/state_processor.go | 3 +- eth/backend.go | 3 +- eth/ethconfig/config.go | 2 + eth/ethconfig/gen_config.go | 69 ++++++++++++++++----- light/trie.go | 8 +++ tests/state_test_util.go | 2 +- trie/database.go | 1 + trie/dummy_trie.go | 96 +++++++++++++++++++++++++++++ 18 files changed, 458 insertions(+), 66 deletions(-) create mode 100644 trie/dummy_trie.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 3576fe2e46..ae1dd6fbe9 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -114,6 +114,7 @@ var ( utils.WhitelistFlag, utils.BloomFilterSizeFlag, utils.TriesInMemoryFlag, + utils.AllowInsecureNoTriesFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 20920a0f94..d829abbc2a 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "encoding/json" "errors" "fmt" "os" @@ -29,13 +30,16 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -129,6 +133,32 @@ geth snapshot verify-state will traverse the whole accounts and storages set based on the specified snapshot and recalculate the root hash of state for verification. In other words, this command does the snapshot to trie conversion. +`, + }, + { + Name: "insecure-prune-all", + Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " + + "who do not need trie storage at all", + ArgsUsage: "", + Action: utils.MigrateFlags(pruneAllState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: ` +will prune all historical trie state data except genesis block. +All trie nodes will be deleted from the database. + +It expects the genesis file as argument. + +WARNING: It's necessary to delete the trie clean cache after the pruning. +If you specify another directory for the trie clean cache via "--cache.trie.journal" +during the use of Geth, please also specify it here for correct deletion. Otherwise +the trie clean cache with default directory will be deleted. `, }, { @@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) { } headHeader := headBlock.Header() //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false) if err != nil { log.Error("snaptree error", "err", err) return nil, err // The relevant snapshot(s) might not exist @@ -362,6 +392,48 @@ func pruneState(ctx *cli.Context) error { return nil } +func pruneAllState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + genesisPath := ctx.Args().First() + if len(genesisPath) == 0 { + utils.Fatalf("Must supply path to genesis JSON file") + } + file, err := os.Open(genesisPath) + if err != nil { + utils.Fatalf("Failed to read genesis file: %v", err) + } + defer file.Close() + + g := new(core.Genesis) + if err := json.NewDecoder(file).Decode(g); err != nil { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + Metrics: metrics.DefaultConfig, + } + + // Load config file. + if err := loadConfig(genesisPath, &cfg); err != nil { + utils.Fatalf("%v", err) + } + g = cfg.Eth.Genesis + } + + chaindb := utils.MakeChainDatabase(ctx, stack, false, false) + pruner, err := pruner.NewAllPruner(chaindb) + if err != nil { + log.Error("Failed to open snapshot tree", "err", err) + return err + } + if err = pruner.PruneAll(g); err != nil { + log.Error("Failed to prune state", "err", err) + return err + } + return nil +} + func verifyState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -372,7 +444,7 @@ func verifyState(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 58f4798aee..9738ec767f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -260,6 +260,10 @@ var ( Usage: "The layer of tries trees that keep in memory", Value: 128, } + AllowInsecureNoTriesFlag = cli.BoolTFlag{ + Name: "allow-insecure-no-tries", + Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`, + } OverrideBerlinFlag = cli.Uint64Flag{ Name: "override.berlin", Usage: "Manually specify Berlin fork-block, overriding the bundled setting", @@ -1659,6 +1663,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(TriesInMemoryFlag.Name) { cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name) } + if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) { + cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name) + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 } diff --git a/core/block_validator.go b/core/block_validator.go index 12fa908cd0..bf2fb40260 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -64,12 +64,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } validateFuns := []func() error{ - func() error { - if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { - return ErrKnownBlock - } - return nil - }, func() error { if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) diff --git a/core/blockchain.go b/core/blockchain.go index f4602bc847..952a496528 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -137,6 +137,7 @@ type CacheConfig struct { SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk TriesInMemory uint64 // How many tries keeps in memory + NoTries bool // Insecure settings. Do not have any tries in databases if enabled. SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -275,6 +276,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Cache: cacheConfig.TrieCleanLimit, Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, + NoTries: cacheConfig.NoTries, }), triesInMemory: cacheConfig.TriesInMemory, quit: make(chan struct{}), @@ -429,7 +431,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer) recover = true } - bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover) + bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.stateCache.NoTries()) } // do options before start any routine for _, option := range options { @@ -1056,6 +1058,9 @@ func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool { // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { + if bc.stateCache.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil + } _, err := bc.stateCache.OpenTrie(hash) return err == nil } @@ -1068,6 +1073,9 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { if block == nil { return false } + if bc.stateCache.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(block.Root()) != nil + } return bc.HasState(block.Root()) } @@ -2054,6 +2062,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if err != nil { return it.index, err } + if statedb.NoTrie() { + statedb.SetCurrentRoot(block.Root()) + } bc.updateHighestVerifiedHeader(block.Header()) // Enable prefetching to pull in trie node paths while processing transactions diff --git a/core/state/database.go b/core/state/database.go index b65dfca158..5f07470d25 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -74,6 +74,9 @@ type Database interface { // Purge cache Purge() + + // NoTries returns whether the database has tries storage. + NoTries() bool } // Trie is a Ethereum Merkle Patricia trie. @@ -134,10 +137,12 @@ func NewDatabase(db ethdb.Database) Database { func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) cc, _ := lru.New(codeCacheSize) + noTries := config != nil && config.NoTries return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: cc, + noTries: noTries, } } @@ -146,6 +151,7 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab cc, _ := lru.New(codeCacheSize) atc, _ := lru.New(accountTrieCacheSize) stc, _ := lru.New(storageTrieCacheSize) + noTries := config != nil && config.NoTries database := &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), @@ -153,8 +159,11 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab codeCache: cc, accountTrieCache: atc, storageTrieCache: stc, + noTries: noTries, + } + if !noTries { + go database.purgeLoop() } - go database.purgeLoop() return database } @@ -164,6 +173,7 @@ type cachingDB struct { codeCache *lru.Cache accountTrieCache *lru.Cache storageTrieCache *lru.Cache + noTries bool } type triePair struct { @@ -187,6 +197,9 @@ func (db *cachingDB) purgeLoop() { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.accountTrieCache != nil { if tr, exist := db.accountTrieCache.Get(root); exist { return tr.(Trie).(*trie.SecureTrie).Copy(), nil @@ -201,6 +214,9 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.storageTrieCache != nil { if tries, exist := db.storageTrieCache.Get(addrHash); exist { triesPairs := tries.([3]*triePair) @@ -246,6 +262,10 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie } } +func (db *cachingDB) NoTries() bool { + return db.noTries +} + func (db *cachingDB) Purge() { if db.storageTrieCache != nil { db.storageTrieCache.Purge() @@ -260,6 +280,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.SecureTrie: return t.Copy() + case *trie.EmptyTrie: + return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 5b070f3afa..a599415aa0 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -103,7 +104,7 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize, trie if headBlock == nil { return nil, errors.New("Failed to load head block") } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false, false) if err != nil { return nil, err // The relevant snapshot(s) might not exist } @@ -138,6 +139,105 @@ func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientP } } +func NewAllPruner(db ethdb.Database) (*Pruner, error) { + headBlock := rawdb.ReadHeadBlock(db) + if headBlock == nil { + return nil, errors.New("Failed to load head block") + } + return &Pruner{ + db: db, + }, nil +} + +func (p *Pruner) PruneAll(genesis *core.Genesis) error { + deleteCleanTrieCache(p.trieCachePath) + return pruneAll(p.db, genesis) +} + +func pruneAll(maindb ethdb.Database, g *core.Genesis) error { + var ( + count int + size common.StorageSize + pstart = time.Now() + logged = time.Now() + batch = maindb.NewBatch() + iter = maindb.NewIterator(nil, nil) + ) + start := time.Now() + for iter.Next() { + key := iter.Key() + if len(key) == common.HashLength { + count += 1 + size += common.StorageSize(len(key) + len(iter.Value())) + batch.Delete(key) + + var eta time.Duration // Realistically will never remain uninited + if done := binary.BigEndian.Uint64(key[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8]) + speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + eta = time.Duration(left/speed) * time.Millisecond + } + if time.Since(logged) > 8*time.Second { + log.Info("Pruning state data", "nodes", count, "size", size, + "elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta)) + logged = time.Now() + } + // Recreate the iterator after every batch commit in order + // to allow the underlying compactor to delete the entries. + if batch.ValueSize() >= ethdb.IdealBatchSize { + batch.Write() + batch.Reset() + + iter.Release() + iter = maindb.NewIterator(nil, key) + } + } + } + if batch.ValueSize() > 0 { + batch.Write() + batch.Reset() + } + iter.Release() + log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart))) + + // Start compactions, will remove the deleted data from the disk immediately. + // Note for small pruning, the compaction is skipped. + if count >= rangeCompactionThreshold { + cstart := time.Now() + for b := 0x00; b <= 0xf0; b += 0x10 { + var ( + start = []byte{byte(b)} + end = []byte{byte(b + 0x10)} + ) + if b == 0xf0 { + end = nil + } + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) + if err := maindb.Compact(start, end); err != nil { + log.Error("Database compaction failed", "error", err) + return err + } + } + log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart))) + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(maindb), nil) + for addr, account := range g.Alloc { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } + root := statedb.IntermediateRoot(false) + statedb.Commit(false) + statedb.Database().TrieDB().Commit(root, true, nil) + log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, middleStateRoots map[common.Hash]struct{}, start time.Time) error { // Delete all stale trie nodes in the disk. With the help of state bloom // the trie nodes(and codes) belong to the active state will be filtered @@ -585,7 +685,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string, tri // - The state HEAD is rewound already because of multiple incomplete `prune-state` // In this case, even the state HEAD is not exactly matched with snapshot, it // still feasible to recover the pruning correctly. - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true, false) if err != nil { return err // The relevant snapshot(s) might not exist } diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 5cfb9a9f2a..6747aed01c 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -126,7 +126,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery, withoutTrie bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { @@ -145,6 +145,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root: baseRoot, } snapshot, generator, err := loadAndParseJournal(diskdb, base) + if err != nil { log.Warn("Failed to load new-format journal", "error", err) return nil, false, err @@ -158,6 +159,11 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // which is below the snapshot. In this case the snapshot can be recovered // by re-executing blocks but right now it's unavailable. if head := snapshot.Root(); head != root { + log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) + + if withoutTrie { + return snapshot, false, nil + } // If it's legacy snapshot, or it's new-format snapshot but // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. @@ -168,7 +174,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // the disk layer is always higher than chain head. It can // be eventually recovered when the chain head beyonds the // disk layer. - log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) } // Everything loaded correctly, resume any suspended operations if !generator.Done { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 46d1b06def..41f38bd255 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -175,7 +175,7 @@ type Tree struct { // store, on a background thread. If the memory layers from the journal is not // continuous with disk layer or the journal is missing, all diffs will be discarded // iff it's in "recovery" mode, otherwise rebuild is mandatory. -func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery bool) (*Tree, error) { +func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery, withoutTrie bool) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ diskdb: diskdb, @@ -188,7 +188,7 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root defer snap.waitBuild() } // Attempt to load a previously persisted snapshot and rebuild one if failed - head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery) + head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery, withoutTrie) if disabled { log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil diff --git a/core/state/statedb.go b/core/state/statedb.go index 6f150915ca..dc48ac0a46 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -73,10 +73,13 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database - prefetcher *triePrefetcher - originalRoot common.Hash // The pre-state root, before any changes were made + db Database + prefetcher *triePrefetcher + originalRoot common.Hash // The pre-state root, before any changes were made + currentRoot common.Hash // only used when noTrie is true + trie Trie + noTrie bool hasher crypto.KeccakState diffLayer *types.DiffLayer diffTries map[common.Address]Trie @@ -158,6 +161,7 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, if err != nil { return nil, err } + _, sdb.noTrie = tr.(*trie.EmptyTrie) sdb.trie = tr if sdb.snaps != nil { if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { @@ -173,6 +177,9 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. func (s *StateDB) StartPrefetcher(namespace string) { + if s.noTrie { + return + } if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -185,6 +192,9 @@ func (s *StateDB) StartPrefetcher(namespace string) { // StopPrefetcher terminates a running prefetcher and reports any leftover stats // from the gathered metrics. func (s *StateDB) StopPrefetcher() { + if s.noTrie { + return + } if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -207,6 +217,14 @@ func (s *StateDB) setError(err error) { } } +func (s *StateDB) NoTrie() bool { + return s.noTrie +} + +func (s *StateDB) SetCurrentRoot(root common.Hash) { + s.currentRoot = root +} + func (s *StateDB) Error() error { return s.dbErr } @@ -504,6 +522,9 @@ func (s *StateDB) Suicide(addr common.Address) bool { // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on updating the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -525,6 +546,9 @@ func (s *StateDB) updateStateObject(obj *StateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on deleting the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -957,6 +981,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // It is called in between transactions to get the root hash that // goes into transaction receipts. func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { + // light process is not allowed when there is no trie if s.lightProcessed { s.StopPrefetcher() return s.trie.Hash() @@ -1042,16 +1067,18 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { s.trie = tr } usedAddrs := make([][]byte, 0, len(s.stateObjectsPending)) - for addr := range s.stateObjectsPending { - if obj := s.stateObjects[addr]; obj.deleted { - s.deleteStateObject(obj) - } else { - s.updateStateObject(obj) + if !s.noTrie { + for addr := range s.stateObjectsPending { + if obj := s.stateObjects[addr]; obj.deleted { + s.deleteStateObject(obj) + } else { + s.updateStateObject(obj) + } + usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure + } + if prefetcher != nil { + prefetcher.used(s.originalRoot, usedAddrs) } - usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure - } - if prefetcher != nil { - prefetcher.used(s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) @@ -1060,8 +1087,11 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) } - root := s.trie.Hash() - return root + if s.noTrie { + return s.currentRoot + } else { + return s.trie.Hash() + } } // Prepare sets the current transaction hash and index and block hash which is @@ -1271,8 +1301,10 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, *types.DiffLayer obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - if err := obj.CommitTrie(s.db); err != nil { - taskResults <- err + if !s.noTrie { + if err := obj.CommitTrie(s.db); err != nil { + taskResults <- err + } } taskResults <- nil } @@ -1299,24 +1331,26 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, *types.DiffLayer } // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. - var account Account - root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { + if !s.noTrie { + var account Account + root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { + if err := rlp.DecodeBytes(leaf, &account); err != nil { + return nil + } + if account.Root != emptyRoot { + s.db.TrieDB().Reference(account.Root, parent) + } return nil + }) + if err != nil { + return err } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) + if metrics.EnabledExpensive { + s.AccountCommits += time.Since(start) + } + if root != emptyRoot { + s.db.CacheAccount(root, s.trie) } - return nil - }) - if err != nil { - return err - } - if metrics.EnabledExpensive { - s.AccountCommits += time.Since(start) - } - if root != emptyRoot { - s.db.CacheAccount(root, s.trie) } wg.Wait() return nil diff --git a/core/state_processor.go b/core/state_processor.go index 5652547db7..a0037559c9 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -399,6 +399,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // usually do have two tx, one for validator set contract, another for system reward contract. systemTxs := make([]*types.Transaction, 0, 2) + for i, tx := range block.Transactions() { if isPoSA { if isSystemTx, err := posa.IsSystemTransaction(tx, block.Header()); err != nil { @@ -414,11 +415,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return statedb, nil, nil, 0, err } statedb.Prepare(tx.Hash(), block.Hash(), i) + receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv, bloomProcessors) if err != nil { return statedb, nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } - commonTxs = append(commonTxs, tx) receipts = append(receipts, receipt) } diff --git a/eth/backend.go b/eth/backend.go index f6599529db..56fd1234a3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -194,13 +194,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieDirtyLimit: config.TrieDirtyCache, TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, + NoTries: config.NoTries, SnapshotLimit: config.SnapshotCache, TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, } ) bcOps := make([]core.BlockChainOption, 0) - if config.DiffSync { + if config.DiffSync && !config.NoTries { bcOps = append(bcOps, core.EnableLightProcessor) } if config.PersistDiff { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 010a1fb923..d3e146aaaf 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -79,6 +79,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, TriesInMemory: 128, + NoTries: false, SnapshotCache: 102, DiffBlock: uint64(86400), Miner: miner.Config{ @@ -174,6 +175,7 @@ type Config struct { TrieTimeout time.Duration SnapshotCache int TriesInMemory uint64 + NoTries bool Preimages bool // Mining options diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index f192a1aace..ba2996279d 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -3,6 +3,7 @@ package ethconfig import ( + "math/big" "time" "github.com/ethereum/go-ethereum/common" @@ -25,6 +26,10 @@ func (c Config) MarshalTOML() (interface{}, error) { SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool + DirectBroadcast bool + DisableSnapProtocol bool + DiffSync bool + RangeLimit bool TxLookupLimit uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` @@ -42,28 +47,30 @@ func (c Config) MarshalTOML() (interface{}, error) { DatabaseCache int DatabaseFreezer string DatabaseDiff string + PersistDiff bool + DiffBlock uint64 TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` TrieCleanCacheRejournal time.Duration `toml:",omitempty"` TrieDirtyCache int TrieTimeout time.Duration - TriesInMemory uint64 `toml:",omitempty"` SnapshotCache int + TriesInMemory uint64 + NoTries bool Preimages bool - PersistDiff bool - DiffBlock uint64 `toml:",omitempty"` Miner miner.Config - Ethash ethash.Config + Ethash ethash.Config `toml:",omitempty"` TxPool core.TxPoolConfig GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` EWASMInterpreter string EVMInterpreter string - RPCGasCap uint64 `toml:",omitempty"` - RPCTxFeeCap float64 `toml:",omitempty"` + RPCGasCap uint64 + RPCTxFeeCap float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -73,6 +80,10 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs enc.NoPruning = c.NoPruning + enc.DirectBroadcast = c.DirectBroadcast + enc.DisableSnapProtocol = c.DisableSnapProtocol + enc.DiffSync = c.DiffSync + enc.RangeLimit = c.RangeLimit enc.TxLookupLimit = c.TxLookupLimit enc.Whitelist = c.Whitelist enc.LightServ = c.LightServ @@ -90,16 +101,17 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DatabaseCache = c.DatabaseCache enc.DatabaseFreezer = c.DatabaseFreezer enc.DatabaseDiff = c.DatabaseDiff + enc.PersistDiff = c.PersistDiff + enc.DiffBlock = c.DiffBlock enc.TrieCleanCache = c.TrieCleanCache enc.TrieCleanCacheJournal = c.TrieCleanCacheJournal enc.TrieCleanCacheRejournal = c.TrieCleanCacheRejournal enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout - enc.TriesInMemory = c.TriesInMemory enc.SnapshotCache = c.SnapshotCache + enc.TriesInMemory = c.TriesInMemory + enc.NoTries = c.NoTries enc.Preimages = c.Preimages - enc.PersistDiff = c.PersistDiff - enc.DiffBlock = c.DiffBlock enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -112,6 +124,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle + enc.OverrideBerlin = c.OverrideBerlin return &enc, nil } @@ -126,6 +139,10 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool + DirectBroadcast *bool + DisableSnapProtocol *bool + DiffSync *bool + RangeLimit *bool TxLookupLimit *uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` @@ -144,27 +161,29 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DatabaseFreezer *string DatabaseDiff *string PersistDiff *bool - DiffBlock *uint64 `toml:",omitempty"` + DiffBlock *uint64 TrieCleanCache *int TrieCleanCacheJournal *string `toml:",omitempty"` TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` TrieDirtyCache *int TrieTimeout *time.Duration - TriesInMemory *uint64 `toml:",omitempty"` SnapshotCache *int + TriesInMemory *uint64 + NoTries *bool Preimages *bool Miner *miner.Config - Ethash *ethash.Config + Ethash *ethash.Config `toml:",omitempty"` TxPool *core.TxPoolConfig GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` EWASMInterpreter *string EVMInterpreter *string - RPCGasCap *uint64 `toml:",omitempty"` - RPCTxFeeCap *float64 `toml:",omitempty"` + RPCGasCap *uint64 + RPCTxFeeCap *float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -191,6 +210,18 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } + if dec.DirectBroadcast != nil { + c.DirectBroadcast = *dec.DirectBroadcast + } + if dec.DisableSnapProtocol != nil { + c.DisableSnapProtocol = *dec.DisableSnapProtocol + } + if dec.DiffSync != nil { + c.DiffSync = *dec.DiffSync + } + if dec.RangeLimit != nil { + c.RangeLimit = *dec.RangeLimit + } if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } @@ -263,11 +294,14 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout } + if dec.SnapshotCache != nil { + c.SnapshotCache = *dec.SnapshotCache + } if dec.TriesInMemory != nil { c.TriesInMemory = *dec.TriesInMemory } - if dec.SnapshotCache != nil { - c.SnapshotCache = *dec.SnapshotCache + if dec.NoTries != nil { + c.NoTries = *dec.NoTries } if dec.Preimages != nil { c.Preimages = *dec.Preimages @@ -308,5 +342,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } + if dec.OverrideBerlin != nil { + c.OverrideBerlin = dec.OverrideBerlin + } return nil } diff --git a/light/trie.go b/light/trie.go index 3896b73c4d..3f942f3607 100644 --- a/light/trie.go +++ b/light/trie.go @@ -49,6 +49,10 @@ type odrDatabase struct { backend OdrBackend } +func (db *odrDatabase) NoTries() bool { + return false +} + func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } @@ -178,6 +182,10 @@ func (t *odrTrie) do(key []byte, fn func() error) error { } } +func (db *odrTrie) NoTries() bool { + return false +} + type nodeIterator struct { trie.NodeIterator t *odrTrie diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 77d4fd08d4..2b22ca0289 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -230,7 +230,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo var snaps *snapshot.Tree if snapshotter { - snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false) + snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false, false) } statedb, _ = state.New(root, sdb, snaps) return snaps, statedb diff --git a/trie/database.go b/trie/database.go index 76f8b26ccd..f15ad3bd64 100644 --- a/trie/database.go +++ b/trie/database.go @@ -282,6 +282,7 @@ type Config struct { Cache int // Memory allowance (MB) to use for caching trie nodes in memory Journal string // Journal of clean cache to survive node restarts Preimages bool // Flag whether the preimage of trie key is recorded + NoTries bool } // NewDatabase creates a new trie database to store ephemeral trie content before diff --git a/trie/dummy_trie.go b/trie/dummy_trie.go new file mode 100644 index 0000000000..99eb79fbd4 --- /dev/null +++ b/trie/dummy_trie.go @@ -0,0 +1,96 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +type EmptyTrie struct{} + +func (t *EmptyTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { + return nil +} + +// NewSecure creates a dummy trie +func NewEmptyTrie() *EmptyTrie { + return &EmptyTrie{} +} + +func (t *EmptyTrie) Get(key []byte) []byte { + return nil +} + +func (t *EmptyTrie) TryGet(key []byte) ([]byte, error) { + return nil, nil +} + +func (t *EmptyTrie) TryGetNode(path []byte) ([]byte, int, error) { + return nil, 0, nil +} +func (t *EmptyTrie) Update(key, value []byte) {} + +func (t *EmptyTrie) TryUpdate(key, value []byte) error { + return nil +} + +// Delete removes any existing value for key from the trie. +func (t *EmptyTrie) Delete(key []byte) { + if err := t.TryDelete(key); err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } +} + +func (t *EmptyTrie) TryDelete(key []byte) error { + + return nil +} + +func (t *EmptyTrie) GetKey(shaKey []byte) []byte { + return nil +} + +func (t *EmptyTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { + + return common.Hash{}, nil +} + +func (t *EmptyTrie) Hash() common.Hash { + return common.Hash{} +} + +// Copy returns a copy of SecureTrie. +func (t *EmptyTrie) Copy() *EmptyTrie { + cpy := *t + return &cpy +} + +func (t *EmptyTrie) ResetCopy() *EmptyTrie { + cpy := *t + return &cpy +} + +// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration +// starts at the key after the given start key. +func (t *EmptyTrie) NodeIterator(start []byte) NodeIterator { + return nil +} From 885aeb94720aa6f7f459b8c798964fb45955e243 Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Tue, 11 Jan 2022 16:17:03 +0800 Subject: [PATCH 02/51] implement trust protocol and verify node --- common/types.go | 15 +++ core/blockchain.go | 114 ++++++++++++++++++++++ core/state/state_object.go | 7 ++ core/state/statedb.go | 6 +- core/types/block.go | 89 ++++++++++++++++- eth/backend.go | 3 + eth/handler.go | 24 ++++- eth/handler_trust.go | 43 +++++++++ eth/peer.go | 25 ++++- eth/peerset.go | 118 +++++++++++++++++++++-- eth/protocols/trust/discovery.go | 14 +++ eth/protocols/trust/handler.go | 159 +++++++++++++++++++++++++++++++ eth/protocols/trust/peer.go | 66 +++++++++++++ eth/protocols/trust/protocol.go | 72 ++++++++++++++ eth/protocols/trust/tracker.go | 10 ++ ethclient/ethclient.go | 6 ++ internal/ethapi/api.go | 4 + p2p/peer.go | 5 + p2p/server.go | 18 ++++ 19 files changed, 785 insertions(+), 13 deletions(-) create mode 100644 eth/handler_trust.go create mode 100644 eth/protocols/trust/discovery.go create mode 100644 eth/protocols/trust/handler.go create mode 100644 eth/protocols/trust/peer.go create mode 100644 eth/protocols/trust/protocol.go create mode 100644 eth/protocols/trust/tracker.go diff --git a/common/types.go b/common/types.go index d715356692..15f2bce681 100644 --- a/common/types.go +++ b/common/types.go @@ -354,6 +354,21 @@ func (a *Address) UnmarshalGraphQL(input interface{}) error { return err } +// AddressSlice is used for sort +type AddressSlice []Address + +func (s AddressSlice) Len() int { + return len(s) +} + +func (s AddressSlice) Less(i, j int) bool { + return s[i].Hex() < s[j].Hex() +} + +func (s AddressSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + // UnprefixedAddress allows marshaling an Address without 0x prefix. type UnprefixedAddress Address diff --git a/core/blockchain.go b/core/blockchain.go index 952a496528..ad9b7db8a7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -28,6 +28,8 @@ import ( "sync/atomic" "time" + "golang.org/x/crypto/sha3" + lru "github.com/hashicorp/golang-lru" "github.com/ethereum/go-ethereum/common" @@ -1687,6 +1689,12 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.Receipts = receipts diffLayer.BlockHash = block.Hash() diffLayer.Number = block.NumberU64() + + sort.Sort(types.DiffCodeSlice(diffLayer.Codes)) + sort.Sort(common.AddressSlice(diffLayer.Destructs)) + sort.Sort(types.DiffAccountSlice(diffLayer.Accounts)) + sort.Sort(types.DiffStorageSlice(diffLayer.Storages)) + bc.cacheDiffLayer(diffLayer) } triedb := bc.stateCache.TrieDB() @@ -3026,3 +3034,109 @@ func EnablePersistDiff(limit uint64) BlockChainOption { return chain } } + +func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { + var res types.VerifyResult + res.BlockNumber = blockNumber + res.BlockHash = blockHash + + if blockNumber > bc.CurrentHeader().Number.Uint64()+11 { + res.Status = types.StatusBlockTooNew + return &res, nil + } else if blockNumber > bc.CurrentHeader().Number.Uint64() { + res.Status = types.StatusBlockNewer + return &res, nil + } + + diff := bc.GetTrustedDiffLayer(blockHash) + if diff != nil { + if diff.DiffHash == (common.Hash{}) { + hash, err := GetTrustedDiffHash(diff) + if err != nil { + res.Status = types.StatusUnexpectedError + return &res, err + } + + diff.DiffHash = hash + } + + if diffHash != diff.DiffHash { + res.Status = types.StatusDiffHashMismatch + return &res, nil + } + + header := bc.GetHeaderByHash(blockHash) + if header == nil { + res.Status = types.StatusUnexpectedError + return &res, fmt.Errorf("unexpected error, header not found") + } + res.Status = types.StatusFullVerified + res.Root = header.Root + return &res, nil + } + + header := bc.GetHeaderByHash(blockHash) + if header == nil { + if blockNumber > bc.CurrentHeader().Number.Uint64()-11 { + res.Status = types.StatusPossibleFork + return &res, nil + } + + res.Status = types.StatusImpossibleFork + return &res, nil + } + + res.Status = types.StatusUntrustedVerified + res.Root = header.Root + return &res, nil +} + +func (bc *BlockChain) GetTrustedDiffLayer(blockHash common.Hash) *types.DiffLayer { + var diff *types.DiffLayer + if cached, ok := bc.diffLayerCache.Get(blockHash); ok { + diff = cached.(*types.DiffLayer) + return diff + } + + diffStore := bc.db.DiffStore() + if diffStore != nil { + diff = rawdb.ReadDiffLayer(diffStore, blockHash) + } + return diff +} + +func GetTrustedDiffHash(d *types.DiffLayer) (common.Hash, error) { + diff := &types.ExtDiffLayer{ + BlockHash: d.BlockHash, + Receipts: make([]*types.ReceiptForStorage, 0), + Number: d.Number, + Codes: d.Codes, + Destructs: d.Destructs, + Accounts: d.Accounts, + Storages: d.Storages, + } + + for index, account := range diff.Accounts { + full, err := snapshot.FullAccount(account.Blob) + if err != nil { + return common.Hash{}, fmt.Errorf("decode full account error: %v", err) + } + // set account root to empty root + diff.Accounts[index].Blob = snapshot.SlimAccountRLP(full.Nonce, full.Balance, common.Hash{}, full.CodeHash) + } + + rawData, err := rlp.EncodeToBytes(diff) + if err != nil { + return common.Hash{}, fmt.Errorf("encode new diff error: %v", err) + } + + hasher := sha3.NewLegacyKeccak256() + _, err = hasher.Write(rawData) + if err != nil { + return common.Hash{}, fmt.Errorf("hasher write error: %v", err) + } + + var hash common.Hash + hasher.Sum(hash[:0]) + return hash, nil +} diff --git a/core/state/state_object.go b/core/state/state_object.go index 298f4305ba..c86585a658 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -391,6 +391,13 @@ func (s *StateObject) updateTrie(db Database) Trie { // UpdateRoot sets the trie root to the current root hash of func (s *StateObject) updateRoot(db Database) { + // If node runs in no trie mode, set root to empty. + defer func() { + if db.NoTries() { + s.data.Root = common.Hash{} + } + }() + // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { return diff --git a/core/state/statedb.go b/core/state/statedb.go index dc48ac0a46..64d06a721a 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1442,8 +1442,12 @@ func (s *StateDB) SnapToDiffLayer() ([]common.Address, []types.DiffAccount, []ty for accountHash, storage := range s.snapStorage { keys := make([]string, 0, len(storage)) values := make([][]byte, 0, len(storage)) - for k, v := range storage { + for k, _ := range storage { keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := storage[k] values = append(values, v) } storages = append(storages, types.DiffStorage{ diff --git a/core/types/block.go b/core/types/block.go index bee5d80cdd..f3c487b684 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -40,6 +40,40 @@ var ( EmptyUncleHash = rlpHash([]*Header(nil)) ) +type VerifyStatus struct { + Code uint16 + Msg string +} + +var ( + // StatusVerified means the processing of request going as expected and found the root correctly. + StatusVerified = VerifyStatus{Code: 0x100} + StatusFullVerified = VerifyStatus{Code: 0x101, Msg: "state root full verified"} + StatusUntrustedVerified = VerifyStatus{Code: 0x102, Msg: "state root untrusted verified, because of difflayer not found"} + + // StatusFailed means the request has something wrong. + StatusFailed = VerifyStatus{Code: 0x200} + StatusDiffHashMismatch = VerifyStatus{Code: 0x201, Msg: "verify failed because of blockhash mismatch with diffhash"} + StatusImpossibleFork = VerifyStatus{Code: 0x202, Msg: "verify failed because of impossible fork detected"} + + // StatusUncertain means verify node can't give a certain result of the request. + StatusUncertain = VerifyStatus{Code: 0x300} + StatusBlockTooNew = VerifyStatus{Code: 0x301, Msg: "can’t verify because of block number larger than current height more than 11"} + StatusBlockNewer = VerifyStatus{Code: 0x302, Msg: "can’t verify because of block number larger than current height"} + StatusPossibleFork = VerifyStatus{Code: 0x303, Msg: "can’t verify because of possible fork detected"} + StatusRequestTooBusy = VerifyStatus{Code: 0x304, Msg: "can’t verify because of request too busy"} + + // StatusUnexpectedError is unexpected internal error. + StatusUnexpectedError = VerifyStatus{Code: 0x400, Msg: "can’t verify because of unexpected internal error"} +) + +type VerifyResult struct { + Status VerifyStatus + BlockNumber uint64 + BlockHash common.Hash + Root common.Hash +} + // A BlockNonce is a 64-bit hash which proves (combined with the // mix-hash) that a sufficient amount of computation has been carried // out on a block. @@ -383,7 +417,7 @@ type DiffLayer struct { DiffHash common.Hash } -type extDiffLayer struct { +type ExtDiffLayer struct { BlockHash common.Hash Number uint64 Receipts []*ReceiptForStorage // Receipts are duplicated stored to simplify the logic @@ -395,7 +429,7 @@ type extDiffLayer struct { // DecodeRLP decodes the Ethereum func (d *DiffLayer) DecodeRLP(s *rlp.Stream) error { - var ed extDiffLayer + var ed ExtDiffLayer if err := s.Decode(&ed); err != nil { return err } @@ -415,7 +449,7 @@ func (d *DiffLayer) EncodeRLP(w io.Writer) error { for i, receipt := range d.Receipts { storageReceipts[i] = (*ReceiptForStorage)(receipt) } - return rlp.Encode(w, extDiffLayer{ + return rlp.Encode(w, ExtDiffLayer{ BlockHash: d.BlockHash, Number: d.Number, Receipts: storageReceipts, @@ -443,17 +477,66 @@ type DiffCode struct { Code []byte } +// DiffCodeSlice is used for sort +type DiffCodeSlice []DiffCode + +func (s DiffCodeSlice) Len() int { + return len(s) +} + +func (s DiffCodeSlice) Less(i, j int) bool { + return s[i].Hash.Hex() < s[j].Hash.Hex() +} + +func (s DiffCodeSlice) Swap(i, j int) { + s[i].Hash, s[j].Hash = s[j].Hash, s[i].Hash + s[i].Code, s[j].Code = s[j].Code, s[i].Code +} + type DiffAccount struct { Account common.Address Blob []byte } +// DiffAccountSlice is used for sort +type DiffAccountSlice []DiffAccount + +func (s DiffAccountSlice) Len() int { + return len(s) +} + +func (s DiffAccountSlice) Less(i, j int) bool { + return s[i].Account.Hex() < s[j].Account.Hex() +} + +func (s DiffAccountSlice) Swap(i, j int) { + s[i].Account, s[j].Account = s[j].Account, s[i].Account + s[i].Blob, s[j].Blob = s[j].Blob, s[i].Blob +} + type DiffStorage struct { Account common.Address Keys []string Vals [][]byte } +// DiffStorageSlice is used for sort +type DiffStorageSlice []DiffStorage + +func (s DiffStorageSlice) Len() int { + return len(s) +} + +func (s DiffStorageSlice) Less(i, j int) bool { + return s[i].Account.Hex() < s[j].Account.Hex() +} + +func (s DiffStorageSlice) Swap(i, j int) { + s[i].Account, s[j].Account = s[j].Account, s[i].Account + s[i].Keys, s[j].Keys = s[j].Keys, s[i].Keys + s[i].Vals, s[j].Vals = s[j].Vals, s[i].Vals +} + type DiffAccountsInTx struct { TxHash common.Hash Accounts map[common.Address]*big.Int diff --git a/eth/backend.go b/eth/backend.go index 56fd1234a3..873980fc14 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -26,6 +26,8 @@ import ( "sync/atomic" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -551,6 +553,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { } // diff protocol can still open without snap protocol protos = append(protos, diff.MakeProtocols((*diffHandler)(s.handler), s.snapDialCandidates)...) + protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.snapDialCandidates)...) return protos } diff --git a/eth/handler.go b/eth/handler.go index cbc6eca809..e47d3eee8d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -24,6 +24,8 @@ import ( "sync/atomic" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -269,6 +271,11 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Error("Diff extension barrier failed", "err", err) return err } + trust, err := h.peers.waitTrustExtension(peer) + if err != nil { + peer.Log().Error("Trust extension barrier failed", "err", err) + return err + } // TODO(karalabe): Not sure why this is needed if !h.chainSync.handlePeerEvent(peer) { return p2p.DiscQuitting @@ -309,7 +316,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Debug("Ethereum peer connected", "name", peer.Name()) // Register the peer locally - if err := h.peers.registerPeer(peer, snap, diff); err != nil { + if err := h.peers.registerPeer(peer, snap, diff, trust); err != nil { peer.Log().Error("Ethereum peer registration failed", "err", err) return err } @@ -395,6 +402,21 @@ func (h *handler) runDiffExtension(peer *diff.Peer, handler diff.Handler) error return handler(peer) } +// runTrustExtension registers a `trust` peer into the joint eth/trust peerset and +// starts handling inbound messages. As `trust` is only a satellite protocol to +// `eth`, all subsystem registrations and lifecycle management will be done by +// the main `eth` handler to prevent strange races. +func (h *handler) runTrustExtension(peer *trust.Peer, handler trust.Handler) error { + h.peerWG.Add(1) + defer h.peerWG.Done() + + if err := h.peers.registerTrustExtension(peer); err != nil { + peer.Log().Error("Trust extension registration failed", "err", err) + return err + } + return handler(peer) +} + // removePeer unregisters a peer from the downloader and fetchers, removes it from // the set of tracked peers and closes the network connection to it. func (h *handler) removePeer(id string) { diff --git a/eth/handler_trust.go b/eth/handler_trust.go new file mode 100644 index 0000000000..6df630a2e8 --- /dev/null +++ b/eth/handler_trust.go @@ -0,0 +1,43 @@ +package eth + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// trustHandler implements the trust.Backend interface to handle the various network +// packets that are sent as replies or broadcasts. +type trustHandler handler + +func (h *trustHandler) Chain() *core.BlockChain { return h.chain } + +// RunPeer is invoked when a peer joins on the `snap` protocol. +func (h *trustHandler) RunPeer(peer *trust.Peer, hand trust.Handler) error { + return (*handler)(h).runTrustExtension(peer, hand) +} + +// PeerInfo retrieves all known `trust` information about a peer. +func (h *trustHandler) PeerInfo(id enode.ID) interface{} { + if p := h.peers.peer(id.String()); p != nil { + if p.trustExt != nil { + return p.trustExt.info() + } + } + return nil +} + +// Handle is invoked from a peer's message handler when it receives a new remote +// message that the handler couldn't consume and serve itself. +func (h *trustHandler) Handle(peer *trust.Peer, packet trust.Packet) error { + switch packet := packet.(type) { + case *trust.RootResponsePacket: + // TODO: h.bc.VerifyManager().HandleRootResponse(peer.ID(), *packet) + return nil + + default: + return fmt.Errorf("unexpected trust packet type: %T", packet) + } +} diff --git a/eth/peer.go b/eth/peer.go index 2fb6fabf26..4d92f4e78f 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -21,6 +21,8 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -37,8 +39,9 @@ type ethPeerInfo struct { // ethPeer is a wrapper around eth.Peer to maintain a few extra metadata. type ethPeer struct { *eth.Peer - snapExt *snapPeer // Satellite `snap` connection - diffExt *diffPeer + snapExt *snapPeer // Satellite `snap` connection + diffExt *diffPeer + trustExt *trustPeer syncDrop *time.Timer // Connection dropper if `eth` sync progress isn't validated in time snapWait chan struct{} // Notification channel for snap connections @@ -69,6 +72,12 @@ type diffPeerInfo struct { DiffSync bool `json:"diff_sync"` } +// trustPeerInfo represents a short summary of the `trust` sub-protocol metadata known +// about a connected peer. +type trustPeerInfo struct { + Version uint `json:"version"` // Trust protocol version negotiated +} + // snapPeer is a wrapper around snap.Peer to maintain a few extra metadata. type snapPeer struct { *snap.Peer @@ -79,6 +88,11 @@ type diffPeer struct { *diff.Peer } +// trustPeer is a wrapper around trust.Peer to maintain a few extra metadata. +type trustPeer struct { + *trust.Peer +} + // info gathers and returns some `diff` protocol metadata known about a peer. func (p *diffPeer) info() *diffPeerInfo { return &diffPeerInfo{ @@ -93,3 +107,10 @@ func (p *snapPeer) info() *snapPeerInfo { Version: p.Version(), } } + +// info gathers and returns some `trust` protocol metadata known about a peer. +func (p *trustPeer) info() *trustPeerInfo { + return &trustPeerInfo{ + Version: p.Version(), + } +} diff --git a/eth/peerset.go b/eth/peerset.go index 0f5245a05e..dc1d7da45e 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -22,6 +22,8 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/diff" @@ -53,6 +55,10 @@ var ( // errDiffWithoutEth is returned if a peer attempts to connect only on the // diff protocol without advertising the eth main protocol. errDiffWithoutEth = errors.New("peer connected on diff without compatible eth support") + + // errTrustWithoutEth is returned if a peer attempts to connect only on the + // trust protocol without advertising the eth main protocol. + errTrustWithoutEth = errors.New("peer connected on trust without compatible eth support") ) const ( @@ -73,6 +79,9 @@ type peerSet struct { diffWait map[string]chan *diff.Peer // Peers connected on `eth` waiting for their diff extension diffPend map[string]*diff.Peer // Peers connected on the `diff` protocol, but not yet on `eth` + trustWait map[string]chan *trust.Peer // Peers connected on `eth` waiting for their trust extension + trustPend map[string]*trust.Peer // Peers connected on the `trust` protocol, but not yet on `eth` + lock sync.RWMutex closed bool } @@ -80,11 +89,13 @@ type peerSet struct { // newPeerSet creates a new peer set to track the active participants. func newPeerSet() *peerSet { return &peerSet{ - peers: make(map[string]*ethPeer), - snapWait: make(map[string]chan *snap.Peer), - snapPend: make(map[string]*snap.Peer), - diffWait: make(map[string]chan *diff.Peer), - diffPend: make(map[string]*diff.Peer), + peers: make(map[string]*ethPeer), + snapWait: make(map[string]chan *snap.Peer), + snapPend: make(map[string]*snap.Peer), + diffWait: make(map[string]chan *diff.Peer), + diffPend: make(map[string]*diff.Peer), + trustWait: make(map[string]chan *trust.Peer), + trustPend: make(map[string]*trust.Peer), } } @@ -148,6 +159,40 @@ func (ps *peerSet) registerDiffExtension(peer *diff.Peer) error { return nil } +// registerTrustExtension unblocks an already connected `eth` peer waiting for its +// `trust` extension, or if no such peer exists, tracks the extension for the time +// being until the `eth` main protocol starts looking for it. +func (ps *peerSet) registerTrustExtension(peer *trust.Peer) error { + // Reject the peer if it advertises `trust` without `eth` as `trust` is only a + // satellite protocol meaningful with the chain selection of `eth` + if !peer.RunningCap(eth.ProtocolName, eth.ProtocolVersions) { + return errTrustWithoutEth + } + // If the peer isn't verify node, don't register trust extension into eth protocol. + if !peer.VerifyNode() { + return nil + } + // Ensure nobody can double connect + ps.lock.Lock() + defer ps.lock.Unlock() + + id := peer.ID() + if _, ok := ps.peers[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as existing ones + } + if _, ok := ps.trustPend[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as pending ones + } + // Inject the peer into an `eth` counterpart is available, otherwise save for later + if wait, ok := ps.trustWait[id]; ok { + delete(ps.trustWait, id) + wait <- peer + return nil + } + ps.trustPend[id] = peer + return nil +} + // waitExtensions blocks until all satellite protocols are connected and tracked // by the peerset. func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { @@ -234,6 +279,53 @@ func (ps *peerSet) waitDiffExtension(peer *eth.Peer) (*diff.Peer, error) { } } +// waitTrustExtension blocks until all satellite protocols are connected and tracked +// by the peerset. +func (ps *peerSet) waitTrustExtension(peer *eth.Peer) (*trust.Peer, error) { + // If the peer does not support a compatible `trust`, don't wait + if !peer.RunningCap(trust.ProtocolName, trust.ProtocolVersions) { + return nil, nil + } + // If the peer isn't verify node, don't register trust extension into eth protocol. + if !peer.VerifyNode() { + return nil, nil + } + // Ensure nobody can double connect + ps.lock.Lock() + + id := peer.ID() + if _, ok := ps.peers[id]; ok { + ps.lock.Unlock() + return nil, errPeerAlreadyRegistered // avoid connections with the same id as existing ones + } + if _, ok := ps.trustWait[id]; ok { + ps.lock.Unlock() + return nil, errPeerAlreadyRegistered // avoid connections with the same id as pending ones + } + // If `trust` already connected, retrieve the peer from the pending set + if trust, ok := ps.trustPend[id]; ok { + delete(ps.trustPend, id) + + ps.lock.Unlock() + return trust, nil + } + // Otherwise wait for `trust` to connect concurrently + wait := make(chan *trust.Peer) + ps.trustWait[id] = wait + ps.lock.Unlock() + + select { + case peer := <-wait: + return peer, nil + + case <-time.After(extensionWaitTimeout): + ps.lock.Lock() + delete(ps.trustWait, id) + ps.lock.Unlock() + return nil, errPeerWaitTimeout + } +} + func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { if p := ps.peer(pid); p != nil && p.diffExt != nil { return p.diffExt @@ -241,9 +333,20 @@ func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { return nil } +// GetVerifyPeers returns an array of verify nodes. +func (ps *peerSet) GetVerifyPeers() []*trustPeer { + res := make([]*trustPeer, 0) + for _, p := range ps.peers { + if p.trustExt != nil { + res = append(res, p.trustExt) + } + } + return res +} + // registerPeer injects a new `eth` peer into the working set, or returns an error // if the peer is already known. -func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer, diffExt *diff.Peer) error { +func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer, diffExt *diff.Peer, trustExt *trust.Peer) error { // Start tracking the new peer ps.lock.Lock() defer ps.lock.Unlock() @@ -265,6 +368,9 @@ func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer, diffExt *diff.Pe if diffExt != nil { eth.diffExt = &diffPeer{diffExt} } + if trustExt != nil { + eth.trustExt = &trustPeer{trustExt} + } ps.peers[id] = eth return nil } diff --git a/eth/protocols/trust/discovery.go b/eth/protocols/trust/discovery.go new file mode 100644 index 0000000000..ce38ec5ed9 --- /dev/null +++ b/eth/protocols/trust/discovery.go @@ -0,0 +1,14 @@ +package trust + +import "github.com/ethereum/go-ethereum/rlp" + +// enrEntry is the ENR entry which advertises `trust` protocol on the discovery. +type enrEntry struct { + // Ignore additional fields (for forward compatibility). + Rest []rlp.RawValue `rlp:"tail"` +} + +// ENRKey implements enr.Entry. +func (e enrEntry) ENRKey() string { + return "trust" +} diff --git a/eth/protocols/trust/handler.go b/eth/protocols/trust/handler.go new file mode 100644 index 0000000000..ce8fc4cd8d --- /dev/null +++ b/eth/protocols/trust/handler.go @@ -0,0 +1,159 @@ +package trust + +import ( + "fmt" + "time" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" +) + +// Handler is a callback to invoke from an outside runner after the boilerplate +// exchanges have passed. +type Handler func(peer *Peer) error + +type Backend interface { + // Chain retrieves the blockchain object to serve data. + Chain() *core.BlockChain + + // RunPeer is invoked when a peer joins on the `eth` protocol. The handler + // should do any peer maintenance work, handshakes and validations. If all + // is passed, control should be given back to the `handler` to process the + // inbound messages going forward. + RunPeer(peer *Peer, handler Handler) error + + PeerInfo(id enode.ID) interface{} + + Handle(peer *Peer, packet Packet) error +} + +// MakeProtocols constructs the P2P protocol definitions for `trust`. +func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { + // Filter the discovery iterator for nodes advertising trust support. + dnsdisc = enode.Filter(dnsdisc, func(n *enode.Node) bool { + var trust enrEntry + return n.Load(&trust) == nil + }) + + protocols := make([]p2p.Protocol, len(ProtocolVersions)) + for i, version := range ProtocolVersions { + version := version // Closure + + protocols[i] = p2p.Protocol{ + Name: ProtocolName, + Version: version, + Length: protocolLengths[version], + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + return backend.RunPeer(NewPeer(version, p, rw), func(peer *Peer) error { + defer peer.Close() + return Handle(backend, peer) + }) + }, + NodeInfo: func() interface{} { + return nodeInfo(backend.Chain()) + }, + PeerInfo: func(id enode.ID) interface{} { + return backend.PeerInfo(id) + }, + Attributes: []enr.Entry{&enrEntry{}}, + DialCandidates: dnsdisc, + } + } + return protocols +} + +// Handle is the callback invoked to manage the life cycle of a `trust` peer. +// When this function terminates, the peer is disconnected. +func Handle(backend Backend, peer *Peer) error { + for { + if err := handleMessage(backend, peer); err != nil { + peer.Log().Debug("Message handling failed in `trust`", "err", err) + return err + } + } +} + +// handleMessage is invoked whenever an inbound message is received from a +// remote peer on the `diff` protocol. The remote connection is torn down upon +// returning any error. +func handleMessage(backend Backend, peer *Peer) error { + // Read the next message from the remote peer, and ensure it's fully consumed + msg, err := peer.rw.ReadMsg() + if err != nil { + return err + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + defer msg.Discard() + + // Track the amount of time it takes to serve the request and run the handler + if metrics.Enabled { + h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) + defer func(start time.Time) { + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) + } + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) + }(time.Now()) + } + // Handle the message depending on its contents + switch { + case msg.Code == RequestRootMsg: + return handleRootRequest(backend, msg, peer) + + case msg.Code == RespondRootMsg: + return handleRootResponse(backend, msg, peer) + + default: + return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) + } +} + +type Decoder interface { + Decode(val interface{}) error + Time() time.Time +} + +func handleRootRequest(backend Backend, msg Decoder, peer *Peer) error { + req := new(RootRequestPacket) + if err := msg.Decode(req); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + + res, err := backend.Chain().GetRootByDiffHash(req.BlockNumber, req.BlockHash, req.DiffHash) + p2p.Send(peer.rw, RespondRootMsg, RootResponsePacket{ + RequestId: req.RequestId, + Status: res.Status, + BlockNumber: req.BlockNumber, + BlockHash: req.BlockHash, + Root: res.Root, + Extra: defaultExtra, + }) + + return err +} + +func handleRootResponse(backend Backend, msg Decoder, peer *Peer) error { + res := new(RootResponsePacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + + requestTracker.Fulfil(peer.id, peer.version, RespondRootMsg, res.RequestId) + return backend.Handle(peer, res) +} + +// NodeInfo represents a short summary of the `trust` sub-protocol metadata +// known about the host peer. +type NodeInfo struct{} + +// nodeInfo retrieves some `trust` protocol metadata about the running host node. +func nodeInfo(chain *core.BlockChain) *NodeInfo { + return &NodeInfo{} +} diff --git a/eth/protocols/trust/peer.go b/eth/protocols/trust/peer.go new file mode 100644 index 0000000000..18ba229914 --- /dev/null +++ b/eth/protocols/trust/peer.go @@ -0,0 +1,66 @@ +package trust + +import ( + "math/rand" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" +) + +// Peer is a collection of relevant information we have about a `trust` peer. +type Peer struct { + id string // Unique ID for the peer, cached + + *p2p.Peer // The embedded P2P package peer + rw p2p.MsgReadWriter // Input/output streams for diff + version uint // Protocol version negotiated + logger log.Logger // Contextual logger with the peer id injected +} + +// NewPeer create a wrapper for a network connection and negotiated protocol +// version. +func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { + id := p.ID().String() + peer := &Peer{ + id: id, + Peer: p, + rw: rw, + version: version, + logger: log.New("peer", id[:8]), + } + return peer +} + +// ID retrieves the peer's unique identifier. +func (p *Peer) ID() string { + return p.id +} + +// Version retrieves the peer's negoatiated `diff` protocol version. +func (p *Peer) Version() uint { + return p.version +} + +// Log overrides the P2P logget with the higher level one containing only the id. +func (p *Peer) Log() log.Logger { + return p.logger +} + +// Close signals the broadcast goroutine to terminate. Only ever call this if +// you created the peer yourself via NewPeer. Otherwise let whoever created it +// clean it up! +func (p *Peer) Close() { +} + +func (p *Peer) RequestRoot(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) error { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, RequestRootMsg, RespondRootMsg, id) + return p2p.Send(p.rw, RequestRootMsg, RootRequestPacket{ + RequestId: id, + BlockNumber: blockNumber, + BlockHash: blockHash, + DiffHash: diffHash, + }) +} diff --git a/eth/protocols/trust/protocol.go b/eth/protocols/trust/protocol.go new file mode 100644 index 0000000000..2bd7fecc15 --- /dev/null +++ b/eth/protocols/trust/protocol.go @@ -0,0 +1,72 @@ +package trust + +import ( + "errors" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +// Constants to match up protocol versions and messages +const ( + Trust1 = 1 +) + +// ProtocolName is the official short name of the `trust` protocol used during +// devp2p capability negotiation. +const ProtocolName = "trust" + +// ProtocolVersions are the supported versions of the `trust` protocol (first +// is primary). +var ProtocolVersions = []uint{Trust1} + +// protocolLengths are the number of implemented message corresponding to +// different protocol versions. +var protocolLengths = map[uint]uint64{Trust1: 2} + +// maxMessageSize is the maximum cap on the size of a protocol message. +const maxMessageSize = 10 * 1024 * 1024 + +const ( + RequestRootMsg = 0x00 + RespondRootMsg = 0x01 +) + +var defaultExtra = []byte{0x00} + +var ( + errMsgTooLarge = errors.New("message too long") + errDecode = errors.New("invalid message") + errInvalidMsgCode = errors.New("invalid message code") + errUnexpectedMsg = errors.New("unexpected message code") +) + +// Packet represents a p2p message in the `trust` protocol. +type Packet interface { + Name() string // Name returns a string corresponding to the message type. + Kind() byte // Kind returns the message type. +} + +type RootRequestPacket struct { + RequestId uint64 + BlockNumber uint64 + BlockHash common.Hash + DiffHash common.Hash +} + +type RootResponsePacket struct { + RequestId uint64 + Status types.VerifyStatus + BlockNumber uint64 + BlockHash common.Hash + Root common.Hash + Extra rlp.RawValue // for extension +} + +func (*RootRequestPacket) Name() string { return "RequestRoot" } +func (*RootRequestPacket) Kind() byte { return RequestRootMsg } + +func (*RootResponsePacket) Name() string { return "RootResponse" } +func (*RootResponsePacket) Kind() byte { return RespondRootMsg } diff --git a/eth/protocols/trust/tracker.go b/eth/protocols/trust/tracker.go new file mode 100644 index 0000000000..ab492b3fb8 --- /dev/null +++ b/eth/protocols/trust/tracker.go @@ -0,0 +1,10 @@ +package trust + +import ( + "time" + + "github.com/ethereum/go-ethereum/p2p/tracker" +) + +// requestTracker is a singleton tracker for request times. +var requestTracker = tracker.New(ProtocolName, time.Minute) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 578b10f09a..395e87fe1d 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -200,6 +200,12 @@ func (ec *Client) GetDiffAccountsWithScope(ctx context.Context, number *big.Int, return &result, err } +func (ec *Client) GetRootByDiffHash(ctx context.Context, blockNr *big.Int, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { + var result types.VerifyResult + err := ec.c.CallContext(ctx, &result, "eth_getRootByDiffHash", toBlockNumArg(blockNr), blockHash, diffHash) + return &result, err +} + type rpcTransaction struct { tx *types.Transaction txExtraInfo diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 091e9e7e82..b01ec8b2e5 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1287,6 +1287,10 @@ func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, bloc return result, err } +func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { + return s.b.Chain().GetRootByDiffHash(uint64(blockNr), blockHash, diffHash) +} + // ExecutionResult groups all structured logs emitted by the EVM // while replaying a transaction in debug mode as well as transaction // execution status, the amount of gas used and the return value diff --git a/p2p/peer.go b/p2p/peer.go index 3b633108db..cdfaf7f21b 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -213,6 +213,11 @@ func (p *Peer) Inbound() bool { return p.rw.is(inboundConn) } +// VerifyNode returns true if the peer is a verification connection +func (p *Peer) VerifyNode() bool { + return p.rw.is(verifyConn) +} + func newPeer(log log.Logger, conn *conn, protocols []Protocol) *Peer { protomap := matchProtocols(protocols, conn.caps, conn) p := &Peer{ diff --git a/p2p/server.go b/p2p/server.go index 2a38550abf..01e56936ab 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -114,6 +114,10 @@ type Config struct { // maintained and re-connected on disconnects. StaticNodes []*enode.Node + // Verify nodes are used as pre-configured connections which are always + // maintained and re-connected on disconnects. + VerifyNodes []*enode.Node + // Trusted nodes are used as pre-configured connections which are always // allowed to connect, even above the peer limit. TrustedNodes []*enode.Node @@ -218,6 +222,7 @@ const ( staticDialedConn inboundConn trustedConn + verifyConn ) // conn wraps a network connection with information gathered @@ -269,6 +274,9 @@ func (f connFlag) String() string { if f&inboundConn != 0 { s += "-inbound" } + if f&verifyConn != 0 { + s += "-verify" + } if s != "" { s = s[1:] } @@ -649,6 +657,9 @@ func (srv *Server) setupDialScheduler() { for _, n := range srv.StaticNodes { srv.dialsched.addStatic(n) } + for _, n := range srv.VerifyNodes { + srv.dialsched.addStatic(n) + } } func (srv *Server) maxInboundConns() int { @@ -934,6 +945,13 @@ func (srv *Server) checkInboundConn(remoteIP net.IP) error { // as a peer. It returns when the connection has been added as a peer // or the handshakes have failed. func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) error { + // If dialDest is verify node, set verifyConn flags. + for _, n := range srv.VerifyNodes { + if dialDest == n { + flags |= verifyConn + } + } + c := &conn{fd: fd, flags: flags, cont: make(chan error)} if dialDest == nil { c.transport = srv.newTransport(fd, nil) From 66dd9ea7fc11c2dae0a2d62e192335900e28d3d0 Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Thu, 20 Jan 2022 15:22:06 +0800 Subject: [PATCH 03/51] testcases for getting root by diff hash --- core/blockchain_diff_test.go | 160 +++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index 451a966589..c213eb157f 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -484,3 +484,163 @@ func TestGetDiffAccounts(t *testing.T) { } } } + +// newTwoForkedBlockchains returns two blockchains, these two chains are generated by different +// generators, they have some same parent blocks, the number of same blocks are determined by +// testBlocks, once chain1 inserted a non-default block, chain1 and chain2 get forked. +func newTwoForkedBlockchains(len1, len2 int) (chain1 *BlockChain, chain2 *BlockChain) { + signer := types.HomesteadSigner{} + // Create a database pre-initialize with a genesis block + db1 := rawdb.NewMemoryDatabase() + db1.SetDiffStore(memorydb.New()) + (&Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db1) + + chain1, _ = NewBlockChain(db1, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000)) + generator1 := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + + for idx, testBlock := range testBlocks { + // Specific block setting, the index in this generator has 1 diff from specified blockNr. + if i+1 == testBlock.blockNr { + for _, testTransaction := range testBlock.txs { + var transaction *types.Transaction + if testTransaction.to == nil { + transaction = types.NewContractCreation(block.TxNonce(testAddr), + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } else { + transaction = types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } + tx, err := types.SignTx(transaction, signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain1, tx) + } + break + } + + // Default block setting. + if idx == len(testBlocks)-1 { + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain1, tx) + } + } + } + + } + bs1, _ := GenerateChain(params.TestChainConfig, chain1.Genesis(), ethash.NewFaker(), db1, len1, generator1) + if _, err := chain1.InsertChain(bs1); err != nil { + panic(err) + } + + // Create a database pre-initialize with a genesis block + db2 := rawdb.NewMemoryDatabase() + db2.SetDiffStore(memorydb.New()) + (&Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db2) + chain2, _ = NewBlockChain(db2, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000)) + generator2 := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain1, tx) + } + } + bs2, _ := GenerateChain(params.TestChainConfig, chain2.Genesis(), ethash.NewFaker(), db2, len2, generator2) + if _, err := chain2.InsertChain(bs2); err != nil { + panic(err) + } + + return chain1, chain2 +} + +func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber uint64, status types.VerifyStatus) { + block2 := chain2.GetBlockByNumber(blockNumber) + if block2 == nil { + t.Fatalf("failed to find block, number: %v", blockNumber) + } + expect := types.VerifyResult{ + Status: status, + BlockNumber: blockNumber, + BlockHash: block2.Hash(), + } + if status.Code&0xff00 == types.StatusVerified.Code { + expect.Root = block2.Root() + } + + diffLayer2 := chain2.GetTrustedDiffLayer(block2.Hash()) + if diffLayer2 == nil { + t.Fatal("failed to find diff layer") + } + diffHash2 := types.EmptyRootHash + if status != types.StatusDiffHashMismatch { + var err error + diffHash2, err = GetTrustedDiffHash(diffLayer2) + if err != nil { + t.Fatalf("failed to compute diff hash: %v", err) + } + } + + if status == types.StatusUntrustedVerified { + block1 := chain1.GetBlockByNumber(blockNumber) + if block1 == nil { + t.Fatalf("failed to find block, number: %v", blockNumber) + } + chain1.diffLayerCache.Remove(block1.Hash()) + } + + result, _ := chain1.GetRootByDiffHash(blockNumber, block2.Hash(), diffHash2) + if result.Status != expect.Status { + t.Fatalf("failed to verify block, number: %v, expect status: %v, real status: %v", blockNumber, expect.Status, result.Status) + } + if result.Root != expect.Root { + t.Fatalf("failed to verify block, number: %v, expect root: %v, real root: %v", blockNumber, expect.Root, result.Root) + } +} + +func TestGetRootByDiffHash(t *testing.T) { + len1 := 23 // length of blockchain1 + len2 := 35 // length of blockchain2 + plen := 11 // length of same parent blocks, which determined by testBlocks. + + chain1, chain2 := newTwoForkedBlockchains(len1, len2) + defer chain1.Stop() + defer chain2.Stop() + + hash1 := chain1.GetBlockByNumber(uint64(plen)).Hash() + hash2 := chain2.GetBlockByNumber(uint64(plen)).Hash() + if hash1 != hash2 { + t.Errorf("chain content mismatch at %d: have hash %v, want hash %v", plen, hash2, hash1) + } + + testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusFullVerified) + testGetRootByDiffHash(t, chain1, chain2, 2, types.StatusUntrustedVerified) + testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusDiffHashMismatch) + testGetRootByDiffHash(t, chain1, chain2, 12, types.StatusImpossibleFork) + testGetRootByDiffHash(t, chain1, chain2, 20, types.StatusPossibleFork) + testGetRootByDiffHash(t, chain1, chain2, 24, types.StatusBlockNewer) + testGetRootByDiffHash(t, chain1, chain2, 35, types.StatusBlockTooNew) +} From 6306002822f69e806207d3e3165bfd60df71720d Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Fri, 21 Jan 2022 18:17:06 +0800 Subject: [PATCH 04/51] generate diff layer by replaying block --- core/blockchain.go | 60 ++++++++++++++++++ core/blockchain_diff_test.go | 117 +++++++++++++++++++++++++++++++++++ core/state/statedb.go | 75 ++++++++++++++++++++++ 3 files changed, 252 insertions(+) diff --git a/core/blockchain.go b/core/blockchain.go index ad9b7db8a7..20c47294bb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -3105,6 +3105,66 @@ func (bc *BlockChain) GetTrustedDiffLayer(blockHash common.Hash) *types.DiffLaye return diff } +// GenerateDiffLayer generates DiffLayer of a specified block by replaying the block's transactions. +// If the block is an empty block, no DiffLayer will be generated. +// The generated DiffLayer whose Receipts are empty, whose DiffAccounts' storage root is empty. +func (bc *BlockChain) GenerateDiffLayer(blockHash common.Hash) (*types.DiffLayer, error) { + if bc.snaps == nil { + return nil, fmt.Errorf("snapshot disabled, can't generate difflayer") + } + + block := bc.GetBlockByHash(blockHash) + if block == nil { + return nil, fmt.Errorf("block not found, block number: %d, blockhash: %v", block.NumberU64(), blockHash) + } + + parent := bc.GetBlockByHash(block.ParentHash()) + if parent == nil { + return nil, fmt.Errorf("block not found, block number: %d, blockhash: %v", block.NumberU64()-1, block.ParentHash()) + } + statedb, err := bc.StateAt(parent.Root()) + if err != nil { + return nil, fmt.Errorf("state not found for block number (%d): %v", parent.NumberU64(), err) + } + + // Empty block, no DiffLayer would be generated. + if block.Header().TxHash == types.EmptyRootHash { + return nil, nil + } + + // Replay transactions. + signer := types.MakeSigner(bc.Config(), block.Number()) + for _, tx := range block.Transactions() { + msg, _ := tx.AsMessage(signer) + txContext := NewEVMTxContext(msg) + context := NewEVMBlockContext(block.Header(), bc, nil) + vmenv := vm.NewEVM(context, txContext, statedb, bc.Config(), vm.Config{}) + + if posa, ok := bc.Engine().(consensus.PoSA); ok { + if isSystem, _ := posa.IsSystemTransaction(tx, block.Header()); isSystem { + balance := statedb.GetBalance(consensus.SystemAddress) + if balance.Cmp(common.Big0) > 0 { + statedb.SetBalance(consensus.SystemAddress, big.NewInt(0)) + statedb.AddBalance(block.Header().Coinbase, balance) + } + } + } + + if _, err := ApplyMessage(vmenv, msg, new(GasPool).AddGas(tx.Gas())); err != nil { + return nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + } + statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + } + + diffLayer := statedb.GenerateDiffLayer() + if diffLayer != nil { + diffLayer.BlockHash = blockHash + diffLayer.Number = block.NumberU64() + } + + return diffLayer, nil +} + func GetTrustedDiffHash(d *types.DiffLayer) (common.Hash, error) { diff := &types.ExtDiffLayer{ BlockHash: d.BlockHash, diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index c213eb157f..df85141a8c 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/consensus/clique" + "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" @@ -644,3 +646,118 @@ func TestGetRootByDiffHash(t *testing.T) { testGetRootByDiffHash(t, chain1, chain2, 24, types.StatusBlockNewer) testGetRootByDiffHash(t, chain1, chain2, 35, types.StatusBlockTooNew) } + +func newBlockChainWithCliqueEngine(blocks int) *BlockChain { + signer := types.HomesteadSigner{} + db := rawdb.NewMemoryDatabase() + engine := clique.New(params.AllCliqueProtocolChanges.Clique, db) + genspec := &Genesis{ + //Config: params.TestChainConfig, + ExtraData: make([]byte, 32+common.AddressLength+65), + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + } + copy(genspec.ExtraData[32:], testAddr[:]) + genesis := genspec.MustCommit(db) + + chain, _ := NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + generator := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + // block.SetCoinbase(testAddr) + block.SetDifficulty(big.NewInt(2)) + + for idx, testBlock := range testBlocks { + // Specific block setting, the index in this generator has 1 diff from specified blockNr. + if i+1 == testBlock.blockNr { + for _, testTransaction := range testBlock.txs { + var transaction *types.Transaction + if testTransaction.to == nil { + transaction = types.NewContractCreation(block.TxNonce(testAddr), + testTransaction.value, uint64(commonGas), nil, testTransaction.data) + } else { + transaction = types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), nil, testTransaction.data) + } + tx, err := types.SignTx(transaction, signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + break + } + + // Default block setting. + if idx == len(testBlocks)-1 { + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), nil, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + } + } + + } + bs, _ := GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, blocks, generator) + for i, block := range bs { + header := block.Header() + if i > 0 { + header.ParentHash = bs[i-1].Hash() + } + header.Extra = make([]byte, 32+65) + header.Difficulty = big.NewInt(2) + + sig, _ := crypto.Sign(clique.SealHash(header).Bytes(), testKey) + copy(header.Extra[len(header.Extra)-65:], sig) + bs[i] = block.WithSeal(header) + } + + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + + return chain +} + +func TestGenerateDiffLayer(t *testing.T) { + blockNum := 32 + chain := newBlockChainWithCliqueEngine(blockNum) + defer chain.Stop() + + for blockNr := 1; blockNr <= blockNum; blockNr++ { + block := chain.GetBlockByNumber(uint64(blockNr)) + if block == nil { + t.Fatal("block should not be nil") + } + + expDiffLayer := chain.GetTrustedDiffLayer(block.Hash()) + if expDiffLayer == nil { + // Skip empty block. + if blockNr == 15 { + continue + } + t.Fatalf("unexpected nil diff layer, block number: %v, block hash: %v", blockNr, block.Hash()) + } + expDiffHash, err := GetTrustedDiffHash(expDiffLayer) + if err != nil { + t.Fatalf("compute diff hash failed: %v", err) + } + + diffLayer, err := chain.GenerateDiffLayer(block.Hash()) + if err != nil || diffLayer == nil { + t.Fatalf("generate diff layer failed: %v", err) + } + diffHash, err := GetTrustedDiffHash(diffLayer) + if err != nil { + t.Fatalf("compute diff hash failed: %v", err) + } + if expDiffHash != diffHash { + t.Fatalf("generated wrong diff layer for block: %d, expected hash: %v, real hash: %v", blockNr, expDiffHash, diffHash) + } + } +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 64d06a721a..f05de2ceb1 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1401,6 +1401,81 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, *types.DiffLayer return root, diffLayer, nil } +// GenerateDiffLayer generates block's DiffLayer after executing the block's txs. +// Attention, the DiffLayer returned include no Receipts, whose accounts' storage root +// is empty, whose BlockHash and Number field is empty, should further process by caller. +func (s *StateDB) GenerateDiffLayer() *types.DiffLayer { + if s.snap == nil { + return nil + } + + for addr := range s.stateObjectsPending { + if obj := s.stateObjects[addr]; !obj.deleted { + // The snapshot storage map for the object + var storage map[string][]byte + obj.finalise(false) + for key, value := range obj.pendingStorage { + // Skip noop changes, persist actual changes + if value == obj.originStorage[key] { + continue + } + obj.originStorage[key] = value + + var v []byte + if (value != common.Hash{}) { + // Encoding []byte cannot fail, ok to ignore the error. + v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) + } + + obj.db.snapMux.Lock() + if storage == nil { + // Retrieve the old storage map, if available, create a new one otherwise + if storage = obj.db.snapStorage[obj.address]; storage == nil { + storage = make(map[string][]byte) + obj.db.snapStorage[obj.address] = storage + } + } + storage[string(key[:])] = v // v will be nil if value is 0x00 + obj.db.snapMux.Unlock() + } + + if !obj.deleted { + s.snapMux.Lock() + // The storage root hasn't been intermediate, pass empty storage root here. + s.snapAccounts[obj.address] = snapshot.SlimAccountRLP(obj.data.Nonce, obj.data.Balance, common.Hash{}, obj.data.CodeHash) + s.snapMux.Unlock() + } + } + } + + var diffLayer = &types.DiffLayer{} + for addr := range s.stateObjectsDirty { + if obj := s.stateObjects[addr]; !obj.deleted { + if obj.code != nil && obj.dirtyCode { + diffLayer.Codes = append(diffLayer.Codes, types.DiffCode{ + Hash: common.BytesToHash(obj.CodeHash()), + Code: obj.code, + }) + } + } + } + + diffLayer.Destructs, diffLayer.Accounts, diffLayer.Storages = s.SnapToDiffLayer() + sort.SliceStable(diffLayer.Codes, func(i, j int) bool { + return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() + }) + sort.SliceStable(diffLayer.Destructs, func(i, j int) bool { + return diffLayer.Destructs[i].Hex() < (diffLayer.Destructs[j].Hex()) + }) + sort.SliceStable(diffLayer.Accounts, func(i, j int) bool { + return diffLayer.Accounts[i].Account.Hex() < diffLayer.Accounts[j].Account.Hex() + }) + sort.SliceStable(diffLayer.Storages, func(i, j int) bool { + return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() + }) + return diffLayer +} + func (s *StateDB) DiffLayerToSnap(diffLayer *types.DiffLayer) (map[common.Address]struct{}, map[common.Address][]byte, map[common.Address]map[string][]byte, error) { snapDestructs := make(map[common.Address]struct{}) snapAccounts := make(map[common.Address][]byte) From 0b296e8a4990f8b798365cb4695821ef5a82650d Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Tue, 25 Jan 2022 21:18:56 +0800 Subject: [PATCH 05/51] fix misc bugs of verify node --- cmd/utils/flags.go | 3 +- common/types.go | 15 ----- core/blockchain.go | 106 +++++++++++++++++++-------------- core/blockchain_diff_test.go | 14 ++--- core/state_processor.go | 2 +- core/types/block.go | 62 ++----------------- eth/backend.go | 18 ++++-- eth/ethconfig/config.go | 5 +- eth/ethconfig/gen_config.go | 6 ++ eth/peerset.go | 3 + eth/protocols/diff/protocol.go | 2 +- eth/protocols/trust/handler.go | 6 +- internal/ethapi/api.go | 2 +- 13 files changed, 105 insertions(+), 139 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 9738ec767f..ae9b6eea32 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1705,7 +1705,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name) } if ctx.GlobalIsSet(NoDiscoverFlag.Name) { - cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs = []string{}, []string{} + cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs, cfg.TrustDiscoveryURLs = []string{}, []string{}, []string{} } else if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) { urls := ctx.GlobalString(DNSDiscoveryFlag.Name) if urls == "" { @@ -1811,6 +1811,7 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) { if url := params.KnownDNSNetwork(genesis, protocol); url != "" { cfg.EthDiscoveryURLs = []string{url} cfg.SnapDiscoveryURLs = cfg.EthDiscoveryURLs + cfg.TrustDiscoveryURLs = cfg.EthDiscoveryURLs } } diff --git a/common/types.go b/common/types.go index 15f2bce681..d715356692 100644 --- a/common/types.go +++ b/common/types.go @@ -354,21 +354,6 @@ func (a *Address) UnmarshalGraphQL(input interface{}) error { return err } -// AddressSlice is used for sort -type AddressSlice []Address - -func (s AddressSlice) Len() int { - return len(s) -} - -func (s AddressSlice) Less(i, j int) bool { - return s[i].Hex() < s[j].Hex() -} - -func (s AddressSlice) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - // UnprefixedAddress allows marshaling an Address without 0x prefix. type UnprefixedAddress Address diff --git a/core/blockchain.go b/core/blockchain.go index 20c47294bb..4aae215508 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -488,7 +488,22 @@ func (bc *BlockChain) cacheReceipts(hash common.Hash, receipts types.Receipts) { bc.receiptsCache.Add(hash, receipts) } -func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer) { +func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, sorted bool) { + if !sorted { + sort.SliceStable(diffLayer.Codes, func(i, j int) bool { + return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() + }) + sort.SliceStable(diffLayer.Destructs, func(i, j int) bool { + return diffLayer.Destructs[i].Hex() < (diffLayer.Destructs[j].Hex()) + }) + sort.SliceStable(diffLayer.Accounts, func(i, j int) bool { + return diffLayer.Accounts[i].Account.Hex() < diffLayer.Accounts[j].Account.Hex() + }) + sort.SliceStable(diffLayer.Storages, func(i, j int) bool { + return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() + }) + } + if bc.diffLayerCache.Len() >= diffLayerCacheLimit { bc.diffLayerCache.RemoveOldest() } @@ -1690,12 +1705,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.BlockHash = block.Hash() diffLayer.Number = block.NumberU64() - sort.Sort(types.DiffCodeSlice(diffLayer.Codes)) - sort.Sort(common.AddressSlice(diffLayer.Destructs)) - sort.Sort(types.DiffAccountSlice(diffLayer.Accounts)) - sort.Sort(types.DiffStorageSlice(diffLayer.Storages)) - - bc.cacheDiffLayer(diffLayer) + go bc.cacheDiffLayer(diffLayer, false) } triedb := bc.stateCache.TrieDB() @@ -2730,9 +2740,14 @@ func (bc *BlockChain) HandleDiffLayer(diffLayer *types.DiffLayer, pid string, fu return nil } + diffHash := common.Hash{} + if diffLayer.DiffHash.Load() != nil { + diffHash = diffLayer.DiffHash.Load().(common.Hash) + } + bc.diffMux.Lock() defer bc.diffMux.Unlock() - if blockHash, exist := bc.diffHashToBlockHash[diffLayer.DiffHash]; exist && blockHash == diffLayer.BlockHash { + if blockHash, exist := bc.diffHashToBlockHash[diffHash]; exist && blockHash == diffLayer.BlockHash { return nil } @@ -2746,28 +2761,28 @@ func (bc *BlockChain) HandleDiffLayer(diffLayer *types.DiffLayer, pid string, fu return nil } if _, exist := bc.diffPeersToDiffHashes[pid]; exist { - if _, alreadyHas := bc.diffPeersToDiffHashes[pid][diffLayer.DiffHash]; alreadyHas { + if _, alreadyHas := bc.diffPeersToDiffHashes[pid][diffHash]; alreadyHas { return nil } } else { bc.diffPeersToDiffHashes[pid] = make(map[common.Hash]struct{}) } - bc.diffPeersToDiffHashes[pid][diffLayer.DiffHash] = struct{}{} + bc.diffPeersToDiffHashes[pid][diffHash] = struct{}{} if _, exist := bc.diffNumToBlockHashes[diffLayer.Number]; !exist { bc.diffNumToBlockHashes[diffLayer.Number] = make(map[common.Hash]struct{}) } bc.diffNumToBlockHashes[diffLayer.Number][diffLayer.BlockHash] = struct{}{} - if _, exist := bc.diffHashToPeers[diffLayer.DiffHash]; !exist { - bc.diffHashToPeers[diffLayer.DiffHash] = make(map[string]struct{}) + if _, exist := bc.diffHashToPeers[diffHash]; !exist { + bc.diffHashToPeers[diffHash] = make(map[string]struct{}) } - bc.diffHashToPeers[diffLayer.DiffHash][pid] = struct{}{} + bc.diffHashToPeers[diffHash][pid] = struct{}{} if _, exist := bc.blockHashToDiffLayers[diffLayer.BlockHash]; !exist { bc.blockHashToDiffLayers[diffLayer.BlockHash] = make(map[common.Hash]*types.DiffLayer) } - bc.blockHashToDiffLayers[diffLayer.BlockHash][diffLayer.DiffHash] = diffLayer - bc.diffHashToBlockHash[diffLayer.DiffHash] = diffLayer.BlockHash + bc.blockHashToDiffLayers[diffLayer.BlockHash][diffHash] = diffLayer + bc.diffHashToBlockHash[diffHash] = diffLayer.BlockHash return nil } @@ -3035,60 +3050,55 @@ func EnablePersistDiff(limit uint64) BlockChainOption { } } -func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { +func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) *types.VerifyResult { var res types.VerifyResult res.BlockNumber = blockNumber res.BlockHash = blockHash - if blockNumber > bc.CurrentHeader().Number.Uint64()+11 { + if blockNumber > bc.CurrentHeader().Number.Uint64()+maxDiffForkDist { res.Status = types.StatusBlockTooNew - return &res, nil + return &res } else if blockNumber > bc.CurrentHeader().Number.Uint64() { res.Status = types.StatusBlockNewer - return &res, nil + return &res + } + + header := bc.GetHeaderByHash(blockHash) + if header == nil { + if blockNumber > bc.CurrentHeader().Number.Uint64()-maxDiffForkDist { + res.Status = types.StatusPossibleFork + return &res + } + + res.Status = types.StatusImpossibleFork + return &res } diff := bc.GetTrustedDiffLayer(blockHash) if diff != nil { - if diff.DiffHash == (common.Hash{}) { - hash, err := GetTrustedDiffHash(diff) + if diff.DiffHash.Load() == nil { + hash, err := CalculateDiffHash(diff) if err != nil { res.Status = types.StatusUnexpectedError - return &res, err + return &res } - diff.DiffHash = hash + diff.DiffHash.Store(hash) } - if diffHash != diff.DiffHash { + if diffHash != diff.DiffHash.Load().(common.Hash) { res.Status = types.StatusDiffHashMismatch - return &res, nil + return &res } - header := bc.GetHeaderByHash(blockHash) - if header == nil { - res.Status = types.StatusUnexpectedError - return &res, fmt.Errorf("unexpected error, header not found") - } res.Status = types.StatusFullVerified res.Root = header.Root - return &res, nil - } - - header := bc.GetHeaderByHash(blockHash) - if header == nil { - if blockNumber > bc.CurrentHeader().Number.Uint64()-11 { - res.Status = types.StatusPossibleFork - return &res, nil - } - - res.Status = types.StatusImpossibleFork - return &res, nil + return &res } - res.Status = types.StatusUntrustedVerified + res.Status = types.StatusPartiallyVerified res.Root = header.Root - return &res, nil + return &res } func (bc *BlockChain) GetTrustedDiffLayer(blockHash common.Hash) *types.DiffLayer { @@ -3160,12 +3170,18 @@ func (bc *BlockChain) GenerateDiffLayer(blockHash common.Hash) (*types.DiffLayer if diffLayer != nil { diffLayer.BlockHash = blockHash diffLayer.Number = block.NumberU64() + + bc.cacheDiffLayer(diffLayer, true) } return diffLayer, nil } -func GetTrustedDiffHash(d *types.DiffLayer) (common.Hash, error) { +func CalculateDiffHash(d *types.DiffLayer) (common.Hash, error) { + if d == nil { + return common.Hash{}, fmt.Errorf("nil diff layer") + } + diff := &types.ExtDiffLayer{ BlockHash: d.BlockHash, Receipts: make([]*types.ReceiptForStorage, 0), diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index df85141a8c..3a190ff2f2 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -288,7 +288,7 @@ func rawDataToDiffLayer(data rlp.RawValue) (*types.DiffLayer, error) { hasher.Write(data) var diffHash common.Hash hasher.Sum(diffHash[:0]) - diff.DiffHash = diffHash + diff.DiffHash.Store(diffHash) hasher.Reset() return &diff, nil } @@ -600,13 +600,13 @@ func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber diffHash2 := types.EmptyRootHash if status != types.StatusDiffHashMismatch { var err error - diffHash2, err = GetTrustedDiffHash(diffLayer2) + diffHash2, err = CalculateDiffHash(diffLayer2) if err != nil { t.Fatalf("failed to compute diff hash: %v", err) } } - if status == types.StatusUntrustedVerified { + if status == types.StatusPartiallyVerified { block1 := chain1.GetBlockByNumber(blockNumber) if block1 == nil { t.Fatalf("failed to find block, number: %v", blockNumber) @@ -614,7 +614,7 @@ func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber chain1.diffLayerCache.Remove(block1.Hash()) } - result, _ := chain1.GetRootByDiffHash(blockNumber, block2.Hash(), diffHash2) + result := chain1.GetRootByDiffHash(blockNumber, block2.Hash(), diffHash2) if result.Status != expect.Status { t.Fatalf("failed to verify block, number: %v, expect status: %v, real status: %v", blockNumber, expect.Status, result.Status) } @@ -639,7 +639,7 @@ func TestGetRootByDiffHash(t *testing.T) { } testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusFullVerified) - testGetRootByDiffHash(t, chain1, chain2, 2, types.StatusUntrustedVerified) + testGetRootByDiffHash(t, chain1, chain2, 2, types.StatusPartiallyVerified) testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusDiffHashMismatch) testGetRootByDiffHash(t, chain1, chain2, 12, types.StatusImpossibleFork) testGetRootByDiffHash(t, chain1, chain2, 20, types.StatusPossibleFork) @@ -743,7 +743,7 @@ func TestGenerateDiffLayer(t *testing.T) { } t.Fatalf("unexpected nil diff layer, block number: %v, block hash: %v", blockNr, block.Hash()) } - expDiffHash, err := GetTrustedDiffHash(expDiffLayer) + expDiffHash, err := CalculateDiffHash(expDiffLayer) if err != nil { t.Fatalf("compute diff hash failed: %v", err) } @@ -752,7 +752,7 @@ func TestGenerateDiffLayer(t *testing.T) { if err != nil || diffLayer == nil { t.Fatalf("generate diff layer failed: %v", err) } - diffHash, err := GetTrustedDiffHash(diffLayer) + diffHash, err := CalculateDiffHash(diffLayer) if err != nil { t.Fatalf("compute diff hash failed: %v", err) } diff --git a/core/state_processor.go b/core/state_processor.go index a0037559c9..0569ea5229 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -118,7 +118,7 @@ func (p *LightStateProcessor) Process(block *types.Block, statedb *state.StateDB return statedb, receipts, logs, gasUsed, nil } log.Error("do light process err at block", "num", block.NumberU64(), "err", err) - p.bc.removeDiffLayers(diffLayer.DiffHash) + p.bc.removeDiffLayers(diffLayer.DiffHash.Load().(common.Hash)) // prepare new statedb statedb.StopPrefetcher() parent := p.bc.GetHeader(block.ParentHash(), block.NumberU64()-1) diff --git a/core/types/block.go b/core/types/block.go index f3c487b684..72cf3408b5 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -49,7 +49,7 @@ var ( // StatusVerified means the processing of request going as expected and found the root correctly. StatusVerified = VerifyStatus{Code: 0x100} StatusFullVerified = VerifyStatus{Code: 0x101, Msg: "state root full verified"} - StatusUntrustedVerified = VerifyStatus{Code: 0x102, Msg: "state root untrusted verified, because of difflayer not found"} + StatusPartiallyVerified = VerifyStatus{Code: 0x102, Msg: "state root partially verified, because of difflayer not found"} // StatusFailed means the request has something wrong. StatusFailed = VerifyStatus{Code: 0x200} @@ -57,11 +57,10 @@ var ( StatusImpossibleFork = VerifyStatus{Code: 0x202, Msg: "verify failed because of impossible fork detected"} // StatusUncertain means verify node can't give a certain result of the request. - StatusUncertain = VerifyStatus{Code: 0x300} - StatusBlockTooNew = VerifyStatus{Code: 0x301, Msg: "can’t verify because of block number larger than current height more than 11"} - StatusBlockNewer = VerifyStatus{Code: 0x302, Msg: "can’t verify because of block number larger than current height"} - StatusPossibleFork = VerifyStatus{Code: 0x303, Msg: "can’t verify because of possible fork detected"} - StatusRequestTooBusy = VerifyStatus{Code: 0x304, Msg: "can’t verify because of request too busy"} + StatusUncertain = VerifyStatus{Code: 0x300} + StatusBlockTooNew = VerifyStatus{Code: 0x301, Msg: "can’t verify because of block number larger than current height more than 11"} + StatusBlockNewer = VerifyStatus{Code: 0x302, Msg: "can’t verify because of block number larger than current height"} + StatusPossibleFork = VerifyStatus{Code: 0x303, Msg: "can’t verify because of possible fork detected"} // StatusUnexpectedError is unexpected internal error. StatusUnexpectedError = VerifyStatus{Code: 0x400, Msg: "can’t verify because of unexpected internal error"} @@ -414,7 +413,7 @@ type DiffLayer struct { Accounts []DiffAccount Storages []DiffStorage - DiffHash common.Hash + DiffHash atomic.Value } type ExtDiffLayer struct { @@ -477,66 +476,17 @@ type DiffCode struct { Code []byte } -// DiffCodeSlice is used for sort -type DiffCodeSlice []DiffCode - -func (s DiffCodeSlice) Len() int { - return len(s) -} - -func (s DiffCodeSlice) Less(i, j int) bool { - return s[i].Hash.Hex() < s[j].Hash.Hex() -} - -func (s DiffCodeSlice) Swap(i, j int) { - s[i].Hash, s[j].Hash = s[j].Hash, s[i].Hash - s[i].Code, s[j].Code = s[j].Code, s[i].Code -} - type DiffAccount struct { Account common.Address Blob []byte } -// DiffAccountSlice is used for sort -type DiffAccountSlice []DiffAccount - -func (s DiffAccountSlice) Len() int { - return len(s) -} - -func (s DiffAccountSlice) Less(i, j int) bool { - return s[i].Account.Hex() < s[j].Account.Hex() -} - -func (s DiffAccountSlice) Swap(i, j int) { - s[i].Account, s[j].Account = s[j].Account, s[i].Account - s[i].Blob, s[j].Blob = s[j].Blob, s[i].Blob -} - type DiffStorage struct { Account common.Address Keys []string Vals [][]byte } -// DiffStorageSlice is used for sort -type DiffStorageSlice []DiffStorage - -func (s DiffStorageSlice) Len() int { - return len(s) -} - -func (s DiffStorageSlice) Less(i, j int) bool { - return s[i].Account.Hex() < s[j].Account.Hex() -} - -func (s DiffStorageSlice) Swap(i, j int) { - s[i].Account, s[j].Account = s[j].Account, s[i].Account - s[i].Keys, s[j].Keys = s[j].Keys, s[i].Keys - s[i].Vals, s[j].Vals = s[j].Vals, s[i].Vals -} - type DiffAccountsInTx struct { TxHash common.Hash Accounts map[common.Address]*big.Int diff --git a/eth/backend.go b/eth/backend.go index 873980fc14..96d559911b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -70,11 +70,12 @@ type Ethereum struct { config *ethconfig.Config // Handlers - txPool *core.TxPool - blockchain *core.BlockChain - handler *handler - ethDialCandidates enode.Iterator - snapDialCandidates enode.Iterator + txPool *core.TxPool + blockchain *core.BlockChain + handler *handler + ethDialCandidates enode.Iterator + snapDialCandidates enode.Iterator + trustDialCandidates enode.Iterator // DB interfaces chainDb ethdb.Database // Block chain database @@ -269,6 +270,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + eth.trustDialCandidates, err = dnsclient.NewIterator(eth.config.TrustDiscoveryURLs...) + if err != nil { + return nil, err + } // Start the RPC service eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer, config.NetworkId) @@ -553,7 +558,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { } // diff protocol can still open without snap protocol protos = append(protos, diff.MakeProtocols((*diffHandler)(s.handler), s.snapDialCandidates)...) - protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.snapDialCandidates)...) + protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.trustDialCandidates)...) return protos } @@ -584,6 +589,7 @@ func (s *Ethereum) Stop() error { // Stop all the peer-related stuff first. s.ethDialCandidates.Close() s.snapDialCandidates.Close() + s.trustDialCandidates.Close() s.handler.Stop() // Then stop everything else. diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index d3e146aaaf..f7c9d9f5f9 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -131,8 +131,9 @@ type Config struct { // This can be set to list of enrtree:// URLs which will be queried for // for nodes to connect to. - EthDiscoveryURLs []string - SnapDiscoveryURLs []string + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + TrustDiscoveryURLs []string NoPruning bool // Whether to disable pruning and flush everything to disk DirectBroadcast bool diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index ba2996279d..c5e46ced41 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -24,6 +24,7 @@ func (c Config) MarshalTOML() (interface{}, error) { DisablePeerTxBroadcast bool EthDiscoveryURLs []string SnapDiscoveryURLs []string + TrustDiscoveryURLs []string NoPruning bool NoPrefetch bool DirectBroadcast bool @@ -79,6 +80,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DisablePeerTxBroadcast = c.DisablePeerTxBroadcast enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs + enc.TrustDiscoveryURLs = c.TrustDiscoveryURLs enc.NoPruning = c.NoPruning enc.DirectBroadcast = c.DirectBroadcast enc.DisableSnapProtocol = c.DisableSnapProtocol @@ -137,6 +139,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DisablePeerTxBroadcast *bool EthDiscoveryURLs []string SnapDiscoveryURLs []string + TrustDiscoveryURLs []string NoPruning *bool NoPrefetch *bool DirectBroadcast *bool @@ -207,6 +210,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SnapDiscoveryURLs != nil { c.SnapDiscoveryURLs = dec.SnapDiscoveryURLs } + if dec.TrustDiscoveryURLs != nil { + c.TrustDiscoveryURLs = dec.TrustDiscoveryURLs + } if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } diff --git a/eth/peerset.go b/eth/peerset.go index dc1d7da45e..5bbaa2dd2b 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -335,6 +335,9 @@ func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { // GetVerifyPeers returns an array of verify nodes. func (ps *peerSet) GetVerifyPeers() []*trustPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + res := make([]*trustPeer, 0) for _, p := range ps.peers { if p.trustExt != nil { diff --git a/eth/protocols/diff/protocol.go b/eth/protocols/diff/protocol.go index 4467d0b327..e6bf5b3e14 100644 --- a/eth/protocols/diff/protocol.go +++ b/eth/protocols/diff/protocol.go @@ -92,7 +92,7 @@ func (p *DiffLayersPacket) Unpack() ([]*types.DiffLayer, error) { var diffHash common.Hash hasher.Sum(diffHash[:0]) hasher.Reset() - diff.DiffHash = diffHash + diff.DiffHash.Store(diffHash) } return diffLayers, nil } diff --git a/eth/protocols/trust/handler.go b/eth/protocols/trust/handler.go index ce8fc4cd8d..9b93d4a228 100644 --- a/eth/protocols/trust/handler.go +++ b/eth/protocols/trust/handler.go @@ -126,8 +126,8 @@ func handleRootRequest(backend Backend, msg Decoder, peer *Peer) error { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - res, err := backend.Chain().GetRootByDiffHash(req.BlockNumber, req.BlockHash, req.DiffHash) - p2p.Send(peer.rw, RespondRootMsg, RootResponsePacket{ + res := backend.Chain().GetRootByDiffHash(req.BlockNumber, req.BlockHash, req.DiffHash) + return p2p.Send(peer.rw, RespondRootMsg, RootResponsePacket{ RequestId: req.RequestId, Status: res.Status, BlockNumber: req.BlockNumber, @@ -135,8 +135,6 @@ func handleRootRequest(backend Backend, msg Decoder, peer *Peer) error { Root: res.Root, Extra: defaultExtra, }) - - return err } func handleRootResponse(backend Backend, msg Decoder, peer *Peer) error { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index b01ec8b2e5..f3bcc3b98d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1287,7 +1287,7 @@ func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, bloc return result, err } -func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { +func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) *types.VerifyResult { return s.b.Chain().GetRootByDiffHash(uint64(blockNr), blockHash, diffHash) } From 6fffa8e298c6f93164fa524c115f3b423c361c42 Mon Sep 17 00:00:00 2001 From: KeefeL <90749943+KeefeL@users.noreply.github.com> Date: Thu, 27 Jan 2022 16:50:59 +0800 Subject: [PATCH 06/51] testcases for trust protocol (#742) --- eth/protocols/trust/handler_test.go | 270 ++++++++++++++++++++++++++++ eth/protocols/trust/peer_test.go | 42 +++++ 2 files changed, 312 insertions(+) create mode 100644 eth/protocols/trust/handler_test.go create mode 100644 eth/protocols/trust/peer_test.go diff --git a/eth/protocols/trust/handler_test.go b/eth/protocols/trust/handler_test.go new file mode 100644 index 0000000000..e594401a2c --- /dev/null +++ b/eth/protocols/trust/handler_test.go @@ -0,0 +1,270 @@ +package trust + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" +) + +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) + +// testBackend is a mock implementation of the live Ethereum message handler. Its +// purpose is to allow testing the request/reply workflows and wire serialization +// in the `eth` protocol without actually doing any data processing. +type testBackend struct { + db ethdb.Database + chain *core.BlockChain + txpool *core.TxPool +} + +// newTestBackend creates an empty chain and wraps it into a mock backend. +func newTestBackend(blocks int) *testBackend { + return newTestBackendWithGenerator(blocks) +} + +// newTestBackend creates a chain with a number of explicitly defined blocks and +// wraps it into a mock backend. +func newTestBackendWithGenerator(blocks int) *testBackend { + signer := types.HomesteadSigner{} + db := rawdb.NewMemoryDatabase() + engine := clique.New(params.AllCliqueProtocolChanges.Clique, db) + genspec := &core.Genesis{ + //Config: params.TestChainConfig, + ExtraData: make([]byte, 32+common.AddressLength+65), + Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + } + copy(genspec.ExtraData[32:], testAddr[:]) + genesis := genspec.MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + generator := func(i int, block *core.BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + // block.SetCoinbase(testAddr) + block.SetDifficulty(big.NewInt(2)) + + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), common.Address{0x01}, big.NewInt(1), params.TxGas, nil, nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + + bs, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, blocks, generator) + for i, block := range bs { + header := block.Header() + if i > 0 { + header.ParentHash = bs[i-1].Hash() + } + header.Extra = make([]byte, 32+65) + header.Difficulty = big.NewInt(2) + + sig, _ := crypto.Sign(clique.SealHash(header).Bytes(), testKey) + copy(header.Extra[len(header.Extra)-65:], sig) + bs[i] = block.WithSeal(header) + } + + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + + txconfig := core.DefaultTxPoolConfig + txconfig.Journal = "" // Don't litter the disk with test journals + + return &testBackend{ + db: db, + chain: chain, + txpool: core.NewTxPool(txconfig, params.AllCliqueProtocolChanges, chain), + } +} + +// close tears down the transaction pool and chain behind the mock backend. +func (b *testBackend) close() { + b.txpool.Stop() + b.chain.Stop() +} + +func (b *testBackend) Chain() *core.BlockChain { return b.chain } + +func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { + // Normally the backend would do peer mainentance and handshakes. All that + // is omitted and we will just give control back to the handler. + return handler(peer) +} +func (b *testBackend) PeerInfo(enode.ID) interface{} { panic("not implemented") } + +func (b *testBackend) Handle(*Peer, Packet) error { + panic("data processing tests should be done in the handler package") +} + +func TestRequestRoot(t *testing.T) { testRequestRoot(t, Trust1) } + +func testRequestRoot(t *testing.T, protocol uint) { + t.Parallel() + + blockNum := 1032 // The latest 1024 blocks' DiffLayer will be cached. + backend := newTestBackend(blockNum) + defer backend.close() + + peer, _ := newTestPeer("peer", protocol, backend) + defer peer.close() + + pairs := []struct { + req RootRequestPacket + res RootResponsePacket + }{ + { + req: RootRequestPacket{ + RequestId: 1, + BlockNumber: 1, + }, + res: RootResponsePacket{ + RequestId: 1, + Status: types.StatusPartiallyVerified, + BlockNumber: 1, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 2, + BlockNumber: 128, + }, + res: RootResponsePacket{ + RequestId: 2, + Status: types.StatusFullVerified, + BlockNumber: 128, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 3, + BlockNumber: 128, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 3, + Status: types.StatusImpossibleFork, + BlockNumber: 128, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 4, + BlockNumber: 128, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 4, + Status: types.StatusDiffHashMismatch, + BlockNumber: 128, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 5, + BlockNumber: 1024, + }, + res: RootResponsePacket{ + RequestId: 5, + Status: types.StatusFullVerified, + BlockNumber: 1024, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 6, + BlockNumber: 1024, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 6, + Status: types.StatusPossibleFork, + BlockNumber: 1024, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 7, + BlockNumber: 1033, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 7, + Status: types.StatusBlockNewer, + BlockNumber: 1033, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 8, + BlockNumber: 1044, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 8, + Status: types.StatusBlockTooNew, + BlockNumber: 1044, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + } + + for idx, pair := range pairs { + header := backend.Chain().GetHeaderByNumber(pair.req.BlockNumber) + if header != nil { + if pair.res.Status.Code&0xFF00 == types.StatusVerified.Code { + pair.req.BlockHash = header.Hash() + pair.req.DiffHash, _ = core.CalculateDiffHash(backend.Chain().GetTrustedDiffLayer(header.Hash())) + pair.res.BlockHash = pair.req.BlockHash + pair.res.Root = header.Root + } else if pair.res.Status.Code == types.StatusDiffHashMismatch.Code { + pair.req.BlockHash = header.Hash() + pair.res.BlockHash = pair.req.BlockHash + } + } + + p2p.Send(peer.app, RequestRootMsg, pair.req) + if err := p2p.ExpectMsg(peer.app, RespondRootMsg, pair.res); err != nil { + t.Errorf("test %d: root response not expected: %v", idx, err) + } + } +} diff --git a/eth/protocols/trust/peer_test.go b/eth/protocols/trust/peer_test.go new file mode 100644 index 0000000000..ab229a1b32 --- /dev/null +++ b/eth/protocols/trust/peer_test.go @@ -0,0 +1,42 @@ +package trust + +import ( + "math/rand" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// testPeer is a simulated peer to allow testing direct network calls. +type testPeer struct { + *Peer + + net p2p.MsgReadWriter // Network layer reader/writer to simulate remote messaging + app *p2p.MsgPipeRW // Application layer reader/writer to simulate the local side +} + +// newTestPeer creates a new peer registered at the given data backend. +func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() + + // Start the peer on a new thread + var id enode.ID + rand.Read(id[:]) + + peer := NewPeer(version, p2p.NewPeer(id, name, nil), net) + errc := make(chan error, 1) + go func() { + errc <- backend.RunPeer(peer, func(peer *Peer) error { + return Handle(backend, peer) + }) + }() + return &testPeer{app: app, net: net, Peer: peer}, errc +} + +// close terminates the local side of the peer, notifying the remote protocol +// manager of termination. +func (p *testPeer) close() { + p.Peer.Close() + p.app.Close() +} From 359906c0dc8d734aea197ff7f8ed98da159b5c05 Mon Sep 17 00:00:00 2001 From: RealUncle Date: Thu, 2 Dec 2021 17:40:47 +0800 Subject: [PATCH 07/51] implement the framework of fast node --- cmd/geth/main.go | 1 + cmd/geth/snapshot.go | 76 ++++++++++++++++++++++- cmd/utils/flags.go | 7 +++ core/block_validator.go | 6 -- core/blockchain.go | 13 +++- core/state/database.go | 24 +++++++- core/state/pruner/pruner.go | 104 +++++++++++++++++++++++++++++++- core/state/snapshot/journal.go | 9 ++- core/state/snapshot/snapshot.go | 4 +- core/state/statedb.go | 87 ++++++++++++++++++-------- core/state_processor.go | 3 +- eth/backend.go | 3 +- eth/ethconfig/config.go | 2 + eth/ethconfig/gen_config.go | 69 ++++++++++++++++----- light/trie.go | 8 +++ tests/state_test_util.go | 2 +- trie/database.go | 1 + trie/dummy_trie.go | 96 +++++++++++++++++++++++++++++ 18 files changed, 454 insertions(+), 61 deletions(-) create mode 100644 trie/dummy_trie.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index e0f29bce77..db00cf19e1 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -115,6 +115,7 @@ var ( utils.WhitelistFlag, utils.BloomFilterSizeFlag, utils.TriesInMemoryFlag, + utils.AllowInsecureNoTriesFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 20920a0f94..d829abbc2a 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "encoding/json" "errors" "fmt" "os" @@ -29,13 +30,16 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -129,6 +133,32 @@ geth snapshot verify-state will traverse the whole accounts and storages set based on the specified snapshot and recalculate the root hash of state for verification. In other words, this command does the snapshot to trie conversion. +`, + }, + { + Name: "insecure-prune-all", + Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " + + "who do not need trie storage at all", + ArgsUsage: "", + Action: utils.MigrateFlags(pruneAllState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: ` +will prune all historical trie state data except genesis block. +All trie nodes will be deleted from the database. + +It expects the genesis file as argument. + +WARNING: It's necessary to delete the trie clean cache after the pruning. +If you specify another directory for the trie clean cache via "--cache.trie.journal" +during the use of Geth, please also specify it here for correct deletion. Otherwise +the trie clean cache with default directory will be deleted. `, }, { @@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) { } headHeader := headBlock.Header() //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false) if err != nil { log.Error("snaptree error", "err", err) return nil, err // The relevant snapshot(s) might not exist @@ -362,6 +392,48 @@ func pruneState(ctx *cli.Context) error { return nil } +func pruneAllState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + genesisPath := ctx.Args().First() + if len(genesisPath) == 0 { + utils.Fatalf("Must supply path to genesis JSON file") + } + file, err := os.Open(genesisPath) + if err != nil { + utils.Fatalf("Failed to read genesis file: %v", err) + } + defer file.Close() + + g := new(core.Genesis) + if err := json.NewDecoder(file).Decode(g); err != nil { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + Metrics: metrics.DefaultConfig, + } + + // Load config file. + if err := loadConfig(genesisPath, &cfg); err != nil { + utils.Fatalf("%v", err) + } + g = cfg.Eth.Genesis + } + + chaindb := utils.MakeChainDatabase(ctx, stack, false, false) + pruner, err := pruner.NewAllPruner(chaindb) + if err != nil { + log.Error("Failed to open snapshot tree", "err", err) + return err + } + if err = pruner.PruneAll(g); err != nil { + log.Error("Failed to prune state", "err", err) + return err + } + return nil +} + func verifyState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -372,7 +444,7 @@ func verifyState(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8f5141907f..fbca81a15d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -264,6 +264,10 @@ var ( Usage: "The layer of tries trees that keep in memory", Value: 128, } + AllowInsecureNoTriesFlag = cli.BoolTFlag{ + Name: "allow-insecure-no-tries", + Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`, + } OverrideBerlinFlag = cli.Uint64Flag{ Name: "override.berlin", Usage: "Manually specify Berlin fork-block, overriding the bundled setting", @@ -1666,6 +1670,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(TriesInMemoryFlag.Name) { cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name) } + if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) { + cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name) + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 } diff --git a/core/block_validator.go b/core/block_validator.go index b109c1e54b..3ea6615b61 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -70,12 +70,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } validateFuns := []func() error{ - func() error { - if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { - return ErrKnownBlock - } - return nil - }, func() error { if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) diff --git a/core/blockchain.go b/core/blockchain.go index 6c87ffc708..e286a1e124 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -141,6 +141,7 @@ type CacheConfig struct { SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk TriesInMemory uint64 // How many tries keeps in memory + NoTries bool // Insecure settings. Do not have any tries in databases if enabled. SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -284,6 +285,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Cache: cacheConfig.TrieCleanLimit, Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, + NoTries: cacheConfig.NoTries, }), triesInMemory: cacheConfig.TriesInMemory, quit: make(chan struct{}), @@ -439,7 +441,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer) recover = true } - bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover) + bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.stateCache.NoTries()) } // do options before start any routine for _, option := range options { @@ -1097,6 +1099,9 @@ func (bc *BlockChain) HasState(hash common.Hash) bool { return true } } + if bc.stateCache.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil + } _, err := bc.stateCache.OpenTrie(hash) return err == nil } @@ -1109,6 +1114,9 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { if block == nil { return false } + if bc.stateCache.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(block.Root()) != nil + } return bc.HasState(block.Root()) } @@ -2105,6 +2113,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if err != nil { return it.index, err } + if statedb.NoTrie() { + statedb.SetCurrentRoot(block.Root()) + } bc.updateHighestVerifiedHeader(block.Header()) // Enable prefetching to pull in trie node paths while processing transactions diff --git a/core/state/database.go b/core/state/database.go index 487589324c..dd114dc6ad 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -74,6 +74,9 @@ type Database interface { // Purge cache Purge() + + // NoTries returns whether the database has tries storage. + NoTries() bool } // Trie is a Ethereum Merkle Patricia trie. @@ -134,10 +137,12 @@ func NewDatabase(db ethdb.Database) Database { func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) cc, _ := lru.New(codeCacheSize) + noTries := config != nil && config.NoTries return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: cc, + noTries: noTries, } } @@ -146,6 +151,7 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab cc, _ := lru.New(codeCacheSize) atc, _ := lru.New(accountTrieCacheSize) stc, _ := lru.New(storageTrieCacheSize) + noTries := config != nil && config.NoTries database := &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), @@ -153,8 +159,11 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab codeCache: cc, accountTrieCache: atc, storageTrieCache: stc, + noTries: noTries, + } + if !noTries { + go database.purgeLoop() } - go database.purgeLoop() return database } @@ -164,6 +173,7 @@ type cachingDB struct { codeCache *lru.Cache accountTrieCache *lru.Cache storageTrieCache *lru.Cache + noTries bool } type triePair struct { @@ -187,6 +197,9 @@ func (db *cachingDB) purgeLoop() { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.accountTrieCache != nil { if tr, exist := db.accountTrieCache.Get(root); exist { return tr.(Trie).(*trie.SecureTrie).Copy(), nil @@ -201,6 +214,9 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.storageTrieCache != nil { if tries, exist := db.storageTrieCache.Get(addrHash); exist { triesPairs := tries.([3]*triePair) @@ -246,6 +262,10 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie } } +func (db *cachingDB) NoTries() bool { + return db.noTries +} + func (db *cachingDB) Purge() { if db.storageTrieCache != nil { db.storageTrieCache.Purge() @@ -263,6 +283,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.SecureTrie: return t.Copy() + case *trie.EmptyTrie: + return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 5b070f3afa..6d89d6b78a 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -103,7 +104,7 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize, trie if headBlock == nil { return nil, errors.New("Failed to load head block") } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false, false) if err != nil { return nil, err // The relevant snapshot(s) might not exist } @@ -138,6 +139,105 @@ func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientP } } +func NewAllPruner(db ethdb.Database) (*Pruner, error) { + headBlock := rawdb.ReadHeadBlock(db) + if headBlock == nil { + return nil, errors.New("Failed to load head block") + } + return &Pruner{ + db: db, + }, nil +} + +func (p *Pruner) PruneAll(genesis *core.Genesis) error { + deleteCleanTrieCache(p.trieCachePath) + return pruneAll(p.db, genesis) +} + +func pruneAll(maindb ethdb.Database, g *core.Genesis) error { + var ( + count int + size common.StorageSize + pstart = time.Now() + logged = time.Now() + batch = maindb.NewBatch() + iter = maindb.NewIterator(nil, nil) + ) + start := time.Now() + for iter.Next() { + key := iter.Key() + if len(key) == common.HashLength { + count += 1 + size += common.StorageSize(len(key) + len(iter.Value())) + batch.Delete(key) + + var eta time.Duration // Realistically will never remain uninited + if done := binary.BigEndian.Uint64(key[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8]) + speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + eta = time.Duration(left/speed) * time.Millisecond + } + if time.Since(logged) > 8*time.Second { + log.Info("Pruning state data", "nodes", count, "size", size, + "elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta)) + logged = time.Now() + } + // Recreate the iterator after every batch commit in order + // to allow the underlying compactor to delete the entries. + if batch.ValueSize() >= ethdb.IdealBatchSize { + batch.Write() + batch.Reset() + + iter.Release() + iter = maindb.NewIterator(nil, key) + } + } + } + if batch.ValueSize() > 0 { + batch.Write() + batch.Reset() + } + iter.Release() + log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart))) + + // Start compactions, will remove the deleted data from the disk immediately. + // Note for small pruning, the compaction is skipped. + if count >= rangeCompactionThreshold { + cstart := time.Now() + for b := 0x00; b <= 0xf0; b += 0x10 { + var ( + start = []byte{byte(b)} + end = []byte{byte(b + 0x10)} + ) + if b == 0xf0 { + end = nil + } + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) + if err := maindb.Compact(start, end); err != nil { + log.Error("Database compaction failed", "error", err) + return err + } + } + log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart))) + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(maindb), nil) + for addr, account := range g.Alloc { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } + root := statedb.IntermediateRoot(false) + statedb.Commit(nil) + statedb.Database().TrieDB().Commit(root, true, nil) + log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, middleStateRoots map[common.Hash]struct{}, start time.Time) error { // Delete all stale trie nodes in the disk. With the help of state bloom // the trie nodes(and codes) belong to the active state will be filtered @@ -585,7 +685,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string, tri // - The state HEAD is rewound already because of multiple incomplete `prune-state` // In this case, even the state HEAD is not exactly matched with snapshot, it // still feasible to recover the pruning correctly. - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true, false) if err != nil { return err // The relevant snapshot(s) might not exist } diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 35c69cfd6b..3c18294cc1 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -126,7 +126,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery, withoutTrie bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { @@ -145,6 +145,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root: baseRoot, } snapshot, generator, err := loadAndParseJournal(diskdb, base) + if err != nil { log.Warn("Failed to load new-format journal", "error", err) return nil, false, err @@ -158,6 +159,11 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // which is below the snapshot. In this case the snapshot can be recovered // by re-executing blocks but right now it's unavailable. if head := snapshot.Root(); head != root { + log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) + + if withoutTrie { + return snapshot, false, nil + } // If it's legacy snapshot, or it's new-format snapshot but // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. @@ -168,7 +174,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // the disk layer is always higher than chain head. It can // be eventually recovered when the chain head beyonds the // disk layer. - log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) } // Everything loaded correctly, resume any suspended operations if !generator.Done { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 8ac93f28e4..38f52acced 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -184,7 +184,7 @@ type Tree struct { // store, on a background thread. If the memory layers from the journal is not // continuous with disk layer or the journal is missing, all diffs will be discarded // iff it's in "recovery" mode, otherwise rebuild is mandatory. -func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery bool) (*Tree, error) { +func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery, withoutTrie bool) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ diskdb: diskdb, @@ -197,7 +197,7 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root defer snap.waitBuild() } // Attempt to load a previously persisted snapshot and rebuild one if failed - head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery) + head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery, withoutTrie) if disabled { log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil diff --git a/core/state/statedb.go b/core/state/statedb.go index 5ea84f4032..1bb1db7ba1 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -77,10 +77,12 @@ type StateDB struct { prefetcherLock sync.Mutex prefetcher *triePrefetcher originalRoot common.Hash // The pre-state root, before any changes were made + currentRoot common.Hash // only used when noTrie is true expectedRoot common.Hash // The state root in the block header stateRoot common.Hash // The calculation result of IntermediateRoot trie Trie + noTrie bool hasher crypto.KeccakState diffLayer *types.DiffLayer diffTries map[common.Address]Trie @@ -160,6 +162,7 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, journal: newJournal(), hasher: crypto.NewKeccakState(), } + if sdb.snaps != nil { if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { sdb.snapDestructs = make(map[common.Address]struct{}) @@ -174,6 +177,7 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, if err != nil && (sdb.snap == nil || snapVerified) { return nil, err } + _, sdb.noTrie = tr.(*trie.EmptyTrie) sdb.trie = tr return sdb, nil } @@ -184,6 +188,9 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, func (s *StateDB) StartPrefetcher(namespace string) { s.prefetcherLock.Lock() defer s.prefetcherLock.Unlock() + if s.noTrie { + return + } if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -198,6 +205,9 @@ func (s *StateDB) StartPrefetcher(namespace string) { func (s *StateDB) StopPrefetcher() { s.prefetcherLock.Lock() defer s.prefetcherLock.Unlock() + if s.noTrie { + return + } if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -237,6 +247,14 @@ func (s *StateDB) setError(err error) { } } +func (s *StateDB) NoTrie() bool { + return s.noTrie +} + +func (s *StateDB) SetCurrentRoot(root common.Hash) { + s.currentRoot = root +} + func (s *StateDB) Error() error { return s.dbErr } @@ -549,6 +567,9 @@ func (s *StateDB) Suicide(addr common.Address) bool { // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on updating the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -570,6 +591,9 @@ func (s *StateDB) updateStateObject(obj *StateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on deleting the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -1013,6 +1037,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // It is called in between transactions to get the root hash that // goes into transaction receipts. func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { + // light process is not allowed when there is no trie if s.lightProcessed { s.StopPrefetcher() return s.trie.Hash() @@ -1109,16 +1134,18 @@ func (s *StateDB) StateIntermediateRoot() common.Hash { s.trie = tr } usedAddrs := make([][]byte, 0, len(s.stateObjectsPending)) - for addr := range s.stateObjectsPending { - if obj := s.stateObjects[addr]; obj.deleted { - s.deleteStateObject(obj) - } else { - s.updateStateObject(obj) + if !s.noTrie { + for addr := range s.stateObjectsPending { + if obj := s.stateObjects[addr]; obj.deleted { + s.deleteStateObject(obj) + } else { + s.updateStateObject(obj) + } + usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure + } + if prefetcher != nil { + prefetcher.used(s.originalRoot, usedAddrs) } - usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure - } - if prefetcher != nil { - prefetcher.used(s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) @@ -1127,8 +1154,11 @@ func (s *StateDB) StateIntermediateRoot() common.Hash { if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) } - root := s.trie.Hash() - return root + if s.noTrie { + return s.currentRoot + } else { + return s.trie.Hash() + } } // Prepare sets the current transaction hash and index and block hash which is @@ -1351,8 +1381,10 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er // Write any contract code associated with the state object tasks <- func() { // Write any storage changes in the state object to its storage trie - if err := obj.CommitTrie(s.db); err != nil { - taskResults <- err + if !s.noTrie { + if err := obj.CommitTrie(s.db); err != nil { + taskResults <- err + } } taskResults <- nil } @@ -1371,24 +1403,27 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. - var account Account - root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { + if !s.noTrie { + var account Account + root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { + if err := rlp.DecodeBytes(leaf, &account); err != nil { + return nil + } + if account.Root != emptyRoot { + s.db.TrieDB().Reference(account.Root, parent) + } return nil + }) + if err != nil { + return err } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) + if root != emptyRoot { + s.db.CacheAccount(root, s.trie) } - return nil - }) - if err != nil { - return err - } - if root != emptyRoot { - s.db.CacheAccount(root, s.trie) } + for _, postFunc := range postCommitFuncs { - err = postFunc() + err := postFunc() if err != nil { return err } diff --git a/core/state_processor.go b/core/state_processor.go index 14fe9b4b92..a21d68c504 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -408,6 +408,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // usually do have two tx, one for validator set contract, another for system reward contract. systemTxs := make([]*types.Transaction, 0, 2) + for i, tx := range block.Transactions() { if isPoSA { if isSystemTx, err := posa.IsSystemTransaction(tx, block.Header()); err != nil { @@ -423,11 +424,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return statedb, nil, nil, 0, err } statedb.Prepare(tx.Hash(), block.Hash(), i) + receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv, bloomProcessors) if err != nil { return statedb, nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } - commonTxs = append(commonTxs, tx) receipts = append(receipts, receipt) } diff --git a/eth/backend.go b/eth/backend.go index ab93006437..285bb6f7a7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -194,13 +194,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieDirtyLimit: config.TrieDirtyCache, TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, + NoTries: config.NoTries, SnapshotLimit: config.SnapshotCache, TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, } ) bcOps := make([]core.BlockChainOption, 0) - if config.DiffSync { + if config.DiffSync && !config.NoTries { bcOps = append(bcOps, core.EnableLightProcessor) } if config.PipeCommit { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 09baad1e1c..4f2c87c52e 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -79,6 +79,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, TriesInMemory: 128, + NoTries: false, SnapshotCache: 102, DiffBlock: uint64(86400), Miner: miner.Config{ @@ -175,6 +176,7 @@ type Config struct { TrieTimeout time.Duration SnapshotCache int TriesInMemory uint64 + NoTries bool Preimages bool // Mining options diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index f192a1aace..ba2996279d 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -3,6 +3,7 @@ package ethconfig import ( + "math/big" "time" "github.com/ethereum/go-ethereum/common" @@ -25,6 +26,10 @@ func (c Config) MarshalTOML() (interface{}, error) { SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool + DirectBroadcast bool + DisableSnapProtocol bool + DiffSync bool + RangeLimit bool TxLookupLimit uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` @@ -42,28 +47,30 @@ func (c Config) MarshalTOML() (interface{}, error) { DatabaseCache int DatabaseFreezer string DatabaseDiff string + PersistDiff bool + DiffBlock uint64 TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` TrieCleanCacheRejournal time.Duration `toml:",omitempty"` TrieDirtyCache int TrieTimeout time.Duration - TriesInMemory uint64 `toml:",omitempty"` SnapshotCache int + TriesInMemory uint64 + NoTries bool Preimages bool - PersistDiff bool - DiffBlock uint64 `toml:",omitempty"` Miner miner.Config - Ethash ethash.Config + Ethash ethash.Config `toml:",omitempty"` TxPool core.TxPoolConfig GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` EWASMInterpreter string EVMInterpreter string - RPCGasCap uint64 `toml:",omitempty"` - RPCTxFeeCap float64 `toml:",omitempty"` + RPCGasCap uint64 + RPCTxFeeCap float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -73,6 +80,10 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs enc.NoPruning = c.NoPruning + enc.DirectBroadcast = c.DirectBroadcast + enc.DisableSnapProtocol = c.DisableSnapProtocol + enc.DiffSync = c.DiffSync + enc.RangeLimit = c.RangeLimit enc.TxLookupLimit = c.TxLookupLimit enc.Whitelist = c.Whitelist enc.LightServ = c.LightServ @@ -90,16 +101,17 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DatabaseCache = c.DatabaseCache enc.DatabaseFreezer = c.DatabaseFreezer enc.DatabaseDiff = c.DatabaseDiff + enc.PersistDiff = c.PersistDiff + enc.DiffBlock = c.DiffBlock enc.TrieCleanCache = c.TrieCleanCache enc.TrieCleanCacheJournal = c.TrieCleanCacheJournal enc.TrieCleanCacheRejournal = c.TrieCleanCacheRejournal enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout - enc.TriesInMemory = c.TriesInMemory enc.SnapshotCache = c.SnapshotCache + enc.TriesInMemory = c.TriesInMemory + enc.NoTries = c.NoTries enc.Preimages = c.Preimages - enc.PersistDiff = c.PersistDiff - enc.DiffBlock = c.DiffBlock enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -112,6 +124,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle + enc.OverrideBerlin = c.OverrideBerlin return &enc, nil } @@ -126,6 +139,10 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool + DirectBroadcast *bool + DisableSnapProtocol *bool + DiffSync *bool + RangeLimit *bool TxLookupLimit *uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` @@ -144,27 +161,29 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DatabaseFreezer *string DatabaseDiff *string PersistDiff *bool - DiffBlock *uint64 `toml:",omitempty"` + DiffBlock *uint64 TrieCleanCache *int TrieCleanCacheJournal *string `toml:",omitempty"` TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` TrieDirtyCache *int TrieTimeout *time.Duration - TriesInMemory *uint64 `toml:",omitempty"` SnapshotCache *int + TriesInMemory *uint64 + NoTries *bool Preimages *bool Miner *miner.Config - Ethash *ethash.Config + Ethash *ethash.Config `toml:",omitempty"` TxPool *core.TxPoolConfig GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` EWASMInterpreter *string EVMInterpreter *string - RPCGasCap *uint64 `toml:",omitempty"` - RPCTxFeeCap *float64 `toml:",omitempty"` + RPCGasCap *uint64 + RPCTxFeeCap *float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -191,6 +210,18 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } + if dec.DirectBroadcast != nil { + c.DirectBroadcast = *dec.DirectBroadcast + } + if dec.DisableSnapProtocol != nil { + c.DisableSnapProtocol = *dec.DisableSnapProtocol + } + if dec.DiffSync != nil { + c.DiffSync = *dec.DiffSync + } + if dec.RangeLimit != nil { + c.RangeLimit = *dec.RangeLimit + } if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } @@ -263,11 +294,14 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout } + if dec.SnapshotCache != nil { + c.SnapshotCache = *dec.SnapshotCache + } if dec.TriesInMemory != nil { c.TriesInMemory = *dec.TriesInMemory } - if dec.SnapshotCache != nil { - c.SnapshotCache = *dec.SnapshotCache + if dec.NoTries != nil { + c.NoTries = *dec.NoTries } if dec.Preimages != nil { c.Preimages = *dec.Preimages @@ -308,5 +342,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } + if dec.OverrideBerlin != nil { + c.OverrideBerlin = dec.OverrideBerlin + } return nil } diff --git a/light/trie.go b/light/trie.go index 3896b73c4d..3f942f3607 100644 --- a/light/trie.go +++ b/light/trie.go @@ -49,6 +49,10 @@ type odrDatabase struct { backend OdrBackend } +func (db *odrDatabase) NoTries() bool { + return false +} + func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } @@ -178,6 +182,10 @@ func (t *odrTrie) do(key []byte, fn func() error) error { } } +func (db *odrTrie) NoTries() bool { + return false +} + type nodeIterator struct { trie.NodeIterator t *odrTrie diff --git a/tests/state_test_util.go b/tests/state_test_util.go index a688254a20..1de4a787dd 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -234,7 +234,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo var snaps *snapshot.Tree if snapshotter { - snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false) + snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false, false) } statedb, _ = state.New(root, sdb, snaps) return snaps, statedb diff --git a/trie/database.go b/trie/database.go index b6a3154d48..fa8b746c58 100644 --- a/trie/database.go +++ b/trie/database.go @@ -282,6 +282,7 @@ type Config struct { Cache int // Memory allowance (MB) to use for caching trie nodes in memory Journal string // Journal of clean cache to survive node restarts Preimages bool // Flag whether the preimage of trie key is recorded + NoTries bool } // NewDatabase creates a new trie database to store ephemeral trie content before diff --git a/trie/dummy_trie.go b/trie/dummy_trie.go new file mode 100644 index 0000000000..99eb79fbd4 --- /dev/null +++ b/trie/dummy_trie.go @@ -0,0 +1,96 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +type EmptyTrie struct{} + +func (t *EmptyTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { + return nil +} + +// NewSecure creates a dummy trie +func NewEmptyTrie() *EmptyTrie { + return &EmptyTrie{} +} + +func (t *EmptyTrie) Get(key []byte) []byte { + return nil +} + +func (t *EmptyTrie) TryGet(key []byte) ([]byte, error) { + return nil, nil +} + +func (t *EmptyTrie) TryGetNode(path []byte) ([]byte, int, error) { + return nil, 0, nil +} +func (t *EmptyTrie) Update(key, value []byte) {} + +func (t *EmptyTrie) TryUpdate(key, value []byte) error { + return nil +} + +// Delete removes any existing value for key from the trie. +func (t *EmptyTrie) Delete(key []byte) { + if err := t.TryDelete(key); err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } +} + +func (t *EmptyTrie) TryDelete(key []byte) error { + + return nil +} + +func (t *EmptyTrie) GetKey(shaKey []byte) []byte { + return nil +} + +func (t *EmptyTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { + + return common.Hash{}, nil +} + +func (t *EmptyTrie) Hash() common.Hash { + return common.Hash{} +} + +// Copy returns a copy of SecureTrie. +func (t *EmptyTrie) Copy() *EmptyTrie { + cpy := *t + return &cpy +} + +func (t *EmptyTrie) ResetCopy() *EmptyTrie { + cpy := *t + return &cpy +} + +// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration +// starts at the key after the given start key. +func (t *EmptyTrie) NodeIterator(start []byte) NodeIterator { + return nil +} From c6e86526dae3bea24f9c56f04d850cd608f065e6 Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Tue, 11 Jan 2022 16:17:03 +0800 Subject: [PATCH 08/51] implement trust protocol and verify node --- common/types.go | 15 +++ core/blockchain.go | 114 ++++++++++++++++++++++ core/state/state_object.go | 7 ++ core/state/statedb.go | 6 +- core/types/block.go | 89 ++++++++++++++++- eth/backend.go | 3 + eth/handler.go | 24 ++++- eth/handler_trust.go | 43 +++++++++ eth/peer.go | 25 ++++- eth/peerset.go | 118 +++++++++++++++++++++-- eth/protocols/trust/discovery.go | 14 +++ eth/protocols/trust/handler.go | 159 +++++++++++++++++++++++++++++++ eth/protocols/trust/peer.go | 66 +++++++++++++ eth/protocols/trust/protocol.go | 72 ++++++++++++++ eth/protocols/trust/tracker.go | 10 ++ ethclient/ethclient.go | 6 ++ internal/ethapi/api.go | 4 + p2p/peer.go | 5 + p2p/server.go | 18 ++++ 19 files changed, 785 insertions(+), 13 deletions(-) create mode 100644 eth/handler_trust.go create mode 100644 eth/protocols/trust/discovery.go create mode 100644 eth/protocols/trust/handler.go create mode 100644 eth/protocols/trust/peer.go create mode 100644 eth/protocols/trust/protocol.go create mode 100644 eth/protocols/trust/tracker.go diff --git a/common/types.go b/common/types.go index d715356692..15f2bce681 100644 --- a/common/types.go +++ b/common/types.go @@ -354,6 +354,21 @@ func (a *Address) UnmarshalGraphQL(input interface{}) error { return err } +// AddressSlice is used for sort +type AddressSlice []Address + +func (s AddressSlice) Len() int { + return len(s) +} + +func (s AddressSlice) Less(i, j int) bool { + return s[i].Hex() < s[j].Hex() +} + +func (s AddressSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + // UnprefixedAddress allows marshaling an Address without 0x prefix. type UnprefixedAddress Address diff --git a/core/blockchain.go b/core/blockchain.go index e286a1e124..1bc5be0b19 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -28,6 +28,8 @@ import ( "sync/atomic" "time" + "golang.org/x/crypto/sha3" + lru "github.com/hashicorp/golang-lru" "github.com/ethereum/go-ethereum/common" @@ -1798,6 +1800,12 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.Receipts = receipts diffLayer.BlockHash = block.Hash() diffLayer.Number = block.NumberU64() + + sort.Sort(types.DiffCodeSlice(diffLayer.Codes)) + sort.Sort(common.AddressSlice(diffLayer.Destructs)) + sort.Sort(types.DiffAccountSlice(diffLayer.Accounts)) + sort.Sort(types.DiffStorageSlice(diffLayer.Storages)) + bc.cacheDiffLayer(diffLayer) } @@ -3111,3 +3119,109 @@ func EnablePersistDiff(limit uint64) BlockChainOption { return chain } } + +func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { + var res types.VerifyResult + res.BlockNumber = blockNumber + res.BlockHash = blockHash + + if blockNumber > bc.CurrentHeader().Number.Uint64()+11 { + res.Status = types.StatusBlockTooNew + return &res, nil + } else if blockNumber > bc.CurrentHeader().Number.Uint64() { + res.Status = types.StatusBlockNewer + return &res, nil + } + + diff := bc.GetTrustedDiffLayer(blockHash) + if diff != nil { + if diff.DiffHash == (common.Hash{}) { + hash, err := GetTrustedDiffHash(diff) + if err != nil { + res.Status = types.StatusUnexpectedError + return &res, err + } + + diff.DiffHash = hash + } + + if diffHash != diff.DiffHash { + res.Status = types.StatusDiffHashMismatch + return &res, nil + } + + header := bc.GetHeaderByHash(blockHash) + if header == nil { + res.Status = types.StatusUnexpectedError + return &res, fmt.Errorf("unexpected error, header not found") + } + res.Status = types.StatusFullVerified + res.Root = header.Root + return &res, nil + } + + header := bc.GetHeaderByHash(blockHash) + if header == nil { + if blockNumber > bc.CurrentHeader().Number.Uint64()-11 { + res.Status = types.StatusPossibleFork + return &res, nil + } + + res.Status = types.StatusImpossibleFork + return &res, nil + } + + res.Status = types.StatusUntrustedVerified + res.Root = header.Root + return &res, nil +} + +func (bc *BlockChain) GetTrustedDiffLayer(blockHash common.Hash) *types.DiffLayer { + var diff *types.DiffLayer + if cached, ok := bc.diffLayerCache.Get(blockHash); ok { + diff = cached.(*types.DiffLayer) + return diff + } + + diffStore := bc.db.DiffStore() + if diffStore != nil { + diff = rawdb.ReadDiffLayer(diffStore, blockHash) + } + return diff +} + +func GetTrustedDiffHash(d *types.DiffLayer) (common.Hash, error) { + diff := &types.ExtDiffLayer{ + BlockHash: d.BlockHash, + Receipts: make([]*types.ReceiptForStorage, 0), + Number: d.Number, + Codes: d.Codes, + Destructs: d.Destructs, + Accounts: d.Accounts, + Storages: d.Storages, + } + + for index, account := range diff.Accounts { + full, err := snapshot.FullAccount(account.Blob) + if err != nil { + return common.Hash{}, fmt.Errorf("decode full account error: %v", err) + } + // set account root to empty root + diff.Accounts[index].Blob = snapshot.SlimAccountRLP(full.Nonce, full.Balance, common.Hash{}, full.CodeHash) + } + + rawData, err := rlp.EncodeToBytes(diff) + if err != nil { + return common.Hash{}, fmt.Errorf("encode new diff error: %v", err) + } + + hasher := sha3.NewLegacyKeccak256() + _, err = hasher.Write(rawData) + if err != nil { + return common.Hash{}, fmt.Errorf("hasher write error: %v", err) + } + + var hash common.Hash + hasher.Sum(hash[:0]) + return hash, nil +} diff --git a/core/state/state_object.go b/core/state/state_object.go index 298f4305ba..c86585a658 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -391,6 +391,13 @@ func (s *StateObject) updateTrie(db Database) Trie { // UpdateRoot sets the trie root to the current root hash of func (s *StateObject) updateRoot(db Database) { + // If node runs in no trie mode, set root to empty. + defer func() { + if db.NoTries() { + s.data.Root = common.Hash{} + } + }() + // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { return diff --git a/core/state/statedb.go b/core/state/statedb.go index 1bb1db7ba1..17eb3bf856 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1575,8 +1575,12 @@ func (s *StateDB) SnapToDiffLayer() ([]common.Address, []types.DiffAccount, []ty for accountHash, storage := range s.snapStorage { keys := make([]string, 0, len(storage)) values := make([][]byte, 0, len(storage)) - for k, v := range storage { + for k, _ := range storage { keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := storage[k] values = append(values, v) } storages = append(storages, types.DiffStorage{ diff --git a/core/types/block.go b/core/types/block.go index bee5d80cdd..f3c487b684 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -40,6 +40,40 @@ var ( EmptyUncleHash = rlpHash([]*Header(nil)) ) +type VerifyStatus struct { + Code uint16 + Msg string +} + +var ( + // StatusVerified means the processing of request going as expected and found the root correctly. + StatusVerified = VerifyStatus{Code: 0x100} + StatusFullVerified = VerifyStatus{Code: 0x101, Msg: "state root full verified"} + StatusUntrustedVerified = VerifyStatus{Code: 0x102, Msg: "state root untrusted verified, because of difflayer not found"} + + // StatusFailed means the request has something wrong. + StatusFailed = VerifyStatus{Code: 0x200} + StatusDiffHashMismatch = VerifyStatus{Code: 0x201, Msg: "verify failed because of blockhash mismatch with diffhash"} + StatusImpossibleFork = VerifyStatus{Code: 0x202, Msg: "verify failed because of impossible fork detected"} + + // StatusUncertain means verify node can't give a certain result of the request. + StatusUncertain = VerifyStatus{Code: 0x300} + StatusBlockTooNew = VerifyStatus{Code: 0x301, Msg: "can’t verify because of block number larger than current height more than 11"} + StatusBlockNewer = VerifyStatus{Code: 0x302, Msg: "can’t verify because of block number larger than current height"} + StatusPossibleFork = VerifyStatus{Code: 0x303, Msg: "can’t verify because of possible fork detected"} + StatusRequestTooBusy = VerifyStatus{Code: 0x304, Msg: "can’t verify because of request too busy"} + + // StatusUnexpectedError is unexpected internal error. + StatusUnexpectedError = VerifyStatus{Code: 0x400, Msg: "can’t verify because of unexpected internal error"} +) + +type VerifyResult struct { + Status VerifyStatus + BlockNumber uint64 + BlockHash common.Hash + Root common.Hash +} + // A BlockNonce is a 64-bit hash which proves (combined with the // mix-hash) that a sufficient amount of computation has been carried // out on a block. @@ -383,7 +417,7 @@ type DiffLayer struct { DiffHash common.Hash } -type extDiffLayer struct { +type ExtDiffLayer struct { BlockHash common.Hash Number uint64 Receipts []*ReceiptForStorage // Receipts are duplicated stored to simplify the logic @@ -395,7 +429,7 @@ type extDiffLayer struct { // DecodeRLP decodes the Ethereum func (d *DiffLayer) DecodeRLP(s *rlp.Stream) error { - var ed extDiffLayer + var ed ExtDiffLayer if err := s.Decode(&ed); err != nil { return err } @@ -415,7 +449,7 @@ func (d *DiffLayer) EncodeRLP(w io.Writer) error { for i, receipt := range d.Receipts { storageReceipts[i] = (*ReceiptForStorage)(receipt) } - return rlp.Encode(w, extDiffLayer{ + return rlp.Encode(w, ExtDiffLayer{ BlockHash: d.BlockHash, Number: d.Number, Receipts: storageReceipts, @@ -443,17 +477,66 @@ type DiffCode struct { Code []byte } +// DiffCodeSlice is used for sort +type DiffCodeSlice []DiffCode + +func (s DiffCodeSlice) Len() int { + return len(s) +} + +func (s DiffCodeSlice) Less(i, j int) bool { + return s[i].Hash.Hex() < s[j].Hash.Hex() +} + +func (s DiffCodeSlice) Swap(i, j int) { + s[i].Hash, s[j].Hash = s[j].Hash, s[i].Hash + s[i].Code, s[j].Code = s[j].Code, s[i].Code +} + type DiffAccount struct { Account common.Address Blob []byte } +// DiffAccountSlice is used for sort +type DiffAccountSlice []DiffAccount + +func (s DiffAccountSlice) Len() int { + return len(s) +} + +func (s DiffAccountSlice) Less(i, j int) bool { + return s[i].Account.Hex() < s[j].Account.Hex() +} + +func (s DiffAccountSlice) Swap(i, j int) { + s[i].Account, s[j].Account = s[j].Account, s[i].Account + s[i].Blob, s[j].Blob = s[j].Blob, s[i].Blob +} + type DiffStorage struct { Account common.Address Keys []string Vals [][]byte } +// DiffStorageSlice is used for sort +type DiffStorageSlice []DiffStorage + +func (s DiffStorageSlice) Len() int { + return len(s) +} + +func (s DiffStorageSlice) Less(i, j int) bool { + return s[i].Account.Hex() < s[j].Account.Hex() +} + +func (s DiffStorageSlice) Swap(i, j int) { + s[i].Account, s[j].Account = s[j].Account, s[i].Account + s[i].Keys, s[j].Keys = s[j].Keys, s[i].Keys + s[i].Vals, s[j].Vals = s[j].Vals, s[i].Vals +} + type DiffAccountsInTx struct { TxHash common.Hash Accounts map[common.Address]*big.Int diff --git a/eth/backend.go b/eth/backend.go index 285bb6f7a7..b9e8e40cb9 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -26,6 +26,8 @@ import ( "sync/atomic" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -554,6 +556,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { } // diff protocol can still open without snap protocol protos = append(protos, diff.MakeProtocols((*diffHandler)(s.handler), s.snapDialCandidates)...) + protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.snapDialCandidates)...) return protos } diff --git a/eth/handler.go b/eth/handler.go index cbc6eca809..e47d3eee8d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -24,6 +24,8 @@ import ( "sync/atomic" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -269,6 +271,11 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Error("Diff extension barrier failed", "err", err) return err } + trust, err := h.peers.waitTrustExtension(peer) + if err != nil { + peer.Log().Error("Trust extension barrier failed", "err", err) + return err + } // TODO(karalabe): Not sure why this is needed if !h.chainSync.handlePeerEvent(peer) { return p2p.DiscQuitting @@ -309,7 +316,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Debug("Ethereum peer connected", "name", peer.Name()) // Register the peer locally - if err := h.peers.registerPeer(peer, snap, diff); err != nil { + if err := h.peers.registerPeer(peer, snap, diff, trust); err != nil { peer.Log().Error("Ethereum peer registration failed", "err", err) return err } @@ -395,6 +402,21 @@ func (h *handler) runDiffExtension(peer *diff.Peer, handler diff.Handler) error return handler(peer) } +// runTrustExtension registers a `trust` peer into the joint eth/trust peerset and +// starts handling inbound messages. As `trust` is only a satellite protocol to +// `eth`, all subsystem registrations and lifecycle management will be done by +// the main `eth` handler to prevent strange races. +func (h *handler) runTrustExtension(peer *trust.Peer, handler trust.Handler) error { + h.peerWG.Add(1) + defer h.peerWG.Done() + + if err := h.peers.registerTrustExtension(peer); err != nil { + peer.Log().Error("Trust extension registration failed", "err", err) + return err + } + return handler(peer) +} + // removePeer unregisters a peer from the downloader and fetchers, removes it from // the set of tracked peers and closes the network connection to it. func (h *handler) removePeer(id string) { diff --git a/eth/handler_trust.go b/eth/handler_trust.go new file mode 100644 index 0000000000..6df630a2e8 --- /dev/null +++ b/eth/handler_trust.go @@ -0,0 +1,43 @@ +package eth + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// trustHandler implements the trust.Backend interface to handle the various network +// packets that are sent as replies or broadcasts. +type trustHandler handler + +func (h *trustHandler) Chain() *core.BlockChain { return h.chain } + +// RunPeer is invoked when a peer joins on the `snap` protocol. +func (h *trustHandler) RunPeer(peer *trust.Peer, hand trust.Handler) error { + return (*handler)(h).runTrustExtension(peer, hand) +} + +// PeerInfo retrieves all known `trust` information about a peer. +func (h *trustHandler) PeerInfo(id enode.ID) interface{} { + if p := h.peers.peer(id.String()); p != nil { + if p.trustExt != nil { + return p.trustExt.info() + } + } + return nil +} + +// Handle is invoked from a peer's message handler when it receives a new remote +// message that the handler couldn't consume and serve itself. +func (h *trustHandler) Handle(peer *trust.Peer, packet trust.Packet) error { + switch packet := packet.(type) { + case *trust.RootResponsePacket: + // TODO: h.bc.VerifyManager().HandleRootResponse(peer.ID(), *packet) + return nil + + default: + return fmt.Errorf("unexpected trust packet type: %T", packet) + } +} diff --git a/eth/peer.go b/eth/peer.go index 2fb6fabf26..4d92f4e78f 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -21,6 +21,8 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -37,8 +39,9 @@ type ethPeerInfo struct { // ethPeer is a wrapper around eth.Peer to maintain a few extra metadata. type ethPeer struct { *eth.Peer - snapExt *snapPeer // Satellite `snap` connection - diffExt *diffPeer + snapExt *snapPeer // Satellite `snap` connection + diffExt *diffPeer + trustExt *trustPeer syncDrop *time.Timer // Connection dropper if `eth` sync progress isn't validated in time snapWait chan struct{} // Notification channel for snap connections @@ -69,6 +72,12 @@ type diffPeerInfo struct { DiffSync bool `json:"diff_sync"` } +// trustPeerInfo represents a short summary of the `trust` sub-protocol metadata known +// about a connected peer. +type trustPeerInfo struct { + Version uint `json:"version"` // Trust protocol version negotiated +} + // snapPeer is a wrapper around snap.Peer to maintain a few extra metadata. type snapPeer struct { *snap.Peer @@ -79,6 +88,11 @@ type diffPeer struct { *diff.Peer } +// trustPeer is a wrapper around trust.Peer to maintain a few extra metadata. +type trustPeer struct { + *trust.Peer +} + // info gathers and returns some `diff` protocol metadata known about a peer. func (p *diffPeer) info() *diffPeerInfo { return &diffPeerInfo{ @@ -93,3 +107,10 @@ func (p *snapPeer) info() *snapPeerInfo { Version: p.Version(), } } + +// info gathers and returns some `trust` protocol metadata known about a peer. +func (p *trustPeer) info() *trustPeerInfo { + return &trustPeerInfo{ + Version: p.Version(), + } +} diff --git a/eth/peerset.go b/eth/peerset.go index 0f5245a05e..dc1d7da45e 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -22,6 +22,8 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/eth/protocols/trust" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/diff" @@ -53,6 +55,10 @@ var ( // errDiffWithoutEth is returned if a peer attempts to connect only on the // diff protocol without advertising the eth main protocol. errDiffWithoutEth = errors.New("peer connected on diff without compatible eth support") + + // errTrustWithoutEth is returned if a peer attempts to connect only on the + // trust protocol without advertising the eth main protocol. + errTrustWithoutEth = errors.New("peer connected on trust without compatible eth support") ) const ( @@ -73,6 +79,9 @@ type peerSet struct { diffWait map[string]chan *diff.Peer // Peers connected on `eth` waiting for their diff extension diffPend map[string]*diff.Peer // Peers connected on the `diff` protocol, but not yet on `eth` + trustWait map[string]chan *trust.Peer // Peers connected on `eth` waiting for their trust extension + trustPend map[string]*trust.Peer // Peers connected on the `trust` protocol, but not yet on `eth` + lock sync.RWMutex closed bool } @@ -80,11 +89,13 @@ type peerSet struct { // newPeerSet creates a new peer set to track the active participants. func newPeerSet() *peerSet { return &peerSet{ - peers: make(map[string]*ethPeer), - snapWait: make(map[string]chan *snap.Peer), - snapPend: make(map[string]*snap.Peer), - diffWait: make(map[string]chan *diff.Peer), - diffPend: make(map[string]*diff.Peer), + peers: make(map[string]*ethPeer), + snapWait: make(map[string]chan *snap.Peer), + snapPend: make(map[string]*snap.Peer), + diffWait: make(map[string]chan *diff.Peer), + diffPend: make(map[string]*diff.Peer), + trustWait: make(map[string]chan *trust.Peer), + trustPend: make(map[string]*trust.Peer), } } @@ -148,6 +159,40 @@ func (ps *peerSet) registerDiffExtension(peer *diff.Peer) error { return nil } +// registerTrustExtension unblocks an already connected `eth` peer waiting for its +// `trust` extension, or if no such peer exists, tracks the extension for the time +// being until the `eth` main protocol starts looking for it. +func (ps *peerSet) registerTrustExtension(peer *trust.Peer) error { + // Reject the peer if it advertises `trust` without `eth` as `trust` is only a + // satellite protocol meaningful with the chain selection of `eth` + if !peer.RunningCap(eth.ProtocolName, eth.ProtocolVersions) { + return errTrustWithoutEth + } + // If the peer isn't verify node, don't register trust extension into eth protocol. + if !peer.VerifyNode() { + return nil + } + // Ensure nobody can double connect + ps.lock.Lock() + defer ps.lock.Unlock() + + id := peer.ID() + if _, ok := ps.peers[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as existing ones + } + if _, ok := ps.trustPend[id]; ok { + return errPeerAlreadyRegistered // avoid connections with the same id as pending ones + } + // Inject the peer into an `eth` counterpart is available, otherwise save for later + if wait, ok := ps.trustWait[id]; ok { + delete(ps.trustWait, id) + wait <- peer + return nil + } + ps.trustPend[id] = peer + return nil +} + // waitExtensions blocks until all satellite protocols are connected and tracked // by the peerset. func (ps *peerSet) waitSnapExtension(peer *eth.Peer) (*snap.Peer, error) { @@ -234,6 +279,53 @@ func (ps *peerSet) waitDiffExtension(peer *eth.Peer) (*diff.Peer, error) { } } +// waitTrustExtension blocks until all satellite protocols are connected and tracked +// by the peerset. +func (ps *peerSet) waitTrustExtension(peer *eth.Peer) (*trust.Peer, error) { + // If the peer does not support a compatible `trust`, don't wait + if !peer.RunningCap(trust.ProtocolName, trust.ProtocolVersions) { + return nil, nil + } + // If the peer isn't verify node, don't register trust extension into eth protocol. + if !peer.VerifyNode() { + return nil, nil + } + // Ensure nobody can double connect + ps.lock.Lock() + + id := peer.ID() + if _, ok := ps.peers[id]; ok { + ps.lock.Unlock() + return nil, errPeerAlreadyRegistered // avoid connections with the same id as existing ones + } + if _, ok := ps.trustWait[id]; ok { + ps.lock.Unlock() + return nil, errPeerAlreadyRegistered // avoid connections with the same id as pending ones + } + // If `trust` already connected, retrieve the peer from the pending set + if trust, ok := ps.trustPend[id]; ok { + delete(ps.trustPend, id) + + ps.lock.Unlock() + return trust, nil + } + // Otherwise wait for `trust` to connect concurrently + wait := make(chan *trust.Peer) + ps.trustWait[id] = wait + ps.lock.Unlock() + + select { + case peer := <-wait: + return peer, nil + + case <-time.After(extensionWaitTimeout): + ps.lock.Lock() + delete(ps.trustWait, id) + ps.lock.Unlock() + return nil, errPeerWaitTimeout + } +} + func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { if p := ps.peer(pid); p != nil && p.diffExt != nil { return p.diffExt @@ -241,9 +333,20 @@ func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { return nil } +// GetVerifyPeers returns an array of verify nodes. +func (ps *peerSet) GetVerifyPeers() []*trustPeer { + res := make([]*trustPeer, 0) + for _, p := range ps.peers { + if p.trustExt != nil { + res = append(res, p.trustExt) + } + } + return res +} + // registerPeer injects a new `eth` peer into the working set, or returns an error // if the peer is already known. -func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer, diffExt *diff.Peer) error { +func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer, diffExt *diff.Peer, trustExt *trust.Peer) error { // Start tracking the new peer ps.lock.Lock() defer ps.lock.Unlock() @@ -265,6 +368,9 @@ func (ps *peerSet) registerPeer(peer *eth.Peer, ext *snap.Peer, diffExt *diff.Pe if diffExt != nil { eth.diffExt = &diffPeer{diffExt} } + if trustExt != nil { + eth.trustExt = &trustPeer{trustExt} + } ps.peers[id] = eth return nil } diff --git a/eth/protocols/trust/discovery.go b/eth/protocols/trust/discovery.go new file mode 100644 index 0000000000..ce38ec5ed9 --- /dev/null +++ b/eth/protocols/trust/discovery.go @@ -0,0 +1,14 @@ +package trust + +import "github.com/ethereum/go-ethereum/rlp" + +// enrEntry is the ENR entry which advertises `trust` protocol on the discovery. +type enrEntry struct { + // Ignore additional fields (for forward compatibility). + Rest []rlp.RawValue `rlp:"tail"` +} + +// ENRKey implements enr.Entry. +func (e enrEntry) ENRKey() string { + return "trust" +} diff --git a/eth/protocols/trust/handler.go b/eth/protocols/trust/handler.go new file mode 100644 index 0000000000..ce8fc4cd8d --- /dev/null +++ b/eth/protocols/trust/handler.go @@ -0,0 +1,159 @@ +package trust + +import ( + "fmt" + "time" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" +) + +// Handler is a callback to invoke from an outside runner after the boilerplate +// exchanges have passed. +type Handler func(peer *Peer) error + +type Backend interface { + // Chain retrieves the blockchain object to serve data. + Chain() *core.BlockChain + + // RunPeer is invoked when a peer joins on the `eth` protocol. The handler + // should do any peer maintenance work, handshakes and validations. If all + // is passed, control should be given back to the `handler` to process the + // inbound messages going forward. + RunPeer(peer *Peer, handler Handler) error + + PeerInfo(id enode.ID) interface{} + + Handle(peer *Peer, packet Packet) error +} + +// MakeProtocols constructs the P2P protocol definitions for `trust`. +func MakeProtocols(backend Backend, dnsdisc enode.Iterator) []p2p.Protocol { + // Filter the discovery iterator for nodes advertising trust support. + dnsdisc = enode.Filter(dnsdisc, func(n *enode.Node) bool { + var trust enrEntry + return n.Load(&trust) == nil + }) + + protocols := make([]p2p.Protocol, len(ProtocolVersions)) + for i, version := range ProtocolVersions { + version := version // Closure + + protocols[i] = p2p.Protocol{ + Name: ProtocolName, + Version: version, + Length: protocolLengths[version], + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + return backend.RunPeer(NewPeer(version, p, rw), func(peer *Peer) error { + defer peer.Close() + return Handle(backend, peer) + }) + }, + NodeInfo: func() interface{} { + return nodeInfo(backend.Chain()) + }, + PeerInfo: func(id enode.ID) interface{} { + return backend.PeerInfo(id) + }, + Attributes: []enr.Entry{&enrEntry{}}, + DialCandidates: dnsdisc, + } + } + return protocols +} + +// Handle is the callback invoked to manage the life cycle of a `trust` peer. +// When this function terminates, the peer is disconnected. +func Handle(backend Backend, peer *Peer) error { + for { + if err := handleMessage(backend, peer); err != nil { + peer.Log().Debug("Message handling failed in `trust`", "err", err) + return err + } + } +} + +// handleMessage is invoked whenever an inbound message is received from a +// remote peer on the `diff` protocol. The remote connection is torn down upon +// returning any error. +func handleMessage(backend Backend, peer *Peer) error { + // Read the next message from the remote peer, and ensure it's fully consumed + msg, err := peer.rw.ReadMsg() + if err != nil { + return err + } + if msg.Size > maxMessageSize { + return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize) + } + defer msg.Discard() + + // Track the amount of time it takes to serve the request and run the handler + if metrics.Enabled { + h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) + defer func(start time.Time) { + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) + } + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(time.Since(start).Microseconds()) + }(time.Now()) + } + // Handle the message depending on its contents + switch { + case msg.Code == RequestRootMsg: + return handleRootRequest(backend, msg, peer) + + case msg.Code == RespondRootMsg: + return handleRootResponse(backend, msg, peer) + + default: + return fmt.Errorf("%w: %v", errInvalidMsgCode, msg.Code) + } +} + +type Decoder interface { + Decode(val interface{}) error + Time() time.Time +} + +func handleRootRequest(backend Backend, msg Decoder, peer *Peer) error { + req := new(RootRequestPacket) + if err := msg.Decode(req); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + + res, err := backend.Chain().GetRootByDiffHash(req.BlockNumber, req.BlockHash, req.DiffHash) + p2p.Send(peer.rw, RespondRootMsg, RootResponsePacket{ + RequestId: req.RequestId, + Status: res.Status, + BlockNumber: req.BlockNumber, + BlockHash: req.BlockHash, + Root: res.Root, + Extra: defaultExtra, + }) + + return err +} + +func handleRootResponse(backend Backend, msg Decoder, peer *Peer) error { + res := new(RootResponsePacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + + requestTracker.Fulfil(peer.id, peer.version, RespondRootMsg, res.RequestId) + return backend.Handle(peer, res) +} + +// NodeInfo represents a short summary of the `trust` sub-protocol metadata +// known about the host peer. +type NodeInfo struct{} + +// nodeInfo retrieves some `trust` protocol metadata about the running host node. +func nodeInfo(chain *core.BlockChain) *NodeInfo { + return &NodeInfo{} +} diff --git a/eth/protocols/trust/peer.go b/eth/protocols/trust/peer.go new file mode 100644 index 0000000000..18ba229914 --- /dev/null +++ b/eth/protocols/trust/peer.go @@ -0,0 +1,66 @@ +package trust + +import ( + "math/rand" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" +) + +// Peer is a collection of relevant information we have about a `trust` peer. +type Peer struct { + id string // Unique ID for the peer, cached + + *p2p.Peer // The embedded P2P package peer + rw p2p.MsgReadWriter // Input/output streams for diff + version uint // Protocol version negotiated + logger log.Logger // Contextual logger with the peer id injected +} + +// NewPeer create a wrapper for a network connection and negotiated protocol +// version. +func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) *Peer { + id := p.ID().String() + peer := &Peer{ + id: id, + Peer: p, + rw: rw, + version: version, + logger: log.New("peer", id[:8]), + } + return peer +} + +// ID retrieves the peer's unique identifier. +func (p *Peer) ID() string { + return p.id +} + +// Version retrieves the peer's negoatiated `diff` protocol version. +func (p *Peer) Version() uint { + return p.version +} + +// Log overrides the P2P logget with the higher level one containing only the id. +func (p *Peer) Log() log.Logger { + return p.logger +} + +// Close signals the broadcast goroutine to terminate. Only ever call this if +// you created the peer yourself via NewPeer. Otherwise let whoever created it +// clean it up! +func (p *Peer) Close() { +} + +func (p *Peer) RequestRoot(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) error { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, RequestRootMsg, RespondRootMsg, id) + return p2p.Send(p.rw, RequestRootMsg, RootRequestPacket{ + RequestId: id, + BlockNumber: blockNumber, + BlockHash: blockHash, + DiffHash: diffHash, + }) +} diff --git a/eth/protocols/trust/protocol.go b/eth/protocols/trust/protocol.go new file mode 100644 index 0000000000..2bd7fecc15 --- /dev/null +++ b/eth/protocols/trust/protocol.go @@ -0,0 +1,72 @@ +package trust + +import ( + "errors" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +// Constants to match up protocol versions and messages +const ( + Trust1 = 1 +) + +// ProtocolName is the official short name of the `trust` protocol used during +// devp2p capability negotiation. +const ProtocolName = "trust" + +// ProtocolVersions are the supported versions of the `trust` protocol (first +// is primary). +var ProtocolVersions = []uint{Trust1} + +// protocolLengths are the number of implemented message corresponding to +// different protocol versions. +var protocolLengths = map[uint]uint64{Trust1: 2} + +// maxMessageSize is the maximum cap on the size of a protocol message. +const maxMessageSize = 10 * 1024 * 1024 + +const ( + RequestRootMsg = 0x00 + RespondRootMsg = 0x01 +) + +var defaultExtra = []byte{0x00} + +var ( + errMsgTooLarge = errors.New("message too long") + errDecode = errors.New("invalid message") + errInvalidMsgCode = errors.New("invalid message code") + errUnexpectedMsg = errors.New("unexpected message code") +) + +// Packet represents a p2p message in the `trust` protocol. +type Packet interface { + Name() string // Name returns a string corresponding to the message type. + Kind() byte // Kind returns the message type. +} + +type RootRequestPacket struct { + RequestId uint64 + BlockNumber uint64 + BlockHash common.Hash + DiffHash common.Hash +} + +type RootResponsePacket struct { + RequestId uint64 + Status types.VerifyStatus + BlockNumber uint64 + BlockHash common.Hash + Root common.Hash + Extra rlp.RawValue // for extension +} + +func (*RootRequestPacket) Name() string { return "RequestRoot" } +func (*RootRequestPacket) Kind() byte { return RequestRootMsg } + +func (*RootResponsePacket) Name() string { return "RootResponse" } +func (*RootResponsePacket) Kind() byte { return RespondRootMsg } diff --git a/eth/protocols/trust/tracker.go b/eth/protocols/trust/tracker.go new file mode 100644 index 0000000000..ab492b3fb8 --- /dev/null +++ b/eth/protocols/trust/tracker.go @@ -0,0 +1,10 @@ +package trust + +import ( + "time" + + "github.com/ethereum/go-ethereum/p2p/tracker" +) + +// requestTracker is a singleton tracker for request times. +var requestTracker = tracker.New(ProtocolName, time.Minute) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 578b10f09a..395e87fe1d 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -200,6 +200,12 @@ func (ec *Client) GetDiffAccountsWithScope(ctx context.Context, number *big.Int, return &result, err } +func (ec *Client) GetRootByDiffHash(ctx context.Context, blockNr *big.Int, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { + var result types.VerifyResult + err := ec.c.CallContext(ctx, &result, "eth_getRootByDiffHash", toBlockNumArg(blockNr), blockHash, diffHash) + return &result, err +} + type rpcTransaction struct { tx *types.Transaction txExtraInfo diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 091e9e7e82..b01ec8b2e5 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1287,6 +1287,10 @@ func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, bloc return result, err } +func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { + return s.b.Chain().GetRootByDiffHash(uint64(blockNr), blockHash, diffHash) +} + // ExecutionResult groups all structured logs emitted by the EVM // while replaying a transaction in debug mode as well as transaction // execution status, the amount of gas used and the return value diff --git a/p2p/peer.go b/p2p/peer.go index 3b633108db..cdfaf7f21b 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -213,6 +213,11 @@ func (p *Peer) Inbound() bool { return p.rw.is(inboundConn) } +// VerifyNode returns true if the peer is a verification connection +func (p *Peer) VerifyNode() bool { + return p.rw.is(verifyConn) +} + func newPeer(log log.Logger, conn *conn, protocols []Protocol) *Peer { protomap := matchProtocols(protocols, conn.caps, conn) p := &Peer{ diff --git a/p2p/server.go b/p2p/server.go index 2a38550abf..01e56936ab 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -114,6 +114,10 @@ type Config struct { // maintained and re-connected on disconnects. StaticNodes []*enode.Node + // Verify nodes are used as pre-configured connections which are always + // maintained and re-connected on disconnects. + VerifyNodes []*enode.Node + // Trusted nodes are used as pre-configured connections which are always // allowed to connect, even above the peer limit. TrustedNodes []*enode.Node @@ -218,6 +222,7 @@ const ( staticDialedConn inboundConn trustedConn + verifyConn ) // conn wraps a network connection with information gathered @@ -269,6 +274,9 @@ func (f connFlag) String() string { if f&inboundConn != 0 { s += "-inbound" } + if f&verifyConn != 0 { + s += "-verify" + } if s != "" { s = s[1:] } @@ -649,6 +657,9 @@ func (srv *Server) setupDialScheduler() { for _, n := range srv.StaticNodes { srv.dialsched.addStatic(n) } + for _, n := range srv.VerifyNodes { + srv.dialsched.addStatic(n) + } } func (srv *Server) maxInboundConns() int { @@ -934,6 +945,13 @@ func (srv *Server) checkInboundConn(remoteIP net.IP) error { // as a peer. It returns when the connection has been added as a peer // or the handshakes have failed. func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) error { + // If dialDest is verify node, set verifyConn flags. + for _, n := range srv.VerifyNodes { + if dialDest == n { + flags |= verifyConn + } + } + c := &conn{fd: fd, flags: flags, cont: make(chan error)} if dialDest == nil { c.transport = srv.newTransport(fd, nil) From 4905aabfffb9a13ef755ed8030e60aea2cae928b Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Thu, 20 Jan 2022 15:22:06 +0800 Subject: [PATCH 09/51] testcases for getting root by diff hash --- core/blockchain_diff_test.go | 160 +++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index 2575843a92..7d19d42ea5 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -484,3 +484,163 @@ func TestGetDiffAccounts(t *testing.T) { } } } + +// newTwoForkedBlockchains returns two blockchains, these two chains are generated by different +// generators, they have some same parent blocks, the number of same blocks are determined by +// testBlocks, once chain1 inserted a non-default block, chain1 and chain2 get forked. +func newTwoForkedBlockchains(len1, len2 int) (chain1 *BlockChain, chain2 *BlockChain) { + signer := types.HomesteadSigner{} + // Create a database pre-initialize with a genesis block + db1 := rawdb.NewMemoryDatabase() + db1.SetDiffStore(memorydb.New()) + (&Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db1) + + chain1, _ = NewBlockChain(db1, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000)) + generator1 := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + + for idx, testBlock := range testBlocks { + // Specific block setting, the index in this generator has 1 diff from specified blockNr. + if i+1 == testBlock.blockNr { + for _, testTransaction := range testBlock.txs { + var transaction *types.Transaction + if testTransaction.to == nil { + transaction = types.NewContractCreation(block.TxNonce(testAddr), + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } else { + transaction = types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } + tx, err := types.SignTx(transaction, signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain1, tx) + } + break + } + + // Default block setting. + if idx == len(testBlocks)-1 { + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain1, tx) + } + } + } + + } + bs1, _ := GenerateChain(params.TestChainConfig, chain1.Genesis(), ethash.NewFaker(), db1, len1, generator1) + if _, err := chain1.InsertChain(bs1); err != nil { + panic(err) + } + + // Create a database pre-initialize with a genesis block + db2 := rawdb.NewMemoryDatabase() + db2.SetDiffStore(memorydb.New()) + (&Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db2) + chain2, _ = NewBlockChain(db2, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000)) + generator2 := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain1, tx) + } + } + bs2, _ := GenerateChain(params.TestChainConfig, chain2.Genesis(), ethash.NewFaker(), db2, len2, generator2) + if _, err := chain2.InsertChain(bs2); err != nil { + panic(err) + } + + return chain1, chain2 +} + +func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber uint64, status types.VerifyStatus) { + block2 := chain2.GetBlockByNumber(blockNumber) + if block2 == nil { + t.Fatalf("failed to find block, number: %v", blockNumber) + } + expect := types.VerifyResult{ + Status: status, + BlockNumber: blockNumber, + BlockHash: block2.Hash(), + } + if status.Code&0xff00 == types.StatusVerified.Code { + expect.Root = block2.Root() + } + + diffLayer2 := chain2.GetTrustedDiffLayer(block2.Hash()) + if diffLayer2 == nil { + t.Fatal("failed to find diff layer") + } + diffHash2 := types.EmptyRootHash + if status != types.StatusDiffHashMismatch { + var err error + diffHash2, err = GetTrustedDiffHash(diffLayer2) + if err != nil { + t.Fatalf("failed to compute diff hash: %v", err) + } + } + + if status == types.StatusUntrustedVerified { + block1 := chain1.GetBlockByNumber(blockNumber) + if block1 == nil { + t.Fatalf("failed to find block, number: %v", blockNumber) + } + chain1.diffLayerCache.Remove(block1.Hash()) + } + + result, _ := chain1.GetRootByDiffHash(blockNumber, block2.Hash(), diffHash2) + if result.Status != expect.Status { + t.Fatalf("failed to verify block, number: %v, expect status: %v, real status: %v", blockNumber, expect.Status, result.Status) + } + if result.Root != expect.Root { + t.Fatalf("failed to verify block, number: %v, expect root: %v, real root: %v", blockNumber, expect.Root, result.Root) + } +} + +func TestGetRootByDiffHash(t *testing.T) { + len1 := 23 // length of blockchain1 + len2 := 35 // length of blockchain2 + plen := 11 // length of same parent blocks, which determined by testBlocks. + + chain1, chain2 := newTwoForkedBlockchains(len1, len2) + defer chain1.Stop() + defer chain2.Stop() + + hash1 := chain1.GetBlockByNumber(uint64(plen)).Hash() + hash2 := chain2.GetBlockByNumber(uint64(plen)).Hash() + if hash1 != hash2 { + t.Errorf("chain content mismatch at %d: have hash %v, want hash %v", plen, hash2, hash1) + } + + testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusFullVerified) + testGetRootByDiffHash(t, chain1, chain2, 2, types.StatusUntrustedVerified) + testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusDiffHashMismatch) + testGetRootByDiffHash(t, chain1, chain2, 12, types.StatusImpossibleFork) + testGetRootByDiffHash(t, chain1, chain2, 20, types.StatusPossibleFork) + testGetRootByDiffHash(t, chain1, chain2, 24, types.StatusBlockNewer) + testGetRootByDiffHash(t, chain1, chain2, 35, types.StatusBlockTooNew) +} From 0405b68b2a7f3953716385ae5ad67121aed76627 Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Fri, 21 Jan 2022 18:17:06 +0800 Subject: [PATCH 10/51] generate diff layer by replaying block --- core/blockchain.go | 60 ++++++++++++++++++ core/blockchain_diff_test.go | 117 +++++++++++++++++++++++++++++++++++ core/state/statedb.go | 75 ++++++++++++++++++++++ 3 files changed, 252 insertions(+) diff --git a/core/blockchain.go b/core/blockchain.go index 1bc5be0b19..78b88b5b1e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -3190,6 +3190,66 @@ func (bc *BlockChain) GetTrustedDiffLayer(blockHash common.Hash) *types.DiffLaye return diff } +// GenerateDiffLayer generates DiffLayer of a specified block by replaying the block's transactions. +// If the block is an empty block, no DiffLayer will be generated. +// The generated DiffLayer whose Receipts are empty, whose DiffAccounts' storage root is empty. +func (bc *BlockChain) GenerateDiffLayer(blockHash common.Hash) (*types.DiffLayer, error) { + if bc.snaps == nil { + return nil, fmt.Errorf("snapshot disabled, can't generate difflayer") + } + + block := bc.GetBlockByHash(blockHash) + if block == nil { + return nil, fmt.Errorf("block not found, block number: %d, blockhash: %v", block.NumberU64(), blockHash) + } + + parent := bc.GetBlockByHash(block.ParentHash()) + if parent == nil { + return nil, fmt.Errorf("block not found, block number: %d, blockhash: %v", block.NumberU64()-1, block.ParentHash()) + } + statedb, err := bc.StateAt(parent.Root()) + if err != nil { + return nil, fmt.Errorf("state not found for block number (%d): %v", parent.NumberU64(), err) + } + + // Empty block, no DiffLayer would be generated. + if block.Header().TxHash == types.EmptyRootHash { + return nil, nil + } + + // Replay transactions. + signer := types.MakeSigner(bc.Config(), block.Number()) + for _, tx := range block.Transactions() { + msg, _ := tx.AsMessage(signer) + txContext := NewEVMTxContext(msg) + context := NewEVMBlockContext(block.Header(), bc, nil) + vmenv := vm.NewEVM(context, txContext, statedb, bc.Config(), vm.Config{}) + + if posa, ok := bc.Engine().(consensus.PoSA); ok { + if isSystem, _ := posa.IsSystemTransaction(tx, block.Header()); isSystem { + balance := statedb.GetBalance(consensus.SystemAddress) + if balance.Cmp(common.Big0) > 0 { + statedb.SetBalance(consensus.SystemAddress, big.NewInt(0)) + statedb.AddBalance(block.Header().Coinbase, balance) + } + } + } + + if _, err := ApplyMessage(vmenv, msg, new(GasPool).AddGas(tx.Gas())); err != nil { + return nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + } + statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + } + + diffLayer := statedb.GenerateDiffLayer() + if diffLayer != nil { + diffLayer.BlockHash = blockHash + diffLayer.Number = block.NumberU64() + } + + return diffLayer, nil +} + func GetTrustedDiffHash(d *types.DiffLayer) (common.Hash, error) { diff := &types.ExtDiffLayer{ BlockHash: d.BlockHash, diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index 7d19d42ea5..1c749be72c 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/consensus/clique" + "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" @@ -644,3 +646,118 @@ func TestGetRootByDiffHash(t *testing.T) { testGetRootByDiffHash(t, chain1, chain2, 24, types.StatusBlockNewer) testGetRootByDiffHash(t, chain1, chain2, 35, types.StatusBlockTooNew) } + +func newBlockChainWithCliqueEngine(blocks int) *BlockChain { + signer := types.HomesteadSigner{} + db := rawdb.NewMemoryDatabase() + engine := clique.New(params.AllCliqueProtocolChanges.Clique, db) + genspec := &Genesis{ + //Config: params.TestChainConfig, + ExtraData: make([]byte, 32+common.AddressLength+65), + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + } + copy(genspec.ExtraData[32:], testAddr[:]) + genesis := genspec.MustCommit(db) + + chain, _ := NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + generator := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + // block.SetCoinbase(testAddr) + block.SetDifficulty(big.NewInt(2)) + + for idx, testBlock := range testBlocks { + // Specific block setting, the index in this generator has 1 diff from specified blockNr. + if i+1 == testBlock.blockNr { + for _, testTransaction := range testBlock.txs { + var transaction *types.Transaction + if testTransaction.to == nil { + transaction = types.NewContractCreation(block.TxNonce(testAddr), + testTransaction.value, uint64(commonGas), nil, testTransaction.data) + } else { + transaction = types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), nil, testTransaction.data) + } + tx, err := types.SignTx(transaction, signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + break + } + + // Default block setting. + if idx == len(testBlocks)-1 { + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), nil, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + } + } + + } + bs, _ := GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, blocks, generator) + for i, block := range bs { + header := block.Header() + if i > 0 { + header.ParentHash = bs[i-1].Hash() + } + header.Extra = make([]byte, 32+65) + header.Difficulty = big.NewInt(2) + + sig, _ := crypto.Sign(clique.SealHash(header).Bytes(), testKey) + copy(header.Extra[len(header.Extra)-65:], sig) + bs[i] = block.WithSeal(header) + } + + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + + return chain +} + +func TestGenerateDiffLayer(t *testing.T) { + blockNum := 32 + chain := newBlockChainWithCliqueEngine(blockNum) + defer chain.Stop() + + for blockNr := 1; blockNr <= blockNum; blockNr++ { + block := chain.GetBlockByNumber(uint64(blockNr)) + if block == nil { + t.Fatal("block should not be nil") + } + + expDiffLayer := chain.GetTrustedDiffLayer(block.Hash()) + if expDiffLayer == nil { + // Skip empty block. + if blockNr == 15 { + continue + } + t.Fatalf("unexpected nil diff layer, block number: %v, block hash: %v", blockNr, block.Hash()) + } + expDiffHash, err := GetTrustedDiffHash(expDiffLayer) + if err != nil { + t.Fatalf("compute diff hash failed: %v", err) + } + + diffLayer, err := chain.GenerateDiffLayer(block.Hash()) + if err != nil || diffLayer == nil { + t.Fatalf("generate diff layer failed: %v", err) + } + diffHash, err := GetTrustedDiffHash(diffLayer) + if err != nil { + t.Fatalf("compute diff hash failed: %v", err) + } + if expDiffHash != diffHash { + t.Fatalf("generated wrong diff layer for block: %d, expected hash: %v, real hash: %v", blockNr, expDiffHash, diffHash) + } + } +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 17eb3bf856..384373c98e 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1534,6 +1534,81 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er return root, diffLayer, nil } +// GenerateDiffLayer generates block's DiffLayer after executing the block's txs. +// Attention, the DiffLayer returned include no Receipts, whose accounts' storage root +// is empty, whose BlockHash and Number field is empty, should further process by caller. +func (s *StateDB) GenerateDiffLayer() *types.DiffLayer { + if s.snap == nil { + return nil + } + + for addr := range s.stateObjectsPending { + if obj := s.stateObjects[addr]; !obj.deleted { + // The snapshot storage map for the object + var storage map[string][]byte + obj.finalise(false) + for key, value := range obj.pendingStorage { + // Skip noop changes, persist actual changes + if value == obj.originStorage[key] { + continue + } + obj.originStorage[key] = value + + var v []byte + if (value != common.Hash{}) { + // Encoding []byte cannot fail, ok to ignore the error. + v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) + } + + obj.db.snapMux.Lock() + if storage == nil { + // Retrieve the old storage map, if available, create a new one otherwise + if storage = obj.db.snapStorage[obj.address]; storage == nil { + storage = make(map[string][]byte) + obj.db.snapStorage[obj.address] = storage + } + } + storage[string(key[:])] = v // v will be nil if value is 0x00 + obj.db.snapMux.Unlock() + } + + if !obj.deleted { + s.snapMux.Lock() + // The storage root hasn't been intermediate, pass empty storage root here. + s.snapAccounts[obj.address] = snapshot.SlimAccountRLP(obj.data.Nonce, obj.data.Balance, common.Hash{}, obj.data.CodeHash) + s.snapMux.Unlock() + } + } + } + + var diffLayer = &types.DiffLayer{} + for addr := range s.stateObjectsDirty { + if obj := s.stateObjects[addr]; !obj.deleted { + if obj.code != nil && obj.dirtyCode { + diffLayer.Codes = append(diffLayer.Codes, types.DiffCode{ + Hash: common.BytesToHash(obj.CodeHash()), + Code: obj.code, + }) + } + } + } + + diffLayer.Destructs, diffLayer.Accounts, diffLayer.Storages = s.SnapToDiffLayer() + sort.SliceStable(diffLayer.Codes, func(i, j int) bool { + return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() + }) + sort.SliceStable(diffLayer.Destructs, func(i, j int) bool { + return diffLayer.Destructs[i].Hex() < (diffLayer.Destructs[j].Hex()) + }) + sort.SliceStable(diffLayer.Accounts, func(i, j int) bool { + return diffLayer.Accounts[i].Account.Hex() < diffLayer.Accounts[j].Account.Hex() + }) + sort.SliceStable(diffLayer.Storages, func(i, j int) bool { + return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() + }) + return diffLayer +} + func (s *StateDB) DiffLayerToSnap(diffLayer *types.DiffLayer) (map[common.Address]struct{}, map[common.Address][]byte, map[common.Address]map[string][]byte, error) { snapDestructs := make(map[common.Address]struct{}) snapAccounts := make(map[common.Address][]byte) From 529e66eb795001754335f305edd86477d5daefdf Mon Sep 17 00:00:00 2001 From: Keefe-Liu Date: Tue, 25 Jan 2022 21:18:56 +0800 Subject: [PATCH 11/51] fix misc bugs of verify node --- cmd/utils/flags.go | 3 +- common/types.go | 15 ----- core/blockchain.go | 106 +++++++++++++++++++-------------- core/blockchain_diff_test.go | 14 ++--- core/state_processor.go | 2 +- core/types/block.go | 62 ++----------------- eth/backend.go | 18 ++++-- eth/ethconfig/config.go | 5 +- eth/ethconfig/gen_config.go | 6 ++ eth/peerset.go | 3 + eth/protocols/diff/protocol.go | 2 +- eth/protocols/trust/handler.go | 6 +- internal/ethapi/api.go | 2 +- 13 files changed, 105 insertions(+), 139 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fbca81a15d..a407098e61 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1712,7 +1712,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name) } if ctx.GlobalIsSet(NoDiscoverFlag.Name) { - cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs = []string{}, []string{} + cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs, cfg.TrustDiscoveryURLs = []string{}, []string{}, []string{} } else if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) { urls := ctx.GlobalString(DNSDiscoveryFlag.Name) if urls == "" { @@ -1818,6 +1818,7 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) { if url := params.KnownDNSNetwork(genesis, protocol); url != "" { cfg.EthDiscoveryURLs = []string{url} cfg.SnapDiscoveryURLs = cfg.EthDiscoveryURLs + cfg.TrustDiscoveryURLs = cfg.EthDiscoveryURLs } } diff --git a/common/types.go b/common/types.go index 15f2bce681..d715356692 100644 --- a/common/types.go +++ b/common/types.go @@ -354,21 +354,6 @@ func (a *Address) UnmarshalGraphQL(input interface{}) error { return err } -// AddressSlice is used for sort -type AddressSlice []Address - -func (s AddressSlice) Len() int { - return len(s) -} - -func (s AddressSlice) Less(i, j int) bool { - return s[i].Hex() < s[j].Hex() -} - -func (s AddressSlice) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - // UnprefixedAddress allows marshaling an Address without 0x prefix. type UnprefixedAddress Address diff --git a/core/blockchain.go b/core/blockchain.go index 78b88b5b1e..6e1a174dbb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -501,7 +501,22 @@ func (bc *BlockChain) cacheReceipts(hash common.Hash, receipts types.Receipts) { bc.receiptsCache.Add(hash, receipts) } -func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer) { +func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, sorted bool) { + if !sorted { + sort.SliceStable(diffLayer.Codes, func(i, j int) bool { + return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() + }) + sort.SliceStable(diffLayer.Destructs, func(i, j int) bool { + return diffLayer.Destructs[i].Hex() < (diffLayer.Destructs[j].Hex()) + }) + sort.SliceStable(diffLayer.Accounts, func(i, j int) bool { + return diffLayer.Accounts[i].Account.Hex() < diffLayer.Accounts[j].Account.Hex() + }) + sort.SliceStable(diffLayer.Storages, func(i, j int) bool { + return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() + }) + } + if bc.diffLayerCache.Len() >= diffLayerCacheLimit { bc.diffLayerCache.RemoveOldest() } @@ -1801,12 +1816,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.BlockHash = block.Hash() diffLayer.Number = block.NumberU64() - sort.Sort(types.DiffCodeSlice(diffLayer.Codes)) - sort.Sort(common.AddressSlice(diffLayer.Destructs)) - sort.Sort(types.DiffAccountSlice(diffLayer.Accounts)) - sort.Sort(types.DiffStorageSlice(diffLayer.Storages)) - - bc.cacheDiffLayer(diffLayer) + go bc.cacheDiffLayer(diffLayer, false) } wg.Wait() @@ -2798,9 +2808,14 @@ func (bc *BlockChain) HandleDiffLayer(diffLayer *types.DiffLayer, pid string, fu return nil } + diffHash := common.Hash{} + if diffLayer.DiffHash.Load() != nil { + diffHash = diffLayer.DiffHash.Load().(common.Hash) + } + bc.diffMux.Lock() defer bc.diffMux.Unlock() - if blockHash, exist := bc.diffHashToBlockHash[diffLayer.DiffHash]; exist && blockHash == diffLayer.BlockHash { + if blockHash, exist := bc.diffHashToBlockHash[diffHash]; exist && blockHash == diffLayer.BlockHash { return nil } @@ -2814,28 +2829,28 @@ func (bc *BlockChain) HandleDiffLayer(diffLayer *types.DiffLayer, pid string, fu return nil } if _, exist := bc.diffPeersToDiffHashes[pid]; exist { - if _, alreadyHas := bc.diffPeersToDiffHashes[pid][diffLayer.DiffHash]; alreadyHas { + if _, alreadyHas := bc.diffPeersToDiffHashes[pid][diffHash]; alreadyHas { return nil } } else { bc.diffPeersToDiffHashes[pid] = make(map[common.Hash]struct{}) } - bc.diffPeersToDiffHashes[pid][diffLayer.DiffHash] = struct{}{} + bc.diffPeersToDiffHashes[pid][diffHash] = struct{}{} if _, exist := bc.diffNumToBlockHashes[diffLayer.Number]; !exist { bc.diffNumToBlockHashes[diffLayer.Number] = make(map[common.Hash]struct{}) } bc.diffNumToBlockHashes[diffLayer.Number][diffLayer.BlockHash] = struct{}{} - if _, exist := bc.diffHashToPeers[diffLayer.DiffHash]; !exist { - bc.diffHashToPeers[diffLayer.DiffHash] = make(map[string]struct{}) + if _, exist := bc.diffHashToPeers[diffHash]; !exist { + bc.diffHashToPeers[diffHash] = make(map[string]struct{}) } - bc.diffHashToPeers[diffLayer.DiffHash][pid] = struct{}{} + bc.diffHashToPeers[diffHash][pid] = struct{}{} if _, exist := bc.blockHashToDiffLayers[diffLayer.BlockHash]; !exist { bc.blockHashToDiffLayers[diffLayer.BlockHash] = make(map[common.Hash]*types.DiffLayer) } - bc.blockHashToDiffLayers[diffLayer.BlockHash][diffLayer.DiffHash] = diffLayer - bc.diffHashToBlockHash[diffLayer.DiffHash] = diffLayer.BlockHash + bc.blockHashToDiffLayers[diffLayer.BlockHash][diffHash] = diffLayer + bc.diffHashToBlockHash[diffHash] = diffLayer.BlockHash return nil } @@ -3120,60 +3135,55 @@ func EnablePersistDiff(limit uint64) BlockChainOption { } } -func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { +func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) *types.VerifyResult { var res types.VerifyResult res.BlockNumber = blockNumber res.BlockHash = blockHash - if blockNumber > bc.CurrentHeader().Number.Uint64()+11 { + if blockNumber > bc.CurrentHeader().Number.Uint64()+maxDiffForkDist { res.Status = types.StatusBlockTooNew - return &res, nil + return &res } else if blockNumber > bc.CurrentHeader().Number.Uint64() { res.Status = types.StatusBlockNewer - return &res, nil + return &res + } + + header := bc.GetHeaderByHash(blockHash) + if header == nil { + if blockNumber > bc.CurrentHeader().Number.Uint64()-maxDiffForkDist { + res.Status = types.StatusPossibleFork + return &res + } + + res.Status = types.StatusImpossibleFork + return &res } diff := bc.GetTrustedDiffLayer(blockHash) if diff != nil { - if diff.DiffHash == (common.Hash{}) { - hash, err := GetTrustedDiffHash(diff) + if diff.DiffHash.Load() == nil { + hash, err := CalculateDiffHash(diff) if err != nil { res.Status = types.StatusUnexpectedError - return &res, err + return &res } - diff.DiffHash = hash + diff.DiffHash.Store(hash) } - if diffHash != diff.DiffHash { + if diffHash != diff.DiffHash.Load().(common.Hash) { res.Status = types.StatusDiffHashMismatch - return &res, nil + return &res } - header := bc.GetHeaderByHash(blockHash) - if header == nil { - res.Status = types.StatusUnexpectedError - return &res, fmt.Errorf("unexpected error, header not found") - } res.Status = types.StatusFullVerified res.Root = header.Root - return &res, nil - } - - header := bc.GetHeaderByHash(blockHash) - if header == nil { - if blockNumber > bc.CurrentHeader().Number.Uint64()-11 { - res.Status = types.StatusPossibleFork - return &res, nil - } - - res.Status = types.StatusImpossibleFork - return &res, nil + return &res } - res.Status = types.StatusUntrustedVerified + res.Status = types.StatusPartiallyVerified res.Root = header.Root - return &res, nil + return &res } func (bc *BlockChain) GetTrustedDiffLayer(blockHash common.Hash) *types.DiffLayer { @@ -3245,12 +3255,18 @@ func (bc *BlockChain) GenerateDiffLayer(blockHash common.Hash) (*types.DiffLayer if diffLayer != nil { diffLayer.BlockHash = blockHash diffLayer.Number = block.NumberU64() + + bc.cacheDiffLayer(diffLayer, true) } return diffLayer, nil } -func GetTrustedDiffHash(d *types.DiffLayer) (common.Hash, error) { +func CalculateDiffHash(d *types.DiffLayer) (common.Hash, error) { + if d == nil { + return common.Hash{}, fmt.Errorf("nil diff layer") + } + diff := &types.ExtDiffLayer{ BlockHash: d.BlockHash, Receipts: make([]*types.ReceiptForStorage, 0), diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index 1c749be72c..ab5af2815c 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -288,7 +288,7 @@ func rawDataToDiffLayer(data rlp.RawValue) (*types.DiffLayer, error) { hasher.Write(data) var diffHash common.Hash hasher.Sum(diffHash[:0]) - diff.DiffHash = diffHash + diff.DiffHash.Store(diffHash) hasher.Reset() return &diff, nil } @@ -600,13 +600,13 @@ func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber diffHash2 := types.EmptyRootHash if status != types.StatusDiffHashMismatch { var err error - diffHash2, err = GetTrustedDiffHash(diffLayer2) + diffHash2, err = CalculateDiffHash(diffLayer2) if err != nil { t.Fatalf("failed to compute diff hash: %v", err) } } - if status == types.StatusUntrustedVerified { + if status == types.StatusPartiallyVerified { block1 := chain1.GetBlockByNumber(blockNumber) if block1 == nil { t.Fatalf("failed to find block, number: %v", blockNumber) @@ -614,7 +614,7 @@ func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber chain1.diffLayerCache.Remove(block1.Hash()) } - result, _ := chain1.GetRootByDiffHash(blockNumber, block2.Hash(), diffHash2) + result := chain1.GetRootByDiffHash(blockNumber, block2.Hash(), diffHash2) if result.Status != expect.Status { t.Fatalf("failed to verify block, number: %v, expect status: %v, real status: %v", blockNumber, expect.Status, result.Status) } @@ -639,7 +639,7 @@ func TestGetRootByDiffHash(t *testing.T) { } testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusFullVerified) - testGetRootByDiffHash(t, chain1, chain2, 2, types.StatusUntrustedVerified) + testGetRootByDiffHash(t, chain1, chain2, 2, types.StatusPartiallyVerified) testGetRootByDiffHash(t, chain1, chain2, 10, types.StatusDiffHashMismatch) testGetRootByDiffHash(t, chain1, chain2, 12, types.StatusImpossibleFork) testGetRootByDiffHash(t, chain1, chain2, 20, types.StatusPossibleFork) @@ -743,7 +743,7 @@ func TestGenerateDiffLayer(t *testing.T) { } t.Fatalf("unexpected nil diff layer, block number: %v, block hash: %v", blockNr, block.Hash()) } - expDiffHash, err := GetTrustedDiffHash(expDiffLayer) + expDiffHash, err := CalculateDiffHash(expDiffLayer) if err != nil { t.Fatalf("compute diff hash failed: %v", err) } @@ -752,7 +752,7 @@ func TestGenerateDiffLayer(t *testing.T) { if err != nil || diffLayer == nil { t.Fatalf("generate diff layer failed: %v", err) } - diffHash, err := GetTrustedDiffHash(diffLayer) + diffHash, err := CalculateDiffHash(diffLayer) if err != nil { t.Fatalf("compute diff hash failed: %v", err) } diff --git a/core/state_processor.go b/core/state_processor.go index a21d68c504..16f6eab63b 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -118,7 +118,7 @@ func (p *LightStateProcessor) Process(block *types.Block, statedb *state.StateDB return statedb, receipts, logs, gasUsed, nil } log.Error("do light process err at block", "num", block.NumberU64(), "err", err) - p.bc.removeDiffLayers(diffLayer.DiffHash) + p.bc.removeDiffLayers(diffLayer.DiffHash.Load().(common.Hash)) // prepare new statedb statedb.StopPrefetcher() parent := p.bc.GetHeader(block.ParentHash(), block.NumberU64()-1) diff --git a/core/types/block.go b/core/types/block.go index f3c487b684..72cf3408b5 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -49,7 +49,7 @@ var ( // StatusVerified means the processing of request going as expected and found the root correctly. StatusVerified = VerifyStatus{Code: 0x100} StatusFullVerified = VerifyStatus{Code: 0x101, Msg: "state root full verified"} - StatusUntrustedVerified = VerifyStatus{Code: 0x102, Msg: "state root untrusted verified, because of difflayer not found"} + StatusPartiallyVerified = VerifyStatus{Code: 0x102, Msg: "state root partially verified, because of difflayer not found"} // StatusFailed means the request has something wrong. StatusFailed = VerifyStatus{Code: 0x200} @@ -57,11 +57,10 @@ var ( StatusImpossibleFork = VerifyStatus{Code: 0x202, Msg: "verify failed because of impossible fork detected"} // StatusUncertain means verify node can't give a certain result of the request. - StatusUncertain = VerifyStatus{Code: 0x300} - StatusBlockTooNew = VerifyStatus{Code: 0x301, Msg: "can’t verify because of block number larger than current height more than 11"} - StatusBlockNewer = VerifyStatus{Code: 0x302, Msg: "can’t verify because of block number larger than current height"} - StatusPossibleFork = VerifyStatus{Code: 0x303, Msg: "can’t verify because of possible fork detected"} - StatusRequestTooBusy = VerifyStatus{Code: 0x304, Msg: "can’t verify because of request too busy"} + StatusUncertain = VerifyStatus{Code: 0x300} + StatusBlockTooNew = VerifyStatus{Code: 0x301, Msg: "can’t verify because of block number larger than current height more than 11"} + StatusBlockNewer = VerifyStatus{Code: 0x302, Msg: "can’t verify because of block number larger than current height"} + StatusPossibleFork = VerifyStatus{Code: 0x303, Msg: "can’t verify because of possible fork detected"} // StatusUnexpectedError is unexpected internal error. StatusUnexpectedError = VerifyStatus{Code: 0x400, Msg: "can’t verify because of unexpected internal error"} @@ -414,7 +413,7 @@ type DiffLayer struct { Accounts []DiffAccount Storages []DiffStorage - DiffHash common.Hash + DiffHash atomic.Value } type ExtDiffLayer struct { @@ -477,66 +476,17 @@ type DiffCode struct { Code []byte } -// DiffCodeSlice is used for sort -type DiffCodeSlice []DiffCode - -func (s DiffCodeSlice) Len() int { - return len(s) -} - -func (s DiffCodeSlice) Less(i, j int) bool { - return s[i].Hash.Hex() < s[j].Hash.Hex() -} - -func (s DiffCodeSlice) Swap(i, j int) { - s[i].Hash, s[j].Hash = s[j].Hash, s[i].Hash - s[i].Code, s[j].Code = s[j].Code, s[i].Code -} - type DiffAccount struct { Account common.Address Blob []byte } -// DiffAccountSlice is used for sort -type DiffAccountSlice []DiffAccount - -func (s DiffAccountSlice) Len() int { - return len(s) -} - -func (s DiffAccountSlice) Less(i, j int) bool { - return s[i].Account.Hex() < s[j].Account.Hex() -} - -func (s DiffAccountSlice) Swap(i, j int) { - s[i].Account, s[j].Account = s[j].Account, s[i].Account - s[i].Blob, s[j].Blob = s[j].Blob, s[i].Blob -} - type DiffStorage struct { Account common.Address Keys []string Vals [][]byte } -// DiffStorageSlice is used for sort -type DiffStorageSlice []DiffStorage - -func (s DiffStorageSlice) Len() int { - return len(s) -} - -func (s DiffStorageSlice) Less(i, j int) bool { - return s[i].Account.Hex() < s[j].Account.Hex() -} - -func (s DiffStorageSlice) Swap(i, j int) { - s[i].Account, s[j].Account = s[j].Account, s[i].Account - s[i].Keys, s[j].Keys = s[j].Keys, s[i].Keys - s[i].Vals, s[j].Vals = s[j].Vals, s[i].Vals -} - type DiffAccountsInTx struct { TxHash common.Hash Accounts map[common.Address]*big.Int diff --git a/eth/backend.go b/eth/backend.go index b9e8e40cb9..0e937c49a0 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -70,11 +70,12 @@ type Ethereum struct { config *ethconfig.Config // Handlers - txPool *core.TxPool - blockchain *core.BlockChain - handler *handler - ethDialCandidates enode.Iterator - snapDialCandidates enode.Iterator + txPool *core.TxPool + blockchain *core.BlockChain + handler *handler + ethDialCandidates enode.Iterator + snapDialCandidates enode.Iterator + trustDialCandidates enode.Iterator // DB interfaces chainDb ethdb.Database // Block chain database @@ -272,6 +273,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + eth.trustDialCandidates, err = dnsclient.NewIterator(eth.config.TrustDiscoveryURLs...) + if err != nil { + return nil, err + } // Start the RPC service eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer, config.NetworkId) @@ -556,7 +561,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { } // diff protocol can still open without snap protocol protos = append(protos, diff.MakeProtocols((*diffHandler)(s.handler), s.snapDialCandidates)...) - protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.snapDialCandidates)...) + protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.trustDialCandidates)...) return protos } @@ -587,6 +592,7 @@ func (s *Ethereum) Stop() error { // Stop all the peer-related stuff first. s.ethDialCandidates.Close() s.snapDialCandidates.Close() + s.trustDialCandidates.Close() s.handler.Stop() // Then stop everything else. diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 4f2c87c52e..ee35d123a3 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -131,8 +131,9 @@ type Config struct { // This can be set to list of enrtree:// URLs which will be queried for // for nodes to connect to. - EthDiscoveryURLs []string - SnapDiscoveryURLs []string + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + TrustDiscoveryURLs []string NoPruning bool // Whether to disable pruning and flush everything to disk DirectBroadcast bool diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index ba2996279d..c5e46ced41 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -24,6 +24,7 @@ func (c Config) MarshalTOML() (interface{}, error) { DisablePeerTxBroadcast bool EthDiscoveryURLs []string SnapDiscoveryURLs []string + TrustDiscoveryURLs []string NoPruning bool NoPrefetch bool DirectBroadcast bool @@ -79,6 +80,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DisablePeerTxBroadcast = c.DisablePeerTxBroadcast enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs + enc.TrustDiscoveryURLs = c.TrustDiscoveryURLs enc.NoPruning = c.NoPruning enc.DirectBroadcast = c.DirectBroadcast enc.DisableSnapProtocol = c.DisableSnapProtocol @@ -137,6 +139,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DisablePeerTxBroadcast *bool EthDiscoveryURLs []string SnapDiscoveryURLs []string + TrustDiscoveryURLs []string NoPruning *bool NoPrefetch *bool DirectBroadcast *bool @@ -207,6 +210,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SnapDiscoveryURLs != nil { c.SnapDiscoveryURLs = dec.SnapDiscoveryURLs } + if dec.TrustDiscoveryURLs != nil { + c.TrustDiscoveryURLs = dec.TrustDiscoveryURLs + } if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } diff --git a/eth/peerset.go b/eth/peerset.go index dc1d7da45e..5bbaa2dd2b 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -335,6 +335,9 @@ func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { // GetVerifyPeers returns an array of verify nodes. func (ps *peerSet) GetVerifyPeers() []*trustPeer { + ps.lock.RLock() + defer ps.lock.RUnlock() + res := make([]*trustPeer, 0) for _, p := range ps.peers { if p.trustExt != nil { diff --git a/eth/protocols/diff/protocol.go b/eth/protocols/diff/protocol.go index 4467d0b327..e6bf5b3e14 100644 --- a/eth/protocols/diff/protocol.go +++ b/eth/protocols/diff/protocol.go @@ -92,7 +92,7 @@ func (p *DiffLayersPacket) Unpack() ([]*types.DiffLayer, error) { var diffHash common.Hash hasher.Sum(diffHash[:0]) hasher.Reset() - diff.DiffHash = diffHash + diff.DiffHash.Store(diffHash) } return diffLayers, nil } diff --git a/eth/protocols/trust/handler.go b/eth/protocols/trust/handler.go index ce8fc4cd8d..9b93d4a228 100644 --- a/eth/protocols/trust/handler.go +++ b/eth/protocols/trust/handler.go @@ -126,8 +126,8 @@ func handleRootRequest(backend Backend, msg Decoder, peer *Peer) error { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - res, err := backend.Chain().GetRootByDiffHash(req.BlockNumber, req.BlockHash, req.DiffHash) - p2p.Send(peer.rw, RespondRootMsg, RootResponsePacket{ + res := backend.Chain().GetRootByDiffHash(req.BlockNumber, req.BlockHash, req.DiffHash) + return p2p.Send(peer.rw, RespondRootMsg, RootResponsePacket{ RequestId: req.RequestId, Status: res.Status, BlockNumber: req.BlockNumber, @@ -135,8 +135,6 @@ func handleRootRequest(backend Backend, msg Decoder, peer *Peer) error { Root: res.Root, Extra: defaultExtra, }) - - return err } func handleRootResponse(backend Backend, msg Decoder, peer *Peer) error { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index b01ec8b2e5..f3bcc3b98d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1287,7 +1287,7 @@ func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, bloc return result, err } -func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { +func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) *types.VerifyResult { return s.b.Chain().GetRootByDiffHash(uint64(blockNr), blockHash, diffHash) } From fed2f3518ec73fcf397abd5106f260d119f1bbc2 Mon Sep 17 00:00:00 2001 From: KeefeL <90749943+KeefeL@users.noreply.github.com> Date: Thu, 27 Jan 2022 16:50:59 +0800 Subject: [PATCH 12/51] testcases for trust protocol (#742) --- eth/protocols/trust/handler_test.go | 270 ++++++++++++++++++++++++++++ eth/protocols/trust/peer_test.go | 42 +++++ 2 files changed, 312 insertions(+) create mode 100644 eth/protocols/trust/handler_test.go create mode 100644 eth/protocols/trust/peer_test.go diff --git a/eth/protocols/trust/handler_test.go b/eth/protocols/trust/handler_test.go new file mode 100644 index 0000000000..e594401a2c --- /dev/null +++ b/eth/protocols/trust/handler_test.go @@ -0,0 +1,270 @@ +package trust + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" +) + +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) + +// testBackend is a mock implementation of the live Ethereum message handler. Its +// purpose is to allow testing the request/reply workflows and wire serialization +// in the `eth` protocol without actually doing any data processing. +type testBackend struct { + db ethdb.Database + chain *core.BlockChain + txpool *core.TxPool +} + +// newTestBackend creates an empty chain and wraps it into a mock backend. +func newTestBackend(blocks int) *testBackend { + return newTestBackendWithGenerator(blocks) +} + +// newTestBackend creates a chain with a number of explicitly defined blocks and +// wraps it into a mock backend. +func newTestBackendWithGenerator(blocks int) *testBackend { + signer := types.HomesteadSigner{} + db := rawdb.NewMemoryDatabase() + engine := clique.New(params.AllCliqueProtocolChanges.Clique, db) + genspec := &core.Genesis{ + //Config: params.TestChainConfig, + ExtraData: make([]byte, 32+common.AddressLength+65), + Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + } + copy(genspec.ExtraData[32:], testAddr[:]) + genesis := genspec.MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + generator := func(i int, block *core.BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + // block.SetCoinbase(testAddr) + block.SetDifficulty(big.NewInt(2)) + + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), common.Address{0x01}, big.NewInt(1), params.TxGas, nil, nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(chain, tx) + } + + bs, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, blocks, generator) + for i, block := range bs { + header := block.Header() + if i > 0 { + header.ParentHash = bs[i-1].Hash() + } + header.Extra = make([]byte, 32+65) + header.Difficulty = big.NewInt(2) + + sig, _ := crypto.Sign(clique.SealHash(header).Bytes(), testKey) + copy(header.Extra[len(header.Extra)-65:], sig) + bs[i] = block.WithSeal(header) + } + + if _, err := chain.InsertChain(bs); err != nil { + panic(err) + } + + txconfig := core.DefaultTxPoolConfig + txconfig.Journal = "" // Don't litter the disk with test journals + + return &testBackend{ + db: db, + chain: chain, + txpool: core.NewTxPool(txconfig, params.AllCliqueProtocolChanges, chain), + } +} + +// close tears down the transaction pool and chain behind the mock backend. +func (b *testBackend) close() { + b.txpool.Stop() + b.chain.Stop() +} + +func (b *testBackend) Chain() *core.BlockChain { return b.chain } + +func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { + // Normally the backend would do peer mainentance and handshakes. All that + // is omitted and we will just give control back to the handler. + return handler(peer) +} +func (b *testBackend) PeerInfo(enode.ID) interface{} { panic("not implemented") } + +func (b *testBackend) Handle(*Peer, Packet) error { + panic("data processing tests should be done in the handler package") +} + +func TestRequestRoot(t *testing.T) { testRequestRoot(t, Trust1) } + +func testRequestRoot(t *testing.T, protocol uint) { + t.Parallel() + + blockNum := 1032 // The latest 1024 blocks' DiffLayer will be cached. + backend := newTestBackend(blockNum) + defer backend.close() + + peer, _ := newTestPeer("peer", protocol, backend) + defer peer.close() + + pairs := []struct { + req RootRequestPacket + res RootResponsePacket + }{ + { + req: RootRequestPacket{ + RequestId: 1, + BlockNumber: 1, + }, + res: RootResponsePacket{ + RequestId: 1, + Status: types.StatusPartiallyVerified, + BlockNumber: 1, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 2, + BlockNumber: 128, + }, + res: RootResponsePacket{ + RequestId: 2, + Status: types.StatusFullVerified, + BlockNumber: 128, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 3, + BlockNumber: 128, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 3, + Status: types.StatusImpossibleFork, + BlockNumber: 128, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 4, + BlockNumber: 128, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 4, + Status: types.StatusDiffHashMismatch, + BlockNumber: 128, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 5, + BlockNumber: 1024, + }, + res: RootResponsePacket{ + RequestId: 5, + Status: types.StatusFullVerified, + BlockNumber: 1024, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 6, + BlockNumber: 1024, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 6, + Status: types.StatusPossibleFork, + BlockNumber: 1024, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 7, + BlockNumber: 1033, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 7, + Status: types.StatusBlockNewer, + BlockNumber: 1033, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + { + req: RootRequestPacket{ + RequestId: 8, + BlockNumber: 1044, + BlockHash: types.EmptyRootHash, + DiffHash: types.EmptyRootHash, + }, + res: RootResponsePacket{ + RequestId: 8, + Status: types.StatusBlockTooNew, + BlockNumber: 1044, + BlockHash: types.EmptyRootHash, + Root: common.Hash{}, + Extra: defaultExtra, + }, + }, + } + + for idx, pair := range pairs { + header := backend.Chain().GetHeaderByNumber(pair.req.BlockNumber) + if header != nil { + if pair.res.Status.Code&0xFF00 == types.StatusVerified.Code { + pair.req.BlockHash = header.Hash() + pair.req.DiffHash, _ = core.CalculateDiffHash(backend.Chain().GetTrustedDiffLayer(header.Hash())) + pair.res.BlockHash = pair.req.BlockHash + pair.res.Root = header.Root + } else if pair.res.Status.Code == types.StatusDiffHashMismatch.Code { + pair.req.BlockHash = header.Hash() + pair.res.BlockHash = pair.req.BlockHash + } + } + + p2p.Send(peer.app, RequestRootMsg, pair.req) + if err := p2p.ExpectMsg(peer.app, RespondRootMsg, pair.res); err != nil { + t.Errorf("test %d: root response not expected: %v", idx, err) + } + } +} diff --git a/eth/protocols/trust/peer_test.go b/eth/protocols/trust/peer_test.go new file mode 100644 index 0000000000..ab229a1b32 --- /dev/null +++ b/eth/protocols/trust/peer_test.go @@ -0,0 +1,42 @@ +package trust + +import ( + "math/rand" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +// testPeer is a simulated peer to allow testing direct network calls. +type testPeer struct { + *Peer + + net p2p.MsgReadWriter // Network layer reader/writer to simulate remote messaging + app *p2p.MsgPipeRW // Application layer reader/writer to simulate the local side +} + +// newTestPeer creates a new peer registered at the given data backend. +func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() + + // Start the peer on a new thread + var id enode.ID + rand.Read(id[:]) + + peer := NewPeer(version, p2p.NewPeer(id, name, nil), net) + errc := make(chan error, 1) + go func() { + errc <- backend.RunPeer(peer, func(peer *Peer) error { + return Handle(backend, peer) + }) + }() + return &testPeer{app: app, net: net, Peer: peer}, errc +} + +// close terminates the local side of the peer, notifying the remote protocol +// manager of termination. +func (p *testPeer) close() { + p.Peer.Close() + p.app.Close() +} From f7ef0160fa9f0734d1e86db812444ced5ba1666a Mon Sep 17 00:00:00 2001 From: kyrie-yl Date: Thu, 24 Feb 2022 17:13:40 +0800 Subject: [PATCH 13/51] fast node verification and fix conflicts Signed-off-by: kyrie-yl --- cmd/geth/main.go | 4 +- cmd/geth/usage.go | 3 + cmd/utils/flags.go | 31 ++- core/block_validator.go | 24 ++- core/blockchain.go | 13 +- core/blockchain_diff_test.go | 2 +- core/remote_state_verifier.go | 392 ++++++++++++++++++++++++++++++++++ core/types.go | 2 + core/types/block.go | 7 - eth/backend.go | 22 +- eth/ethconfig/config.go | 6 +- eth/ethconfig/gen_config.go | 22 +- eth/handler.go | 3 +- eth/handler_trust.go | 14 +- eth/peerset.go | 12 +- ethclient/ethclient.go | 5 +- internal/ethapi/api.go | 2 +- 17 files changed, 517 insertions(+), 47 deletions(-) create mode 100644 core/remote_state_verifier.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index db00cf19e1..97b3b5b268 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -71,6 +71,8 @@ var ( utils.NoUSBFlag, utils.DirectBroadcastFlag, utils.DisableSnapProtocolFlag, + utils.DisableDiffProtocolFlag, + utils.EnableTrustProtocolFlag, utils.DiffSyncFlag, utils.PipeCommitFlag, utils.RangeLimitFlag, @@ -98,6 +100,7 @@ var ( utils.TxPoolLifetimeFlag, utils.TxPoolReannounceTimeFlag, utils.SyncModeFlag, + utils.TriesVerifyModeFlag, utils.ExitWhenSyncedFlag, utils.GCModeFlag, utils.SnapshotFlag, @@ -115,7 +118,6 @@ var ( utils.WhitelistFlag, utils.BloomFilterSizeFlag, utils.TriesInMemoryFlag, - utils.AllowInsecureNoTriesFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index a1a0a7d0b4..6dd878f9e9 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -41,6 +41,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.NoUSBFlag, utils.DirectBroadcastFlag, utils.DisableSnapProtocolFlag, + utils.DisableDiffProtocolFlag, + utils.EnableTrustProtocolFlag, utils.RangeLimitFlag, utils.SmartCardDaemonPathFlag, utils.NetworkIdFlag, @@ -50,6 +52,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.YoloV3Flag, utils.RopstenFlag, utils.SyncModeFlag, + utils.TriesVerifyModeFlag, utils.ExitWhenSyncedFlag, utils.GCModeFlag, utils.TxLookupLimitFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a407098e61..8d24d23d84 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -122,6 +122,14 @@ var ( Name: "disablesnapprotocol", Usage: "Disable snap protocol", } + DisableDiffProtocolFlag = cli.BoolFlag{ + Name: "disablediffprotocol", + Usage: "Disable diff protocol", + } + EnableTrustProtocolFlag = cli.BoolFlag{ + Name: "enabletrustprotocol", + Usage: "Enable trust protocol", + } DiffSyncFlag = cli.BoolFlag{ Name: "diffsync", Usage: "Enable diffy sync, Please note that enable diffsync will improve the syncing speed, " + @@ -264,9 +272,11 @@ var ( Usage: "The layer of tries trees that keep in memory", Value: 128, } - AllowInsecureNoTriesFlag = cli.BoolTFlag{ - Name: "allow-insecure-no-tries", - Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`, + defaultVerifyMode = ethconfig.Defaults.TriesVerifyMode + TriesVerifyModeFlag = TextMarshalerFlag{ + Name: "tries-verify-mode", + Usage: `tries verify mode: "local", "full", "insecure", "none"`, + Value: &defaultVerifyMode, } OverrideBerlinFlag = cli.Uint64Flag{ Name: "override.berlin", @@ -1637,6 +1647,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(DisableSnapProtocolFlag.Name) { cfg.DisableSnapProtocol = ctx.GlobalBool(DisableSnapProtocolFlag.Name) } + if ctx.GlobalIsSet(DisableDiffProtocolFlag.Name) { + cfg.DisableDiffProtocol = ctx.GlobalIsSet(DisableDiffProtocolFlag.Name) + } + if ctx.GlobalIsSet(EnableTrustProtocolFlag.Name) { + cfg.EnableTrustProtocol = ctx.GlobalIsSet(EnableTrustProtocolFlag.Name) + } if ctx.GlobalIsSet(DiffSyncFlag.Name) { cfg.DiffSync = ctx.GlobalBool(DiffSyncFlag.Name) } @@ -1670,8 +1686,13 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(TriesInMemoryFlag.Name) { cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name) } - if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) { - cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name) + if ctx.GlobalIsSet(TriesVerifyModeFlag.Name) { + cfg.TriesVerifyMode = *GlobalTextMarshaler(ctx, TriesVerifyModeFlag.Name).(*core.VerifyMode) + // If a node sets verify mode to full or light, it's a fast node and need + // to verify blocks from verify nodes, then it should enable trust protocol. + if cfg.TriesVerifyMode.NeedRemoteVerify() { + cfg.EnableTrustProtocol = true + } } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 diff --git a/core/block_validator.go b/core/block_validator.go index 3ea6615b61..4cc5c67c16 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -34,18 +34,23 @@ const badBlockCacheExpire = 30 * time.Second // // BlockValidator implements Validator. type BlockValidator struct { - config *params.ChainConfig // Chain configuration options - bc *BlockChain // Canonical block chain - engine consensus.Engine // Consensus engine used for validating + config *params.ChainConfig // Chain configuration options + bc *BlockChain // Canonical block chain + engine consensus.Engine // Consensus engine used for validating + remoteValidator *remoteVerifyManager } // NewBlockValidator returns a new block validator which is safe for re-use -func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine) *BlockValidator { +func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine, mode VerifyMode, peers verifyPeers) *BlockValidator { validator := &BlockValidator{ config: config, engine: engine, bc: blockchain, } + if mode.NeedRemoteVerify() { + validator.remoteValidator = NewVerifyManager(blockchain, peers, mode == InsecureVerify) + go validator.remoteValidator.mainLoop() + } return validator } @@ -85,6 +90,13 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } return nil }, + // for fast node which verify trie from remote verify peers, a block's H-11 ancestor should have been verify. + func() error { + if v.remoteValidator != nil && !v.remoteValidator.AncestorVerified(v.bc.GetHeaderByNumber(header.Number.Uint64())) { + return fmt.Errorf("block's ancessor %x has not been verified", block.Hash()) + } + return nil + }, } validateRes := make(chan error, len(validateFuns)) for _, f := range validateFuns { @@ -164,6 +176,10 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD return err } +func (v *BlockValidator) RemoteVerifyManager() *remoteVerifyManager { + return v.remoteValidator +} + // CalcGasLimit computes the gas limit of the next block after parent. It aims // to keep the baseline gas above the provided floor, and increase it towards the // ceil if the blocks are full. If the ceil is exceeded, it will always decrease diff --git a/core/blockchain.go b/core/blockchain.go index 6e1a174dbb..ffcfe444b6 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -311,8 +311,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par diffNumToBlockHashes: make(map[uint64]map[common.Hash]struct{}), diffPeersToDiffHashes: make(map[string]map[common.Hash]struct{}), } + bc.prefetcher = NewStatePrefetcher(chainConfig, bc, engine) - bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.processor = NewStateProcessor(chainConfig, bc, engine) var err error @@ -3135,8 +3135,15 @@ func EnablePersistDiff(limit uint64) BlockChainOption { } } -func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) *types.VerifyResult { - var res types.VerifyResult +func EnableBlockValidator(chainConfig *params.ChainConfig, engine consensus.Engine, mode VerifyMode, peers verifyPeers) BlockChainOption { + return func(bc *BlockChain) *BlockChain { + bc.validator = NewBlockValidator(chainConfig, bc, engine, mode, peers) + return bc + } +} + +func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) *VerifyResult { + var res VerifyResult res.BlockNumber = blockNumber res.BlockHash = blockHash diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index ab5af2815c..c4eeccda45 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -584,7 +584,7 @@ func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber if block2 == nil { t.Fatalf("failed to find block, number: %v", blockNumber) } - expect := types.VerifyResult{ + expect := VerifyResult{ Status: status, BlockNumber: blockNumber, BlockHash: block2.Hash(), diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go new file mode 100644 index 0000000000..224d9b009b --- /dev/null +++ b/core/remote_state_verifier.go @@ -0,0 +1,392 @@ +package core + +import ( + "fmt" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/metrics" + "math/rand" + "time" + + lru "github.com/hashicorp/golang-lru" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +const ( + verifiedCacheSize = 256 + maxForkHeight = 11 + + // defaultPeerNumber is default number of verify peers + defaultPeerNumber = 3 + // pruneHeightDiff indicates that if the height difference between current block and task's + // corresponding block is larger than it, the task should be pruned. + pruneHeightDiff = 15 + pruneInterval = 5 * time.Second + resendInterval = 2 * time.Second + // tryAllPeersTime is the time that a block has not been verified and then try all the valid verify peers. + tryAllPeersTime = 15 * time.Second +) + +var ( + remoteVerifyTaskCounter = metrics.NewRegisteredCounter("remote/state/verify/task/total", nil) + + statusFullVerifiedMeter = metrics.NewRegisteredMeter("status/full/verified/messages/total", nil) + statusUntrustedVerifiedMeter = metrics.NewRegisteredMeter("status/untrusted/verified/messages/total", nil) + + statusDiffHashMismatchMeter = metrics.NewRegisteredMeter("status/diffhash/mismatch/messages/total", nil) + statusImpossibleForkMeter = metrics.NewRegisteredMeter("status/impossible/fork/messages/total", nil) + + statusBlockTooNewMeter = metrics.NewRegisteredMeter("status/block/too/new/messages/total", nil) + statusBlockNewerMeter = metrics.NewRegisteredMeter("status/block/newer/messages/total", nil) + statusPossibleForkMeter = metrics.NewRegisteredMeter("status/possible/fork/messages/total", nil) + statusUnexpectedErrorMeter = metrics.NewRegisteredMeter("status/unexpected/error/total", nil) + + codeMap = map[uint16]metrics.Meter{ + 0x101: statusFullVerifiedMeter, + 0x102: statusUntrustedVerifiedMeter, + 0x201: statusDiffHashMismatchMeter, + 0x202: statusImpossibleForkMeter, + 0x301: statusBlockTooNewMeter, + 0x302: statusBlockNewerMeter, + 0x303: statusPossibleForkMeter, + 0x400: statusUnexpectedErrorMeter, + } +) + +type remoteVerifyManager struct { + bc *BlockChain + tasks map[common.Hash]*verifyTask + peers verifyPeers + verifiedCache *lru.Cache + allowInsecure bool + + // Subscription + chainHeadCh chan ChainHeadEvent + chainHeadSub event.Subscription + + // Channels + verifyCh chan common.Hash + messageCh chan verifyMessage +} + +func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowUntrusted bool) *remoteVerifyManager { + verifiedCache, _ := lru.New(verifiedCacheSize) + vm := &remoteVerifyManager{ + bc: blockchain, + tasks: make(map[common.Hash]*verifyTask), + peers: peers, + verifiedCache: verifiedCache, + allowInsecure: allowUntrusted, + + chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), + verifyCh: make(chan common.Hash, maxForkHeight), + messageCh: make(chan verifyMessage), + } + vm.chainHeadSub = blockchain.SubscribeChainHeadEvent(vm.chainHeadCh) + return vm +} + +func (vm *remoteVerifyManager) mainLoop() { + defer vm.chainHeadSub.Unsubscribe() + + // load unverified blocks in a normalized chain and start a batch of verify task + header := vm.bc.CurrentHeader() + // Start verify task from H to H-11 if need. + vm.NewBlockVerifyTask(header) + pruneTicker := time.NewTicker(pruneInterval) + defer pruneTicker.Stop() + for { + select { + case h := <-vm.chainHeadCh: + vm.NewBlockVerifyTask(h.Block.Header()) + case hash := <-vm.verifyCh: + vm.cacheBlockVerified(hash) + if task, ok := vm.tasks[hash]; ok { + delete(vm.tasks, hash) + remoteVerifyTaskCounter.Dec(1) + close(task.terminalCh) + } + case <-pruneTicker.C: + for hash, task := range vm.tasks { + if vm.bc.CurrentHeader().Number.Cmp(task.blockHeader.Number) == 1 && + vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff { + delete(vm.tasks, hash) + remoteVerifyTaskCounter.Dec(1) + close(task.terminalCh) + } + } + case message := <-vm.messageCh: + if vt, ok := vm.tasks[message.verifyResult.BlockHash]; ok { + vt.messageCh <- message + } + + // System stopped + case <-vm.bc.quit: + for _, task := range vm.tasks { + close(task.terminalCh) + } + return + case <-vm.chainHeadSub.Err(): + return + } + } +} + +func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { + for i := 0; header != nil && i <= maxForkHeight; i++ { + func(hash common.Hash) { + // if verified cache record that this block has been verified, skip. + if _, ok := vm.verifiedCache.Get(hash); ok { + return + } + // if there already has a verify task for this block, skip. + if _, ok := vm.tasks[hash]; ok { + return + } + diffLayer := vm.bc.GetTrustedDiffLayer(hash) + // if this block has no diff, there is no need to verify it. + var err error + if diffLayer == nil { + if diffLayer, err = vm.bc.GenerateDiffLayer(hash); err != nil { + log.Error("failed to get diff layer", "block", hash, "number", header.Number, "error", err) + return + } else { + log.Info("this is an empty block:", "block", hash, "number", header.Number) + return + } + } + diffHash, err := CalculateDiffHash(diffLayer) + if err != nil { + log.Error("failed to get diff hash", "block", hash, "number", header.Number, "error", err) + return + } + verifyTask := NewVerifyTask(diffHash, header, vm.peers, vm.verifyCh, vm.allowInsecure) + vm.tasks[hash] = verifyTask + remoteVerifyTaskCounter.Inc(1) + }(header.Hash()) + header = vm.bc.GetHeaderByHash(header.ParentHash) + } +} + +func (vm *remoteVerifyManager) cacheBlockVerified(hash common.Hash) { + if vm.verifiedCache.Len() >= verifiedCacheSize { + vm.verifiedCache.RemoveOldest() + } + vm.verifiedCache.Add(hash, true) +} + +// AncestorVerified function check block has been verified or it's a empty block. +func (vm *remoteVerifyManager) AncestorVerified(header *types.Header) bool { + // find header of H-11 block. + header = vm.bc.GetHeaderByNumber(header.Number.Uint64() - maxForkHeight) + // If start from genesis block, there has not a H-11 block. + if header == nil { + return true + } + // check whether H-11 block is a empty block. + if header.TxHash == types.EmptyRootHash { + parent := vm.bc.GetHeaderByHash(header.ParentHash) + return header.Root == parent.Root + } + hash := header.Hash() + _, exist := vm.verifiedCache.Get(hash) + return exist +} + +func (vm *remoteVerifyManager) HandleRootResponse(vr *VerifyResult, pid string) error { + vm.messageCh <- verifyMessage{verifyResult: vr, peerId: pid} + return nil +} + +type VerifyResult struct { + Status types.VerifyStatus + BlockNumber uint64 + BlockHash common.Hash + Root common.Hash +} + +type verifyMessage struct { + verifyResult *VerifyResult + peerId string +} + +type verifyTask struct { + diffhash common.Hash + blockHeader *types.Header + candidatePeers verifyPeers + BadPeers map[string]struct{} + startAt time.Time + allowUntrusted bool + + messageCh chan verifyMessage + terminalCh chan struct{} +} + +func NewVerifyTask(diffhash common.Hash, header *types.Header, peers verifyPeers, verifyCh chan common.Hash, allowUntrusted bool) *verifyTask { + vt := &verifyTask{ + diffhash: diffhash, + blockHeader: header, + candidatePeers: peers, + BadPeers: make(map[string]struct{}), + allowUntrusted: allowUntrusted, + messageCh: make(chan verifyMessage), + terminalCh: make(chan struct{}), + } + go vt.Start(verifyCh) + return vt +} + +func (vt *verifyTask) Start(verifyCh chan common.Hash) { + vt.startAt = time.Now() + + vt.sendVerifyRequest(defaultPeerNumber) + resend := time.NewTicker(resendInterval) + defer resend.Stop() + for { + select { + case msg := <-vt.messageCh: + if metric, exist := codeMap[msg.verifyResult.Status.Code]; exist { + metric.Mark(1) + } + switch msg.verifyResult.Status { + case types.StatusFullVerified: + statusFullVerifiedMeter.Mark(1) + vt.compareRootHashAndWrite(msg, verifyCh) + case types.StatusPartiallyVerified: + statusUntrustedVerifiedMeter.Mark(1) + log.Warn("block %s , num= %s is untrusted verified", msg.verifyResult.BlockHash, msg.verifyResult.BlockNumber) + if vt.allowUntrusted { + vt.compareRootHashAndWrite(msg, verifyCh) + } + case types.StatusUnexpectedError, types.StatusImpossibleFork, types.StatusDiffHashMismatch: + vt.BadPeers[msg.peerId] = struct{}{} + log.Info("peer %s is not available: code %d, msg %s,", msg.peerId, msg.verifyResult.Status.Code, msg.verifyResult.Status.Msg) + case types.StatusBlockTooNew, types.StatusBlockNewer, types.StatusPossibleFork: + log.Info("return msg from peer %s for block %s is %s", msg.peerId, msg.verifyResult.BlockHash, msg.verifyResult.Status.Msg) + } + case <-resend.C: + // if a task has run over 15s, try all the vaild peers to verify. + if time.Since(vt.startAt) < tryAllPeersTime { + vt.sendVerifyRequest(1) + } else { + vt.sendVerifyRequest(-1) + } + case <-vt.terminalCh: + return + } + } +} + +// sendVerifyRequest func select at most n peers from (candidatePeers-badPeers) randomly and send verify request. +// when n<0, send to all the peers exclude badPeers. +func (vt *verifyTask) sendVerifyRequest(n int) { + var validPeers []VerifyPeer + candidatePeers := vt.candidatePeers.GetVerifyPeers() + for _, p := range candidatePeers { + if _, ok := vt.BadPeers[p.ID()]; !ok { + validPeers = append(validPeers, p) + } + } + // if has not valid peer, log warning. + if len(validPeers) == 0 { + log.Warn("there is no valid peer for block", vt.blockHeader.Number) + } + if n < 0 || n >= len(validPeers) { + for _, p := range validPeers { + p.RequestRoot(vt.blockHeader.Number.Uint64(), vt.blockHeader.Hash(), vt.diffhash) + } + return + } + + // if n < len(validPeers), select n peers from validPeers randomly. + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(validPeers), func(i, j int) { validPeers[i], validPeers[j] = validPeers[j], validPeers[i] }) + for i := 0; i < n; i++ { + p := validPeers[i] + p.RequestRoot(vt.blockHeader.Number.Uint64(), vt.blockHeader.Hash(), vt.diffhash) + } +} + +func (vt *verifyTask) compareRootHashAndWrite(msg verifyMessage, verifyCh chan common.Hash) { + if msg.verifyResult.Root == vt.blockHeader.Root { + blockhash := msg.verifyResult.BlockHash + // write back to manager so that manager can cache the result and delete this task. + verifyCh <- blockhash + } else { + vt.BadPeers[msg.peerId] = struct{}{} + } +} + +type VerifyPeer interface { + RequestRoot(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) error + ID() string +} + +type verifyPeers interface { + GetVerifyPeers() []VerifyPeer +} + +type VerifyMode uint32 + +const ( + LocalVerify VerifyMode = iota + FullVerify + InsecureVerify + NoneVerify +) + +func (mode VerifyMode) IsValid() bool { + return mode >= LocalVerify && mode <= NoneVerify +} + +func (mode VerifyMode) String() string { + switch mode { + case LocalVerify: + return "local" + case FullVerify: + return "full" + case InsecureVerify: + return "insecure" + case NoneVerify: + return "none" + default: + return "unknown" + } +} + +func (mode VerifyMode) MarshalText() ([]byte, error) { + switch mode { + case LocalVerify: + return []byte("local"), nil + case FullVerify: + return []byte("full"), nil + case InsecureVerify: + return []byte("insecure"), nil + case NoneVerify: + return []byte("none"), nil + default: + return nil, fmt.Errorf("unknown verify mode %d", mode) + } +} + +func (mode *VerifyMode) UnmarshalText(text []byte) error { + switch string(text) { + case "local": + *mode = LocalVerify + case "full": + *mode = FullVerify + case "insecure": + *mode = InsecureVerify + case "none": + *mode = NoneVerify + default: + return fmt.Errorf(`unknown sync mode %q, want "full", "light" or "insecure"`, text) + } + return nil +} + +func (mode VerifyMode) NeedRemoteVerify() bool { + return mode == FullVerify || mode == InsecureVerify +} diff --git a/core/types.go b/core/types.go index 5ed4817e68..0a0633103e 100644 --- a/core/types.go +++ b/core/types.go @@ -32,6 +32,8 @@ type Validator interface { // ValidateState validates the given statedb and optionally the receipts and // gas used. ValidateState(block *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64, skipHeavyVerify bool) error + // RemoteVerifyManager return remoteVerifyManager of validator. + RemoteVerifyManager() *remoteVerifyManager } // Prefetcher is an interface for pre-caching transaction signatures and state. diff --git a/core/types/block.go b/core/types/block.go index 72cf3408b5..da3bf163bf 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -66,13 +66,6 @@ var ( StatusUnexpectedError = VerifyStatus{Code: 0x400, Msg: "can’t verify because of unexpected internal error"} ) -type VerifyResult struct { - Status VerifyStatus - BlockNumber uint64 - BlockHash common.Hash - Root common.Hash -} - // A BlockNonce is a 64-bit hash which proves (combined with the // mix-hash) that a sufficient amount of computation has been carried // out on a block. diff --git a/eth/backend.go b/eth/backend.go index 0e937c49a0..a062dd834d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -26,8 +26,6 @@ import ( "sync/atomic" "time" - "github.com/ethereum/go-ethereum/eth/protocols/trust" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -47,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/eth/protocols/trust" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -112,6 +111,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if !config.SyncMode.IsValid() { return nil, fmt.Errorf("invalid sync mode %d", config.SyncMode) } + if !config.TriesVerifyMode.IsValid() { + return nil, fmt.Errorf("invalid tries verify mode %d", config.TriesVerifyMode) + } if config.Miner.GasPrice == nil || config.Miner.GasPrice.Cmp(common.Big0) <= 0 { log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", ethconfig.Defaults.Miner.GasPrice) config.Miner.GasPrice = new(big.Int).Set(ethconfig.Defaults.Miner.GasPrice) @@ -197,14 +199,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieDirtyLimit: config.TrieDirtyCache, TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, - NoTries: config.NoTries, + NoTries: config.TriesVerifyMode != core.LocalVerify, SnapshotLimit: config.SnapshotCache, TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, } ) bcOps := make([]core.BlockChainOption, 0) - if config.DiffSync && !config.NoTries { + if config.DiffSync && config.TriesVerifyMode == core.LocalVerify { bcOps = append(bcOps, core.EnableLightProcessor) } if config.PipeCommit { @@ -213,6 +215,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.PersistDiff { bcOps = append(bcOps, core.EnablePersistDiff(config.DiffBlock)) } + + peers := newPeerSet() + bcOps = append(bcOps, core.EnableBlockValidator(chainConfig, eth.engine, config.TriesVerifyMode, peers)) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit, bcOps...) if err != nil { return nil, err @@ -250,6 +255,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { DirectBroadcast: config.DirectBroadcast, DiffSync: config.DiffSync, DisablePeerTxBroadcast: config.DisablePeerTxBroadcast, + PeerSet: peers, }); err != nil { return nil, err } @@ -560,8 +566,12 @@ func (s *Ethereum) Protocols() []p2p.Protocol { protos = append(protos, snap.MakeProtocols((*snapHandler)(s.handler), s.snapDialCandidates)...) } // diff protocol can still open without snap protocol - protos = append(protos, diff.MakeProtocols((*diffHandler)(s.handler), s.snapDialCandidates)...) - protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.trustDialCandidates)...) + if !s.config.DisableDiffProtocol { + protos = append(protos, diff.MakeProtocols((*diffHandler)(s.handler), s.snapDialCandidates)...) + } + if s.config.EnableTrustProtocol { + protos = append(protos, trust.MakeProtocols((*trustHandler)(s.handler), s.snapDialCandidates)...) + } return protos } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index ee35d123a3..8dbef35442 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -79,7 +79,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, TriesInMemory: 128, - NoTries: false, + TriesVerifyMode: core.LocalVerify, SnapshotCache: 102, DiffBlock: uint64(86400), Miner: miner.Config{ @@ -138,6 +138,8 @@ type Config struct { NoPruning bool // Whether to disable pruning and flush everything to disk DirectBroadcast bool DisableSnapProtocol bool //Whether disable snap protocol + DisableDiffProtocol bool //Whether disable diff protocol + EnableTrustProtocol bool //Whether enable trust protocol DiffSync bool // Whether support diff sync PipeCommit bool RangeLimit bool @@ -177,7 +179,7 @@ type Config struct { TrieTimeout time.Duration SnapshotCache int TriesInMemory uint64 - NoTries bool + TriesVerifyMode core.VerifyMode Preimages bool // Mining options diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index c5e46ced41..df60b498bd 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -29,6 +29,8 @@ func (c Config) MarshalTOML() (interface{}, error) { NoPrefetch bool DirectBroadcast bool DisableSnapProtocol bool + DisableDiffProtocol bool + EnableTrustProtocol bool DiffSync bool RangeLimit bool TxLookupLimit uint64 `toml:",omitempty"` @@ -57,7 +59,7 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieTimeout time.Duration SnapshotCache int TriesInMemory uint64 - NoTries bool + TriesVerifyMode core.VerifyMode Preimages bool Miner miner.Config Ethash ethash.Config `toml:",omitempty"` @@ -84,6 +86,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPruning = c.NoPruning enc.DirectBroadcast = c.DirectBroadcast enc.DisableSnapProtocol = c.DisableSnapProtocol + enc.DisableDiffProtocol = c.DisableDiffProtocol + enc.EnableTrustProtocol = c.EnableTrustProtocol enc.DiffSync = c.DiffSync enc.RangeLimit = c.RangeLimit enc.TxLookupLimit = c.TxLookupLimit @@ -112,7 +116,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache enc.TriesInMemory = c.TriesInMemory - enc.NoTries = c.NoTries + enc.TriesVerifyMode = c.TriesVerifyMode enc.Preimages = c.Preimages enc.Miner = c.Miner enc.Ethash = c.Ethash @@ -144,6 +148,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NoPrefetch *bool DirectBroadcast *bool DisableSnapProtocol *bool + DisableDiffProtocol *bool + EnableTrustProtocol *bool DiffSync *bool RangeLimit *bool TxLookupLimit *uint64 `toml:",omitempty"` @@ -172,7 +178,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieTimeout *time.Duration SnapshotCache *int TriesInMemory *uint64 - NoTries *bool + TriesVerifyMode *core.VerifyMode Preimages *bool Miner *miner.Config Ethash *ethash.Config `toml:",omitempty"` @@ -222,6 +228,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DisableSnapProtocol != nil { c.DisableSnapProtocol = *dec.DisableSnapProtocol } + if dec.DisableDiffProtocol != nil { + c.DisableDiffProtocol = *dec.DisableDiffProtocol + } + if dec.EnableTrustProtocol != nil { + c.EnableTrustProtocol = *dec.EnableTrustProtocol + } if dec.DiffSync != nil { c.DiffSync = *dec.DiffSync } @@ -306,8 +318,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TriesInMemory != nil { c.TriesInMemory = *dec.TriesInMemory } - if dec.NoTries != nil { - c.NoTries = *dec.NoTries + if dec.TriesVerifyMode != nil { + c.TriesVerifyMode = *dec.TriesVerifyMode } if dec.Preimages != nil { c.Preimages = *dec.Preimages diff --git a/eth/handler.go b/eth/handler.go index e47d3eee8d..f9854766c4 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -96,6 +96,7 @@ type handlerConfig struct { Whitelist map[uint64]common.Hash // Hard coded whitelist for sync challenged DirectBroadcast bool DisablePeerTxBroadcast bool + PeerSet *peerSet } type handler struct { @@ -155,7 +156,7 @@ func newHandler(config *handlerConfig) (*handler, error) { database: config.Database, txpool: config.TxPool, chain: config.Chain, - peers: newPeerSet(), + peers: config.PeerSet, whitelist: config.Whitelist, directBroadcast: config.DirectBroadcast, diffSync: config.DiffSync, diff --git a/eth/handler_trust.go b/eth/handler_trust.go index 6df630a2e8..52100144e2 100644 --- a/eth/handler_trust.go +++ b/eth/handler_trust.go @@ -2,7 +2,6 @@ package eth import ( "fmt" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth/protocols/trust" "github.com/ethereum/go-ethereum/p2p/enode" @@ -34,8 +33,17 @@ func (h *trustHandler) PeerInfo(id enode.ID) interface{} { func (h *trustHandler) Handle(peer *trust.Peer, packet trust.Packet) error { switch packet := packet.(type) { case *trust.RootResponsePacket: - // TODO: h.bc.VerifyManager().HandleRootResponse(peer.ID(), *packet) - return nil + verifyResult := &core.VerifyResult{ + Status: packet.Status, + BlockNumber: packet.BlockNumber, + BlockHash: packet.BlockHash, + Root: packet.Root, + } + if vm := h.Chain().Validator().RemoteVerifyManager(); vm != nil { + vm.HandleRootResponse(verifyResult, peer.ID()) + return nil + } + return fmt.Errorf("verify manager is nil which is unexpected") default: return fmt.Errorf("unexpected trust packet type: %T", packet) diff --git a/eth/peerset.go b/eth/peerset.go index 5bbaa2dd2b..b68d0e7783 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -22,13 +22,13 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/eth/protocols/trust" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/eth/protocols/trust" "github.com/ethereum/go-ethereum/p2p" ) @@ -334,14 +334,14 @@ func (ps *peerSet) GetDiffPeer(pid string) downloader.IDiffPeer { } // GetVerifyPeers returns an array of verify nodes. -func (ps *peerSet) GetVerifyPeers() []*trustPeer { +func (ps *peerSet) GetVerifyPeers() []core.VerifyPeer { ps.lock.RLock() defer ps.lock.RUnlock() - res := make([]*trustPeer, 0) + res := make([]core.VerifyPeer, 0) for _, p := range ps.peers { - if p.trustExt != nil { - res = append(res, p.trustExt) + if p.trustExt != nil && p.trustExt.Peer != nil { + res = append(res, p.trustExt.Peer) } } return res diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 395e87fe1d..78f5c6bb14 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ethereum/go-ethereum/core" "math/big" "github.com/ethereum/go-ethereum" @@ -200,8 +201,8 @@ func (ec *Client) GetDiffAccountsWithScope(ctx context.Context, number *big.Int, return &result, err } -func (ec *Client) GetRootByDiffHash(ctx context.Context, blockNr *big.Int, blockHash common.Hash, diffHash common.Hash) (*types.VerifyResult, error) { - var result types.VerifyResult +func (ec *Client) GetRootByDiffHash(ctx context.Context, blockNr *big.Int, blockHash common.Hash, diffHash common.Hash) (*core.VerifyResult, error) { + var result core.VerifyResult err := ec.c.CallContext(ctx, &result, "eth_getRootByDiffHash", toBlockNumArg(blockNr), blockHash, diffHash) return &result, err } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index f3bcc3b98d..3f3122a37e 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1287,7 +1287,7 @@ func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, bloc return result, err } -func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) *types.VerifyResult { +func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) *core.VerifyResult { return s.b.Chain().GetRootByDiffHash(uint64(blockNr), blockHash, diffHash) } From 2ed75aba82d301b0d79327f74d2a45d687a1ec0c Mon Sep 17 00:00:00 2001 From: kyrie-yl Date: Tue, 1 Mar 2022 13:58:20 +0800 Subject: [PATCH 14/51] add metrics Signed-off-by: kyrie-yl --- core/block_validator.go | 2 + core/remote_state_verifier.go | 79 +++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 4cc5c67c16..656be57806 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) @@ -48,6 +49,7 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin bc: blockchain, } if mode.NeedRemoteVerify() { + log.Info("this node is a fast node with remote state verifier.") validator.remoteValidator = NewVerifyManager(blockchain, peers, mode == InsecureVerify) go validator.remoteValidator.mainLoop() } diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index 224d9b009b..5eb92b3caf 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -30,29 +30,11 @@ const ( ) var ( - remoteVerifyTaskCounter = metrics.NewRegisteredCounter("remote/state/verify/task/total", nil) - - statusFullVerifiedMeter = metrics.NewRegisteredMeter("status/full/verified/messages/total", nil) - statusUntrustedVerifiedMeter = metrics.NewRegisteredMeter("status/untrusted/verified/messages/total", nil) - - statusDiffHashMismatchMeter = metrics.NewRegisteredMeter("status/diffhash/mismatch/messages/total", nil) - statusImpossibleForkMeter = metrics.NewRegisteredMeter("status/impossible/fork/messages/total", nil) - - statusBlockTooNewMeter = metrics.NewRegisteredMeter("status/block/too/new/messages/total", nil) - statusBlockNewerMeter = metrics.NewRegisteredMeter("status/block/newer/messages/total", nil) - statusPossibleForkMeter = metrics.NewRegisteredMeter("status/possible/fork/messages/total", nil) - statusUnexpectedErrorMeter = metrics.NewRegisteredMeter("status/unexpected/error/total", nil) - - codeMap = map[uint16]metrics.Meter{ - 0x101: statusFullVerifiedMeter, - 0x102: statusUntrustedVerifiedMeter, - 0x201: statusDiffHashMismatchMeter, - 0x202: statusImpossibleForkMeter, - 0x301: statusBlockTooNewMeter, - 0x302: statusBlockNewerMeter, - 0x303: statusPossibleForkMeter, - 0x400: statusUnexpectedErrorMeter, - } + remoteVerifyTaskCounter = metrics.NewRegisteredCounter("remote/state/verify/task/total", nil) + succeedRemoteVerifyTaskMeter = metrics.NewRegisteredMeter("succeed/remote/verify/task", nil) + failedRemoteVerifyTaskMeter = metrics.NewRegisteredMeter("failed/remote/verify/task", nil) + + succeedTaskExecutionTimer = metrics.NewRegisteredTimer("succeed/task/execution", nil) ) type remoteVerifyManager struct { @@ -106,6 +88,8 @@ func (vm *remoteVerifyManager) mainLoop() { if task, ok := vm.tasks[hash]; ok { delete(vm.tasks, hash) remoteVerifyTaskCounter.Dec(1) + succeedRemoteVerifyTaskMeter.Mark(1) + succeedTaskExecutionTimer.Update(time.Since(task.startAt)) close(task.terminalCh) } case <-pruneTicker.C: @@ -114,6 +98,7 @@ func (vm *remoteVerifyManager) mainLoop() { vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff { delete(vm.tasks, hash) remoteVerifyTaskCounter.Dec(1) + failedRemoteVerifyTaskMeter.Mark(1) close(task.terminalCh) } } @@ -247,24 +232,41 @@ func (vt *verifyTask) Start(verifyCh chan common.Hash) { for { select { case msg := <-vt.messageCh: - if metric, exist := codeMap[msg.verifyResult.Status.Code]; exist { - metric.Mark(1) - } - switch msg.verifyResult.Status { - case types.StatusFullVerified: - statusFullVerifiedMeter.Mark(1) - vt.compareRootHashAndWrite(msg, verifyCh) - case types.StatusPartiallyVerified: - statusUntrustedVerifiedMeter.Mark(1) - log.Warn("block %s , num= %s is untrusted verified", msg.verifyResult.BlockHash, msg.verifyResult.BlockNumber) - if vt.allowUntrusted { + switch msg.verifyResult.Status.Code / 100 { + case 1: + switch msg.verifyResult.Status { + case types.StatusFullVerified: vt.compareRootHashAndWrite(msg, verifyCh) + newRecievedMsgTypeGauge("fullVerified", msg.peerId).Inc(1) + case types.StatusPartiallyVerified: + log.Warn("block %s , num= %s is untrusted verified", msg.verifyResult.BlockHash, msg.verifyResult.BlockNumber) + if vt.allowUntrusted { + vt.compareRootHashAndWrite(msg, verifyCh) + } + newRecievedMsgTypeGauge("partialVerified", msg.peerId).Inc(1) } - case types.StatusUnexpectedError, types.StatusImpossibleFork, types.StatusDiffHashMismatch: + + case 2, 4: vt.BadPeers[msg.peerId] = struct{}{} log.Info("peer %s is not available: code %d, msg %s,", msg.peerId, msg.verifyResult.Status.Code, msg.verifyResult.Status.Msg) - case types.StatusBlockTooNew, types.StatusBlockNewer, types.StatusPossibleFork: + switch msg.verifyResult.Status { + case types.StatusDiffHashMismatch: + newRecievedMsgTypeGauge("diffHashMismatch", msg.peerId).Inc(1) + case types.StatusImpossibleFork: + newRecievedMsgTypeGauge("impossibleFork", msg.peerId).Inc(1) + case types.StatusUnexpectedError: + newRecievedMsgTypeGauge("unexpectedError", msg.peerId).Inc(1) + } + case 3: log.Info("return msg from peer %s for block %s is %s", msg.peerId, msg.verifyResult.BlockHash, msg.verifyResult.Status.Msg) + switch msg.verifyResult.Status { + case types.StatusBlockTooNew: + newRecievedMsgTypeGauge("blockTooNew", msg.peerId).Inc(1) + case types.StatusBlockNewer: + newRecievedMsgTypeGauge("blockNewer", msg.peerId).Inc(1) + case types.StatusPossibleFork: + newRecievedMsgTypeGauge("possibleFork", msg.peerId).Inc(1) + } } case <-resend.C: // if a task has run over 15s, try all the vaild peers to verify. @@ -390,3 +392,8 @@ func (mode *VerifyMode) UnmarshalText(text []byte) error { func (mode VerifyMode) NeedRemoteVerify() bool { return mode == FullVerify || mode == InsecureVerify } + +func newRecievedMsgTypeGauge(msgType, peerId string) metrics.Gauge { + m := fmt.Sprintf("recieved/%s/message/from/%s", msgType, peerId) + return metrics.GetOrRegisterGauge(m, nil) +} From 7b070c02bbb906e17565c1bd866e5081cb4eadbe Mon Sep 17 00:00:00 2001 From: kyrie-yl Date: Wed, 2 Mar 2022 10:07:14 +0800 Subject: [PATCH 15/51] resolve comments Signed-off-by: kyrie-yl --- core/block_validator.go | 2 +- core/remote_state_verifier.go | 109 ++++++++++++++-------------------- 2 files changed, 44 insertions(+), 67 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 656be57806..cf956dccbb 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -94,7 +94,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { }, // for fast node which verify trie from remote verify peers, a block's H-11 ancestor should have been verify. func() error { - if v.remoteValidator != nil && !v.remoteValidator.AncestorVerified(v.bc.GetHeaderByNumber(header.Number.Uint64())) { + if v.remoteValidator != nil && !v.remoteValidator.AncestorVerified(header) { return fmt.Errorf("block's ancessor %x has not been verified", block.Hash()) } return nil diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index 5eb92b3caf..a61bb79944 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -2,8 +2,6 @@ package core import ( "fmt" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/metrics" "math/rand" "time" @@ -11,7 +9,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" ) const ( @@ -30,11 +30,11 @@ const ( ) var ( - remoteVerifyTaskCounter = metrics.NewRegisteredCounter("remote/state/verify/task/total", nil) - succeedRemoteVerifyTaskMeter = metrics.NewRegisteredMeter("succeed/remote/verify/task", nil) - failedRemoteVerifyTaskMeter = metrics.NewRegisteredMeter("failed/remote/verify/task", nil) + verifyTaskCounter = metrics.NewRegisteredCounter("verifymanager/task/total", nil) + verifyTaskSucceedMeter = metrics.NewRegisteredMeter("verifymanager/task/result/succeed", nil) + verifyTaskFailedMeter = metrics.NewRegisteredMeter("verifymanager/task/result/failed", nil) - succeedTaskExecutionTimer = metrics.NewRegisteredTimer("succeed/task/execution", nil) + verifyTaskExecutionTimer = metrics.NewRegisteredTimer("verifymanager/task/execution", nil) ) type remoteVerifyManager struct { @@ -53,14 +53,14 @@ type remoteVerifyManager struct { messageCh chan verifyMessage } -func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowUntrusted bool) *remoteVerifyManager { +func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure bool) *remoteVerifyManager { verifiedCache, _ := lru.New(verifiedCacheSize) vm := &remoteVerifyManager{ bc: blockchain, tasks: make(map[common.Hash]*verifyTask), peers: peers, verifiedCache: verifiedCache, - allowInsecure: allowUntrusted, + allowInsecure: allowInsecure, chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), verifyCh: make(chan common.Hash, maxForkHeight), @@ -87,9 +87,9 @@ func (vm *remoteVerifyManager) mainLoop() { vm.cacheBlockVerified(hash) if task, ok := vm.tasks[hash]; ok { delete(vm.tasks, hash) - remoteVerifyTaskCounter.Dec(1) - succeedRemoteVerifyTaskMeter.Mark(1) - succeedTaskExecutionTimer.Update(time.Since(task.startAt)) + verifyTaskCounter.Dec(1) + verifyTaskSucceedMeter.Mark(1) + verifyTaskExecutionTimer.Update(time.Since(task.startAt)) close(task.terminalCh) } case <-pruneTicker.C: @@ -97,8 +97,8 @@ func (vm *remoteVerifyManager) mainLoop() { if vm.bc.CurrentHeader().Number.Cmp(task.blockHeader.Number) == 1 && vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff { delete(vm.tasks, hash) - remoteVerifyTaskCounter.Dec(1) - failedRemoteVerifyTaskMeter.Mark(1) + verifyTaskCounter.Dec(1) + verifyTaskFailedMeter.Mark(1) close(task.terminalCh) } } @@ -137,7 +137,7 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { if diffLayer, err = vm.bc.GenerateDiffLayer(hash); err != nil { log.Error("failed to get diff layer", "block", hash, "number", header.Number, "error", err) return - } else { + } else if diffLayer == nil { log.Info("this is an empty block:", "block", hash, "number", header.Number) return } @@ -149,7 +149,7 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { } verifyTask := NewVerifyTask(diffHash, header, vm.peers, vm.verifyCh, vm.allowInsecure) vm.tasks[hash] = verifyTask - remoteVerifyTaskCounter.Inc(1) + verifyTaskCounter.Inc(1) }(header.Hash()) header = vm.bc.GetHeaderByHash(header.ParentHash) } @@ -173,7 +173,7 @@ func (vm *remoteVerifyManager) AncestorVerified(header *types.Header) bool { // check whether H-11 block is a empty block. if header.TxHash == types.EmptyRootHash { parent := vm.bc.GetHeaderByHash(header.ParentHash) - return header.Root == parent.Root + return parent == nil || header.Root == parent.Root } hash := header.Hash() _, exist := vm.verifiedCache.Get(hash) @@ -201,21 +201,21 @@ type verifyTask struct { diffhash common.Hash blockHeader *types.Header candidatePeers verifyPeers - BadPeers map[string]struct{} + badPeers map[string]struct{} startAt time.Time - allowUntrusted bool + allowInsecure bool messageCh chan verifyMessage terminalCh chan struct{} } -func NewVerifyTask(diffhash common.Hash, header *types.Header, peers verifyPeers, verifyCh chan common.Hash, allowUntrusted bool) *verifyTask { +func NewVerifyTask(diffhash common.Hash, header *types.Header, peers verifyPeers, verifyCh chan common.Hash, allowInsecure bool) *verifyTask { vt := &verifyTask{ diffhash: diffhash, blockHeader: header, candidatePeers: peers, - BadPeers: make(map[string]struct{}), - allowUntrusted: allowUntrusted, + badPeers: make(map[string]struct{}), + allowInsecure: allowInsecure, messageCh: make(chan verifyMessage), terminalCh: make(chan struct{}), } @@ -232,42 +232,21 @@ func (vt *verifyTask) Start(verifyCh chan common.Hash) { for { select { case msg := <-vt.messageCh: - switch msg.verifyResult.Status.Code / 100 { - case 1: - switch msg.verifyResult.Status { - case types.StatusFullVerified: - vt.compareRootHashAndWrite(msg, verifyCh) - newRecievedMsgTypeGauge("fullVerified", msg.peerId).Inc(1) - case types.StatusPartiallyVerified: - log.Warn("block %s , num= %s is untrusted verified", msg.verifyResult.BlockHash, msg.verifyResult.BlockNumber) - if vt.allowUntrusted { - vt.compareRootHashAndWrite(msg, verifyCh) - } - newRecievedMsgTypeGauge("partialVerified", msg.peerId).Inc(1) + switch msg.verifyResult.Status { + case types.StatusFullVerified: + vt.compareRootHashAndMark(msg, verifyCh) + case types.StatusPartiallyVerified: + log.Warn("block %s , num= %s is insecure verified", msg.verifyResult.BlockHash, msg.verifyResult.BlockNumber) + if vt.allowInsecure { + vt.compareRootHashAndMark(msg, verifyCh) } - - case 2, 4: - vt.BadPeers[msg.peerId] = struct{}{} + case types.StatusDiffHashMismatch, types.StatusImpossibleFork, types.StatusUnexpectedError: + vt.badPeers[msg.peerId] = struct{}{} log.Info("peer %s is not available: code %d, msg %s,", msg.peerId, msg.verifyResult.Status.Code, msg.verifyResult.Status.Msg) - switch msg.verifyResult.Status { - case types.StatusDiffHashMismatch: - newRecievedMsgTypeGauge("diffHashMismatch", msg.peerId).Inc(1) - case types.StatusImpossibleFork: - newRecievedMsgTypeGauge("impossibleFork", msg.peerId).Inc(1) - case types.StatusUnexpectedError: - newRecievedMsgTypeGauge("unexpectedError", msg.peerId).Inc(1) - } - case 3: + case types.StatusBlockTooNew, types.StatusBlockNewer, types.StatusPossibleFork: log.Info("return msg from peer %s for block %s is %s", msg.peerId, msg.verifyResult.BlockHash, msg.verifyResult.Status.Msg) - switch msg.verifyResult.Status { - case types.StatusBlockTooNew: - newRecievedMsgTypeGauge("blockTooNew", msg.peerId).Inc(1) - case types.StatusBlockNewer: - newRecievedMsgTypeGauge("blockNewer", msg.peerId).Inc(1) - case types.StatusPossibleFork: - newRecievedMsgTypeGauge("possibleFork", msg.peerId).Inc(1) - } } + newVerifyMsgTypeGauge(msg.verifyResult.Status.Code, msg.peerId).Inc(1) case <-resend.C: // if a task has run over 15s, try all the vaild peers to verify. if time.Since(vt.startAt) < tryAllPeersTime { @@ -287,37 +266,35 @@ func (vt *verifyTask) sendVerifyRequest(n int) { var validPeers []VerifyPeer candidatePeers := vt.candidatePeers.GetVerifyPeers() for _, p := range candidatePeers { - if _, ok := vt.BadPeers[p.ID()]; !ok { + if _, ok := vt.badPeers[p.ID()]; !ok { validPeers = append(validPeers, p) } } // if has not valid peer, log warning. if len(validPeers) == 0 { log.Warn("there is no valid peer for block", vt.blockHeader.Number) - } - if n < 0 || n >= len(validPeers) { - for _, p := range validPeers { - p.RequestRoot(vt.blockHeader.Number.Uint64(), vt.blockHeader.Hash(), vt.diffhash) - } return } - // if n < len(validPeers), select n peers from validPeers randomly. - rand.Seed(time.Now().UnixNano()) - rand.Shuffle(len(validPeers), func(i, j int) { validPeers[i], validPeers[j] = validPeers[j], validPeers[i] }) + if n < len(validPeers) && n > 0 { + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(validPeers), func(i, j int) { validPeers[i], validPeers[j] = validPeers[j], validPeers[i] }) + } else { + n = len(validPeers) + } for i := 0; i < n; i++ { p := validPeers[i] p.RequestRoot(vt.blockHeader.Number.Uint64(), vt.blockHeader.Hash(), vt.diffhash) } } -func (vt *verifyTask) compareRootHashAndWrite(msg verifyMessage, verifyCh chan common.Hash) { +func (vt *verifyTask) compareRootHashAndMark(msg verifyMessage, verifyCh chan common.Hash) { if msg.verifyResult.Root == vt.blockHeader.Root { blockhash := msg.verifyResult.BlockHash // write back to manager so that manager can cache the result and delete this task. verifyCh <- blockhash } else { - vt.BadPeers[msg.peerId] = struct{}{} + vt.badPeers[msg.peerId] = struct{}{} } } @@ -393,7 +370,7 @@ func (mode VerifyMode) NeedRemoteVerify() bool { return mode == FullVerify || mode == InsecureVerify } -func newRecievedMsgTypeGauge(msgType, peerId string) metrics.Gauge { - m := fmt.Sprintf("recieved/%s/message/from/%s", msgType, peerId) +func newVerifyMsgTypeGauge(msgType uint16, peerId string) metrics.Gauge { + m := fmt.Sprintf("verifymanager/message/%d/peer/%s", msgType, peerId) return metrics.GetOrRegisterGauge(m, nil) } From 1aaab766e369802586400ecaa41b55acea3232fa Mon Sep 17 00:00:00 2001 From: kyrie-yl Date: Thu, 7 Apr 2022 00:21:26 +0800 Subject: [PATCH 16/51] verify task get difflayer cache synchronously Signed-off-by: kyrie-yl --- core/block_validator.go | 9 ------ core/blockchain.go | 29 +++++++++++++++++- core/remote_state_verifier.go | 56 ++++++++++++++++++++++------------- 3 files changed, 64 insertions(+), 30 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index cf956dccbb..29ee2ec348 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) @@ -49,7 +48,6 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin bc: blockchain, } if mode.NeedRemoteVerify() { - log.Info("this node is a fast node with remote state verifier.") validator.remoteValidator = NewVerifyManager(blockchain, peers, mode == InsecureVerify) go validator.remoteValidator.mainLoop() } @@ -92,13 +90,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } return nil }, - // for fast node which verify trie from remote verify peers, a block's H-11 ancestor should have been verify. - func() error { - if v.remoteValidator != nil && !v.remoteValidator.AncestorVerified(header) { - return fmt.Errorf("block's ancessor %x has not been verified", block.Hash()) - } - return nil - }, } validateRes := make(chan error, len(validateFuns)) for _, f := range validateFuns { diff --git a/core/blockchain.go b/core/blockchain.go index ffcfe444b6..0776a2d637 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -203,6 +203,7 @@ type BlockChain struct { chainFeed event.Feed chainSideFeed event.Feed chainHeadFeed event.Feed + chainBlockFeed event.Feed logsFeed event.Feed blockProcFeed event.Feed scope event.SubscriptionScope @@ -226,6 +227,7 @@ type BlockChain struct { // trusted diff layers diffLayerCache *lru.Cache // Cache for the diffLayers diffLayerRLPCache *lru.Cache // Cache for the rlp encoded diffLayers + diffLayerChanCache *lru.Cache // Cache for diffQueue *prque.Prque // A Priority queue to store recent diff layer diffQueueBuffer chan *types.DiffLayer diffLayerFreezerBlockLimit uint64 @@ -277,6 +279,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par futureBlocks, _ := lru.New(maxFutureBlocks) diffLayerCache, _ := lru.New(diffLayerCacheLimit) diffLayerRLPCache, _ := lru.New(diffLayerRLPCacheLimit) + diffLayerChanCache, _ := lru.New(diffLayerCacheLimit) bc := &BlockChain{ chainConfig: chainConfig, @@ -299,6 +302,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par badBlockCache: badBlockCache, diffLayerCache: diffLayerCache, diffLayerRLPCache: diffLayerRLPCache, + diffLayerChanCache: diffLayerChanCache, txLookupCache: txLookupCache, futureBlocks: futureBlocks, engine: engine, @@ -520,7 +524,13 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, sorted bool) { if bc.diffLayerCache.Len() >= diffLayerCacheLimit { bc.diffLayerCache.RemoveOldest() } + + //json.MarshalIndent() bc.diffLayerCache.Add(diffLayer.BlockHash, diffLayer) + if cached, ok := bc.diffLayerChanCache.Get(diffLayer.BlockHash); ok { + diffLayerCh := cached.(chan struct{}) + close(diffLayerCh) + } if bc.db.DiffStore() != nil { // push to priority queue before persisting bc.diffQueueBuffer <- diffLayer @@ -1816,6 +1826,9 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.BlockHash = block.Hash() diffLayer.Number = block.NumberU64() + diffLayerCh := make(chan struct{}) + bc.diffLayerChanCache.Add(diffLayer.BlockHash, diffLayerCh) + go bc.cacheDiffLayer(diffLayer, false) } @@ -2072,6 +2085,15 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er }() for ; block != nil && err == nil || err == ErrKnownBlock; block, err = it.next() { + if bc.validator.RemoteVerifyManager() != nil { + for !bc.Validator().RemoteVerifyManager().AncestorVerified(block.Header()) { + if bc.insertStopped() { + break + } + log.Info("block ancestor has not been verified", "number", block.Number(), "hash", block.Hash()) + time.Sleep(100 * time.Millisecond) + } + } // If the chain is terminating, stop processing blocks if bc.insertStopped() { log.Debug("Abort during block processing") @@ -2231,6 +2253,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er stats.processed++ stats.usedGas += usedGas + bc.chainBlockFeed.Send(ChainHeadEvent{block}) dirty, _ := bc.stateCache.TrieDB().Size() stats.report(chain, it.index, dirty) } @@ -3101,6 +3124,10 @@ func (bc *BlockChain) SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Su return bc.scope.Track(bc.chainHeadFeed.Subscribe(ch)) } +func (bc *BlockChain) SubscribeChainBlockEvent(ch chan<- ChainHeadEvent) event.Subscription { + return bc.scope.Track(bc.chainBlockFeed.Subscribe(ch)) +} + // SubscribeChainSideEvent registers a subscription of ChainSideEvent. func (bc *BlockChain) SubscribeChainSideEvent(ch chan<- ChainSideEvent) event.Subscription { return bc.scope.Track(bc.chainSideFeed.Subscribe(ch)) @@ -3278,7 +3305,7 @@ func CalculateDiffHash(d *types.DiffLayer) (common.Hash, error) { BlockHash: d.BlockHash, Receipts: make([]*types.ReceiptForStorage, 0), Number: d.Number, - Codes: d.Codes, + //Codes: d.Codes, Destructs: d.Destructs, Accounts: d.Accounts, Storages: d.Storages, diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index a61bb79944..3481113112 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -45,7 +45,7 @@ type remoteVerifyManager struct { allowInsecure bool // Subscription - chainHeadCh chan ChainHeadEvent + chainBlockCh chan ChainHeadEvent chainHeadSub event.Subscription // Channels @@ -62,11 +62,11 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b verifiedCache: verifiedCache, allowInsecure: allowInsecure, - chainHeadCh: make(chan ChainHeadEvent, chainHeadChanSize), + chainBlockCh: make(chan ChainHeadEvent, chainHeadChanSize), verifyCh: make(chan common.Hash, maxForkHeight), messageCh: make(chan verifyMessage), } - vm.chainHeadSub = blockchain.SubscribeChainHeadEvent(vm.chainHeadCh) + vm.chainHeadSub = blockchain.SubscribeChainBlockEvent(vm.chainBlockCh) return vm } @@ -81,7 +81,7 @@ func (vm *remoteVerifyManager) mainLoop() { defer pruneTicker.Stop() for { select { - case h := <-vm.chainHeadCh: + case h := <-vm.chainBlockCh: vm.NewBlockVerifyTask(h.Block.Header()) case hash := <-vm.verifyCh: vm.cacheBlockVerified(hash) @@ -121,6 +121,11 @@ func (vm *remoteVerifyManager) mainLoop() { func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { for i := 0; header != nil && i <= maxForkHeight; i++ { + // if is genesis block, mark it as verified and break. + if header.Number.Uint64() == 0 { + vm.cacheBlockVerified(header.Hash()) + break + } func(hash common.Hash) { // if verified cache record that this block has been verified, skip. if _, ok := vm.verifiedCache.Get(hash); ok { @@ -130,17 +135,32 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { if _, ok := vm.tasks[hash]; ok { return } - diffLayer := vm.bc.GetTrustedDiffLayer(hash) + + if header.TxHash == types.EmptyRootHash { + log.Debug("this is an empty block:", "block", hash, "number", header.Number) + vm.cacheBlockVerified(hash) + return + } + + var diffLayer *types.DiffLayer + if cached, ok := vm.bc.diffLayerChanCache.Get(hash); ok { + diffLayerCh := cached.(chan struct{}) + <-diffLayerCh + vm.bc.diffLayerChanCache.Remove(hash) + diffLayer = vm.bc.GetTrustedDiffLayer(hash) + } // if this block has no diff, there is no need to verify it. var err error if diffLayer == nil { - if diffLayer, err = vm.bc.GenerateDiffLayer(hash); err != nil { - log.Error("failed to get diff layer", "block", hash, "number", header.Number, "error", err) - return - } else if diffLayer == nil { - log.Info("this is an empty block:", "block", hash, "number", header.Number) - return - } + log.Info("block's trusted diffLayer is nil", "hash", hash, "number", header.Number) + //if diffLayer, err = vm.bc.GenerateDiffLayer(hash); err != nil { + // log.Error("failed to get diff layer", "block", hash, "number", header.Number, "error", err) + // return + //} else if diffLayer == nil { + // log.Info("this is an empty block:", "block", hash, "number", header.Number) + // vm.cacheBlockVerified(hash) + // return + //} } diffHash, err := CalculateDiffHash(diffLayer) if err != nil { @@ -170,11 +190,7 @@ func (vm *remoteVerifyManager) AncestorVerified(header *types.Header) bool { if header == nil { return true } - // check whether H-11 block is a empty block. - if header.TxHash == types.EmptyRootHash { - parent := vm.bc.GetHeaderByHash(header.ParentHash) - return parent == nil || header.Root == parent.Root - } + hash := header.Hash() _, exist := vm.verifiedCache.Get(hash) return exist @@ -203,7 +219,7 @@ type verifyTask struct { candidatePeers verifyPeers badPeers map[string]struct{} startAt time.Time - allowInsecure bool + allowInsecure bool messageCh chan verifyMessage terminalCh chan struct{} @@ -236,13 +252,13 @@ func (vt *verifyTask) Start(verifyCh chan common.Hash) { case types.StatusFullVerified: vt.compareRootHashAndMark(msg, verifyCh) case types.StatusPartiallyVerified: - log.Warn("block %s , num= %s is insecure verified", msg.verifyResult.BlockHash, msg.verifyResult.BlockNumber) + log.Warn("block is insecure verified", "hash", msg.verifyResult.BlockHash, "number", msg.verifyResult.BlockNumber) if vt.allowInsecure { vt.compareRootHashAndMark(msg, verifyCh) } case types.StatusDiffHashMismatch, types.StatusImpossibleFork, types.StatusUnexpectedError: vt.badPeers[msg.peerId] = struct{}{} - log.Info("peer %s is not available: code %d, msg %s,", msg.peerId, msg.verifyResult.Status.Code, msg.verifyResult.Status.Msg) + log.Info("peer is not available", "hash", msg.verifyResult.BlockHash, "number", msg.verifyResult.BlockNumber, "peer", msg.peerId, "reason", msg.verifyResult.Status.Msg) case types.StatusBlockTooNew, types.StatusBlockNewer, types.StatusPossibleFork: log.Info("return msg from peer %s for block %s is %s", msg.peerId, msg.verifyResult.BlockHash, msg.verifyResult.Status.Msg) } From b978ed3d0351c962fd2696824eca39f46729f103 Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 13 Apr 2022 18:40:49 +0800 Subject: [PATCH 17/51] put difflayer into verifyManage cache when node restart --- core/block_validator.go | 10 ++++--- core/blockchain.go | 49 +++++++++++++++++++---------------- core/blockchain_diff_test.go | 9 ++++--- core/blockchain_test.go | 2 +- core/error.go | 3 +++ core/remote_state_verifier.go | 34 +++++++++++++++++++----- 6 files changed, 70 insertions(+), 37 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 29ee2ec348..56f609f41a 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -41,17 +41,21 @@ type BlockValidator struct { } // NewBlockValidator returns a new block validator which is safe for re-use -func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine, mode VerifyMode, peers verifyPeers) *BlockValidator { +func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine, mode VerifyMode, peers verifyPeers) (*BlockValidator, error) { validator := &BlockValidator{ config: config, engine: engine, bc: blockchain, } if mode.NeedRemoteVerify() { - validator.remoteValidator = NewVerifyManager(blockchain, peers, mode == InsecureVerify) + remoteValidator, err := NewVerifyManager(blockchain, peers, mode == InsecureVerify) + if err != nil { + return nil, err + } + validator.remoteValidator = remoteValidator go validator.remoteValidator.mainLoop() } - return validator + return validator, nil } // ValidateBody validates the given block's uncles and verifies the block diff --git a/core/blockchain.go b/core/blockchain.go index 0776a2d637..3097d9d537 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -164,7 +164,7 @@ var defaultCacheConfig = &CacheConfig{ SnapshotWait: true, } -type BlockChainOption func(*BlockChain) *BlockChain +type BlockChainOption func(*BlockChain) (*BlockChain, error) // BlockChain represents the canonical chain given a database with a genesis // block. The Blockchain manages chain imports, reverts, chain reorganisations. @@ -198,16 +198,16 @@ type BlockChain struct { txLookupLimit uint64 triesInMemory uint64 - hc *HeaderChain - rmLogsFeed event.Feed - chainFeed event.Feed - chainSideFeed event.Feed - chainHeadFeed event.Feed + hc *HeaderChain + rmLogsFeed event.Feed + chainFeed event.Feed + chainSideFeed event.Feed + chainHeadFeed event.Feed chainBlockFeed event.Feed - logsFeed event.Feed - blockProcFeed event.Feed - scope event.SubscriptionScope - genesisBlock *types.Block + logsFeed event.Feed + blockProcFeed event.Feed + scope event.SubscriptionScope + genesisBlock *types.Block chainmu sync.RWMutex // blockchain insertion lock @@ -451,7 +451,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } // do options before start any routine for _, option := range options { - bc = option(bc) + bc, err = option(bc) + if err != nil { + return nil, err + } } // Take ownership of this particular state go bc.update() @@ -525,7 +528,6 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, sorted bool) { bc.diffLayerCache.RemoveOldest() } - //json.MarshalIndent() bc.diffLayerCache.Add(diffLayer.BlockHash, diffLayer) if cached, ok := bc.diffLayerChanCache.Get(diffLayer.BlockHash); ok { diffLayerCh := cached.(chan struct{}) @@ -3145,27 +3147,31 @@ func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscr } // Options -func EnableLightProcessor(bc *BlockChain) *BlockChain { +func EnableLightProcessor(bc *BlockChain) (*BlockChain, error) { bc.processor = NewLightStateProcessor(bc.Config(), bc, bc.engine) - return bc + return bc, nil } -func EnablePipelineCommit(bc *BlockChain) *BlockChain { +func EnablePipelineCommit(bc *BlockChain) (*BlockChain, error) { bc.pipeCommit = true - return bc + return bc, nil } func EnablePersistDiff(limit uint64) BlockChainOption { - return func(chain *BlockChain) *BlockChain { + return func(chain *BlockChain) (*BlockChain, error) { chain.diffLayerFreezerBlockLimit = limit - return chain + return chain, nil } } func EnableBlockValidator(chainConfig *params.ChainConfig, engine consensus.Engine, mode VerifyMode, peers verifyPeers) BlockChainOption { - return func(bc *BlockChain) *BlockChain { - bc.validator = NewBlockValidator(chainConfig, bc, engine, mode, peers) - return bc + return func(bc *BlockChain) (*BlockChain, error) { + validator, err := NewBlockValidator(chainConfig, bc, engine, mode, peers) + if err != nil { + return bc, err + } + bc.validator = validator + return bc, nil } } @@ -3289,7 +3295,6 @@ func (bc *BlockChain) GenerateDiffLayer(blockHash common.Hash) (*types.DiffLayer if diffLayer != nil { diffLayer.BlockHash = blockHash diffLayer.Number = block.NumberU64() - bc.cacheDiffLayer(diffLayer, true) } diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index c4eeccda45..c245d71883 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -499,8 +499,8 @@ func newTwoForkedBlockchains(len1, len2 int) (chain1 *BlockChain, chain2 *BlockC Config: params.TestChainConfig, Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, }).MustCommit(db1) - - chain1, _ = NewBlockChain(db1, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000)) + engine1 := ethash.NewFaker() + chain1, _ = NewBlockChain(db1, nil, params.TestChainConfig, engine1, vm.Config{}, nil, nil, EnablePersistDiff(860000), EnableBlockValidator(params.TestChainConfig, engine1, 0, nil)) generator1 := func(i int, block *BlockGen) { // The chain maker doesn't have access to a chain, so the difficulty will be // lets unset (nil). Set it here to the correct value. @@ -555,7 +555,8 @@ func newTwoForkedBlockchains(len1, len2 int) (chain1 *BlockChain, chain2 *BlockC Config: params.TestChainConfig, Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, }).MustCommit(db2) - chain2, _ = NewBlockChain(db2, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000)) + engine2 := ethash.NewFaker() + chain2, _ = NewBlockChain(db2, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, EnablePersistDiff(860000), EnableBlockValidator(params.TestChainConfig, engine2, 0, nil)) generator2 := func(i int, block *BlockGen) { // The chain maker doesn't have access to a chain, so the difficulty will be // lets unset (nil). Set it here to the correct value. @@ -659,7 +660,7 @@ func newBlockChainWithCliqueEngine(blocks int) *BlockChain { copy(genspec.ExtraData[32:], testAddr[:]) genesis := genspec.MustCommit(db) - chain, _ := NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + chain, _ := NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, EnableBlockValidator(params.AllCliqueProtocolChanges, engine, 0 /*LocalVerify*/, nil)) generator := func(i int, block *BlockGen) { // The chain maker doesn't have access to a chain, so the difficulty will be // lets unset (nil). Set it here to the correct value. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 50d02e0acc..8a7581016a 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -252,7 +252,7 @@ func TestBlockImportVerification(t *testing.T) { } defer processor.Stop() // Start fork from current height - processor = EnablePipelineCommit(processor) + processor, _ = EnablePipelineCommit(processor) testInvalidStateRootBlockImport(t, processor, length, 10, true) } diff --git a/core/error.go b/core/error.go index 0830a699fe..75fa6a205e 100644 --- a/core/error.go +++ b/core/error.go @@ -35,6 +35,9 @@ var ( // ErrDiffLayerNotFound is returned when diff layer not found. ErrDiffLayerNotFound = errors.New("diff layer not found") + // ErrCurrentBlockNotFound is returned when current block not found. + ErrCurrentBlockNotFound = errors.New("current block not found") + // ErrKnownBadBlock is return when the block is a known bad block ErrKnownBadBlock = errors.New("already known bad block") ) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index 3481113112..bd85b46eff 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "math/big" "math/rand" "time" @@ -45,7 +46,7 @@ type remoteVerifyManager struct { allowInsecure bool // Subscription - chainBlockCh chan ChainHeadEvent + chainBlockCh chan ChainHeadEvent chainHeadSub event.Subscription // Channels @@ -53,8 +54,27 @@ type remoteVerifyManager struct { messageCh chan verifyMessage } -func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure bool) *remoteVerifyManager { +func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure bool) (*remoteVerifyManager, error) { verifiedCache, _ := lru.New(verifiedCacheSize) + block := blockchain.CurrentBlock() + if block == nil { + return nil, ErrCurrentBlockNotFound + } + number := block.Number() + for i := maxForkHeight; i >= 0; i-- { + if new(big.Int).Sub(number, big.NewInt(int64(i))).Cmp(common.Big0) <= 0 { + continue + } + oldBlock := blockchain.GetBlockByNumber(number.Uint64() - uint64(i)) + if oldBlock == nil { + return nil, fmt.Errorf("block is nil, number: %d", number) + } + _, err := blockchain.GenerateDiffLayer(oldBlock.Hash()) + if err != nil { + return nil, err + } + } + vm := &remoteVerifyManager{ bc: blockchain, tasks: make(map[common.Hash]*verifyTask), @@ -63,11 +83,11 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b allowInsecure: allowInsecure, chainBlockCh: make(chan ChainHeadEvent, chainHeadChanSize), - verifyCh: make(chan common.Hash, maxForkHeight), - messageCh: make(chan verifyMessage), + verifyCh: make(chan common.Hash, maxForkHeight), + messageCh: make(chan verifyMessage), } vm.chainHeadSub = blockchain.SubscribeChainBlockEvent(vm.chainBlockCh) - return vm + return vm, nil } func (vm *remoteVerifyManager) mainLoop() { @@ -219,7 +239,7 @@ type verifyTask struct { candidatePeers verifyPeers badPeers map[string]struct{} startAt time.Time - allowInsecure bool + allowInsecure bool messageCh chan verifyMessage terminalCh chan struct{} @@ -288,7 +308,7 @@ func (vt *verifyTask) sendVerifyRequest(n int) { } // if has not valid peer, log warning. if len(validPeers) == 0 { - log.Warn("there is no valid peer for block", vt.blockHeader.Number) + log.Warn("there is no valid peer for block", "number", vt.blockHeader.Number) return } From 34e8ce062603abf5a8acd27d5cbcaeab2c140c45 Mon Sep 17 00:00:00 2001 From: j75689 Date: Thu, 14 Apr 2022 00:19:41 +0800 Subject: [PATCH 18/51] remove testing code --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 221b9a707f..20a57bdb3e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -3315,7 +3315,7 @@ func CalculateDiffHash(d *types.DiffLayer) (common.Hash, error) { BlockHash: d.BlockHash, Receipts: make([]*types.ReceiptForStorage, 0), Number: d.Number, - //Codes: d.Codes, + Codes: d.Codes, Destructs: d.Destructs, Accounts: d.Accounts, Storages: d.Storages, From 56759163491262a7db5d49e1cc3c88a8e28a4507 Mon Sep 17 00:00:00 2001 From: j75689 Date: Thu, 14 Apr 2022 01:16:00 +0800 Subject: [PATCH 19/51] fixup! put difflayer into verifyManage cache when node restart --- core/remote_state_verifier.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index bd85b46eff..c190671850 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -69,10 +69,14 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b if oldBlock == nil { return nil, fmt.Errorf("block is nil, number: %d", number) } - _, err := blockchain.GenerateDiffLayer(oldBlock.Hash()) + blockHash := oldBlock.Hash() + _, err := blockchain.GenerateDiffLayer(blockHash) if err != nil { return nil, err } + diffLayerCh := make(chan struct{}) + close(diffLayerCh) + blockchain.diffLayerChanCache.Add(blockHash, diffLayerCh) } vm := &remoteVerifyManager{ From f8c6a4a7b9f4565a3f7694d29bd1e15a87fe4db9 Mon Sep 17 00:00:00 2001 From: j75689 Date: Tue, 19 Apr 2022 12:46:01 +0800 Subject: [PATCH 20/51] change type of diffLayerChanCache to sync.Map --- core/blockchain.go | 8 ++++---- core/remote_state_verifier.go | 14 +++----------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 20a57bdb3e..c1bfbff616 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -227,7 +227,7 @@ type BlockChain struct { // trusted diff layers diffLayerCache *lru.Cache // Cache for the diffLayers diffLayerRLPCache *lru.Cache // Cache for the rlp encoded diffLayers - diffLayerChanCache *lru.Cache // Cache for + diffLayerChanCache *sync.Map // Cache for diffQueue *prque.Prque // A Priority queue to store recent diff layer diffQueueBuffer chan *types.DiffLayer diffLayerFreezerBlockLimit uint64 @@ -279,7 +279,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par futureBlocks, _ := lru.New(maxFutureBlocks) diffLayerCache, _ := lru.New(diffLayerCacheLimit) diffLayerRLPCache, _ := lru.New(diffLayerRLPCacheLimit) - diffLayerChanCache, _ := lru.New(diffLayerCacheLimit) + diffLayerChanCache := new(sync.Map) bc := &BlockChain{ chainConfig: chainConfig, @@ -529,7 +529,7 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, sorted bool) { } bc.diffLayerCache.Add(diffLayer.BlockHash, diffLayer) - if cached, ok := bc.diffLayerChanCache.Get(diffLayer.BlockHash); ok { + if cached, ok := bc.diffLayerChanCache.Load(diffLayer.BlockHash); ok { diffLayerCh := cached.(chan struct{}) close(diffLayerCh) } @@ -1834,7 +1834,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.Number = block.NumberU64() diffLayerCh := make(chan struct{}) - bc.diffLayerChanCache.Add(diffLayer.BlockHash, diffLayerCh) + bc.diffLayerChanCache.Store(diffLayer.BlockHash, diffLayerCh) go bc.cacheDiffLayer(diffLayer, false) } diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index c190671850..ed768e0a94 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -76,7 +76,7 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b } diffLayerCh := make(chan struct{}) close(diffLayerCh) - blockchain.diffLayerChanCache.Add(blockHash, diffLayerCh) + blockchain.diffLayerChanCache.Store(blockHash, diffLayerCh) } vm := &remoteVerifyManager{ @@ -167,24 +167,16 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { } var diffLayer *types.DiffLayer - if cached, ok := vm.bc.diffLayerChanCache.Get(hash); ok { + if cached, ok := vm.bc.diffLayerChanCache.Load(hash); ok { diffLayerCh := cached.(chan struct{}) <-diffLayerCh - vm.bc.diffLayerChanCache.Remove(hash) + vm.bc.diffLayerChanCache.Delete(hash) diffLayer = vm.bc.GetTrustedDiffLayer(hash) } // if this block has no diff, there is no need to verify it. var err error if diffLayer == nil { log.Info("block's trusted diffLayer is nil", "hash", hash, "number", header.Number) - //if diffLayer, err = vm.bc.GenerateDiffLayer(hash); err != nil { - // log.Error("failed to get diff layer", "block", hash, "number", header.Number, "error", err) - // return - //} else if diffLayer == nil { - // log.Info("this is an empty block:", "block", hash, "number", header.Number) - // vm.cacheBlockVerified(hash) - // return - //} } diffHash, err := CalculateDiffHash(diffLayer) if err != nil { From b3a701d36a5a279e6cf23884d54c0909b8b06e55 Mon Sep 17 00:00:00 2001 From: j75689 Date: Tue, 19 Apr 2022 13:40:17 +0800 Subject: [PATCH 21/51] refactor block remote validation code --- core/block_validator.go | 7 +++++++ core/blockchain.go | 9 --------- core/error.go | 3 +++ core/remote_state_verifier.go | 28 ++++++++++++++++++++++++++-- eth/downloader/downloader.go | 4 ++++ 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 1ce8cc70a7..2afe81ce0b 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -94,6 +94,13 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } return nil }, + func() error { + if v.remoteValidator != nil && !v.remoteValidator.AncestorVerified(block.Header()) { + return fmt.Errorf("%w, number: %s, hash: %s", ErrAncestorHasNotBeenVerified, block.Number(), block.Hash()) + } + + return nil + }, } validateRes := make(chan error, len(validateFuns)) for _, f := range validateFuns { diff --git a/core/blockchain.go b/core/blockchain.go index c1bfbff616..b63730a319 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2092,15 +2092,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er }() for ; block != nil && err == nil || err == ErrKnownBlock; block, err = it.next() { - if bc.validator.RemoteVerifyManager() != nil { - for !bc.Validator().RemoteVerifyManager().AncestorVerified(block.Header()) { - if bc.insertStopped() { - break - } - log.Info("block ancestor has not been verified", "number", block.Number(), "hash", block.Hash()) - time.Sleep(100 * time.Millisecond) - } - } // If the chain is terminating, stop processing blocks if bc.insertStopped() { log.Debug("Abort during block processing") diff --git a/core/error.go b/core/error.go index 75fa6a205e..8934ef719c 100644 --- a/core/error.go +++ b/core/error.go @@ -35,6 +35,9 @@ var ( // ErrDiffLayerNotFound is returned when diff layer not found. ErrDiffLayerNotFound = errors.New("diff layer not found") + // ErrDiffLayerNotFound is returned when block - 11 has not been verified by the remote verifier. + ErrAncestorHasNotBeenVerified = errors.New("block ancestor has not been verified") + // ErrCurrentBlockNotFound is returned when current block not found. ErrCurrentBlockNotFound = errors.New("current block not found") diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index ed768e0a94..c9e137f300 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -4,6 +4,7 @@ import ( "fmt" "math/big" "math/rand" + "sync" "time" lru "github.com/hashicorp/golang-lru" @@ -40,6 +41,7 @@ var ( type remoteVerifyManager struct { bc *BlockChain + taskLock sync.RWMutex tasks map[common.Hash]*verifyTask peers verifyPeers verifiedCache *lru.Cache @@ -109,6 +111,7 @@ func (vm *remoteVerifyManager) mainLoop() { vm.NewBlockVerifyTask(h.Block.Header()) case hash := <-vm.verifyCh: vm.cacheBlockVerified(hash) + vm.taskLock.Lock() if task, ok := vm.tasks[hash]; ok { delete(vm.tasks, hash) verifyTaskCounter.Dec(1) @@ -116,7 +119,9 @@ func (vm *remoteVerifyManager) mainLoop() { verifyTaskExecutionTimer.Update(time.Since(task.startAt)) close(task.terminalCh) } + vm.taskLock.Unlock() case <-pruneTicker.C: + vm.taskLock.Lock() for hash, task := range vm.tasks { if vm.bc.CurrentHeader().Number.Cmp(task.blockHeader.Number) == 1 && vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff { @@ -126,16 +131,21 @@ func (vm *remoteVerifyManager) mainLoop() { close(task.terminalCh) } } + vm.taskLock.Unlock() case message := <-vm.messageCh: + vm.taskLock.RLock() if vt, ok := vm.tasks[message.verifyResult.BlockHash]; ok { vt.messageCh <- message } + vm.taskLock.RUnlock() // System stopped case <-vm.bc.quit: + vm.taskLock.RLock() for _, task := range vm.tasks { close(task.terminalCh) } + vm.taskLock.RUnlock() return case <-vm.chainHeadSub.Err(): return @@ -156,7 +166,10 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { return } // if there already has a verify task for this block, skip. - if _, ok := vm.tasks[hash]; ok { + vm.taskLock.RLock() + _, ok := vm.tasks[hash] + vm.taskLock.RUnlock() + if ok { return } @@ -184,7 +197,9 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { return } verifyTask := NewVerifyTask(diffHash, header, vm.peers, vm.verifyCh, vm.allowInsecure) + vm.taskLock.Lock() vm.tasks[hash] = verifyTask + vm.taskLock.Unlock() verifyTaskCounter.Inc(1) }(header.Hash()) header = vm.bc.GetHeaderByHash(header.ParentHash) @@ -208,7 +223,16 @@ func (vm *remoteVerifyManager) AncestorVerified(header *types.Header) bool { } hash := header.Hash() - _, exist := vm.verifiedCache.Get(hash) + + // Check if the task is complete + vm.taskLock.RLock() + task, exist := vm.tasks[hash] + vm.taskLock.RUnlock() + if exist { + <-task.terminalCh + } + + _, exist = vm.verifiedCache.Get(hash) return exist } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index e1225e7a1c..dcbeb1eea7 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" @@ -1812,6 +1813,9 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { // of the blocks delivered from the downloader, and the indexing will be off. log.Debug("Downloaded item processing failed on sidechain import", "index", index, "err", err) } + if errors.Is(err, core.ErrAncestorHasNotBeenVerified) { + return err + } return fmt.Errorf("%w: %v", errInvalidChain, err) } return nil From be531535669013c44125a2272bc0df2ccc5c12bb Mon Sep 17 00:00:00 2001 From: j75689 Date: Tue, 19 Apr 2022 14:39:12 +0800 Subject: [PATCH 22/51] rewind to last non verified block when restart fast node --- core/blockchain.go | 61 ------------------ core/blockchain_diff_test.go | 117 ---------------------------------- core/remote_state_verifier.go | 26 ++++++-- 3 files changed, 19 insertions(+), 185 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index b63730a319..946fb78c40 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -3236,67 +3236,6 @@ func (bc *BlockChain) GetTrustedDiffLayer(blockHash common.Hash) *types.DiffLaye return diff } -// GenerateDiffLayer generates DiffLayer of a specified block by replaying the block's transactions. -// If the block is an empty block, no DiffLayer will be generated. -// The generated DiffLayer whose Receipts are empty, whose DiffAccounts' storage root is empty. -func (bc *BlockChain) GenerateDiffLayer(blockHash common.Hash) (*types.DiffLayer, error) { - if bc.snaps == nil { - return nil, fmt.Errorf("snapshot disabled, can't generate difflayer") - } - - block := bc.GetBlockByHash(blockHash) - if block == nil { - return nil, fmt.Errorf("block not found, block number: %d, blockhash: %v", block.NumberU64(), blockHash) - } - - parent := bc.GetBlockByHash(block.ParentHash()) - if parent == nil { - return nil, fmt.Errorf("block not found, block number: %d, blockhash: %v", block.NumberU64()-1, block.ParentHash()) - } - statedb, err := bc.StateAt(parent.Root()) - if err != nil { - return nil, fmt.Errorf("state not found for block number (%d): %v", parent.NumberU64(), err) - } - - // Empty block, no DiffLayer would be generated. - if block.Header().TxHash == types.EmptyRootHash { - return nil, nil - } - - // Replay transactions. - signer := types.MakeSigner(bc.Config(), block.Number()) - for _, tx := range block.Transactions() { - msg, _ := tx.AsMessage(signer) - txContext := NewEVMTxContext(msg) - context := NewEVMBlockContext(block.Header(), bc, nil) - vmenv := vm.NewEVM(context, txContext, statedb, bc.Config(), vm.Config{}) - - if posa, ok := bc.Engine().(consensus.PoSA); ok { - if isSystem, _ := posa.IsSystemTransaction(tx, block.Header()); isSystem { - balance := statedb.GetBalance(consensus.SystemAddress) - if balance.Cmp(common.Big0) > 0 { - statedb.SetBalance(consensus.SystemAddress, big.NewInt(0)) - statedb.AddBalance(block.Header().Coinbase, balance) - } - } - } - - if _, err := ApplyMessage(vmenv, msg, new(GasPool).AddGas(tx.Gas())); err != nil { - return nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) - } - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) - } - - diffLayer := statedb.GenerateDiffLayer() - if diffLayer != nil { - diffLayer.BlockHash = blockHash - diffLayer.Number = block.NumberU64() - bc.cacheDiffLayer(diffLayer, true) - } - - return diffLayer, nil -} - func CalculateDiffHash(d *types.DiffLayer) (common.Hash, error) { if d == nil { return common.Hash{}, fmt.Errorf("nil diff layer") diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index c245d71883..fad8fe353a 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -27,8 +27,6 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/consensus/clique" - "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" @@ -647,118 +645,3 @@ func TestGetRootByDiffHash(t *testing.T) { testGetRootByDiffHash(t, chain1, chain2, 24, types.StatusBlockNewer) testGetRootByDiffHash(t, chain1, chain2, 35, types.StatusBlockTooNew) } - -func newBlockChainWithCliqueEngine(blocks int) *BlockChain { - signer := types.HomesteadSigner{} - db := rawdb.NewMemoryDatabase() - engine := clique.New(params.AllCliqueProtocolChanges.Clique, db) - genspec := &Genesis{ - //Config: params.TestChainConfig, - ExtraData: make([]byte, 32+common.AddressLength+65), - Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, - } - copy(genspec.ExtraData[32:], testAddr[:]) - genesis := genspec.MustCommit(db) - - chain, _ := NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, EnableBlockValidator(params.AllCliqueProtocolChanges, engine, 0 /*LocalVerify*/, nil)) - generator := func(i int, block *BlockGen) { - // The chain maker doesn't have access to a chain, so the difficulty will be - // lets unset (nil). Set it here to the correct value. - // block.SetCoinbase(testAddr) - block.SetDifficulty(big.NewInt(2)) - - for idx, testBlock := range testBlocks { - // Specific block setting, the index in this generator has 1 diff from specified blockNr. - if i+1 == testBlock.blockNr { - for _, testTransaction := range testBlock.txs { - var transaction *types.Transaction - if testTransaction.to == nil { - transaction = types.NewContractCreation(block.TxNonce(testAddr), - testTransaction.value, uint64(commonGas), nil, testTransaction.data) - } else { - transaction = types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, - testTransaction.value, uint64(commonGas), nil, testTransaction.data) - } - tx, err := types.SignTx(transaction, signer, testKey) - if err != nil { - panic(err) - } - block.AddTxWithChain(chain, tx) - } - break - } - - // Default block setting. - if idx == len(testBlocks)-1 { - // We want to simulate an empty middle block, having the same state as the - // first one. The last is needs a state change again to force a reorg. - for _, testTransaction := range testBlocks[0].txs { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, - testTransaction.value, uint64(commonGas), nil, testTransaction.data), signer, testKey) - if err != nil { - panic(err) - } - block.AddTxWithChain(chain, tx) - } - } - } - - } - bs, _ := GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, blocks, generator) - for i, block := range bs { - header := block.Header() - if i > 0 { - header.ParentHash = bs[i-1].Hash() - } - header.Extra = make([]byte, 32+65) - header.Difficulty = big.NewInt(2) - - sig, _ := crypto.Sign(clique.SealHash(header).Bytes(), testKey) - copy(header.Extra[len(header.Extra)-65:], sig) - bs[i] = block.WithSeal(header) - } - - if _, err := chain.InsertChain(bs); err != nil { - panic(err) - } - - return chain -} - -func TestGenerateDiffLayer(t *testing.T) { - blockNum := 32 - chain := newBlockChainWithCliqueEngine(blockNum) - defer chain.Stop() - - for blockNr := 1; blockNr <= blockNum; blockNr++ { - block := chain.GetBlockByNumber(uint64(blockNr)) - if block == nil { - t.Fatal("block should not be nil") - } - - expDiffLayer := chain.GetTrustedDiffLayer(block.Hash()) - if expDiffLayer == nil { - // Skip empty block. - if blockNr == 15 { - continue - } - t.Fatalf("unexpected nil diff layer, block number: %v, block hash: %v", blockNr, block.Hash()) - } - expDiffHash, err := CalculateDiffHash(expDiffLayer) - if err != nil { - t.Fatalf("compute diff hash failed: %v", err) - } - - diffLayer, err := chain.GenerateDiffLayer(block.Hash()) - if err != nil || diffLayer == nil { - t.Fatalf("generate diff layer failed: %v", err) - } - diffHash, err := CalculateDiffHash(diffLayer) - if err != nil { - t.Fatalf("compute diff hash failed: %v", err) - } - if expDiffHash != diffHash { - t.Fatalf("generated wrong diff layer for block: %d, expected hash: %v, real hash: %v", blockNr, expDiffHash, diffHash) - } - } -} diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index c9e137f300..ee567d359e 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -71,14 +71,26 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b if oldBlock == nil { return nil, fmt.Errorf("block is nil, number: %d", number) } - blockHash := oldBlock.Hash() - _, err := blockchain.GenerateDiffLayer(blockHash) - if err != nil { - return nil, err + + // rewind to last non verified block + blockchain.SetHead(oldBlock.NumberU64()) + block = oldBlock + break + } + + number = block.Number() + for i := maxForkHeight; i >= 0; i-- { + if new(big.Int).Sub(number, big.NewInt(int64(i))).Cmp(common.Big0) <= 0 { + continue + } + oldBlock := blockchain.GetBlockByNumber(number.Uint64() - uint64(i)) + if oldBlock == nil { + return nil, fmt.Errorf("block is nil, number: %d", number) } - diffLayerCh := make(chan struct{}) - close(diffLayerCh) - blockchain.diffLayerChanCache.Store(blockHash, diffLayerCh) + // When inserting a block, + // the block before 11 blocks will be verified, + // so the parent block of 11-22 will directly write the verification information. + verifiedCache.Add(oldBlock.Hash(), true) } vm := &remoteVerifyManager{ From 8c6d5a8c7bcb8ca3aa2ab9ee711348c60c80e82b Mon Sep 17 00:00:00 2001 From: j75689 Date: Tue, 19 Apr 2022 17:37:04 +0800 Subject: [PATCH 23/51] fixup! rewind to last non verified block when restart fast node --- core/remote_state_verifier.go | 44 ++++++++++++++--------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index ee567d359e..a8e99e4c09 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -62,35 +62,25 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b if block == nil { return nil, ErrCurrentBlockNotFound } - number := block.Number() - for i := maxForkHeight; i >= 0; i-- { - if new(big.Int).Sub(number, big.NewInt(int64(i))).Cmp(common.Big0) <= 0 { - continue - } - oldBlock := blockchain.GetBlockByNumber(number.Uint64() - uint64(i)) - if oldBlock == nil { - return nil, fmt.Errorf("block is nil, number: %d", number) - } - - // rewind to last non verified block - blockchain.SetHead(oldBlock.NumberU64()) - block = oldBlock - break - } - number = block.Number() - for i := maxForkHeight; i >= 0; i-- { - if new(big.Int).Sub(number, big.NewInt(int64(i))).Cmp(common.Big0) <= 0 { - continue - } - oldBlock := blockchain.GetBlockByNumber(number.Uint64() - uint64(i)) - if oldBlock == nil { - return nil, fmt.Errorf("block is nil, number: %d", number) + // rewind to last non verified block + number := new(big.Int).Sub(block.Number(), big.NewInt(int64(maxForkHeight))) + if number.Cmp(common.Big0) < 0 { + blockchain.SetHead(0) + } else { + numberU64 := number.Uint64() + blockchain.SetHead(numberU64) + block := blockchain.GetBlockByNumber(numberU64) + for i := 0; i < maxForkHeight; i++ { + // When inserting a block, + // the block before 11 blocks will be verified, + // so the parent block of 11-22 will directly write the verification information. + verifiedCache.Add(block.Hash(), true) + block = blockchain.GetBlockByHash(block.ParentHash()) + if block == nil { + return nil, fmt.Errorf("block is nil, number: %d", number) + } } - // When inserting a block, - // the block before 11 blocks will be verified, - // so the parent block of 11-22 will directly write the verification information. - verifiedCache.Add(oldBlock.Hash(), true) } vm := &remoteVerifyManager{ From 1e44aa6859e37817f6d25fb70ab507e4423b284e Mon Sep 17 00:00:00 2001 From: j75689 Date: Tue, 19 Apr 2022 17:54:37 +0800 Subject: [PATCH 24/51] fixup! rewind to last non verified block when restart fast node --- core/remote_state_verifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index a8e99e4c09..7c1687fd83 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -71,7 +71,7 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b numberU64 := number.Uint64() blockchain.SetHead(numberU64) block := blockchain.GetBlockByNumber(numberU64) - for i := 0; i < maxForkHeight; i++ { + for i := 0; i < maxForkHeight && block.NumberU64() > 0; i++ { // When inserting a block, // the block before 11 blocks will be verified, // so the parent block of 11-22 will directly write the verification information. From 7ad4d024cbdd2ca3510fa1394ce31d83e67c8642 Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 20 Apr 2022 15:13:51 +0800 Subject: [PATCH 25/51] Revert "change type of diffLayerChanCache to sync.Map" This reverts commit f8c6a4a7b9f4565a3f7694d29bd1e15a87fe4db9. --- core/blockchain.go | 8 ++++---- core/remote_state_verifier.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 946fb78c40..3b1c3c96b4 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -227,7 +227,7 @@ type BlockChain struct { // trusted diff layers diffLayerCache *lru.Cache // Cache for the diffLayers diffLayerRLPCache *lru.Cache // Cache for the rlp encoded diffLayers - diffLayerChanCache *sync.Map // Cache for + diffLayerChanCache *lru.Cache // Cache for diffQueue *prque.Prque // A Priority queue to store recent diff layer diffQueueBuffer chan *types.DiffLayer diffLayerFreezerBlockLimit uint64 @@ -279,7 +279,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par futureBlocks, _ := lru.New(maxFutureBlocks) diffLayerCache, _ := lru.New(diffLayerCacheLimit) diffLayerRLPCache, _ := lru.New(diffLayerRLPCacheLimit) - diffLayerChanCache := new(sync.Map) + diffLayerChanCache, _ := lru.New(diffLayerCacheLimit) bc := &BlockChain{ chainConfig: chainConfig, @@ -529,7 +529,7 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, sorted bool) { } bc.diffLayerCache.Add(diffLayer.BlockHash, diffLayer) - if cached, ok := bc.diffLayerChanCache.Load(diffLayer.BlockHash); ok { + if cached, ok := bc.diffLayerChanCache.Get(diffLayer.BlockHash); ok { diffLayerCh := cached.(chan struct{}) close(diffLayerCh) } @@ -1834,7 +1834,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.Number = block.NumberU64() diffLayerCh := make(chan struct{}) - bc.diffLayerChanCache.Store(diffLayer.BlockHash, diffLayerCh) + bc.diffLayerChanCache.Add(diffLayer.BlockHash, diffLayerCh) go bc.cacheDiffLayer(diffLayer, false) } diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index 7c1687fd83..02cbd40f39 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -182,16 +182,24 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { } var diffLayer *types.DiffLayer - if cached, ok := vm.bc.diffLayerChanCache.Load(hash); ok { + if cached, ok := vm.bc.diffLayerChanCache.Get(hash); ok { diffLayerCh := cached.(chan struct{}) <-diffLayerCh - vm.bc.diffLayerChanCache.Delete(hash) + vm.bc.diffLayerChanCache.Remove(hash) diffLayer = vm.bc.GetTrustedDiffLayer(hash) } // if this block has no diff, there is no need to verify it. var err error if diffLayer == nil { log.Info("block's trusted diffLayer is nil", "hash", hash, "number", header.Number) + //if diffLayer, err = vm.bc.GenerateDiffLayer(hash); err != nil { + // log.Error("failed to get diff layer", "block", hash, "number", header.Number, "error", err) + // return + //} else if diffLayer == nil { + // log.Info("this is an empty block:", "block", hash, "number", header.Number) + // vm.cacheBlockVerified(hash) + // return + //} } diffHash, err := CalculateDiffHash(diffLayer) if err != nil { From c57fdd7cefc4e7a9652b26ef89679fa8743e5b6b Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 20 Apr 2022 15:27:30 +0800 Subject: [PATCH 26/51] fix comments --- core/blockchain.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 3b1c3c96b4..1ca43ec7c8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -227,7 +227,7 @@ type BlockChain struct { // trusted diff layers diffLayerCache *lru.Cache // Cache for the diffLayers diffLayerRLPCache *lru.Cache // Cache for the rlp encoded diffLayers - diffLayerChanCache *lru.Cache // Cache for + diffLayerChanCache *lru.Cache // Cache for the difflayer channel diffQueue *prque.Prque // A Priority queue to store recent diff layer diffQueueBuffer chan *types.DiffLayer diffLayerFreezerBlockLimit uint64 @@ -1834,6 +1834,9 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. diffLayer.Number = block.NumberU64() diffLayerCh := make(chan struct{}) + if bc.diffLayerChanCache.Len() >= diffLayerCacheLimit { + bc.diffLayerChanCache.RemoveOldest() + } bc.diffLayerChanCache.Add(diffLayer.BlockHash, diffLayerCh) go bc.cacheDiffLayer(diffLayer, false) From e0cf707675b3127bfb6f6ecbf579ada79d6e71ca Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 20 Apr 2022 15:48:47 +0800 Subject: [PATCH 27/51] remove unused code --- core/remote_state_verifier.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index 02cbd40f39..af775918fd 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -189,17 +189,9 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { diffLayer = vm.bc.GetTrustedDiffLayer(hash) } // if this block has no diff, there is no need to verify it. - var err error if diffLayer == nil { log.Info("block's trusted diffLayer is nil", "hash", hash, "number", header.Number) - //if diffLayer, err = vm.bc.GenerateDiffLayer(hash); err != nil { - // log.Error("failed to get diff layer", "block", hash, "number", header.Number, "error", err) - // return - //} else if diffLayer == nil { - // log.Info("this is an empty block:", "block", hash, "number", header.Number) - // vm.cacheBlockVerified(hash) - // return - //} + return } diffHash, err := CalculateDiffHash(diffLayer) if err != nil { From 2e6c33bae5bc4f35506129d621ebd754118b58a9 Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 20 Apr 2022 15:52:24 +0800 Subject: [PATCH 28/51] don't need to send verify task when node start --- core/remote_state_verifier.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index af775918fd..8b6d05ce05 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -101,10 +101,6 @@ func NewVerifyManager(blockchain *BlockChain, peers verifyPeers, allowInsecure b func (vm *remoteVerifyManager) mainLoop() { defer vm.chainHeadSub.Unsubscribe() - // load unverified blocks in a normalized chain and start a batch of verify task - header := vm.bc.CurrentHeader() - // Start verify task from H to H-11 if need. - vm.NewBlockVerifyTask(header) pruneTicker := time.NewTicker(pruneInterval) defer pruneTicker.Stop() for { From 23913cf1adf4b1fb66ed17da1110672d2188c5ed Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 20 Apr 2022 17:53:52 +0800 Subject: [PATCH 29/51] remove unused code --- core/state/statedb.go | 75 ------------------------------------------- 1 file changed, 75 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index ee7ae7020a..f3d0478482 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1553,81 +1553,6 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er return root, diffLayer, nil } -// GenerateDiffLayer generates block's DiffLayer after executing the block's txs. -// Attention, the DiffLayer returned include no Receipts, whose accounts' storage root -// is empty, whose BlockHash and Number field is empty, should further process by caller. -func (s *StateDB) GenerateDiffLayer() *types.DiffLayer { - if s.snap == nil { - return nil - } - - for addr := range s.stateObjectsPending { - if obj := s.stateObjects[addr]; !obj.deleted { - // The snapshot storage map for the object - var storage map[string][]byte - obj.finalise(false) - for key, value := range obj.pendingStorage { - // Skip noop changes, persist actual changes - if value == obj.originStorage[key] { - continue - } - obj.originStorage[key] = value - - var v []byte - if (value != common.Hash{}) { - // Encoding []byte cannot fail, ok to ignore the error. - v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) - } - - obj.db.snapMux.Lock() - if storage == nil { - // Retrieve the old storage map, if available, create a new one otherwise - if storage = obj.db.snapStorage[obj.address]; storage == nil { - storage = make(map[string][]byte) - obj.db.snapStorage[obj.address] = storage - } - } - storage[string(key[:])] = v // v will be nil if value is 0x00 - obj.db.snapMux.Unlock() - } - - if !obj.deleted { - s.snapMux.Lock() - // The storage root hasn't been intermediate, pass empty storage root here. - s.snapAccounts[obj.address] = snapshot.SlimAccountRLP(obj.data.Nonce, obj.data.Balance, common.Hash{}, obj.data.CodeHash) - s.snapMux.Unlock() - } - } - } - - var diffLayer = &types.DiffLayer{} - for addr := range s.stateObjectsDirty { - if obj := s.stateObjects[addr]; !obj.deleted { - if obj.code != nil && obj.dirtyCode { - diffLayer.Codes = append(diffLayer.Codes, types.DiffCode{ - Hash: common.BytesToHash(obj.CodeHash()), - Code: obj.code, - }) - } - } - } - - diffLayer.Destructs, diffLayer.Accounts, diffLayer.Storages = s.SnapToDiffLayer() - sort.SliceStable(diffLayer.Codes, func(i, j int) bool { - return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() - }) - sort.SliceStable(diffLayer.Destructs, func(i, j int) bool { - return diffLayer.Destructs[i].Hex() < (diffLayer.Destructs[j].Hex()) - }) - sort.SliceStable(diffLayer.Accounts, func(i, j int) bool { - return diffLayer.Accounts[i].Account.Hex() < diffLayer.Accounts[j].Account.Hex() - }) - sort.SliceStable(diffLayer.Storages, func(i, j int) bool { - return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() - }) - return diffLayer -} - func (s *StateDB) DiffLayerToSnap(diffLayer *types.DiffLayer) (map[common.Address]struct{}, map[common.Address][]byte, map[common.Address]map[string][]byte, error) { snapDestructs := make(map[common.Address]struct{}) snapAccounts := make(map[common.Address][]byte) From a73aad6c034589bbb100e27f9af0d7312f4c99cd Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 20 Apr 2022 18:00:32 +0800 Subject: [PATCH 30/51] fix validateBody --- core/block_validator.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/block_validator.go b/core/block_validator.go index 2afe81ce0b..6f2c40b2ec 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -79,6 +79,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } validateFuns := []func() error{ + func() error { + if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { + return ErrKnownBlock + } + return nil + }, func() error { if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) From 734d17ff67e303a972b016729c48c985afc0ed9f Mon Sep 17 00:00:00 2001 From: j75689 Date: Sat, 23 Apr 2022 06:01:23 +0800 Subject: [PATCH 31/51] fix gracefull shutdown issue --- core/remote_state_verifier.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index 8b6d05ce05..b9067db1e0 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -121,6 +121,10 @@ func (vm *remoteVerifyManager) mainLoop() { case <-pruneTicker.C: vm.taskLock.Lock() for hash, task := range vm.tasks { + if vm.bc.insertStopped() { + close(task.terminalCh) + continue + } if vm.bc.CurrentHeader().Number.Cmp(task.blockHeader.Number) == 1 && vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff { delete(vm.tasks, hash) @@ -136,7 +140,6 @@ func (vm *remoteVerifyManager) mainLoop() { vt.messageCh <- message } vm.taskLock.RUnlock() - // System stopped case <-vm.bc.quit: vm.taskLock.RLock() From c840e7f34a4330a4dc4dcb71193538a248095a48 Mon Sep 17 00:00:00 2001 From: j75689 Date: Sat, 23 Apr 2022 06:02:29 +0800 Subject: [PATCH 32/51] fix diffhash mismatch issue --- core/state/statedb.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index f3d0478482..b82a8ae0a3 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1517,11 +1517,9 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-(n-1) layer(bottom-most diff layer) is paired with HEAD-(n-1)state - go func() { - if err := s.snaps.Cap(s.expectedRoot, s.snaps.CapLimit()); err != nil { - log.Warn("Failed to cap snapshot tree", "root", s.expectedRoot, "layers", s.snaps.CapLimit(), "err", err) - } - }() + if err := s.snaps.Cap(s.expectedRoot, s.snaps.CapLimit()); err != nil { + log.Warn("Failed to cap snapshot tree", "root", s.expectedRoot, "layers", s.snaps.CapLimit(), "err", err) + } } } return nil From 315075564f5264c58aa3fc4383092039d080119d Mon Sep 17 00:00:00 2001 From: j75689 Date: Sat, 23 Apr 2022 16:29:41 +0800 Subject: [PATCH 33/51] fix log format --- core/remote_state_verifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index b9067db1e0..ff0a23018b 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -301,7 +301,7 @@ func (vt *verifyTask) Start(verifyCh chan common.Hash) { vt.badPeers[msg.peerId] = struct{}{} log.Info("peer is not available", "hash", msg.verifyResult.BlockHash, "number", msg.verifyResult.BlockNumber, "peer", msg.peerId, "reason", msg.verifyResult.Status.Msg) case types.StatusBlockTooNew, types.StatusBlockNewer, types.StatusPossibleFork: - log.Info("return msg from peer %s for block %s is %s", msg.peerId, msg.verifyResult.BlockHash, msg.verifyResult.Status.Msg) + log.Info("return msg from peer", "peerId", msg.peerId, "hash", msg.verifyResult.BlockHash, "msg", msg.verifyResult.Status.Msg) } newVerifyMsgTypeGauge(msg.verifyResult.Status.Code, msg.peerId).Inc(1) case <-resend.C: From dfff21934cfdab95481ced817668243ed29159bc Mon Sep 17 00:00:00 2001 From: j75689 Date: Mon, 25 Apr 2022 10:25:23 +0800 Subject: [PATCH 34/51] fix close of closed channel issue --- core/remote_state_verifier.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index ff0a23018b..cef98e5b9a 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -121,12 +121,8 @@ func (vm *remoteVerifyManager) mainLoop() { case <-pruneTicker.C: vm.taskLock.Lock() for hash, task := range vm.tasks { - if vm.bc.insertStopped() { - close(task.terminalCh) - continue - } - if vm.bc.CurrentHeader().Number.Cmp(task.blockHeader.Number) == 1 && - vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff { + if vm.bc.insertStopped() || (vm.bc.CurrentHeader().Number.Cmp(task.blockHeader.Number) == 1 && + vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff) { delete(vm.tasks, hash) verifyTaskCounter.Dec(1) verifyTaskFailedMeter.Mark(1) From b56c19c2f9a8cd396114e8886868fab9b32a216c Mon Sep 17 00:00:00 2001 From: j75689 Date: Tue, 26 Apr 2022 11:18:03 +0800 Subject: [PATCH 35/51] fix diffhash issue --- core/state/state_object.go | 6 ++++++ core/state/statedb.go | 14 +++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/core/state/state_object.go b/core/state/state_object.go index 538d5e5da8..b454de272d 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -18,6 +18,7 @@ package state import ( "bytes" + "errors" "fmt" "io" "math/big" @@ -25,6 +26,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" @@ -274,6 +276,10 @@ func (s *StateObject) GetCommittedState(db Database, key common.Hash) common.Has } enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) } + // ErrSnapshotStale may occur due to diff layers in the update, so we should try again in noTrie mode. + if s.db.NoTrie() && err != nil && errors.Is(err, snapshot.ErrSnapshotStale) { + return s.GetCommittedState(db, key) + } // If snapshot unavailable or reading from it failed, load from the database if s.db.snap == nil || err != nil { if meter != nil { diff --git a/core/state/statedb.go b/core/state/statedb.go index b82a8ae0a3..7ceb8e0e47 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -674,6 +674,12 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *StateObject { } } } + + // ErrSnapshotStale may occur due to diff layers in the update, so we should try again in noTrie mode. + if s.NoTrie() && err != nil && errors.Is(err, snapshot.ErrSnapshotStale) { + return s.getDeletedStateObject(addr) + } + // If snapshot unavailable or reading from it failed, load from the database if s.snap == nil || err != nil { if s.trie == nil { @@ -1517,9 +1523,11 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-(n-1) layer(bottom-most diff layer) is paired with HEAD-(n-1)state - if err := s.snaps.Cap(s.expectedRoot, s.snaps.CapLimit()); err != nil { - log.Warn("Failed to cap snapshot tree", "root", s.expectedRoot, "layers", s.snaps.CapLimit(), "err", err) - } + go func() { + if err := s.snaps.Cap(s.expectedRoot, s.snaps.CapLimit()); err != nil { + log.Warn("Failed to cap snapshot tree", "root", s.expectedRoot, "layers", s.snaps.CapLimit(), "err", err) + } + }() } } return nil From 5a254c48e33d9d4996af642e24b66db75c2c531e Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 27 Apr 2022 10:20:27 +0800 Subject: [PATCH 36/51] add interval when getting ErrSnapshotStale --- core/state/state_object.go | 8 ++++++++ core/state/statedb.go | 13 ++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/core/state/state_object.go b/core/state/state_object.go index b454de272d..0263d8d64b 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -278,6 +278,14 @@ func (s *StateObject) GetCommittedState(db Database, key common.Hash) common.Has } // ErrSnapshotStale may occur due to diff layers in the update, so we should try again in noTrie mode. if s.db.NoTrie() && err != nil && errors.Is(err, snapshot.ErrSnapshotStale) { + // This error occurs when statedb.snaps.Cap changes the state of the merged difflayer + // to stale during the refresh of the difflayer, indicating that it is about to be discarded. + // Since the difflayer is refreshed in parallel, + // there is a small chance that the difflayer of the stale will be read while reading, + // resulting in an empty array being returned here. + // Therefore, noTrie mode must retry here, + // and add a time interval when retrying to avoid stacking too much and causing OOM. + time.Sleep(snapshotStaleRetryInterval) return s.GetCommittedState(db, key) } // If snapshot unavailable or reading from it failed, load from the database diff --git a/core/state/statedb.go b/core/state/statedb.go index 7ceb8e0e47..c294c73667 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -39,7 +39,10 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -const defaultNumOfSlots = 100 +const ( + defaultNumOfSlots = 100 + snapshotStaleRetryInterval = time.Millisecond * 100 +) type revision struct { id int @@ -677,6 +680,14 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *StateObject { // ErrSnapshotStale may occur due to diff layers in the update, so we should try again in noTrie mode. if s.NoTrie() && err != nil && errors.Is(err, snapshot.ErrSnapshotStale) { + // This error occurs when statedb.snaps.Cap changes the state of the merged difflayer + // to stale during the refresh of the difflayer, indicating that it is about to be discarded. + // Since the difflayer is refreshed in parallel, + // there is a small chance that the difflayer of the stale will be read while reading, + // resulting in an empty array being returned here. + // Therefore, noTrie mode must retry here, + // and add a time interval when retrying to avoid stacking too much and causing OOM. + time.Sleep(snapshotStaleRetryInterval) return s.getDeletedStateObject(addr) } From 5f22105303183e482f5c45bd43668e4f186bb158 Mon Sep 17 00:00:00 2001 From: j75689 Date: Wed, 27 Apr 2022 16:20:47 +0800 Subject: [PATCH 37/51] add unit-test for fastnode --- core/blockchain_notries_test.go | 187 ++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 core/blockchain_notries_test.go diff --git a/core/blockchain_notries_test.go b/core/blockchain_notries_test.go new file mode 100644 index 0000000000..19481a044f --- /dev/null +++ b/core/blockchain_notries_test.go @@ -0,0 +1,187 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Tests that abnormal program termination (i.e.crash) and restart doesn't leave +// the database in some strange state with gaps in the chain, nor with block data +// dangling in the future. + +package core + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/params" +) + +func newMockVerifyPeer() *mockVerifyPeer { + return &mockVerifyPeer{} +} + +type requestRoot struct { + blockNumber uint64 + blockHash common.Hash + diffHash common.Hash +} + +// mockVerifyPeer is a mocking struct that simulates p2p signals for verification tasks. +type mockVerifyPeer struct { + callback func(*requestRoot) +} + +func (peer *mockVerifyPeer) setCallBack(callback func(*requestRoot)) { + peer.callback = callback +} + +func (peer *mockVerifyPeer) RequestRoot(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) error { + if peer.callback != nil { + peer.callback(&requestRoot{blockNumber, blockHash, diffHash}) + } + return nil +} + +func (peer *mockVerifyPeer) ID() string { + return "mock_peer" +} + +type mockVerifyPeers struct { + peers []VerifyPeer +} + +func (peers *mockVerifyPeers) GetVerifyPeers() []VerifyPeer { + return peers.peers +} + +func newMockRemoteVerifyPeer(peers []VerifyPeer) *mockVerifyPeers { + return &mockVerifyPeers{peers} +} + +func makeTestBackendWithRemoteValidator(blocks int) (*testBackend, *testBackend, []*types.Block, error) { + signer := types.HomesteadSigner{} + + // Create a database pre-initialize with a genesis block + db := rawdb.NewMemoryDatabase() + db.SetDiffStore(memorydb.New()) + (&Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db) + engine := ethash.NewFaker() + + db2 := rawdb.NewMemoryDatabase() + db2.SetDiffStore(memorydb.New()) + (&Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testAddr: {Balance: big.NewInt(100000000000000000)}}, + }).MustCommit(db2) + engine2 := ethash.NewFaker() + + peer := newMockVerifyPeer() + peers := []VerifyPeer{peer} + + verifier, err := NewBlockChain(db, nil, params.TestChainConfig, engine, vm.Config{}, + nil, nil, EnablePersistDiff(100000), EnableBlockValidator(params.TestChainConfig, engine2, LocalVerify, nil)) + if err != nil { + return nil, nil, nil, err + } + + fastnode, err := NewBlockChain(db2, nil, params.TestChainConfig, engine2, vm.Config{}, + nil, nil, EnableBlockValidator(params.TestChainConfig, engine2, FullVerify, newMockRemoteVerifyPeer(peers))) + if err != nil { + return nil, nil, nil, err + } + + generator := func(i int, block *BlockGen) { + // The chain maker doesn't have access to a chain, so the difficulty will be + // lets unset (nil). Set it here to the correct value. + block.SetCoinbase(testAddr) + + for idx, testBlock := range testBlocks { + // Specific block setting, the index in this generator has 1 diff from specified blockNr. + if i+1 == testBlock.blockNr { + for _, testTransaction := range testBlock.txs { + var transaction *types.Transaction + if testTransaction.to == nil { + transaction = types.NewContractCreation(block.TxNonce(testAddr), + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } else { + transaction = types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data) + } + tx, err := types.SignTx(transaction, signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(verifier, tx) + } + break + } + + // Default block setting. + if idx == len(testBlocks)-1 { + // We want to simulate an empty middle block, having the same state as the + // first one. The last is needs a state change again to force a reorg. + for _, testTransaction := range testBlocks[0].txs { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), *testTransaction.to, + testTransaction.value, uint64(commonGas), testTransaction.gasPrice, testTransaction.data), signer, testKey) + if err != nil { + panic(err) + } + block.AddTxWithChain(verifier, tx) + } + } + } + } + + bs, _ := GenerateChain(params.TestChainConfig, verifier.Genesis(), ethash.NewFaker(), db, blocks, generator) + + peer.setCallBack(func(req *requestRoot) { + if fastnode.validator != nil && fastnode.validator.RemoteVerifyManager() != nil { + fastnode.validator.RemoteVerifyManager(). + HandleRootResponse( + verifier.GetRootByDiffHash(req.blockNumber, req.blockHash, req.diffHash), "mock") + } + }) + if _, err := verifier.InsertChain(bs); err != nil { + return nil, nil, nil, err + } + + return &testBackend{ + db: db, + chain: verifier, + }, + &testBackend{ + db: db2, + chain: fastnode, + }, bs, nil +} + +func TestFastNode(t *testing.T) { + _, fastnode, blocks, err := makeTestBackendWithRemoteValidator(10240) + if err != nil { + t.Fatalf(err.Error()) + } + _, err = fastnode.chain.InsertChain(blocks) + if err != nil { + t.Fatalf(err.Error()) + } +} From ef3ec13eeeea3060e0128cc1716848dbd10cdbe9 Mon Sep 17 00:00:00 2001 From: dean65 Date: Thu, 28 Apr 2022 14:49:38 +0800 Subject: [PATCH 38/51] fix lint error --- cmd/utils/flags.go | 8 ++++---- core/state/statedb.go | 2 +- eth/handler_trust.go | 1 + eth/protocols/trust/protocol.go | 1 - ethclient/ethclient.go | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e1669fcb12..79ad12dc33 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -123,11 +123,11 @@ var ( Usage: "Disable snap protocol", } DisableDiffProtocolFlag = cli.BoolFlag{ - Name: "disablediffprotocol", + Name: "disablediffprotocol", Usage: "Disable diff protocol", } EnableTrustProtocolFlag = cli.BoolFlag{ - Name: "enabletrustprotocol", + Name: "enabletrustprotocol", Usage: "Enable trust protocol", } DiffSyncFlag = cli.BoolFlag{ @@ -272,9 +272,9 @@ var ( Usage: "The layer of tries trees that keep in memory", Value: 128, } - defaultVerifyMode = ethconfig.Defaults.TriesVerifyMode + defaultVerifyMode = ethconfig.Defaults.TriesVerifyMode TriesVerifyModeFlag = TextMarshalerFlag{ - Name: "tries-verify-mode", + Name: "tries-verify-mode", Usage: `tries verify mode: "local", "full", "insecure", "none"`, Value: &defaultVerifyMode, } diff --git a/core/state/statedb.go b/core/state/statedb.go index c294c73667..4c2e12f0b6 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1611,7 +1611,7 @@ func (s *StateDB) SnapToDiffLayer() ([]common.Address, []types.DiffAccount, []ty for accountHash, storage := range s.snapStorage { keys := make([]string, 0, len(storage)) values := make([][]byte, 0, len(storage)) - for k, _ := range storage { + for k := range storage { keys = append(keys, k) } sort.Strings(keys) diff --git a/eth/handler_trust.go b/eth/handler_trust.go index 52100144e2..0b116b9255 100644 --- a/eth/handler_trust.go +++ b/eth/handler_trust.go @@ -2,6 +2,7 @@ package eth import ( "fmt" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth/protocols/trust" "github.com/ethereum/go-ethereum/p2p/enode" diff --git a/eth/protocols/trust/protocol.go b/eth/protocols/trust/protocol.go index 2bd7fecc15..e4f98dd324 100644 --- a/eth/protocols/trust/protocol.go +++ b/eth/protocols/trust/protocol.go @@ -40,7 +40,6 @@ var ( errMsgTooLarge = errors.New("message too long") errDecode = errors.New("invalid message") errInvalidMsgCode = errors.New("invalid message code") - errUnexpectedMsg = errors.New("unexpected message code") ) // Packet represents a p2p message in the `trust` protocol. diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 78f5c6bb14..d4d19ff5f0 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -22,12 +22,12 @@ import ( "encoding/json" "errors" "fmt" - "github.com/ethereum/go-ethereum/core" "math/big" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rpc" ) From 749049942616832f662119358f318cf7aa58442a Mon Sep 17 00:00:00 2001 From: dean65 Date: Thu, 28 Apr 2022 15:22:06 +0800 Subject: [PATCH 39/51] setup default validator for blockchain --- core/block_validator.go | 24 +++++++++++++++--------- core/blockchain.go | 12 ++++++++---- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 6f2c40b2ec..876c6e43a2 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -29,6 +29,15 @@ import ( const badBlockCacheExpire = 30 * time.Second +type BlockValidatorOption func(*BlockValidator) *BlockValidator + +func EnableRemoteVerifyManager(remoteValidator *remoteVerifyManager) BlockValidatorOption { + return func(bv *BlockValidator) *BlockValidator { + bv.remoteValidator = remoteValidator + return bv + } +} + // BlockValidator is responsible for validating block headers, uncles and // processed state. // @@ -41,21 +50,18 @@ type BlockValidator struct { } // NewBlockValidator returns a new block validator which is safe for re-use -func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine, mode VerifyMode, peers verifyPeers) (*BlockValidator, error) { +func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine, opts ...BlockValidatorOption) *BlockValidator { validator := &BlockValidator{ config: config, engine: engine, bc: blockchain, } - if mode.NeedRemoteVerify() { - remoteValidator, err := NewVerifyManager(blockchain, peers, mode == InsecureVerify) - if err != nil { - return nil, err - } - validator.remoteValidator = remoteValidator - go validator.remoteValidator.mainLoop() + + for _, opt := range opts { + validator = opt(validator) } - return validator, nil + + return validator } // ValidateBody validates the given block's uncles and verifies the block diff --git a/core/blockchain.go b/core/blockchain.go index 1ca43ec7c8..465235fdd2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -317,6 +317,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } bc.prefetcher = NewStatePrefetcher(chainConfig, bc, engine) + bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.processor = NewStateProcessor(chainConfig, bc, engine) var err error @@ -3165,11 +3166,14 @@ func EnablePersistDiff(limit uint64) BlockChainOption { func EnableBlockValidator(chainConfig *params.ChainConfig, engine consensus.Engine, mode VerifyMode, peers verifyPeers) BlockChainOption { return func(bc *BlockChain) (*BlockChain, error) { - validator, err := NewBlockValidator(chainConfig, bc, engine, mode, peers) - if err != nil { - return bc, err + if mode.NeedRemoteVerify() { + vm, err := NewVerifyManager(bc, peers, mode == InsecureVerify) + if err != nil { + return nil, err + } + go vm.mainLoop() + bc.validator = NewBlockValidator(chainConfig, bc, engine, EnableRemoteVerifyManager(vm)) } - bc.validator = validator return bc, nil } } From 89295108f195961fb5b35d65c2ba36cd0dcc6330 Mon Sep 17 00:00:00 2001 From: dean65 Date: Thu, 28 Apr 2022 17:01:05 +0800 Subject: [PATCH 40/51] fix tests --- eth/handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/handler.go b/eth/handler.go index 8844944b11..82f7cd611e 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -148,6 +148,9 @@ func newHandler(config *handlerConfig) (*handler, error) { if config.EventMux == nil { config.EventMux = new(event.TypeMux) // Nicety initialization for tests } + if config.PeerSet == nil { + config.PeerSet = newPeerSet() // Nicety initialization for tests + } h := &handler{ networkID: config.Network, forkFilter: forkid.NewFilter(config.Chain), From e9be0d4400b0251e7357a9733639d315fad202d3 Mon Sep 17 00:00:00 2001 From: dean65 Date: Fri, 29 Apr 2022 10:01:16 +0800 Subject: [PATCH 41/51] close verifyTask when length of verifyPeers is 0 --- core/remote_state_verifier.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index cef98e5b9a..d23715bea1 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -115,7 +115,7 @@ func (vm *remoteVerifyManager) mainLoop() { verifyTaskCounter.Dec(1) verifyTaskSucceedMeter.Mark(1) verifyTaskExecutionTimer.Update(time.Since(task.startAt)) - close(task.terminalCh) + task.Close() } vm.taskLock.Unlock() case <-pruneTicker.C: @@ -126,7 +126,7 @@ func (vm *remoteVerifyManager) mainLoop() { delete(vm.tasks, hash) verifyTaskCounter.Dec(1) verifyTaskFailedMeter.Mark(1) - close(task.terminalCh) + task.Close() } } vm.taskLock.Unlock() @@ -140,7 +140,7 @@ func (vm *remoteVerifyManager) mainLoop() { case <-vm.bc.quit: vm.taskLock.RLock() for _, task := range vm.tasks { - close(task.terminalCh) + task.Close() } vm.taskLock.RUnlock() return @@ -276,6 +276,15 @@ func NewVerifyTask(diffhash common.Hash, header *types.Header, peers verifyPeers return vt } +func (vt *verifyTask) Close() { + // It is safe to call close multiple + select { + case <-vt.terminalCh: + default: + close(vt.terminalCh) + } +} + func (vt *verifyTask) Start(verifyCh chan common.Hash) { vt.startAt = time.Now() @@ -326,7 +335,7 @@ func (vt *verifyTask) sendVerifyRequest(n int) { // if has not valid peer, log warning. if len(validPeers) == 0 { log.Warn("there is no valid peer for block", "number", vt.blockHeader.Number) - return + vt.Close() } if n < len(validPeers) && n > 0 { From 0132f13b869d6c42c245bf782f44e9a0b289d9ca Mon Sep 17 00:00:00 2001 From: kyrie-yl Date: Fri, 29 Apr 2022 14:39:46 +0800 Subject: [PATCH 42/51] add UT tests --- core/blockchain_notries_test.go | 49 ++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/core/blockchain_notries_test.go b/core/blockchain_notries_test.go index 19481a044f..b4fa305888 100644 --- a/core/blockchain_notries_test.go +++ b/core/blockchain_notries_test.go @@ -43,6 +43,11 @@ type requestRoot struct { diffHash common.Hash } +type verifFailedStatus struct { + status types.VerifyStatus + blockNumber uint64 +} + // mockVerifyPeer is a mocking struct that simulates p2p signals for verification tasks. type mockVerifyPeer struct { callback func(*requestRoot) @@ -75,7 +80,7 @@ func newMockRemoteVerifyPeer(peers []VerifyPeer) *mockVerifyPeers { return &mockVerifyPeers{peers} } -func makeTestBackendWithRemoteValidator(blocks int) (*testBackend, *testBackend, []*types.Block, error) { +func makeTestBackendWithRemoteValidator(blocks int, mode VerifyMode, failed *verifFailedStatus) (*testBackend, *testBackend, []*types.Block, error) { signer := types.HomesteadSigner{} // Create a database pre-initialize with a genesis block @@ -105,7 +110,7 @@ func makeTestBackendWithRemoteValidator(blocks int) (*testBackend, *testBackend, } fastnode, err := NewBlockChain(db2, nil, params.TestChainConfig, engine2, vm.Config{}, - nil, nil, EnableBlockValidator(params.TestChainConfig, engine2, FullVerify, newMockRemoteVerifyPeer(peers))) + nil, nil, EnableBlockValidator(params.TestChainConfig, engine2, mode, newMockRemoteVerifyPeer(peers))) if err != nil { return nil, nil, nil, err } @@ -156,9 +161,15 @@ func makeTestBackendWithRemoteValidator(blocks int) (*testBackend, *testBackend, peer.setCallBack(func(req *requestRoot) { if fastnode.validator != nil && fastnode.validator.RemoteVerifyManager() != nil { + resp := verifier.GetRootByDiffHash(req.blockNumber, req.blockHash, req.diffHash) + if failed != nil && req.blockNumber == failed.blockNumber { + resp.Status = failed.status + } else { + resp.Status = types.StatusFullVerified + } fastnode.validator.RemoteVerifyManager(). HandleRootResponse( - verifier.GetRootByDiffHash(req.blockNumber, req.blockHash, req.diffHash), "mock") + resp, peer.ID()) } }) if _, err := verifier.InsertChain(bs); err != nil { @@ -176,7 +187,8 @@ func makeTestBackendWithRemoteValidator(blocks int) (*testBackend, *testBackend, } func TestFastNode(t *testing.T) { - _, fastnode, blocks, err := makeTestBackendWithRemoteValidator(10240) + // test full mode and succeed + _, fastnode, blocks, err := makeTestBackendWithRemoteValidator(10240, FullVerify, nil) if err != nil { t.Fatalf(err.Error()) } @@ -184,4 +196,33 @@ func TestFastNode(t *testing.T) { if err != nil { t.Fatalf(err.Error()) } + // test full mode and failed + failed := &verifFailedStatus{status: types.StatusDiffHashMismatch, blockNumber: 2048} + _, fastnode, blocks, err = makeTestBackendWithRemoteValidator(10240, FullVerify, failed) + if err != nil { + t.Fatalf(err.Error()) + } + _, err = fastnode.chain.InsertChain(blocks) + if err == nil || fastnode.chain.CurrentBlock().NumberU64() != failed.blockNumber+10 { + t.Fatalf("blocks insert should be failed at height %d", failed.blockNumber+11) + } + // test insecure mode and succeed + _, fastnode, blocks, err = makeTestBackendWithRemoteValidator(10240, InsecureVerify, nil) + if err != nil { + t.Fatalf(err.Error()) + } + _, err = fastnode.chain.InsertChain(blocks) + if err != nil { + t.Fatalf(err.Error()) + } + // test insecure mode and failed + failed = &verifFailedStatus{status: types.StatusImpossibleFork, blockNumber: 2048} + _, fastnode, blocks, err = makeTestBackendWithRemoteValidator(10240, FullVerify, failed) + if err != nil { + t.Fatalf(err.Error()) + } + _, err = fastnode.chain.InsertChain(blocks) + if err == nil || fastnode.chain.CurrentBlock().NumberU64() != failed.blockNumber+10 { + t.Fatalf("blocks insert should be failed at height %d", failed.blockNumber+11) + } } From ae3bdea794402dd22d74a307bef07bf4d6850027 Mon Sep 17 00:00:00 2001 From: cryyl <1226241521@qq.com> Date: Thu, 5 May 2022 19:10:52 +0800 Subject: [PATCH 43/51] fix to resolve comments Signed-off-by: cryyl <1226241521@qq.com> --- cmd/utils/flags.go | 4 +-- core/block_validator.go | 1 - core/blockchain.go | 49 +++++++++++++++------------------ core/blockchain_diff_test.go | 2 +- core/blockchain_notries_test.go | 2 +- core/remote_state_verifier.go | 31 +++++++++++++-------- core/state/state_object.go | 4 ++- core/state/statedb.go | 12 ++------ eth/handler.go | 3 +- eth/protocols/trust/handler.go | 2 +- internal/ethapi/api.go | 4 +-- 11 files changed, 54 insertions(+), 60 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 79ad12dc33..f3c37b8f38 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -275,7 +275,7 @@ var ( defaultVerifyMode = ethconfig.Defaults.TriesVerifyMode TriesVerifyModeFlag = TextMarshalerFlag{ Name: "tries-verify-mode", - Usage: `tries verify mode: "local", "full", "insecure", "none"`, + Usage: `tries verify mode: "local: a normal full node", "full: state verification by verify node which has diffLayer of blocks", "insecure: state verification by verify node which has no diffLayer of blocks", "none: no state verification"`, Value: &defaultVerifyMode, } OverrideBerlinFlag = cli.Uint64Flag{ @@ -1688,7 +1688,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } if ctx.GlobalIsSet(TriesVerifyModeFlag.Name) { cfg.TriesVerifyMode = *GlobalTextMarshaler(ctx, TriesVerifyModeFlag.Name).(*core.VerifyMode) - // If a node sets verify mode to full or light, it's a fast node and need + // If a node sets verify mode to full or insecure, it's a fast node and need // to verify blocks from verify nodes, then it should enable trust protocol. if cfg.TriesVerifyMode.NeedRemoteVerify() { cfg.EnableTrustProtocol = true diff --git a/core/block_validator.go b/core/block_validator.go index 876c6e43a2..85e016df84 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -110,7 +110,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if v.remoteValidator != nil && !v.remoteValidator.AncestorVerified(block.Header()) { return fmt.Errorf("%w, number: %s, hash: %s", ErrAncestorHasNotBeenVerified, block.Number(), block.Hash()) } - return nil }, } diff --git a/core/blockchain.go b/core/blockchain.go index 465235fdd2..a11d08b05d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -28,9 +28,8 @@ import ( "sync/atomic" "time" - "golang.org/x/crypto/sha3" - lru "github.com/hashicorp/golang-lru" + "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" @@ -509,31 +508,27 @@ func (bc *BlockChain) cacheReceipts(hash common.Hash, receipts types.Receipts) { bc.receiptsCache.Add(hash, receipts) } -func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, sorted bool) { - if !sorted { - sort.SliceStable(diffLayer.Codes, func(i, j int) bool { - return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() - }) - sort.SliceStable(diffLayer.Destructs, func(i, j int) bool { - return diffLayer.Destructs[i].Hex() < (diffLayer.Destructs[j].Hex()) - }) - sort.SliceStable(diffLayer.Accounts, func(i, j int) bool { - return diffLayer.Accounts[i].Account.Hex() < diffLayer.Accounts[j].Account.Hex() - }) - sort.SliceStable(diffLayer.Storages, func(i, j int) bool { - return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() - }) - } +func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, diffLayerCh chan struct{}) { + sort.SliceStable(diffLayer.Codes, func(i, j int) bool { + return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() + }) + sort.SliceStable(diffLayer.Destructs, func(i, j int) bool { + return diffLayer.Destructs[i].Hex() < (diffLayer.Destructs[j].Hex()) + }) + sort.SliceStable(diffLayer.Accounts, func(i, j int) bool { + return diffLayer.Accounts[i].Account.Hex() < diffLayer.Accounts[j].Account.Hex() + }) + sort.SliceStable(diffLayer.Storages, func(i, j int) bool { + return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() + }) if bc.diffLayerCache.Len() >= diffLayerCacheLimit { bc.diffLayerCache.RemoveOldest() } bc.diffLayerCache.Add(diffLayer.BlockHash, diffLayer) - if cached, ok := bc.diffLayerChanCache.Get(diffLayer.BlockHash); ok { - diffLayerCh := cached.(chan struct{}) - close(diffLayerCh) - } + close(diffLayerCh) + if bc.db.DiffStore() != nil { // push to priority queue before persisting bc.diffQueueBuffer <- diffLayer @@ -1840,7 +1835,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } bc.diffLayerChanCache.Add(diffLayer.BlockHash, diffLayerCh) - go bc.cacheDiffLayer(diffLayer, false) + go bc.cacheDiffLayer(diffLayer, diffLayerCh) } wg.Wait() @@ -2156,7 +2151,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er return it.index, err } if statedb.NoTrie() { - statedb.SetCurrentRoot(block.Root()) + statedb.SetExpectedStateRoot(block.Root()) } bc.updateHighestVerifiedHeader(block.Header()) @@ -2833,10 +2828,10 @@ func (bc *BlockChain) HandleDiffLayer(diffLayer *types.DiffLayer, pid string, fu return nil } - diffHash := common.Hash{} - if diffLayer.DiffHash.Load() != nil { - diffHash = diffLayer.DiffHash.Load().(common.Hash) + if diffLayer.DiffHash.Load() == nil { + return fmt.Errorf("unexpected difflayer which diffHash is nil from peeer %s", pid) } + diffHash := diffLayer.DiffHash.Load().(common.Hash) bc.diffMux.Lock() defer bc.diffMux.Unlock() @@ -3178,7 +3173,7 @@ func EnableBlockValidator(chainConfig *params.ChainConfig, engine consensus.Engi } } -func (bc *BlockChain) GetRootByDiffHash(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) *VerifyResult { +func (bc *BlockChain) GetVerifyResult(blockNumber uint64, blockHash common.Hash, diffHash common.Hash) *VerifyResult { var res VerifyResult res.BlockNumber = blockNumber res.BlockHash = blockHash diff --git a/core/blockchain_diff_test.go b/core/blockchain_diff_test.go index fad8fe353a..7e65ac6d79 100644 --- a/core/blockchain_diff_test.go +++ b/core/blockchain_diff_test.go @@ -613,7 +613,7 @@ func testGetRootByDiffHash(t *testing.T, chain1, chain2 *BlockChain, blockNumber chain1.diffLayerCache.Remove(block1.Hash()) } - result := chain1.GetRootByDiffHash(blockNumber, block2.Hash(), diffHash2) + result := chain1.GetVerifyResult(blockNumber, block2.Hash(), diffHash2) if result.Status != expect.Status { t.Fatalf("failed to verify block, number: %v, expect status: %v, real status: %v", blockNumber, expect.Status, result.Status) } diff --git a/core/blockchain_notries_test.go b/core/blockchain_notries_test.go index b4fa305888..e439fd709f 100644 --- a/core/blockchain_notries_test.go +++ b/core/blockchain_notries_test.go @@ -161,7 +161,7 @@ func makeTestBackendWithRemoteValidator(blocks int, mode VerifyMode, failed *ver peer.setCallBack(func(req *requestRoot) { if fastnode.validator != nil && fastnode.validator.RemoteVerifyManager() != nil { - resp := verifier.GetRootByDiffHash(req.blockNumber, req.blockHash, req.diffHash) + resp := verifier.GetVerifyResult(req.blockNumber, req.blockHash, req.diffHash) if failed != nil && req.blockNumber == failed.blockNumber { resp.Status = failed.status } else { diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index d23715bea1..3875a25cea 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -29,6 +29,8 @@ const ( resendInterval = 2 * time.Second // tryAllPeersTime is the time that a block has not been verified and then try all the valid verify peers. tryAllPeersTime = 15 * time.Second + // maxWaitVerifyResultTime is the max time of waiting for ancestor's verify result. + maxWaitVerifyResultTime = 30 * time.Second ) var ( @@ -111,22 +113,18 @@ func (vm *remoteVerifyManager) mainLoop() { vm.cacheBlockVerified(hash) vm.taskLock.Lock() if task, ok := vm.tasks[hash]; ok { - delete(vm.tasks, hash) - verifyTaskCounter.Dec(1) + vm.CloseTask(task) verifyTaskSucceedMeter.Mark(1) verifyTaskExecutionTimer.Update(time.Since(task.startAt)) - task.Close() } vm.taskLock.Unlock() case <-pruneTicker.C: vm.taskLock.Lock() - for hash, task := range vm.tasks { + for _, task := range vm.tasks { if vm.bc.insertStopped() || (vm.bc.CurrentHeader().Number.Cmp(task.blockHeader.Number) == 1 && vm.bc.CurrentHeader().Number.Uint64()-task.blockHeader.Number.Uint64() > pruneHeightDiff) { - delete(vm.tasks, hash) - verifyTaskCounter.Dec(1) + vm.CloseTask(task) verifyTaskFailedMeter.Mark(1) - task.Close() } } vm.taskLock.Unlock() @@ -180,7 +178,6 @@ func (vm *remoteVerifyManager) NewBlockVerifyTask(header *types.Header) { if cached, ok := vm.bc.diffLayerChanCache.Get(hash); ok { diffLayerCh := cached.(chan struct{}) <-diffLayerCh - vm.bc.diffLayerChanCache.Remove(hash) diffLayer = vm.bc.GetTrustedDiffLayer(hash) } // if this block has no diff, there is no need to verify it. @@ -225,8 +222,13 @@ func (vm *remoteVerifyManager) AncestorVerified(header *types.Header) bool { vm.taskLock.RLock() task, exist := vm.tasks[hash] vm.taskLock.RUnlock() + timeout := time.After(maxWaitVerifyResultTime) if exist { - <-task.terminalCh + select { + case <-task.terminalCh: + case <-timeout: + return false + } } _, exist = vm.verifiedCache.Get(hash) @@ -238,6 +240,12 @@ func (vm *remoteVerifyManager) HandleRootResponse(vr *VerifyResult, pid string) return nil } +func (vm *remoteVerifyManager) CloseTask(task *verifyTask) { + delete(vm.tasks, task.blockHeader.Hash()) + task.Close() + verifyTaskCounter.Dec(1) +} + type VerifyResult struct { Status types.VerifyStatus BlockNumber uint64 @@ -335,7 +343,7 @@ func (vt *verifyTask) sendVerifyRequest(n int) { // if has not valid peer, log warning. if len(validPeers) == 0 { log.Warn("there is no valid peer for block", "number", vt.blockHeader.Number) - vt.Close() + return } if n < len(validPeers) && n > 0 { @@ -352,9 +360,8 @@ func (vt *verifyTask) sendVerifyRequest(n int) { func (vt *verifyTask) compareRootHashAndMark(msg verifyMessage, verifyCh chan common.Hash) { if msg.verifyResult.Root == vt.blockHeader.Root { - blockhash := msg.verifyResult.BlockHash // write back to manager so that manager can cache the result and delete this task. - verifyCh <- blockhash + verifyCh <- msg.verifyResult.BlockHash } else { vt.badPeers[msg.peerId] = struct{}{} } diff --git a/core/state/state_object.go b/core/state/state_object.go index 0263d8d64b..b6b550de4d 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -32,6 +32,8 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +const snapshotStaleRetryInterval = time.Millisecond * 10 + var emptyCodeHash = crypto.Keccak256(nil) type Code []byte @@ -284,7 +286,7 @@ func (s *StateObject) GetCommittedState(db Database, key common.Hash) common.Has // there is a small chance that the difflayer of the stale will be read while reading, // resulting in an empty array being returned here. // Therefore, noTrie mode must retry here, - // and add a time interval when retrying to avoid stacking too much and causing OOM. + // and add a time interval when retrying to avoid stacking too much and causing stack overflow. time.Sleep(snapshotStaleRetryInterval) return s.GetCommittedState(db, key) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 4c2e12f0b6..3179a2c17c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -39,10 +39,7 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -const ( - defaultNumOfSlots = 100 - snapshotStaleRetryInterval = time.Millisecond * 100 -) +const defaultNumOfSlots = 100 type revision struct { id int @@ -81,7 +78,6 @@ type StateDB struct { prefetcherLock sync.Mutex prefetcher *triePrefetcher originalRoot common.Hash // The pre-state root, before any changes were made - currentRoot common.Hash // only used when noTrie is true expectedRoot common.Hash // The state root in the block header stateRoot common.Hash // The calculation result of IntermediateRoot @@ -276,10 +272,6 @@ func (s *StateDB) NoTrie() bool { return s.noTrie } -func (s *StateDB) SetCurrentRoot(root common.Hash) { - s.currentRoot = root -} - func (s *StateDB) Error() error { return s.dbErr } @@ -1184,7 +1176,7 @@ func (s *StateDB) StateIntermediateRoot() common.Hash { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) } if s.noTrie { - return s.currentRoot + return s.expectedRoot } else { return s.trie.Hash() } diff --git a/eth/handler.go b/eth/handler.go index 82f7cd611e..8db505b343 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -24,8 +24,6 @@ import ( "sync/atomic" "time" - "github.com/ethereum/go-ethereum/eth/protocols/trust" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" @@ -35,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/eth/protocols/diff" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/eth/protocols/trust" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" diff --git a/eth/protocols/trust/handler.go b/eth/protocols/trust/handler.go index 9b93d4a228..f10aff5178 100644 --- a/eth/protocols/trust/handler.go +++ b/eth/protocols/trust/handler.go @@ -126,7 +126,7 @@ func handleRootRequest(backend Backend, msg Decoder, peer *Peer) error { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } - res := backend.Chain().GetRootByDiffHash(req.BlockNumber, req.BlockHash, req.DiffHash) + res := backend.Chain().GetVerifyResult(req.BlockNumber, req.BlockHash, req.DiffHash) return p2p.Send(peer.rw, RespondRootMsg, RootResponsePacket{ RequestId: req.RequestId, Status: res.Status, diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index db16a05338..3309542581 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1287,8 +1287,8 @@ func (s *PublicBlockChainAPI) GetDiffAccountsWithScope(ctx context.Context, bloc return result, err } -func (s *PublicBlockChainAPI) GetRootByDiffHash(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) *core.VerifyResult { - return s.b.Chain().GetRootByDiffHash(uint64(blockNr), blockHash, diffHash) +func (s *PublicBlockChainAPI) GetVerifyResult(ctx context.Context, blockNr rpc.BlockNumber, blockHash common.Hash, diffHash common.Hash) *core.VerifyResult { + return s.b.Chain().GetVerifyResult(uint64(blockNr), blockHash, diffHash) } // ExecutionResult groups all structured logs emitted by the EVM From f027818e6050153697be49fed22536df9661f40b Mon Sep 17 00:00:00 2001 From: cryyl <1226241521@qq.com> Date: Thu, 5 May 2022 19:10:52 +0800 Subject: [PATCH 44/51] fix to resolve comments Signed-off-by: cryyl <1226241521@qq.com> --- core/blockchain.go | 9 +++++++++ core/state/statedb.go | 6 +----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a11d08b05d..a6f20cf6b7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -521,6 +521,15 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, diffLayerCh cha sort.SliceStable(diffLayer.Storages, func(i, j int) bool { return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() }) + for _, storage := range diffLayer.Storages { + // Sort keys and vals by key. + sort.SliceStable(storage.Keys, func(i, j int) bool { + return storage.Keys[i] < storage.Keys[j] + }) + sort.SliceStable(storage.Vals, func(i, j int) bool { + return storage.Keys[i] < storage.Keys[j] + }) + } if bc.diffLayerCache.Len() >= diffLayerCacheLimit { bc.diffLayerCache.RemoveOldest() diff --git a/core/state/statedb.go b/core/state/statedb.go index 3179a2c17c..8aad0689df 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1603,12 +1603,8 @@ func (s *StateDB) SnapToDiffLayer() ([]common.Address, []types.DiffAccount, []ty for accountHash, storage := range s.snapStorage { keys := make([]string, 0, len(storage)) values := make([][]byte, 0, len(storage)) - for k := range storage { + for k, v := range storage { keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - v := storage[k] values = append(values, v) } storages = append(storages, types.DiffStorage{ From e93cced175ac664be7ee12237cc29ab66adb4a8a Mon Sep 17 00:00:00 2001 From: dean65 Date: Fri, 6 May 2022 01:06:14 +0800 Subject: [PATCH 45/51] use NewTimer instead of time.After --- core/remote_state_verifier.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/remote_state_verifier.go b/core/remote_state_verifier.go index 3875a25cea..cb8d2366bb 100644 --- a/core/remote_state_verifier.go +++ b/core/remote_state_verifier.go @@ -222,11 +222,12 @@ func (vm *remoteVerifyManager) AncestorVerified(header *types.Header) bool { vm.taskLock.RLock() task, exist := vm.tasks[hash] vm.taskLock.RUnlock() - timeout := time.After(maxWaitVerifyResultTime) + timeout := time.NewTimer(maxWaitVerifyResultTime) + defer timeout.Stop() if exist { select { case <-task.terminalCh: - case <-timeout: + case <-timeout.C: return false } } From 4001f37005db590e2b9aa9d36f8c827fb8bde3d2 Mon Sep 17 00:00:00 2001 From: dean65 Date: Fri, 6 May 2022 02:16:57 +0800 Subject: [PATCH 46/51] fix sorting difflayer.storage --- core/blockchain.go | 9 ++------- core/types/block.go | 7 +++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a6f20cf6b7..ac57b8af0e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -521,14 +521,9 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, diffLayerCh cha sort.SliceStable(diffLayer.Storages, func(i, j int) bool { return diffLayer.Storages[i].Account.Hex() < diffLayer.Storages[j].Account.Hex() }) - for _, storage := range diffLayer.Storages { + for index := range diffLayer.Storages { // Sort keys and vals by key. - sort.SliceStable(storage.Keys, func(i, j int) bool { - return storage.Keys[i] < storage.Keys[j] - }) - sort.SliceStable(storage.Vals, func(i, j int) bool { - return storage.Keys[i] < storage.Keys[j] - }) + sort.Sort(&diffLayer.Storages[index]) } if bc.diffLayerCache.Len() >= diffLayerCacheLimit { diff --git a/core/types/block.go b/core/types/block.go index da3bf163bf..7be5b2a3ed 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -480,6 +480,13 @@ type DiffStorage struct { Vals [][]byte } +func (storage *DiffStorage) Len() int { return len(storage.Keys) } +func (storage *DiffStorage) Swap(i, j int) { + storage.Keys[i], storage.Keys[j] = storage.Keys[j], storage.Keys[i] + storage.Vals[i], storage.Vals[j] = storage.Vals[j], storage.Vals[i] +} +func (storage *DiffStorage) Less(i, j int) bool { return storage.Keys[i] < storage.Keys[j] } + type DiffAccountsInTx struct { TxHash common.Hash Accounts map[common.Address]*big.Int From e4408801b0fba74da8378a11c03104484647cc94 Mon Sep 17 00:00:00 2001 From: dean65 Date: Fri, 6 May 2022 02:17:24 +0800 Subject: [PATCH 47/51] fix TestFastNode --- core/blockchain_notries_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/blockchain_notries_test.go b/core/blockchain_notries_test.go index e439fd709f..7b7a977971 100644 --- a/core/blockchain_notries_test.go +++ b/core/blockchain_notries_test.go @@ -164,8 +164,6 @@ func makeTestBackendWithRemoteValidator(blocks int, mode VerifyMode, failed *ver resp := verifier.GetVerifyResult(req.blockNumber, req.blockHash, req.diffHash) if failed != nil && req.blockNumber == failed.blockNumber { resp.Status = failed.status - } else { - resp.Status = types.StatusFullVerified } fastnode.validator.RemoteVerifyManager(). HandleRootResponse( From cc1940c536f7bc8021ebd4c80d58fa02fb2cc262 Mon Sep 17 00:00:00 2001 From: zjubfd <296179868@qq.com> Date: Fri, 6 May 2022 16:38:59 +0800 Subject: [PATCH 48/51] change the flag description --- cmd/utils/flags.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f3c37b8f38..dba2214c2d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -274,8 +274,16 @@ var ( } defaultVerifyMode = ethconfig.Defaults.TriesVerifyMode TriesVerifyModeFlag = TextMarshalerFlag{ - Name: "tries-verify-mode", - Usage: `tries verify mode: "local: a normal full node", "full: state verification by verify node which has diffLayer of blocks", "insecure: state verification by verify node which has no diffLayer of blocks", "none: no state verification"`, + Name: "tries-verify-mode", + Usage: `tries verify mode: + "local(default): a normal full node with complete state world(both MPT and snapshot), merkle state root will + be verified against the block header.", + "full: a fast node with only snapshot state world. Merkle state root is verified by the trustworthy remote verify node + by comparing the diffhash(an identify of difflayer generated by the block) and state root.", + "insecure: same as full mode, except that it can tolerate without verifying the diffhash when verify node does not have it.", + "none: no merkle state root verification at all, there is no need to setup or connect remote verify node at all, + it is more light comparing to full and insecure mode, but get a very little chance that the state is not consistent + with other peers."`, Value: &defaultVerifyMode, } OverrideBerlinFlag = cli.Uint64Flag{ From 84bb85c57d7e18e0bef99531ae179b339aba16f6 Mon Sep 17 00:00:00 2001 From: zjubfd <296179868@qq.com> Date: Fri, 6 May 2022 16:42:40 +0800 Subject: [PATCH 49/51] refine the description --- cmd/utils/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index dba2214c2d..7765475ea4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -282,7 +282,7 @@ var ( by comparing the diffhash(an identify of difflayer generated by the block) and state root.", "insecure: same as full mode, except that it can tolerate without verifying the diffhash when verify node does not have it.", "none: no merkle state root verification at all, there is no need to setup or connect remote verify node at all, - it is more light comparing to full and insecure mode, but get a very little chance that the state is not consistent + it is more light comparing to full and insecure mode, but get a very small chance that the state is not consistent with other peers."`, Value: &defaultVerifyMode, } From 363bb96623dd628ddce5ce712199d2c157c78a9b Mon Sep 17 00:00:00 2001 From: dean65 Date: Thu, 30 Jun 2022 14:28:33 +0800 Subject: [PATCH 50/51] fix comments --- core/state/statedb.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 8ff67893a9..6ef45c2861 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -203,11 +203,11 @@ func (s *StateDB) EnableWriteOnSharedStorage() { // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. func (s *StateDB) StartPrefetcher(namespace string) { - s.prefetcherLock.Lock() - defer s.prefetcherLock.Unlock() if s.noTrie { return } + s.prefetcherLock.Lock() + defer s.prefetcherLock.Unlock() if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -220,11 +220,11 @@ func (s *StateDB) StartPrefetcher(namespace string) { // StopPrefetcher terminates a running prefetcher and reports any leftover stats // from the gathered metrics. func (s *StateDB) StopPrefetcher() { - s.prefetcherLock.Lock() - defer s.prefetcherLock.Unlock() if s.noTrie { return } + s.prefetcherLock.Lock() + defer s.prefetcherLock.Unlock() if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil From 149718f340b856f5815323f68e36d753743697a4 Mon Sep 17 00:00:00 2001 From: dean65 Date: Fri, 1 Jul 2022 15:38:56 +0800 Subject: [PATCH 51/51] fix comments --- core/blockchain.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 324c89064a..1555d19e78 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -511,6 +511,10 @@ func (bc *BlockChain) cacheReceipts(hash common.Hash, receipts types.Receipts) { } func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, diffLayerCh chan struct{}) { + // The difflayer in the system is stored by the map structure, + // so it will be out of order. + // It must be sorted first and then cached, + // otherwise the DiffHash calculated by different nodes will be inconsistent sort.SliceStable(diffLayer.Codes, func(i, j int) bool { return diffLayer.Codes[i].Hash.Hex() < diffLayer.Codes[j].Hash.Hex() }) @@ -2163,9 +2167,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if err != nil { return it.index, err } - if statedb.NoTrie() { - statedb.SetExpectedStateRoot(block.Root()) - } bc.updateHighestVerifiedHeader(block.Header()) // Enable prefetching to pull in trie node paths while processing transactions