Skip to content

Commit

Permalink
state/stateobject: fix revive in diff & delete shrink miss node bug;
Browse files Browse the repository at this point in the history 
trie: support revive from prefix;
  • Loading branch information
@0xbundler
0xbundler committed Nov 2, 2023
1 parent 9c3d512 commit 9825ce3
Show file tree
Hide file tree
Showing 6 changed files with 110 additions and 104 deletions.
21 changes: 11 additions & 10 deletions core/state/state_expiry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,22 +33,23 @@ func defaultStateExpiryMeta() *stateExpiryMeta {
return &stateExpiryMeta{enableStateExpiry: false}
}

// fetchExpiredStorageFromRemote request expired state from remote full state node;
func fetchExpiredStorageFromRemote(meta *stateExpiryMeta, addr common.Address, root common.Hash, tr Trie, prefixKey []byte, key common.Hash) (map[string][]byte, error) {
// tryReviveState request expired state from remote full state node;
func tryReviveState(meta *stateExpiryMeta, addr common.Address, root common.Hash, tr Trie, prefixKey []byte, key common.Hash, force bool) (map[string][]byte, error) {
if !meta.enableStateExpiry {
return nil, nil
}
//log.Debug("fetching expired storage from remoteDB", "addr", addr, "prefix", prefixKey, "key", key)
reviveTrieMeter.Mark(1)
if meta.enableLocalRevive {
if meta.enableLocalRevive && !force {
// if there need revive expired state, try to revive locally, when the node is not being pruned, just renew the epoch
val, err := tr.TryLocalRevive(addr, key.Bytes())
//log.Debug("fetchExpiredStorageFromRemote TryLocalRevive", "addr", addr, "key", key, "val", val, "err", err)
//log.Debug("tryReviveState TryLocalRevive", "addr", addr, "key", key, "val", val, "err", err)
switch err.(type) {
case *trie.MissingNodeError:
// cannot revive locally, request from remote
case nil:
reviveFromLocalMeter.Mark(1)
ret := make(map[string][]byte, 1)
ret[key.String()] = val
return ret, nil
return map[string][]byte{key.String(): val}, nil
default:
return nil, err
}
Expand All @@ -57,7 +58,7 @@ func fetchExpiredStorageFromRemote(meta *stateExpiryMeta, addr common.Address, r
reviveFromRemoteMeter.Mark(1)
// cannot revive locally, fetch remote proof
proofs, err := meta.fullStateDB.GetStorageReviveProof(meta.originalRoot, addr, root, []string{common.Bytes2Hex(prefixKey)}, []string{common.Bytes2Hex(key[:])})
//log.Debug("fetchExpiredStorageFromRemote GetStorageReviveProof", "addr", addr, "key", key, "proofs", len(proofs), "err", err)
//log.Debug("tryReviveState GetStorageReviveProof", "addr", addr, "key", key, "proofs", len(proofs), "err", err)
if err != nil {
return nil, err
}
Expand All @@ -82,7 +83,7 @@ func batchFetchExpiredFromRemote(expiryMeta *stateExpiryMeta, addr common.Addres
var expiredPrefixKeys [][]byte
for i, key := range keys {
val, err := tr.TryLocalRevive(addr, key.Bytes())
//log.Debug("fetchExpiredStorageFromRemote TryLocalRevive", "addr", addr, "key", key, "val", val, "err", err)
//log.Debug("tryReviveState TryLocalRevive", "addr", addr, "key", key, "val", val, "err", err)
switch err.(type) {
case *trie.MissingNodeError:
expiredKeys = append(expiredKeys, key)
Expand Down Expand Up @@ -119,7 +120,7 @@ func batchFetchExpiredFromRemote(expiryMeta *stateExpiryMeta, addr common.Addres
// cannot revive locally, fetch remote proof
reviveFromRemoteMeter.Mark(int64(len(keysStr)))
proofs, err := expiryMeta.fullStateDB.GetStorageReviveProof(expiryMeta.originalRoot, addr, root, prefixKeysStr, keysStr)
//log.Debug("fetchExpiredStorageFromRemote GetStorageReviveProof", "addr", addr, "keys", keysStr, "prefixKeys", prefixKeysStr, "proofs", len(proofs), "err", err)
//log.Debug("tryReviveState GetStorageReviveProof", "addr", addr, "keys", keysStr, "prefixKeys", prefixKeysStr, "proofs", len(proofs), "err", err)
if err != nil {
return nil, err
}
Expand Down
70 changes: 47 additions & 23 deletions core/state/state_object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,9 +316,9 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash {
}
// handle state expiry situation
if s.db.EnableExpire() {
if enErr, ok := err.(*trie.ExpiredNodeError); ok {
if path, ok := trie.ParseExpiredNodeErr(err); ok {
//log.Debug("GetCommittedState expired in trie", "addr", s.address, "key", key, "err", err)
val, err = s.fetchExpiredFromRemote(enErr.Path, key, false)
val, err = s.tryReviveState(path, key, false)
getCommittedStorageExpiredMeter.Mark(1)
} else if err != nil {
getCommittedStorageUnexpiredMeter.Mark(1)
Expand Down Expand Up @@ -490,13 +490,14 @@ func (s *stateObject) updateTrie() (Trie, error) {
if err == nil {
continue
}
enErr, ok := err.(*trie.ExpiredNodeError)
path, ok := trie.ParseExpiredNodeErr(err)
if !ok {
s.db.setError(fmt.Errorf("state object pendingFutureReviveState err, contract: %v, key: %v, err: %v", s.address, key, err))
s.db.setError(fmt.Errorf("updateTrie pendingFutureReviveState err, contract: %v, key: %v, err: %v", s.address, key, err))
//log.Debug("updateTrie pendingFutureReviveState", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s), "err", err)
continue
}
if _, err = fetchExpiredStorageFromRemote(s.db.expiryMeta, s.address, s.data.Root, tr, enErr.Path, key); err != nil {
s.db.setError(fmt.Errorf("state object pendingFutureReviveState fetchExpiredStorageFromRemote err, contract: %v, key: %v, path: %v, err: %v", s.address, key, enErr.Path, err))
if _, err = tryReviveState(s.db.expiryMeta, s.address, s.data.Root, tr, path, key, false); err != nil {
s.db.setError(fmt.Errorf("updateTrie pendingFutureReviveState tryReviveState err, contract: %v, key: %v, path: %v, err: %v", s.address, key, path, err))
}
//log.Debug("updateTrie pendingFutureReviveState", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
}
Expand All @@ -507,35 +508,57 @@ func (s *stateObject) updateTrie() (Trie, error) {
// continue
// }
// log.Error("EnableExpire GetStorage error", "addr", s.address, "key", key, "val", val, "origin", s.originStorage[key], "err", err)
// enErr, ok := err.(*trie.ExpiredNodeError)
// path, ok := trie.ParseExpiredNodeErr(err)
// if !ok {
// s.db.setError(fmt.Errorf("state object dirtyStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
// continue
// }
// if _, err = fetchExpiredStorageFromRemote(s.db.expiryMeta, s.address, s.data.Root, tr, enErr.Path, key); err != nil {
// log.Error("EnableExpire GetStorage fetchExpiredStorageFromRemote error", "addr", s.address, "key", key, "val", val, "origin", s.originStorage[key], "err", err)
// s.db.setError(fmt.Errorf("state object dirtyStorage fetchExpiredStorageFromRemote err, contract: %v, key: %v, path: %v, err: %v", s.address, key, enErr.Path, err))
// if _, err = tryReviveState(s.db.expiryMeta, s.address, s.data.Root, tr, path key); err != nil {
// log.Error("EnableExpire GetStorage tryReviveState error", "addr", s.address, "key", key, "val", val, "origin", s.originStorage[key], "err", err)
// s.db.setError(fmt.Errorf("state object dirtyStorage tryReviveState err, contract: %v, key: %v, path: %v, err: %v", s.address, key, enErr.Path, err))
// }
// //log.Debug("updateTrie dirtyStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
//}
}
touchExpiredStorage := make(map[common.Hash][]byte)
for key, value := range dirtyStorage {
if len(value) == 0 {
if err := tr.DeleteStorage(s.address, key[:]); err != nil {
s.db.setError(fmt.Errorf("state object update trie DeleteStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
err := tr.DeleteStorage(s.address, key[:])
if path, ok := trie.ParseExpiredNodeErr(err); ok {
touchExpiredStorage[key] = value
if _, err = tryReviveState(s.db.expiryMeta, s.address, s.data.Root, tr, path, key, true); err != nil {
s.db.setError(fmt.Errorf("updateTrie DeleteStorage tryReviveState err, contract: %v, key: %v, path: %v, err: %v", s.address, key, path, err))
}
} else if err != nil {
s.db.setError(fmt.Errorf("updateTrie DeleteStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
}
//log.Debug("updateTrie DeleteStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
//log.Debug("updateTrie DeleteStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "err", err, "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
s.db.StorageDeleted += 1
} else {
if err := tr.UpdateStorage(s.address, key[:], value); err != nil {
s.db.setError(fmt.Errorf("state object update trie UpdateStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
s.db.setError(fmt.Errorf("updateTrie UpdateStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
}
//log.Debug("updateTrie UpdateStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
//log.Debug("updateTrie UpdateStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "err", err, "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
s.db.StorageUpdated += 1
}
// Cache the items for preloading
usedStorage = append(usedStorage, common.CopyBytes(key[:]))
}

// re-execute touched expired storage
for key, value := range touchExpiredStorage {
if len(value) == 0 {
if err := tr.DeleteStorage(s.address, key[:]); err != nil {
s.db.setError(fmt.Errorf("updateTrie DeleteStorage in touchExpiredStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
}
//log.Debug("updateTrie DeleteStorage in touchExpiredStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "err", err, "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
} else {
if err := tr.UpdateStorage(s.address, key[:], value); err != nil {
s.db.setError(fmt.Errorf("updateTrie UpdateStorage in touchExpiredStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
}
//log.Debug("updateTrie UpdateStorage in touchExpiredStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "err", err, "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
}
}
}()
// If state snapshotting is active, cache the data til commit
wg.Add(1)
Expand Down Expand Up @@ -873,8 +896,8 @@ func (s *stateObject) queryFromReviveState(reviveState map[string]common.Hash, k
return val, ok
}

// fetchExpiredStorageFromRemote request expired state from remote full state node;
func (s *stateObject) fetchExpiredFromRemote(prefixKey []byte, key common.Hash, resolvePath bool) ([]byte, error) {
// tryReviveState request expired state from remote full state node;
func (s *stateObject) tryReviveState(prefixKey []byte, key common.Hash, resolvePath bool) ([]byte, error) {
tr, err := s.getPendingReviveTrie()
if err != nil {
return nil, err
Expand All @@ -883,19 +906,19 @@ func (s *stateObject) fetchExpiredFromRemote(prefixKey []byte, key common.Hash,
// if no prefix, query from revive trie, got the newest expired info
if resolvePath {
val, err := tr.GetStorage(s.address, key.Bytes())
// TODO(asyukii): temporary fix snap expired, but trie not expire, may investigate more later.
if val != nil {
if err == nil {
// TODO(asyukii): temporary fix snap expired, but trie not expire, may investigate more later.
s.pendingReviveState[string(crypto.Keccak256(key[:]))] = common.BytesToHash(val)
return val, nil
}
enErr, ok := err.(*trie.ExpiredNodeError)
path, ok := trie.ParseExpiredNodeErr(err)
if !ok {
return nil, fmt.Errorf("cannot find expired state from trie, err: %v", err)
}
prefixKey = enErr.Path
prefixKey = path
}

kvs, err := fetchExpiredStorageFromRemote(s.db.expiryMeta, s.address, s.data.Root, tr, prefixKey, key)
kvs, err := tryReviveState(s.db.expiryMeta, s.address, s.data.Root, tr, prefixKey, key, false)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -924,6 +947,7 @@ func (s *stateObject) getExpirySnapStorage(key common.Hash) ([]byte, error, erro

if val == nil {
// record access empty kv, try touch in updateTrie for duplication
//log.Debug("getExpirySnapStorage nil val", "addr", s.address, "key", key, "val", val)
s.futureReviveState(key)
return nil, nil, nil
}
Expand All @@ -945,7 +969,7 @@ func (s *stateObject) getExpirySnapStorage(key common.Hash) ([]byte, error, erro

//log.Debug("GetCommittedState expired in snapshot", "addr", s.address, "key", key, "val", val, "enc", enc, "err", err)
// handle from remoteDB, if got err just setError, or return to revert in consensus version.
valRaw, err := s.fetchExpiredFromRemote(nil, key, true)
valRaw, err := s.tryReviveState(nil, key, true)
if err != nil {
return nil, nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions core/state/trie_prefetcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -576,11 +576,11 @@ func (sf *subfetcher) loop() {
// handle expired state
if sf.expiryMeta.enableStateExpiry {
// TODO(0xbundler): revert to single fetch, because tasks is a channel
if exErr, match := err.(*trie2.ExpiredNodeError); match {
if path, match := trie2.ParseExpiredNodeErr(err); match {
key := common.BytesToHash(task)
_, err = fetchExpiredStorageFromRemote(sf.expiryMeta, sf.addr, sf.root, sf.trie, exErr.Path, key)
_, err = tryReviveState(sf.expiryMeta, sf.addr, sf.root, sf.trie, path, key, false)
if err != nil {
log.Error("subfetcher fetchExpiredStorageFromRemote err", "addr", sf.addr, "path", exErr.Path, "err", err)
log.Error("subfetcher tryReviveState err", "addr", sf.addr, "path", path, "err", err)
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions eth/protocols/snap/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,8 +445,8 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
}
proof := light.NewNodeSet()
if err := stTrie.Prove(origin[:], proof); err != nil {
if enErr, ok := err.(*trie.ExpiredNodeError); ok {
err := reviveAndGetProof(chain.FullStateDB(), stTrie, req.Root, common.BytesToAddress(account[:]), acc.Root, enErr.Path, origin, proof)
if path, ok := trie.ParseExpiredNodeErr(err); ok {
err := reviveAndGetProof(chain.FullStateDB(), stTrie, req.Root, common.BytesToAddress(account[:]), acc.Root, path, origin, proof)
if err != nil {
log.Warn("Failed to prove storage range", "origin", origin, "err", err)
return nil, nil
Expand All @@ -455,8 +455,8 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
}
if last != (common.Hash{}) {
if err := stTrie.Prove(last[:], proof); err != nil {
if enErr, ok := err.(*trie.ExpiredNodeError); ok {
err := reviveAndGetProof(chain.FullStateDB(), stTrie, req.Root, common.BytesToAddress(account[:]), acc.Root, enErr.Path, last, proof)
if path, ok := trie.ParseExpiredNodeErr(err); ok {
err := reviveAndGetProof(chain.FullStateDB(), stTrie, req.Root, common.BytesToAddress(account[:]), acc.Root, path, last, proof)
if err != nil {
log.Warn("Failed to prove storage range", "origin", origin, "err", err)
return nil, nil
Expand Down
14 changes: 14 additions & 0 deletions trie/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,17 @@ func NewExpiredNodeError(path []byte, epoch types.StateEpoch, n node) error {
func (err *ExpiredNodeError) Error() string {
return fmt.Sprintf("expired trie node, path: %v, epoch: %v, node: %v", err.Path, err.Epoch, err.Node.fstring(""))
}

func ParseExpiredNodeErr(err error) ([]byte, bool) {
var path []byte
switch enErr := err.(type) {
case *ExpiredNodeError:
path = enErr.Path
case *MissingNodeError: // when meet MissingNodeError, try revive or fail
path = enErr.Path
default:
return nil, false
}

return path, true
}
Loading

0 comments on commit 9825ce3

Please sign in to comment.