Add missing functions from db package.

db/base_db.go

@@ -36,7 +36,7 @@ type BaseDB interface {
 	// until a final write is called.
 	NewBatch() Batch
 
-	// newIterator creates a binary-alphabetical iterator over a subset
+	// NewIterator creates a binary-alphabetical iterator over a subset
 	// of database content with a particular key prefix, starting at a particular
 	// initial key (or after, if it does not exist).
 	//
@@ -64,6 +64,9 @@ type BaseDB interface {
 	// It is valid to call Close multiple times.
 	// Other methods should not be called after the DB has been closed.
 	Close() error
+
+	// getBackend returns the database backend.
+	getBackend() *leveldb.DB
 }
 
 // NewDefaultBaseDB creates new instance of BaseDB with default options.
@@ -77,6 +80,10 @@ func NewBaseDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOp
 	return newBaseDB(path, o, wo, ro)
 }
 
+func MakeDefaultBaseDBFromBaseDB(db BaseDB) BaseDB {
+	return &baseDB{backend: db.getBackend()}
+}
+
 func newBaseDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*baseDB, error) {
 	b, err := leveldb.OpenFile(path, o)
 	if err != nil {
@@ -96,6 +103,10 @@ type baseDB struct {
 	ro      *opt.ReadOptions
 }
 
+func (db *baseDB) getBackend() *leveldb.DB {
+	return db.backend
+}
+
 func (db *baseDB) Put(key []byte, value []byte) error {
 	return db.backend.Put(key, value, db.wo)
 }
@@ -141,3 +152,43 @@ func (db *baseDB) Stat(property string) (string, error) {
 func (db *baseDB) Compact(start []byte, limit []byte) error {
 	return db.backend.CompactRange(util.Range{Start: start, Limit: limit})
 }
+
+func (db *baseDB) hasKeyValuesFor(prefix []byte, start []byte) bool {
+	iter := db.NewIterator(prefix, start)
+	defer iter.Release()
+	return iter.Next()
+}
+
+func (db *baseDB) binarySearchForLastPrefixKey(lastKeyPrefix []byte) (byte, error) {
+	var min uint16 = 0
+	var max uint16 = 255
+
+	startIndex := make([]byte, 1)
+
+	for max-min > 1 {
+		searchHalf := (max + min) / 2
+		startIndex[0] = byte(searchHalf)
+		if db.hasKeyValuesFor(lastKeyPrefix, startIndex) {
+			min = searchHalf
+		} else {
+			max = searchHalf
+		}
+	}
+
+	// shouldn't occure
+	if max-min == 0 {
+		return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search; max - min == 0")
+	}
+
+	startIndex[0] = byte(min)
+	if db.hasKeyValuesFor(lastKeyPrefix, startIndex) {
+		startIndex[0] = byte(max)
+		if db.hasKeyValuesFor(lastKeyPrefix, startIndex) {
+			return byte(max), nil
+		} else {
+			return byte(min), nil
+		}
+	} else {
+		return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search")
+	}
+}

db/code_db.go

@@ -40,6 +40,10 @@ func NewCodeDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOp
 	return newCodeDB(path, o, wo, ro)
 }
 
+func MakeDefaultCodeDBFromBaseDB(db BaseDB) CodeDB {
+	return &codeDB{&baseDB{backend: db.getBackend()}}
+}
+
 func newCodeDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*codeDB, error) {
 	base, err := newBaseDB(path, o, wo, ro)
 	if err != nil {

db/destroyed_account_db.go

@@ -27,6 +27,10 @@ func OpenDestroyedAccountDBReadOnly(destroyedAccountDir string) (*DestroyedAccou
 	return openDestroyedAccountDB(destroyedAccountDir, &opt.Options{ReadOnly: true}, nil, nil)
 }
 
+func MakeDestroyedAccountDBFromBaseDB(db BaseDB) *DestroyedAccountDB {
+	return &DestroyedAccountDB{db}
+}
+
 func openDestroyedAccountDB(destroyedAccountDir string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*DestroyedAccountDB, error) {
 	log.Println("substate: OpenDestroyedAccountDB")
 	backend, err := newBaseDB(destroyedAccountDir, o, wo, ro)
@@ -59,6 +63,9 @@ func (db *DestroyedAccountDB) GetDestroyedAccounts(block uint64, tx int) ([]type
 	if err != nil {
 		return nil, nil, err
 	}
+	if data == nil {
+		return nil, nil, nil
+	}
 	list, err := DecodeAddressList(data)
 	return list.DestroyedAccounts, list.ResurrectedAccounts, err
 }

db/substate_db.go

@@ -10,6 +10,7 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
+	"github.com/urfave/cli/v2"
 )
 
 const SubstateDBPrefix = "1s" // SubstateDBPrefix + block (64-bit) + tx (64-bit) -> substateRLP
@@ -34,6 +35,14 @@ type SubstateDB interface {
 	DeleteSubstate(block uint64, tx int) error
 
 	NewSubstateIterator(start int, numWorkers int) Iterator[*substate.Substate]
+
+	NewSubstateTaskPool(name string, taskFunc SubstateTaskFunc, first, last uint64, ctx *cli.Context) *SubstateTaskPool
+
+	// GetFirstSubstate returns last substate (block and transaction wise) inside given DB.
+	GetFirstSubstate() *substate.Substate
+
+	// GetLastSubstate returns last substate (block and transaction wise) inside given DB.
+	GetLastSubstate() (*substate.Substate, error)
 }
 
 // NewDefaultSubstateDB creates new instance of SubstateDB with default options.
@@ -47,10 +56,14 @@ func NewSubstateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.Re
 	return newSubstateDB(path, o, wo, ro)
 }
 
-func MakeDefaultSubstateDb(db *leveldb.DB) SubstateDB {
+func MakeDefaultSubstateDB(db *leveldb.DB) SubstateDB {
 	return &substateDB{&codeDB{&baseDB{backend: db}}}
 }
 
+func MakeDefaultSubstateDBFromBaseDB(db BaseDB) SubstateDB {
+	return &substateDB{&codeDB{&baseDB{backend: db.getBackend()}}}
+}
+
 func MakeSubstateDb(db *leveldb.DB, wo *opt.WriteOptions, ro *opt.ReadOptions) SubstateDB {
 	return &substateDB{&codeDB{&baseDB{backend: db, wo: wo, ro: ro}}}
 }
@@ -67,6 +80,18 @@ type substateDB struct {
 	*codeDB
 }
 
+func (db *substateDB) GetFirstSubstate() *substate.Substate {
+	iter := db.NewSubstateIterator(0, 1)
+
+	defer iter.Release()
+
+	if iter.Next() {
+		return iter.Value()
+	}
+
+	return nil
+}
+
 func (db *substateDB) HasSubstate(block uint64, tx int) (bool, error) {
 	return db.Has(SubstateDBKey(block, tx))
 }
@@ -182,6 +207,98 @@ func (db *substateDB) NewSubstateIterator(start int, numWorkers int) Iterator[*s
 	return iter
 }
 
+func (db *substateDB) NewSubstateTaskPool(name string, taskFunc SubstateTaskFunc, first, last uint64, ctx *cli.Context) *SubstateTaskPool {
+	return &SubstateTaskPool{
+		Name:     name,
+		TaskFunc: taskFunc,
+
+		First: first,
+		Last:  last,
+
+		Workers:         ctx.Int(WorkersFlag.Name),
+		SkipTransferTxs: ctx.Bool(SkipTransferTxsFlag.Name),
+		SkipCallTxs:     ctx.Bool(SkipCallTxsFlag.Name),
+		SkipCreateTxs:   ctx.Bool(SkipCreateTxsFlag.Name),
+
+		Ctx: ctx,
+
+		DB: db,
+	}
+}
+
+// getLongestEncodedKeyZeroPrefixLength returns longest index of biggest block number to be search for in its search
+func (db *substateDB) getLongestEncodedKeyZeroPrefixLength() (byte, error) {
+	var i byte
+	for i = 0; i < 8; i++ {
+		startingIndex := make([]byte, 8)
+		startingIndex[i] = 1
+		if db.hasKeyValuesFor([]byte(SubstateDBPrefix), startingIndex) {
+			return i, nil
+		}
+	}
+
+	return 0, fmt.Errorf("unable to find prefix of substate with biggest block")
+}
+
+// getLastBlock returns block number of last substate
+func (db *substateDB) getLastBlock() (uint64, error) {
+	zeroBytes, err := db.getLongestEncodedKeyZeroPrefixLength()
+	if err != nil {
+		return 0, err
+	}
+
+	var lastKeyPrefix []byte
+	if zeroBytes > 0 {
+		blockBytes := make([]byte, zeroBytes)
+
+		lastKeyPrefix = append([]byte(SubstateDBPrefix), blockBytes...)
+	} else {
+		lastKeyPrefix = []byte(SubstateDBPrefix)
+	}
+
+	substatePrefixSize := len([]byte(SubstateDBPrefix))
+
+	// binary search for biggest key
+	for {
+		nextBiggestPrefixValue, err := db.binarySearchForLastPrefixKey(lastKeyPrefix)
+		if err != nil {
+			return 0, err
+		}
+		lastKeyPrefix = append(lastKeyPrefix, nextBiggestPrefixValue)
+		// we have all 8 bytes of uint64 encoded block
+		if len(lastKeyPrefix) == (substatePrefixSize + 8) {
+			// full key is already found
+			substateBlockValue := lastKeyPrefix[substatePrefixSize:]
+
+			if len(substateBlockValue) != 8 {
+				return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search; retrieved block bytes can't be converted")
+			}
+			return binary.BigEndian.Uint64(substateBlockValue), nil
+		}
+	}
+}
+
+func (db *substateDB) GetLastSubstate() (*substate.Substate, error) {
+	block, err := db.getLastBlock()
+	if err != nil {
+		return nil, err
+	}
+	substates, err := db.GetBlockSubstates(block)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get block substates; %w", err)
+	}
+	if len(substates) == 0 {
+		return nil, fmt.Errorf("block %v doesn't have any substates", block)
+	}
+	maxTx := 0
+	for txIdx := range substates {
+		if txIdx > maxTx {
+			maxTx = txIdx
+		}
+	}
+	return substates[maxTx], nil
+}
+
 // BlockToBytes returns binary BigEndian representation of given block number.
 func BlockToBytes(block uint64) []byte {
 	blockBytes := make([]byte, 8)