From 55aa025aaf67cddbe2ec4bf453e68aa8900ffea9 Mon Sep 17 00:00:00 2001 From: Petr Hanzl Date: Tue, 9 Apr 2024 12:49:34 +0200 Subject: [PATCH 1/6] Add missing functions from db package. --- db/base_db.go | 108 ++++++++++++++++++++++++++++++++++++- db/code_db.go | 4 ++ db/destroyed_account_db.go | 7 +++ db/substate_db.go | 67 ++++++++++++++++++++++- db/substate_task.go | 20 +------ db/update_db.go | 6 +++ 6 files changed, 191 insertions(+), 21 deletions(-) diff --git a/db/base_db.go b/db/base_db.go index 23b7ebd..0af7041 100644 --- a/db/base_db.go +++ b/db/base_db.go @@ -1,6 +1,7 @@ package db import ( + "encoding/binary" "errors" "fmt" "io" @@ -36,7 +37,7 @@ type BaseDB interface { // until a final write is called. NewBatch() Batch - // newIterator creates a binary-alphabetical iterator over a subset + // NewIterator creates a binary-alphabetical iterator over a subset // of database content with a particular key prefix, starting at a particular // initial key (or after, if it does not exist). // @@ -64,6 +65,12 @@ type BaseDB interface { // It is valid to call Close multiple times. // Other methods should not be called after the DB has been closed. Close() error + + // GetLastBlock returns last block of given Database. + GetLastBlock() (uint64, error) + + // getBackend returns the database backend. + getBackend() *leveldb.DB } // NewDefaultBaseDB creates new instance of BaseDB with default options. @@ -77,6 +84,10 @@ func NewBaseDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOp return newBaseDB(path, o, wo, ro) } +func MakeDefaultBaseDBFromBaseDB(db BaseDB) BaseDB { + return &baseDB{backend: db.getBackend()} +} + func newBaseDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*baseDB, error) { b, err := leveldb.OpenFile(path, o) if err != nil { @@ -96,6 +107,10 @@ type baseDB struct { ro *opt.ReadOptions } +func (db *baseDB) getBackend() *leveldb.DB { + return db.backend +} + func (db *baseDB) Put(key []byte, value []byte) error { return db.backend.Put(key, value, db.wo) } @@ -141,3 +156,94 @@ func (db *baseDB) Stat(property string) (string, error) { func (db *baseDB) Compact(start []byte, limit []byte) error { return db.backend.CompactRange(util.Range{Start: start, Limit: limit}) } + +func (db *baseDB) GetLastBlock() (uint64, error) { + zeroBytes, err := db.getLongestEncodedKeyZeroPrefixLength() + if err != nil { + return 0, err + } + + var lastKeyPrefix []byte + if zeroBytes > 0 { + blockBytes := make([]byte, zeroBytes) + + lastKeyPrefix = append([]byte(SubstateDBPrefix), blockBytes...) + } else { + lastKeyPrefix = []byte(SubstateDBPrefix) + } + + substatePrefixSize := len([]byte(SubstateDBPrefix)) + + // binary search for biggest key + for { + nextBiggestPrefixValue, err := db.binarySearchForLastPrefixKey(lastKeyPrefix) + if err != nil { + return 0, err + } + lastKeyPrefix = append(lastKeyPrefix, nextBiggestPrefixValue) + // we have all 8 bytes of uint64 encoded block + if len(lastKeyPrefix) == (substatePrefixSize + 8) { + // full key is already found + substateBlockValue := lastKeyPrefix[substatePrefixSize:] + + if len(substateBlockValue) != 8 { + return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search; retrieved block bytes can't be converted") + } + return binary.BigEndian.Uint64(substateBlockValue), nil + } + } +} + +func (db *baseDB) hasKeyValuesFor(prefix []byte, start []byte) bool { + iter := db.NewIterator(prefix, start) + defer iter.Release() + return iter.Next() +} + +// getLongestEncodedValue returns longest index of biggest block number to be search for in its search +func (db *baseDB) getLongestEncodedKeyZeroPrefixLength() (byte, error) { + var i byte + for i = 0; i < 8; i++ { + startingIndex := make([]byte, 8) + startingIndex[i] = 1 + if db.hasKeyValuesFor([]byte(SubstateDBPrefix), startingIndex) { + return i, nil + } + } + + return 0, fmt.Errorf("unable to find prefix of substate with biggest block") +} + +func (db *baseDB) binarySearchForLastPrefixKey(lastKeyPrefix []byte) (byte, error) { + var min uint16 = 0 + var max uint16 = 255 + + startIndex := make([]byte, 1) + + for max-min > 1 { + searchHalf := (max + min) / 2 + startIndex[0] = byte(searchHalf) + if db.hasKeyValuesFor(lastKeyPrefix, startIndex) { + min = searchHalf + } else { + max = searchHalf + } + } + + // shouldn't occure + if max-min == 0 { + return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search; max - min == 0") + } + + startIndex[0] = byte(min) + if db.hasKeyValuesFor(lastKeyPrefix, startIndex) { + startIndex[0] = byte(max) + if db.hasKeyValuesFor(lastKeyPrefix, startIndex) { + return byte(max), nil + } else { + return byte(min), nil + } + } else { + return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search") + } +} diff --git a/db/code_db.go b/db/code_db.go index 262189f..3a0a79d 100644 --- a/db/code_db.go +++ b/db/code_db.go @@ -40,6 +40,10 @@ func NewCodeDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOp return newCodeDB(path, o, wo, ro) } +func MakeDefaultCodeDBFromBaseDB(db BaseDB) CodeDB { + return &codeDB{&baseDB{backend: db.getBackend()}} +} + func newCodeDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*codeDB, error) { base, err := newBaseDB(path, o, wo, ro) if err != nil { diff --git a/db/destroyed_account_db.go b/db/destroyed_account_db.go index f8a4752..ccf0789 100644 --- a/db/destroyed_account_db.go +++ b/db/destroyed_account_db.go @@ -27,6 +27,10 @@ func OpenDestroyedAccountDBReadOnly(destroyedAccountDir string) (*DestroyedAccou return openDestroyedAccountDB(destroyedAccountDir, &opt.Options{ReadOnly: true}, nil, nil) } +func MakeDestroyedAccountDBFromBaseDB(db BaseDB) *DestroyedAccountDB { + return &DestroyedAccountDB{db} +} + func openDestroyedAccountDB(destroyedAccountDir string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*DestroyedAccountDB, error) { log.Println("substate: OpenDestroyedAccountDB") backend, err := newBaseDB(destroyedAccountDir, o, wo, ro) @@ -59,6 +63,9 @@ func (db *DestroyedAccountDB) GetDestroyedAccounts(block uint64, tx int) ([]type if err != nil { return nil, nil, err } + if data == nil { + return nil, nil, nil + } list, err := DecodeAddressList(data) return list.DestroyedAccounts, list.ResurrectedAccounts, err } diff --git a/db/substate_db.go b/db/substate_db.go index 3784b80..d7efd5d 100644 --- a/db/substate_db.go +++ b/db/substate_db.go @@ -10,6 +10,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" + "github.com/urfave/cli/v2" ) const SubstateDBPrefix = "1s" // SubstateDBPrefix + block (64-bit) + tx (64-bit) -> substateRLP @@ -34,6 +35,14 @@ type SubstateDB interface { DeleteSubstate(block uint64, tx int) error NewSubstateIterator(start int, numWorkers int) Iterator[*substate.Substate] + + NewSubstateTaskPool(name string, taskFunc SubstateTaskFunc, first, last uint64, ctx *cli.Context) *SubstateTaskPool + + // GetFirstSubstate returns last substate (block and transaction wise) inside given DB. + GetFirstSubstate() *substate.Substate + + // GetLastSubstate returns last substate (block and transaction wise) inside given DB. + GetLastSubstate() (*substate.Substate, error) } // NewDefaultSubstateDB creates new instance of SubstateDB with default options. @@ -47,10 +56,14 @@ func NewSubstateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.Re return newSubstateDB(path, o, wo, ro) } -func MakeDefaultSubstateDb(db *leveldb.DB) SubstateDB { +func MakeDefaultSubstateDB(db *leveldb.DB) SubstateDB { return &substateDB{&codeDB{&baseDB{backend: db}}} } +func MakeDefaultSubstateDBFromBaseDB(db BaseDB) SubstateDB { + return &substateDB{&codeDB{&baseDB{backend: db.getBackend()}}} +} + func MakeSubstateDb(db *leveldb.DB, wo *opt.WriteOptions, ro *opt.ReadOptions) SubstateDB { return &substateDB{&codeDB{&baseDB{backend: db, wo: wo, ro: ro}}} } @@ -67,6 +80,18 @@ type substateDB struct { *codeDB } +func (db *substateDB) GetFirstSubstate() *substate.Substate { + iter := db.NewSubstateIterator(0, 1) + + defer iter.Release() + + if iter.Next() { + return iter.Value() + } + + return nil +} + func (db *substateDB) HasSubstate(block uint64, tx int) (bool, error) { return db.Has(SubstateDBKey(block, tx)) } @@ -182,6 +207,46 @@ func (db *substateDB) NewSubstateIterator(start int, numWorkers int) Iterator[*s return iter } +func (db *substateDB) NewSubstateTaskPool(name string, taskFunc SubstateTaskFunc, first, last uint64, ctx *cli.Context) *SubstateTaskPool { + return &SubstateTaskPool{ + Name: name, + TaskFunc: taskFunc, + + First: first, + Last: last, + + Workers: ctx.Int(WorkersFlag.Name), + SkipTransferTxs: ctx.Bool(SkipTransferTxsFlag.Name), + SkipCallTxs: ctx.Bool(SkipCallTxsFlag.Name), + SkipCreateTxs: ctx.Bool(SkipCreateTxsFlag.Name), + + Ctx: ctx, + + DB: db, + } +} + +func (db *substateDB) GetLastSubstate() (*substate.Substate, error) { + block, err := db.GetLastBlock() + if err != nil { + return nil, err + } + substates, err := db.GetBlockSubstates(block) + if err != nil { + return nil, fmt.Errorf("cannot get block substates; %w", err) + } + if len(substates) == 0 { + return nil, fmt.Errorf("block %v doesn't have any substates.", block) + } + maxTx := 0 + for txIdx, _ := range substates { + if txIdx > maxTx { + maxTx = txIdx + } + } + return substates[maxTx], nil +} + // BlockToBytes returns binary BigEndian representation of given block number. func BlockToBytes(block uint64) []byte { blockBytes := make([]byte, 8) diff --git a/db/substate_task.go b/db/substate_task.go index f202971..2589cc0 100644 --- a/db/substate_task.go +++ b/db/substate_task.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Fantom-foundation/Substate/substate" + "github.com/urfave/cli/v2" ) @@ -53,25 +54,6 @@ type SubstateTaskPool struct { DB SubstateDB } -func NewSubstateTaskPool(name string, taskFunc SubstateTaskFunc, first, last uint64, ctx *cli.Context, database SubstateDB) *SubstateTaskPool { - return &SubstateTaskPool{ - Name: name, - TaskFunc: taskFunc, - - First: first, - Last: last, - - Workers: ctx.Int(WorkersFlag.Name), - SkipTransferTxs: ctx.Bool(SkipTransferTxsFlag.Name), - SkipCallTxs: ctx.Bool(SkipCallTxsFlag.Name), - SkipCreateTxs: ctx.Bool(SkipCreateTxsFlag.Name), - - Ctx: ctx, - - DB: database, - } -} - // ExecuteBlock function iterates on substates of a given block call TaskFunc func (pool *SubstateTaskPool) ExecuteBlock(block uint64) (numTx int64, gas int64, err error) { transactions, err := pool.DB.GetBlockSubstates(block) diff --git a/db/update_db.go b/db/update_db.go index 652c727..30f5bfb 100644 --- a/db/update_db.go +++ b/db/update_db.go @@ -40,6 +40,8 @@ type UpdateDB interface { DeleteUpdateSet(block uint64) error NewUpdateSetIterator(start, end uint64) Iterator[*updateset.UpdateSet] + + PutMetadata(interval, size uint64) error } // NewDefaultUpdateDB creates new instance of UpdateDB with default options. @@ -53,6 +55,10 @@ func NewUpdateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.Read return newUpdateDB(path, o, wo, ro) } +func MakeDefaultUpdateDBFromBaseDB(db BaseDB) UpdateDB { + return &updateDB{&codeDB{&baseDB{backend: db.getBackend()}}} +} + func newUpdateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*updateDB, error) { base, err := newCodeDB(path, o, wo, ro) if err != nil { From 98bf04e31e85ae7b098affbe5d9f60eee89ec744 Mon Sep 17 00:00:00 2001 From: Petr Hanzl Date: Tue, 9 Apr 2024 15:47:36 +0200 Subject: [PATCH 2/6] Fix formatting --- db/substate_db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/substate_db.go b/db/substate_db.go index d7efd5d..31ffc99 100644 --- a/db/substate_db.go +++ b/db/substate_db.go @@ -236,7 +236,7 @@ func (db *substateDB) GetLastSubstate() (*substate.Substate, error) { return nil, fmt.Errorf("cannot get block substates; %w", err) } if len(substates) == 0 { - return nil, fmt.Errorf("block %v doesn't have any substates.", block) + return nil, fmt.Errorf("block %v doesn't have any substates", block) } maxTx := 0 for txIdx, _ := range substates { From 5f9d016bd429ec7eff8ec577f587ecf709e49a53 Mon Sep 17 00:00:00 2001 From: Petr Hanzl Date: Tue, 9 Apr 2024 16:13:50 +0200 Subject: [PATCH 3/6] Move getLastBlock to substateDB. Add unit tests. Sort imports. --- db/base_db.go | 55 ---------------------- db/substate_db.go | 54 ++++++++++++++++++++- db/substate_db_test.go | 104 +++++++++++++++++++++++++++++++++++------ db/substate_task.go | 1 - 4 files changed, 143 insertions(+), 71 deletions(-) diff --git a/db/base_db.go b/db/base_db.go index 0af7041..bcb6304 100644 --- a/db/base_db.go +++ b/db/base_db.go @@ -1,7 +1,6 @@ package db import ( - "encoding/binary" "errors" "fmt" "io" @@ -66,9 +65,6 @@ type BaseDB interface { // Other methods should not be called after the DB has been closed. Close() error - // GetLastBlock returns last block of given Database. - GetLastBlock() (uint64, error) - // getBackend returns the database backend. getBackend() *leveldb.DB } @@ -157,63 +153,12 @@ func (db *baseDB) Compact(start []byte, limit []byte) error { return db.backend.CompactRange(util.Range{Start: start, Limit: limit}) } -func (db *baseDB) GetLastBlock() (uint64, error) { - zeroBytes, err := db.getLongestEncodedKeyZeroPrefixLength() - if err != nil { - return 0, err - } - - var lastKeyPrefix []byte - if zeroBytes > 0 { - blockBytes := make([]byte, zeroBytes) - - lastKeyPrefix = append([]byte(SubstateDBPrefix), blockBytes...) - } else { - lastKeyPrefix = []byte(SubstateDBPrefix) - } - - substatePrefixSize := len([]byte(SubstateDBPrefix)) - - // binary search for biggest key - for { - nextBiggestPrefixValue, err := db.binarySearchForLastPrefixKey(lastKeyPrefix) - if err != nil { - return 0, err - } - lastKeyPrefix = append(lastKeyPrefix, nextBiggestPrefixValue) - // we have all 8 bytes of uint64 encoded block - if len(lastKeyPrefix) == (substatePrefixSize + 8) { - // full key is already found - substateBlockValue := lastKeyPrefix[substatePrefixSize:] - - if len(substateBlockValue) != 8 { - return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search; retrieved block bytes can't be converted") - } - return binary.BigEndian.Uint64(substateBlockValue), nil - } - } -} - func (db *baseDB) hasKeyValuesFor(prefix []byte, start []byte) bool { iter := db.NewIterator(prefix, start) defer iter.Release() return iter.Next() } -// getLongestEncodedValue returns longest index of biggest block number to be search for in its search -func (db *baseDB) getLongestEncodedKeyZeroPrefixLength() (byte, error) { - var i byte - for i = 0; i < 8; i++ { - startingIndex := make([]byte, 8) - startingIndex[i] = 1 - if db.hasKeyValuesFor([]byte(SubstateDBPrefix), startingIndex) { - return i, nil - } - } - - return 0, fmt.Errorf("unable to find prefix of substate with biggest block") -} - func (db *baseDB) binarySearchForLastPrefixKey(lastKeyPrefix []byte) (byte, error) { var min uint16 = 0 var max uint16 = 255 diff --git a/db/substate_db.go b/db/substate_db.go index 31ffc99..aa18962 100644 --- a/db/substate_db.go +++ b/db/substate_db.go @@ -226,8 +226,60 @@ func (db *substateDB) NewSubstateTaskPool(name string, taskFunc SubstateTaskFunc } } +// getLongestEncodedKeyZeroPrefixLength returns longest index of biggest block number to be search for in its search +func (db *substateDB) getLongestEncodedKeyZeroPrefixLength() (byte, error) { + var i byte + for i = 0; i < 8; i++ { + startingIndex := make([]byte, 8) + startingIndex[i] = 1 + if db.hasKeyValuesFor([]byte(SubstateDBPrefix), startingIndex) { + return i, nil + } + } + + return 0, fmt.Errorf("unable to find prefix of substate with biggest block") +} + +// getLastBlock returns block number of last substate +func (db *substateDB) getLastBlock() (uint64, error) { + zeroBytes, err := db.getLongestEncodedKeyZeroPrefixLength() + if err != nil { + return 0, err + } + + var lastKeyPrefix []byte + if zeroBytes > 0 { + blockBytes := make([]byte, zeroBytes) + + lastKeyPrefix = append([]byte(SubstateDBPrefix), blockBytes...) + } else { + lastKeyPrefix = []byte(SubstateDBPrefix) + } + + substatePrefixSize := len([]byte(SubstateDBPrefix)) + + // binary search for biggest key + for { + nextBiggestPrefixValue, err := db.binarySearchForLastPrefixKey(lastKeyPrefix) + if err != nil { + return 0, err + } + lastKeyPrefix = append(lastKeyPrefix, nextBiggestPrefixValue) + // we have all 8 bytes of uint64 encoded block + if len(lastKeyPrefix) == (substatePrefixSize + 8) { + // full key is already found + substateBlockValue := lastKeyPrefix[substatePrefixSize:] + + if len(substateBlockValue) != 8 { + return 0, fmt.Errorf("undefined behaviour in GetLastSubstate search; retrieved block bytes can't be converted") + } + return binary.BigEndian.Uint64(substateBlockValue), nil + } + } +} + func (db *substateDB) GetLastSubstate() (*substate.Substate, error) { - block, err := db.GetLastBlock() + block, err := db.getLastBlock() if err != nil { return nil, err } diff --git a/db/substate_db_test.go b/db/substate_db_test.go index fd610f2..1d56e2c 100644 --- a/db/substate_db_test.go +++ b/db/substate_db_test.go @@ -15,13 +15,12 @@ var testSubstate = &substate.Substate{ InputSubstate: substate.NewWorldState(), OutputSubstate: substate.NewWorldState(), Env: &substate.Env{ - Coinbase: types.Address{1}, - Difficulty: new(big.Int).SetUint64(1), - GasLimit: 1, - Number: 1, - Timestamp: 1, - BlockHashes: make(map[uint64]types.Hash), - BaseFee: new(big.Int).SetUint64(1), + Coinbase: types.Address{1}, + Difficulty: new(big.Int).SetUint64(1), + GasLimit: 1, + Number: 1, + Timestamp: 1, + BaseFee: new(big.Int).SetUint64(1), }, Message: substate.NewMessage(1, true, new(big.Int).SetUint64(1), 1, types.Address{1}, new(types.Address), new(big.Int).SetUint64(1), []byte{1}, nil, types.AccessList{}, new(big.Int).SetUint64(1), new(big.Int).SetUint64(1)), Result: substate.NewResult(1, []byte{}, []*types.Log{}, types.Address{1}, 1), @@ -108,12 +107,94 @@ func TestSubstateDB_DeleteSubstate(t *testing.T) { } } +func TestSubstateDB_getLastBlock(t *testing.T) { + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + // add one more substate + if err = addSubstate(db, 2); err != nil { + t.Fatal(err) + } + + block, err := db.getLastBlock() + if err != nil { + t.Fatal(err) + } + + if block != 2 { + t.Fatalf("incorrect block number\ngot: %v\nwant: %v", block, 2) + } + +} + +func TestSubstateDB_GetFirstSubstate(t *testing.T) { + // save data for comparison + want := *testSubstate + want.Block = 1 + + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + // add one more substate + if err = addSubstate(db, 2); err != nil { + t.Fatal(err) + } + + got := db.GetFirstSubstate() + + if err = got.Equal(&want); err != nil { + t.Fatalf("substates are different\nerr: %v\ngot: %s\nwant: %s", err, got, &want) + } + +} + +func TestSubstateDB_GetLastSubstate(t *testing.T) { + // save data for comparison + want := *testSubstate + want.Block = 2 + + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + // add one more substate + if err = addSubstate(db, 2); err != nil { + t.Fatal(err) + } + + got, err := db.GetLastSubstate() + if err != nil { + t.Fatal(err) + } + + if err = got.Equal(&want); err != nil { + t.Fatalf("substates are different\nerr: %v\ngot: %s\nwant: %s", err, got, &want) + } + +} + func createDbAndPutSubstate(dbPath string) (*substateDB, error) { db, err := newSubstateDB(dbPath, nil, nil, nil) if err != nil { return nil, fmt.Errorf("cannot open db; %v", err) } + if err = addSubstate(db, 1); err != nil { + return nil, err + } + + return db, nil +} + +func addSubstate(db *substateDB, blk uint64) error { h1 := types.Hash{} h1.SetBytes(nil) @@ -122,12 +203,7 @@ func createDbAndPutSubstate(dbPath string) (*substateDB, error) { testSubstate.InputSubstate[types.Address{1}] = substate.NewAccount(1, new(big.Int).SetUint64(1), h1[:]) testSubstate.OutputSubstate[types.Address{2}] = substate.NewAccount(2, new(big.Int).SetUint64(2), h2[:]) - testSubstate.Env.BlockHashes[1] = types.BytesToHash([]byte{1}) + testSubstate.Block = blk - err = db.PutSubstate(testSubstate) - if err != nil { - return nil, err - } - - return db, nil + return db.PutSubstate(testSubstate) } diff --git a/db/substate_task.go b/db/substate_task.go index 2589cc0..78b0690 100644 --- a/db/substate_task.go +++ b/db/substate_task.go @@ -9,7 +9,6 @@ import ( "time" "github.com/Fantom-foundation/Substate/substate" - "github.com/urfave/cli/v2" ) From f322ec938586ae93767b7d52b6e6de9396d2fea3 Mon Sep 17 00:00:00 2001 From: Petr Hanzl Date: Thu, 11 Apr 2024 09:14:27 +0200 Subject: [PATCH 4/6] Fix formatting --- db/substate_db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/substate_db.go b/db/substate_db.go index aa18962..2397839 100644 --- a/db/substate_db.go +++ b/db/substate_db.go @@ -291,7 +291,7 @@ func (db *substateDB) GetLastSubstate() (*substate.Substate, error) { return nil, fmt.Errorf("block %v doesn't have any substates", block) } maxTx := 0 - for txIdx, _ := range substates { + for txIdx := range substates { if txIdx > maxTx { maxTx = txIdx } From 4e98693974576c485104b07a064700adb70043fb Mon Sep 17 00:00:00 2001 From: Petr Hanzl Date: Thu, 11 Apr 2024 09:23:51 +0200 Subject: [PATCH 5/6] Copy test substate instead of rewritting its values. --- db/substate_db_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/db/substate_db_test.go b/db/substate_db_test.go index 1d56e2c..cdea81d 100644 --- a/db/substate_db_test.go +++ b/db/substate_db_test.go @@ -115,7 +115,7 @@ func TestSubstateDB_getLastBlock(t *testing.T) { } // add one more substate - if err = addSubstate(db, 2); err != nil { + if err = addSubstate(db, testSubstate.Block+1); err != nil { t.Fatal(err) } @@ -124,8 +124,8 @@ func TestSubstateDB_getLastBlock(t *testing.T) { t.Fatal(err) } - if block != 2 { - t.Fatalf("incorrect block number\ngot: %v\nwant: %v", block, 2) + if block != 37534835 { + t.Fatalf("incorrect block number\ngot: %v\nwant: %v", block, testSubstate.Block+1) } } @@ -187,7 +187,7 @@ func createDbAndPutSubstate(dbPath string) (*substateDB, error) { return nil, fmt.Errorf("cannot open db; %v", err) } - if err = addSubstate(db, 1); err != nil { + if err = addSubstate(db, testSubstate.Block); err != nil { return nil, err } @@ -201,9 +201,11 @@ func addSubstate(db *substateDB, blk uint64) error { h2 := types.Hash{} h2.SetBytes(nil) - testSubstate.InputSubstate[types.Address{1}] = substate.NewAccount(1, new(big.Int).SetUint64(1), h1[:]) - testSubstate.OutputSubstate[types.Address{2}] = substate.NewAccount(2, new(big.Int).SetUint64(2), h2[:]) - testSubstate.Block = blk + s := *testSubstate + + s.InputSubstate[types.Address{1}] = substate.NewAccount(1, new(big.Int).SetUint64(1), h1[:]) + s.OutputSubstate[types.Address{2}] = substate.NewAccount(2, new(big.Int).SetUint64(2), h2[:]) + s.Block = blk - return db.PutSubstate(testSubstate) + return db.PutSubstate(&s) } From 605eefcb8f6131d1b02c4ed41a2490338e0ac80b Mon Sep 17 00:00:00 2001 From: Eugene Shevchuk <70735938+evgensheff@users.noreply.github.com> Date: Fri, 12 Apr 2024 13:31:59 +0700 Subject: [PATCH 6/6] Unit tests for task pool (#70) --- db/substate_task.go | 6 +- db/substate_task_test.go | 175 +++++++++++++++++++++++++++++++++++++++ go.mod | 4 + go.sum | 8 ++ 4 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 db/substate_task_test.go diff --git a/db/substate_task.go b/db/substate_task.go index 78b0690..f7432b5 100644 --- a/db/substate_task.go +++ b/db/substate_task.go @@ -57,13 +57,13 @@ type SubstateTaskPool struct { func (pool *SubstateTaskPool) ExecuteBlock(block uint64) (numTx int64, gas int64, err error) { transactions, err := pool.DB.GetBlockSubstates(block) if err != nil { - return numTx, gas, err + return 0, 0, err } if pool.BlockFunc != nil { err := pool.BlockFunc(block, transactions, pool) if err != nil { - return numTx, gas, fmt.Errorf("%s: block %v: %v", pool.Name, block, err) + return 0, 0, fmt.Errorf("%s: block %v: %v", pool.Name, block, err) } } if pool.TaskFunc == nil { @@ -103,7 +103,7 @@ func (pool *SubstateTaskPool) ExecuteBlock(block uint64) (numTx int64, gas int64 } err = pool.TaskFunc(block, tx, substate, pool) if err != nil { - return numTx, gas, fmt.Errorf("%s: %v_%v: %v", pool.Name, block, tx, err) + return 0, 0, fmt.Errorf("%s: %v_%v: %v", pool.Name, block, tx, err) } numTx++ diff --git a/db/substate_task_test.go b/db/substate_task_test.go new file mode 100644 index 0000000..ded2c11 --- /dev/null +++ b/db/substate_task_test.go @@ -0,0 +1,175 @@ +package db + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Fantom-foundation/Substate/substate" +) + +func TestSubstateTaskPool_Execute(t *testing.T) { + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + // add one more substate + if err = addSubstate(db, testSubstate.Block+1); err != nil { + t.Fatal(err) + } + + stPool := SubstateTaskPool{ + Name: "test", + + TaskFunc: func(block uint64, tx int, substate *substate.Substate, taskPool *SubstateTaskPool) error { + return nil + }, + + First: testSubstate.Block, + Last: testSubstate.Block + 1, + + Workers: 1, + DB: db, + } + + err = stPool.Execute() + if err != nil { + t.Fatal(err) + } +} + +func TestSubstateTaskPool_ExecuteBlock(t *testing.T) { + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + stPool := SubstateTaskPool{ + Name: "test", + + TaskFunc: func(block uint64, tx int, substate *substate.Substate, taskPool *SubstateTaskPool) error { + return nil + }, + + First: testSubstate.Block, + Last: testSubstate.Block + 1, + + Workers: 1, + DB: db, + } + + numTx, gas, err := stPool.ExecuteBlock(testSubstate.Block) + require.Nil(t, err) + require.Equal(t, int64(1), numTx) + require.Equal(t, testSubstate.Message.Gas, uint64(gas)) +} + +func TestSubstateTaskPool_ExecuteBlock_TaskFuncErr(t *testing.T) { + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + stPool := SubstateTaskPool{ + Name: "test", + + TaskFunc: func(block uint64, tx int, substate *substate.Substate, taskPool *SubstateTaskPool) error { + return errors.New("test error") + }, + + First: testSubstate.Block, + Last: testSubstate.Block + 1, + + Workers: 1, + DB: db, + } + + _, _, err = stPool.ExecuteBlock(testSubstate.Block) + require.Error(t, err) +} + +func TestSubstateTaskPool_ExecuteBlockNilTaskFunc(t *testing.T) { + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + stPool := SubstateTaskPool{ + Name: "test", + + BlockFunc: func(block uint64, transactions map[int]*substate.Substate, taskPool *SubstateTaskPool) error { + return nil + }, + + First: testSubstate.Block, + Last: testSubstate.Block + 1, + + Workers: 1, + DB: db, + } + + numTx, gas, err := stPool.ExecuteBlock(testSubstate.Block) + require.Nil(t, err) + require.Equal(t, int64(1), numTx) + require.Equal(t, int64(0), gas) +} + +func TestSubstateTaskPool_ExecuteBlockDBError(t *testing.T) { + dbPath := t.TempDir() + "test-db" + db, err := newSubstateDB(dbPath, nil, nil, nil) + if err != nil { + t.Fatalf("cannot open db; %v", err) + } + + stPool := SubstateTaskPool{ + Name: "test", + + BlockFunc: func(block uint64, transactions map[int]*substate.Substate, taskPool *SubstateTaskPool) error { + return errors.New("test error") + }, + + First: testSubstate.Block, + Last: testSubstate.Block + 1, + + Workers: 1, + DB: db, + } + + _, _, err = stPool.ExecuteBlock(testSubstate.Block) + require.Error(t, err) +} + +func TestSubstateTaskPool_ExecuteBlockSkipTransferTx(t *testing.T) { + dbPath := t.TempDir() + "test-db" + db, err := createDbAndPutSubstate(dbPath) + if err != nil { + t.Fatal(err) + } + + stPool := SubstateTaskPool{ + Name: "test", + + TaskFunc: func(block uint64, tx int, substate *substate.Substate, taskPool *SubstateTaskPool) error { + return nil + }, + + First: testSubstate.Block, + Last: testSubstate.Block + 1, + + SkipTransferTxs: true, + + Workers: 1, + DB: db, + } + + numTx, gas, err := stPool.ExecuteBlock(testSubstate.Block) + require.Nil(t, err) + require.Equal(t, int64(0), numTx) + require.Equal(t, int64(0), gas) +} diff --git a/go.mod b/go.mod index 2b9ac05..60d0c28 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,13 @@ require ( require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/snappy v0.0.3 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/sys v0.15.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e4d782a..bd658ba 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -26,8 +28,12 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= @@ -79,3 +85,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=