Skip to content

Commit

Permalink
Merge pull request #6791 from onflow/leo/v0.37-cmd-add-verify-executi…
Browse files Browse the repository at this point in the history
…on-result

Backport v0.37 cmd add verify execution result
  • Loading branch information
zhangchiqing authored Dec 10, 2024
2 parents e94c022 + d3d193e commit eaa65ca
Show file tree
Hide file tree
Showing 9 changed files with 473 additions and 117 deletions.
32 changes: 4 additions & 28 deletions cmd/scaffold.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ import (
"github.com/onflow/flow-go/cmd/build"
"github.com/onflow/flow-go/config"
"github.com/onflow/flow-go/consensus/hotstuff/persister"
"github.com/onflow/flow-go/fvm"
"github.com/onflow/flow-go/fvm/environment"
"github.com/onflow/flow-go/fvm/initialize"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/model/flow/filter"
"github.com/onflow/flow-go/module"
Expand Down Expand Up @@ -1520,32 +1519,9 @@ func (fnb *FlowNodeBuilder) initLocal() error {
}

func (fnb *FlowNodeBuilder) initFvmOptions() {
blockFinder := environment.NewBlockFinder(fnb.Storage.Headers)
vmOpts := []fvm.Option{
fvm.WithChain(fnb.RootChainID.Chain()),
fvm.WithBlocks(blockFinder),
fvm.WithAccountStorageLimit(true),
}
switch fnb.RootChainID {
case flow.Testnet,
flow.Sandboxnet,
flow.Previewnet,
flow.Mainnet:
vmOpts = append(vmOpts,
fvm.WithTransactionFeesEnabled(true),
)
}
switch fnb.RootChainID {
case flow.Testnet,
flow.Sandboxnet,
flow.Previewnet,
flow.Localnet,
flow.Benchnet:
vmOpts = append(vmOpts,
fvm.WithContractDeploymentRestricted(false),
)
}
fnb.FvmOptions = vmOpts
fnb.FvmOptions = initialize.InitFvmOptions(
fnb.RootChainID, fnb.Storage.Headers,
)
}

// handleModules initializes the given module.
Expand Down
2 changes: 2 additions & 0 deletions cmd/util/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/onflow/flow-go/cmd/util/cmd/snapshot"
system_addresses "github.com/onflow/flow-go/cmd/util/cmd/system-addresses"
truncate_database "github.com/onflow/flow-go/cmd/util/cmd/truncate-database"
verify_execution_result "github.com/onflow/flow-go/cmd/util/cmd/verify_execution_result"
"github.com/onflow/flow-go/cmd/util/cmd/version"
"github.com/onflow/flow-go/module/profiler"
)
Expand Down Expand Up @@ -120,6 +121,7 @@ func addCommands() {
rootCmd.AddCommand(system_addresses.Cmd)
rootCmd.AddCommand(check_storage.Cmd)
rootCmd.AddCommand(generate_authorization_fixes.Cmd)
rootCmd.AddCommand(verify_execution_result.Cmd)
}

func initConfig() {
Expand Down
101 changes: 101 additions & 0 deletions cmd/util/cmd/verify_execution_result/cmd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package verify

import (
"fmt"
"strconv"
"strings"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"

"github.com/onflow/flow-go/engine/verification/verifier"
"github.com/onflow/flow-go/model/flow"
)

var (
flagLastK uint64
flagDatadir string
flagChunkDataPackDir string
flagChain string
flagFromTo string
)

// # verify the last 100 sealed blocks
// ./util verify_execution_result --chain flow-testnet --datadir /var/flow/data/protocol --chunk_data_pack_dir /var/flow/data/chunk_data_pack --lastk 100
// # verify the blocks from height 2000 to 3000
// ./util verify_execution_result --chain flow-testnet --datadir /var/flow/data/protocol --chunk_data_pack_dir /var/flow/data/chunk_data_pack --from_to 2000-3000
var Cmd = &cobra.Command{
Use: "verify-execution-result",
Short: "verify block execution by verifying all chunks in the result",
Run: run,
}

func init() {
Cmd.Flags().StringVar(&flagChain, "chain", "", "Chain name")
_ = Cmd.MarkFlagRequired("chain")

Cmd.Flags().StringVar(&flagDatadir, "datadir", "/var/flow/data/protocol",
"directory that stores the protocol state")
_ = Cmd.MarkFlagRequired("datadir")

Cmd.Flags().StringVar(&flagChunkDataPackDir, "chunk_data_pack_dir", "/var/flow/data/chunk_data_pack",
"directory that stores the protocol state")
_ = Cmd.MarkFlagRequired("chunk_data_pack_dir")

Cmd.Flags().Uint64Var(&flagLastK, "lastk", 1,
"last k sealed blocks to verify")

Cmd.Flags().StringVar(&flagFromTo, "from_to", "",
"the height range to verify blocks (inclusive), i.e, 1-1000, 1000-2000, 2000-3000, etc.")
}

func run(*cobra.Command, []string) {
chainID := flow.ChainID(flagChain)
_ = chainID.Chain()

if flagFromTo != "" {
from, to, err := parseFromTo(flagFromTo)
if err != nil {
log.Fatal().Err(err).Msg("could not parse from_to")
}

log.Info().Msgf("verifying range from %d to %d", from, to)
err = verifier.VerifyRange(from, to, chainID, flagDatadir, flagChunkDataPackDir)
if err != nil {
log.Fatal().Err(err).Msgf("could not verify range from %d to %d", from, to)
}
log.Info().Msgf("successfully verified range from %d to %d", from, to)

} else {
log.Info().Msgf("verifying last %d sealed blocks", flagLastK)
err := verifier.VerifyLastKHeight(flagLastK, chainID, flagDatadir, flagChunkDataPackDir)
if err != nil {
log.Fatal().Err(err).Msg("could not verify last k height")
}

log.Info().Msgf("successfully verified last %d sealed blocks", flagLastK)
}
}

func parseFromTo(fromTo string) (from, to uint64, err error) {
parts := strings.Split(fromTo, "-")
if len(parts) != 2 {
return 0, 0, fmt.Errorf("invalid format: expected 'from-to', got '%s'", fromTo)
}

from, err = strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("invalid 'from' value: %w", err)
}

to, err = strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("invalid 'to' value: %w", err)
}

if from > to {
return 0, 0, fmt.Errorf("'from' value (%d) must be less than or equal to 'to' value (%d)", from, to)
}

return from, to, nil
}
28 changes: 14 additions & 14 deletions engine/execution/computation/execution_verification_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"github.com/onflow/flow-go/engine/execution/testutil"
"github.com/onflow/flow-go/engine/execution/utils"
"github.com/onflow/flow-go/engine/testutil/mocklocal"
"github.com/onflow/flow-go/engine/verification/fetcher"
"github.com/onflow/flow-go/fvm"
"github.com/onflow/flow-go/fvm/blueprints"
"github.com/onflow/flow-go/fvm/environment"
Expand All @@ -36,6 +35,7 @@ import (
"github.com/onflow/flow-go/ledger/complete/wal/fixtures"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/model/verification"
"github.com/onflow/flow-go/model/verification/convert"
"github.com/onflow/flow-go/module/chunks"
"github.com/onflow/flow-go/module/executiondatasync/execution_data"
exedataprovider "github.com/onflow/flow-go/module/executiondatasync/provider"
Expand Down Expand Up @@ -69,7 +69,7 @@ func Test_ExecutionMatchesVerification(t *testing.T) {
`access(all) contract Foo {
access(all) event FooEvent(x: Int, y: Int)
access(all) fun emitEvent() {
access(all) fun emitEvent() {
emit FooEvent(x: 2, y: 1)
}
}`), "Foo")
Expand Down Expand Up @@ -113,7 +113,7 @@ func Test_ExecutionMatchesVerification(t *testing.T) {
`access(all) contract Foo {
access(all) event FooEvent(x: Int, y: Int)
access(all) fun emitEvent() {
access(all) fun emitEvent() {
emit FooEvent(x: 2, y: 1)
}
}`), "Foo")
Expand Down Expand Up @@ -585,34 +585,34 @@ func TestTransactionFeeDeduction(t *testing.T) {
//
// The withdraw amount and the account from getAccount
// would be the parameters to the transaction
import FungibleToken from 0x%s
import FlowToken from 0x%s
transaction(amount: UFix64, to: Address) {
// The Vault resource that holds the tokens that are being transferred
let sentVault: @{FungibleToken.Vault}
prepare(signer: auth(BorrowValue) &Account) {
// Get a reference to the signer's stored vault
let vaultRef = signer.storage.borrow<auth(FungibleToken.Withdraw) &FlowToken.Vault>(from: /storage/flowTokenVault)
?? panic("Could not borrow reference to the owner's Vault!")
// Withdraw tokens from the signer's stored vault
self.sentVault <- vaultRef.withdraw(amount: amount)
}
execute {
// Get the recipient's public account object
let recipient = getAccount(to)
// Get a reference to the recipient's Receiver
let receiverRef = recipient.capabilities.borrow<&{FungibleToken.Receiver}>(/public/flowTokenReceiver)
?? panic("Could not borrow receiver reference to the recipient's Vault")
// Deposit the withdrawn tokens in the recipient's receiver
receiverRef.deposit(from: <-self.sentVault)
}
Expand Down Expand Up @@ -840,7 +840,7 @@ func executeBlockAndVerifyWithParameters(t *testing.T,

for i, chunk := range er.Chunks {
isSystemChunk := i == er.Chunks.Len()-1
offsetForChunk, err := fetcher.TransactionOffsetForChunk(er.Chunks, chunk.Index)
offsetForChunk, err := convert.TransactionOffsetForChunk(er.Chunks, chunk.Index)
require.NoError(t, err)

vcds[i] = &verification.VerifiableChunkData{
Expand Down
74 changes: 10 additions & 64 deletions engine/verification/fetcher/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/model/flow/filter"
"github.com/onflow/flow-go/model/verification"
"github.com/onflow/flow-go/model/verification/convert"
"github.com/onflow/flow-go/module"
"github.com/onflow/flow-go/module/mempool"
"github.com/onflow/flow-go/module/trace"
Expand Down Expand Up @@ -259,7 +260,7 @@ func (e *Engine) HandleChunkDataPack(originID flow.Identifier, response *verific
Uint64("block_height", status.BlockHeight).
Hex("result_id", logging.ID(resultID)).
Uint64("chunk_index", status.ChunkIndex).
Bool("system_chunk", IsSystemChunk(status.ChunkIndex, status.ExecutionResult)).
Bool("system_chunk", convert.IsSystemChunk(status.ChunkIndex, status.ExecutionResult)).
Logger()

span, ctx := e.tracer.StartBlockSpan(context.Background(), status.ExecutionResult.BlockID, trace.VERFetcherHandleChunkDataPack)
Expand Down Expand Up @@ -413,7 +414,7 @@ func (e Engine) validateCollectionID(
result *flow.ExecutionResult,
chunk *flow.Chunk) error {

if IsSystemChunk(chunk.Index, result) {
if convert.IsSystemChunk(chunk.Index, result) {
return e.validateSystemChunkCollection(chunkDataPack)
}

Expand Down Expand Up @@ -550,29 +551,13 @@ func (e *Engine) makeVerifiableChunkData(chunk *flow.Chunk,
chunkDataPack *flow.ChunkDataPack,
) (*verification.VerifiableChunkData, error) {

// system chunk is the last chunk
isSystemChunk := IsSystemChunk(chunk.Index, result)

endState, err := EndStateCommitment(result, chunk.Index, isSystemChunk)
if err != nil {
return nil, fmt.Errorf("could not compute end state of chunk: %w", err)
}

transactionOffset, err := TransactionOffsetForChunk(result.Chunks, chunk.Index)
if err != nil {
return nil, fmt.Errorf("cannot compute transaction offset for chunk: %w", err)
}

return &verification.VerifiableChunkData{
IsSystemChunk: isSystemChunk,
Chunk: chunk,
Header: header,
Snapshot: snapshot,
Result: result,
ChunkDataPack: chunkDataPack,
EndState: endState,
TransactionOffset: transactionOffset,
}, nil
return convert.FromChunkDataPack(
chunk,
chunkDataPack,
header,
snapshot,
result,
)
}

// requestChunkDataPack creates and dispatches a chunk data pack request to the requester engine.
Expand Down Expand Up @@ -661,42 +646,3 @@ func executorsOf(receipts []*flow.ExecutionReceipt, resultID flow.Identifier) (f

return agrees, disagrees
}

// EndStateCommitment computes the end state of the given chunk.
func EndStateCommitment(result *flow.ExecutionResult, chunkIndex uint64, systemChunk bool) (flow.StateCommitment, error) {
var endState flow.StateCommitment
if systemChunk {
var err error
// last chunk in a result is the system chunk and takes final state commitment
endState, err = result.FinalStateCommitment()
if err != nil {
return flow.DummyStateCommitment, fmt.Errorf("can not read final state commitment, likely a bug:%w", err)
}
} else {
// any chunk except last takes the subsequent chunk's start state
endState = result.Chunks[chunkIndex+1].StartState
}

return endState, nil
}

// TransactionOffsetForChunk calculates transaction offset for a given chunk which is the index of the first
// transaction of this chunk within the whole block
func TransactionOffsetForChunk(chunks flow.ChunkList, chunkIndex uint64) (uint32, error) {
if int(chunkIndex) > len(chunks)-1 {
return 0, fmt.Errorf("chunk list out of bounds, len %d asked for chunk %d", len(chunks), chunkIndex)
}
var offset uint32 = 0
for i := 0; i < int(chunkIndex); i++ {
offset += uint32(chunks[i].NumberOfTransactions)
}
return offset, nil
}

// IsSystemChunk returns true if `chunkIndex` points to a system chunk in `result`.
// Otherwise, it returns false.
// In the current version, a chunk is a system chunk if it is the last chunk of the
// execution result.
func IsSystemChunk(chunkIndex uint64, result *flow.ExecutionResult) bool {
return chunkIndex == uint64(len(result.Chunks)-1)
}
Loading

0 comments on commit eaa65ca

Please sign in to comment.