Add initial FOCIL implementation

beacon/engine/errors.go

@@ -74,6 +74,10 @@ var (
 	//   - newPayloadV1: if the payload was accepted, but not processed (side chain)
 	ACCEPTED = "ACCEPTED"
 
+	// INVALID_INCLUSION_LIST is returned by the engine API in the following calls:
+	//   - newPayloadV5: if the payload failed to satisfy the inclusion list constraints
+	INVALID_INCLUSION_LIST = "INVALID_INCLUSION_LIST"
+
 	GenericServerError       = &EngineAPIError{code: -32000, msg: "Server error"}
 	UnknownPayload           = &EngineAPIError{code: -38001, msg: "Unknown payload"}
 	InvalidForkChoiceState   = &EngineAPIError{code: -38002, msg: "Invalid forkchoice state"}

beacon/engine/types.go

@@ -129,6 +129,9 @@ type executionPayloadEnvelopeMarshaling struct {
 	Requests   []hexutil.Bytes
 }
 
+// Max size of inclusion list in bytes.
+const MaxBytesPerInclusionList = uint64(8192)
+
 type PayloadStatusV1 struct {
 	Status          string         `json:"status"`
 	Witness         *hexutil.Bytes `json:"witness"`
@@ -344,6 +347,19 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.
 	}
 }
 
+func InclusionListToTransactions(inclusionList [][]byte) ([]*types.Transaction, error) {
+	txs, err := decodeTransactions(inclusionList)
+	if err != nil {
+		return nil, err
+	}
+
+	return txs, nil
+}
+
+func TransactionsToInclusionList(txs []*types.Transaction) [][]byte {
+	return encodeTransactions(txs)
+}
+
 // ExecutionPayloadBody is used in the response to GetPayloadBodiesByHash and GetPayloadBodiesByRange
 type ExecutionPayloadBody struct {
 	TransactionData []hexutil.Bytes     `json:"transactions"`

eth/catalyst/api.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/stateless"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
@@ -94,10 +95,13 @@ var caps = []string{
 	"engine_getPayloadV3",
 	"engine_getPayloadV4",
 	"engine_getBlobsV1",
+	"engine_getInclusionListV1",
+	"engine_updatePayloadWithInclusionListV1",
 	"engine_newPayloadV1",
 	"engine_newPayloadV2",
 	"engine_newPayloadV3",
 	"engine_newPayloadV4",
+	"engine_newPayloadV5",
 	"engine_newPayloadWithWitnessV1",
 	"engine_newPayloadWithWitnessV2",
 	"engine_newPayloadWithWitnessV3",
@@ -116,8 +120,9 @@ var caps = []string{
 type ConsensusAPI struct {
 	eth *eth.Ethereum
 
-	remoteBlocks *headerQueue  // Cache of remote payloads received
-	localBlocks  *payloadQueue // Cache of local payloads generated
+	remoteBlocks        *headerQueue        // Cache of remote payloads received
+	localBlocks         *payloadQueue       // Cache of local payloads generated
+	localInclusionLists *inclusionListQueue // Cache of inclusion list generated
 
 	// The forkchoice update and new payload method require us to return the
 	// latest valid hash in an invalid chain. To support that return, we need
@@ -170,11 +175,12 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI {
 		log.Warn("Engine API started but chain not configured for merge yet")
 	}
 	api := &ConsensusAPI{
-		eth:               eth,
-		remoteBlocks:      newHeaderQueue(),
-		localBlocks:       newPayloadQueue(),
-		invalidBlocksHits: make(map[common.Hash]int),
-		invalidTipsets:    make(map[common.Hash]*types.Header),
+		eth:                 eth,
+		remoteBlocks:        newHeaderQueue(),
+		localBlocks:         newPayloadQueue(),
+		localInclusionLists: newInclusionListQueue(),
+		invalidBlocksHits:   make(map[common.Hash]int),
+		invalidTipsets:      make(map[common.Hash]*types.Header),
 	}
 	eth.Downloader().SetBadBlockCallback(api.setInvalidAncestor)
 	return api
@@ -556,12 +562,47 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo
 	return res, nil
 }
 
+func (api *ConsensusAPI) GetInclusionListV1(parentHash common.Hash) ([][]byte, error) {
+	if inclusionList := api.localInclusionLists.get(parentHash); inclusionList != nil {
+		return inclusionList, nil
+	}
+
+	args := &miner.BuildInclusionListArgs{
+		Parent: parentHash,
+	}
+	inclusionList, err := api.eth.Miner().BuildInclusionList(args)
+	if err != nil {
+		log.Error("Failed to build inclusion list", "err", err)
+		return nil, err
+	}
+
+	api.localInclusionLists.put(parentHash, inclusionList)
+
+	return inclusionList, nil
+}
+
+func (api *ConsensusAPI) UpdatePayloadWithInclusionListV1(payloadID engine.PayloadID, inclusionList [][]byte) (*engine.PayloadID, error) {
+	payload := api.localBlocks.peak(payloadID)
+	if payload == nil {
+		return nil, nil
+	}
+
+	inclusionListTxs, err := engine.InclusionListToTransactions(inclusionList)
+	if err != nil {
+		return nil, err
+	}
+
+	payload.UpdateWithInclusionList(inclusionListTxs)
+
+	return &payloadID, nil
+}
+
 // NewPayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
 func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) {
 	if params.Withdrawals != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1"))
 	}
-	return api.newPayload(params, nil, nil, nil, false)
+	return api.newPayload(params, nil, nil, nil, nil, false)
 }
 
 // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
@@ -584,7 +625,7 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl
 	if params.BlobGasUsed != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun"))
 	}
-	return api.newPayload(params, nil, nil, nil, false)
+	return api.newPayload(params, nil, nil, nil, nil, false)
 }
 
 // NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
@@ -609,7 +650,7 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas
 	if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadV3 must only be called for cancun payloads"))
 	}
-	return api.newPayload(params, versionedHashes, beaconRoot, nil, false)
+	return api.newPayload(params, versionedHashes, beaconRoot, nil, nil, false)
 }
 
 // NewPayloadV4 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
@@ -641,7 +682,36 @@ func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHas
 	if err := validateRequests(requests); err != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err)
 	}
-	return api.newPayload(params, versionedHashes, beaconRoot, requests, false)
+	return api.newPayload(params, versionedHashes, beaconRoot, requests, nil, false)
+}
+
+// NewPayloadV5 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
+func (api *ConsensusAPI) NewPayloadV5(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes, inclusionList [][]byte) (engine.PayloadStatusV1, error) {
+	if params.Withdrawals == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil withdrawals post-shanghai"))
+	}
+	if params.ExcessBlobGas == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun"))
+	}
+	if params.BlobGasUsed == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun"))
+	}
+
+	if versionedHashes == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun"))
+	}
+	if beaconRoot == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun"))
+	}
+	if executionRequests == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil executionRequests post-prague"))
+	}
+
+	if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Prague {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadV5 must only be called for prague payloads"))
+	}
+	requests := convertRequests(executionRequests)
+	return api.newPayload(params, versionedHashes, beaconRoot, requests, inclusionList, false)
 }
 
 // NewPayloadWithWitnessV1 is analogous to NewPayloadV1, only it also generates
@@ -650,7 +720,7 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV1(params engine.ExecutableData) (
 	if params.Withdrawals != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1"))
 	}
-	return api.newPayload(params, nil, nil, nil, true)
+	return api.newPayload(params, nil, nil, nil, nil, true)
 }
 
 // NewPayloadWithWitnessV2 is analogous to NewPayloadV2, only it also generates
@@ -674,7 +744,7 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV2(params engine.ExecutableData) (
 	if params.BlobGasUsed != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun"))
 	}
-	return api.newPayload(params, nil, nil, nil, true)
+	return api.newPayload(params, nil, nil, nil, nil, true)
 }
 
 // NewPayloadWithWitnessV3 is analogous to NewPayloadV3, only it also generates
@@ -700,7 +770,7 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV3(params engine.ExecutableData, v
 	if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadWithWitnessV3 must only be called for cancun payloads"))
 	}
-	return api.newPayload(params, versionedHashes, beaconRoot, nil, true)
+	return api.newPayload(params, versionedHashes, beaconRoot, nil, nil, true)
 }
 
 // NewPayloadWithWitnessV4 is analogous to NewPayloadV4, only it also generates
@@ -733,7 +803,40 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV4(params engine.ExecutableData, v
 	if err := validateRequests(requests); err != nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err)
 	}
-	return api.newPayload(params, versionedHashes, beaconRoot, requests, true)
+	return api.newPayload(params, versionedHashes, beaconRoot, requests, nil, true)
+}
+
+// NewPayloadWithWitnessV5 is analogous to NewPayloadV5, only it also generates
+// and returns a stateless witness after running the payload.
+func (api *ConsensusAPI) NewPayloadWithWitnessV5(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes, inclusionList [][]byte) (engine.PayloadStatusV1, error) {
+	if params.Withdrawals == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil withdrawals post-shanghai"))
+	}
+	if params.ExcessBlobGas == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun"))
+	}
+	if params.BlobGasUsed == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun"))
+	}
+
+	if versionedHashes == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun"))
+	}
+	if beaconRoot == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun"))
+	}
+	if executionRequests == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil executionRequests post-prague"))
+	}
+	if inclusionList == nil {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil inclusionList post-prague"))
+	}
+
+	if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Prague {
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.UnsupportedFork.With(errors.New("newPayloadWithWitnessV5 must only be called for prague payloads"))
+	}
+	requests := convertRequests(executionRequests)
+	return api.newPayload(params, versionedHashes, beaconRoot, requests, inclusionList, true)
 }
 
 // ExecuteStatelessPayloadV1 is analogous to NewPayloadV1, only it operates in
@@ -825,7 +928,7 @@ func (api *ConsensusAPI) ExecuteStatelessPayloadV4(params engine.ExecutableData,
 	return api.executeStatelessPayload(params, versionedHashes, beaconRoot, requests, opaqueWitness)
 }
 
-func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, witness bool) (engine.PayloadStatusV1, error) {
+func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, inclusionList [][]byte, witness bool) (engine.PayloadStatusV1, error) {
 	// The locking here is, strictly, not required. Without these locks, this can happen:
 	//
 	// 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to
@@ -933,17 +1036,34 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe
 		return engine.PayloadStatusV1{Status: engine.ACCEPTED}, nil
 	}
 	log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number())
-	proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(block, witness)
-	if err != nil {
-		log.Warn("NewPayload: inserting block failed", "error", err)
-
+	markInvalidBlock := func() {
 		api.invalidLock.Lock()
 		api.invalidBlocksHits[block.Hash()] = 1
 		api.invalidTipsets[block.Hash()] = block.Header()
 		api.invalidLock.Unlock()
-
+	}
+	proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(block, witness)
+	if err != nil {
+		log.Warn("NewPayload: inserting block failed", "error", err)
+		markInvalidBlock()
 		return api.invalid(err, parent.Header()), nil
 	}
+	// Verify if the block satisfies the inclusion list constraints.
+	if inclusionList != nil {
+		statedb, err := state.New(block.Root(), api.eth.BlockChain().StateCache())
+		if err != nil {
+			return api.invalid(err, parent.Header()), nil
+		}
+		inclusionListTxs, err := engine.InclusionListToTransactions(inclusionList)
+		if err != nil {
+			return api.invalid(err, parent.Header()), nil
+		}
+		if res := api.validateInclusionListConstraints(block, statedb, inclusionListTxs); res != nil {
+			log.Warn("NewPayload: satisfying the inclusion list constraints failed", "error", err)
+			markInvalidBlock()
+			return *res, nil
+		}
+	}
 	hash := block.Hash()
 
 	// If witness collection was requested, inject that into the result too
@@ -1112,6 +1232,49 @@ func (api *ConsensusAPI) checkInvalidAncestor(check common.Hash, head common.Has
 	}
 }
 
+// validateInclusionListConstraints verifies that all transactions in the inclusion list
+// are either included in the block or cannot be appended at the end of the block.
+// If any appendable transaction is found, the block fails to meet the inclusion list constraints.
+func (api *ConsensusAPI) validateInclusionListConstraints(block *types.Block, statedb *state.StateDB, inclusionListTxs []*types.Transaction) *engine.PayloadStatusV1 {
+	// Create a map of transaction hashes present in the block.
+	includedTxHashes := make(map[common.Hash]bool)
+	for _, tx := range block.Transactions() {
+		includedTxHashes[tx.Hash()] = true
+	}
+
+	// Get the block's gas limit and gas left.
+	gasLimit := block.GasLimit()
+	gasLeft := gasLimit - block.GasUsed()
+
+	// Iterate over each transaction in the inclusion list and check if it is either included in the block or cannot be placed at the end of the block.
+	for _, tx := range inclusionListTxs {
+		// Check if the transaction is included in the block.
+		if _, exists := includedTxHashes[tx.Hash()]; exists {
+			continue
+		}
+
+		// Check if there is enough gas left to execute the transaction.
+		if tx.Gas() > gasLeft {
+			continue
+		}
+
+		// Check if the transaction could have been appended at the end of the block. If yes, the block fails to satisfy the inclusion list constraints.
+		snapshot := statedb.Copy()
+		evm := vm.NewEVM(core.NewEVMBlockContext(block.Header(), api.eth.BlockChain(), &block.Header().Coinbase), snapshot, api.eth.BlockChain().Config(), vm.Config{})
+		gasPool := new(core.GasPool).AddGas(gasLeft)
+
+		if _, err := core.ApplyTransaction(evm, gasPool, snapshot, block.Header(), tx, new(uint64)); err == nil {
+			return &engine.PayloadStatusV1{
+				Status:          engine.INVALID_INCLUSION_LIST,
+				LatestValidHash: nil,
+				ValidationError: nil,
+			}
+		}
+	}
+
+	return nil
+}
+
 // invalid returns a response "INVALID" with the latest valid hash supplied by latest.
 func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) engine.PayloadStatusV1 {
 	var currentHash *common.Hash