Merge branch 'petera/fix-proxy-unittest' into auto-update-onflow-cade…

…nce-v1.3.0

.github/workflows/builds.yml

@@ -43,6 +43,10 @@ on:
         type: boolean
         description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images'
         required: false
+      private_build:
+        type: boolean
+        description: 'Build private images'
+        required: false
 
 jobs:
   # matrix_builder generates a matrix that includes the roles selected in the input
@@ -109,16 +113,30 @@ jobs:
         credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }}
     - name: Set up Google Cloud SDK
       uses: google-github-actions/setup-gcloud@v1
-    - name: Authenticate docker with gcloud
+
+    - name: Authenticate Docker with gcloud
       run: |
-        gcloud auth configure-docker
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          gcloud auth configure-docker us-central1-docker.pkg.dev
+        else
+          gcloud auth configure-docker
+        fi
+
+    - name: Set CONTAINER_REGISTRY
+      id: set-registry
+      run: |
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          echo "CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }}" >> $GITHUB_ENV
+        else
+          echo "CONTAINER_REGISTRY=${{ vars.PUBLIC_REGISTRY }}" >> $GITHUB_ENV
+        fi
 
     - name: Build/Push ${{ matrix.role }} amd64 images with adx (default)
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx
+        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
     - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images
       if: ${{ inputs.include_alternative_builds }}
@@ -128,7 +146,7 @@ jobs:
       run: |
         make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \
           docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \
-          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm
+          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
 
         

Makefile

@@ -205,6 +205,7 @@ generate-mocks: install-mock-generators
 	mockery --name 'BlockTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock"  --outpkg="mock"
 	mockery --name 'DataProvider' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
 	mockery --name 'DataProviderFactory' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
+	mockery --name 'WebsocketConnection' --dir="./engine/access/rest/websockets" --case=underscore --output="./engine/access/rest/websockets/mock"  --outpkg="mock"
 	mockery --name 'ExecutionDataTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock"  --outpkg="mock"
 	mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock"
 	mockery --name 'Communicator' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock"
@@ -215,6 +216,7 @@ generate-mocks: install-mock-generators
 	mockery --name 'Storage' --dir=module/executiondatasync/tracker --case=underscore --output="module/executiondatasync/tracker/mock" --outpkg="mocktracker"
 	mockery --name 'ScriptExecutor' --dir=module/execution --case=underscore --output="module/execution/mock" --outpkg="mock"
 	mockery --name 'StorageSnapshot' --dir=fvm/storage/snapshot --case=underscore --output="fvm/storage/snapshot/mock" --outpkg="mock"
+	mockery --name 'WebsocketConnection' --dir=engine/access/rest/websockets --case=underscore --output="engine/access/rest/websockets/mock" --outpkg="mock"
 
 	#temporarily make insecure/ a non-module to allow mockery to create mocks
 	mv insecure/go.mod insecure/go2.mod
@@ -859,6 +861,7 @@ docker-all-tools: tool-util tool-remove-execution-fork
 PHONY: docker-build-util
 docker-build-util:
 	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
 		-t "$(CONTAINER_REGISTRY)/util:latest"  \
 		-t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" .
 

cmd/bootstrap/utils/md5.go

@@ -2,7 +2,8 @@ package utils
 
 // The google storage API only provides md5 and crc32 hence overriding the linter flag for md5
 import (
-	"crypto/md5" //nolint:gosec
+	// #nosec
+	"crypto/md5"
 	"io"
 	"os"
 )

cmd/util/cmd/export-evm-state/cmd.go

@@ -3,13 +3,17 @@ package evm_exporter
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
+	"github.com/onflow/atree"
+
 	"github.com/onflow/flow-go/cmd/util/ledger/util"
 	"github.com/onflow/flow-go/fvm/evm"
 	"github.com/onflow/flow-go/fvm/evm/emulator/state"
+	"github.com/onflow/flow-go/fvm/evm/testutils"
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/common/convert"
 	"github.com/onflow/flow-go/model/flow"
@@ -20,6 +24,8 @@ var (
 	flagExecutionStateDir string
 	flagOutputDir         string
 	flagStateCommitment   string
+	flagEVMStateGobDir    string
+	flagEVMStateGobHeight uint64
 )
 
 var Cmd = &cobra.Command{
@@ -34,21 +40,33 @@ func init() {
 
 	Cmd.Flags().StringVar(&flagExecutionStateDir, "execution-state-dir", "",
 		"Execution Node state dir (where WAL logs are written")
-	_ = Cmd.MarkFlagRequired("execution-state-dir")
 
 	Cmd.Flags().StringVar(&flagOutputDir, "output-dir", "",
 		"Directory to write new Execution State to")
 	_ = Cmd.MarkFlagRequired("output-dir")
 
 	Cmd.Flags().StringVar(&flagStateCommitment, "state-commitment", "",
 		"State commitment (hex-encoded, 64 characters)")
+
+	Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob",
+		"directory that stores the evm state gob files as checkpoint")
+
+	Cmd.Flags().Uint64Var(&flagEVMStateGobHeight, "evm_state_gob_height", 0,
+		"the flow height of the evm state gob files")
 }
 
 func run(*cobra.Command, []string) {
 	log.Info().Msg("start exporting evm state")
-	err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir)
-	if err != nil {
-		log.Fatal().Err(err).Msg("cannot get export evm state")
+	if flagExecutionStateDir != "" {
+		err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get export evm state")
+		}
+	} else if flagEVMStateGobDir != "" {
+		err := ExportEVMStateFromGob(flagChain, flagEVMStateGobDir, flagEVMStateGobHeight, flagOutputDir)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get export evm state from gob files")
+		}
 	}
 }
 
@@ -83,7 +101,40 @@ func ExportEVMState(
 
 	payloadsLedger := util.NewPayloadsLedger(filteredPayloads)
 
-	exporter, err := state.NewExporter(payloadsLedger, storageRoot)
+	return ExportEVMStateFromPayloads(payloadsLedger, storageRoot, outputPath)
+}
+
+func ExportEVMStateFromGob(
+	chainName string,
+	evmStateGobDir string,
+	flowHeight uint64,
+	outputPath string) error {
+
+	valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, flowHeight)
+	chainID := flow.ChainID(chainName)
+
+	storageRoot := evm.StorageAccountAddress(chainID)
+	valuesGob, err := testutils.DeserializeState(valueFileName)
+	if err != nil {
+		return err
+	}
+
+	allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName)
+	if err != nil {
+		return err
+	}
+
+	store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs)
+
+	return ExportEVMStateFromPayloads(store, storageRoot, outputPath)
+}
+
+func ExportEVMStateFromPayloads(
+	ledger atree.Ledger,
+	storageRoot flow.Address,
+	outputPath string,
+) error {
+	exporter, err := state.NewExporter(ledger, storageRoot)
 	if err != nil {
 		return fmt.Errorf("failed to create exporter: %w", err)
 	}
@@ -95,15 +146,15 @@ func ExportEVMState(
 		}
 	}
 
-	fi, err := os.Create(outputPath)
-	if err != nil {
-		return err
-	}
-	defer fi.Close()
-
-	err = exporter.Export(outputPath)
+	err = exporter.ExportGob(outputPath)
 	if err != nil {
 		return fmt.Errorf("failed to export: %w", err)
 	}
 	return nil
 }
+
+func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) {
+	valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight))
+	allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight))
+	return valueFileName, allocatorFileName
+}

cmd/util/cmd/verify_execution_result/cmd.go

@@ -53,27 +53,33 @@ func run(*cobra.Command, []string) {
 	chainID := flow.ChainID(flagChain)
 	_ = chainID.Chain()
 
+	lg := log.With().
+		Str("chain", string(chainID)).
+		Str("datadir", flagDatadir).
+		Str("chunk_data_pack_dir", flagChunkDataPackDir).
+		Logger()
+
 	if flagFromTo != "" {
 		from, to, err := parseFromTo(flagFromTo)
 		if err != nil {
-			log.Fatal().Err(err).Msg("could not parse from_to")
+			lg.Fatal().Err(err).Msg("could not parse from_to")
 		}
 
-		log.Info().Msgf("verifying range from %d to %d", from, to)
+		lg.Info().Msgf("verifying range from %d to %d", from, to)
 		err = verifier.VerifyRange(from, to, chainID, flagDatadir, flagChunkDataPackDir)
 		if err != nil {
-			log.Fatal().Err(err).Msgf("could not verify range from %d to %d", from, to)
+			lg.Fatal().Err(err).Msgf("could not verify range from %d to %d", from, to)
 		}
-		log.Info().Msgf("successfully verified range from %d to %d", from, to)
+		lg.Info().Msgf("successfully verified range from %d to %d", from, to)
 
 	} else {
-		log.Info().Msgf("verifying last %d sealed blocks", flagLastK)
+		lg.Info().Msgf("verifying last %d sealed blocks", flagLastK)
 		err := verifier.VerifyLastKHeight(flagLastK, chainID, flagDatadir, flagChunkDataPackDir)
 		if err != nil {
-			log.Fatal().Err(err).Msg("could not verify last k height")
+			lg.Fatal().Err(err).Msg("could not verify last k height")
 		}
 
-		log.Info().Msgf("successfully verified last %d sealed blocks", flagLastK)
+		lg.Info().Msgf("successfully verified last %d sealed blocks", flagLastK)
 	}
 }
 

engine/access/rest/websockets/config.go

@@ -4,6 +4,33 @@ import (
 	"time"
 )
 
+const (
+	// PingPeriod defines the interval at which ping messages are sent to the client.
+	// This value must be less than pongWait, cause it that case the server ensures it sends a ping well before the PongWait
+	// timeout elapses. Each new pong message resets the server's read deadline, keeping the connection alive as long as
+	// the client is responsive.
+	//
+	// Example:
+	// At t=9, the server sends a ping, initial read deadline is t=10 (for the first message)
+	// At t=10, the client responds with a pong. The server resets its read deadline to t=20.
+	// At t=18, the server sends another ping. If the client responds with a pong at t=19, the read deadline is extended to t=29.
+	//
+	// In case of failure:
+	// If the client stops responding, the server will send a ping at t=9 but won't receive a pong by t=10. The server then closes the connection.
+	PingPeriod = (PongWait * 9) / 10
+
+	// PongWait specifies the maximum time to wait for a pong response message from the peer
+	// after sending a ping
+	PongWait = 10 * time.Second
+
+	// WriteWait specifies a timeout for the write operation. If the write
+	// isn't completed within this duration, it fails with a timeout error.
+	// SetWriteDeadline ensures the write operation does not block indefinitely
+	// if the client is slow or unresponsive. This prevents resource exhaustion
+	// and allows the server to gracefully handle timeouts for delayed writes.
+	WriteWait = 10 * time.Second
+)
+
 type Config struct {
 	MaxSubscriptionsPerConnection uint64
 	MaxResponsesPerSecond         uint64

engine/access/rest/websockets/connection.go

@@ -0,0 +1,57 @@
+package websockets
+
+import (
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+type WebsocketConnection interface {
+	ReadJSON(v interface{}) error
+	WriteJSON(v interface{}) error
+	WriteControl(messageType int, deadline time.Time) error
+	Close() error
+	SetReadDeadline(deadline time.Time) error
+	SetWriteDeadline(deadline time.Time) error
+	SetPongHandler(h func(string) error)
+}
+
+type WebsocketConnectionImpl struct {
+	conn *websocket.Conn
+}
+
+func NewWebsocketConnection(conn *websocket.Conn) *WebsocketConnectionImpl {
+	return &WebsocketConnectionImpl{
+		conn: conn,
+	}
+}
+
+var _ WebsocketConnection = (*WebsocketConnectionImpl)(nil)
+
+func (c *WebsocketConnectionImpl) ReadJSON(v interface{}) error {
+	return c.conn.ReadJSON(v)
+}
+
+func (c *WebsocketConnectionImpl) WriteJSON(v interface{}) error {
+	return c.conn.WriteJSON(v)
+}
+
+func (c *WebsocketConnectionImpl) WriteControl(messageType int, deadline time.Time) error {
+	return c.conn.WriteControl(messageType, nil, deadline)
+}
+
+func (c *WebsocketConnectionImpl) Close() error {
+	return c.conn.Close()
+}
+
+func (c *WebsocketConnectionImpl) SetReadDeadline(deadline time.Time) error {
+	return c.conn.SetReadDeadline(deadline)
+}
+
+func (c *WebsocketConnectionImpl) SetWriteDeadline(deadline time.Time) error {
+	return c.conn.SetWriteDeadline(deadline)
+}
+
+func (c *WebsocketConnectionImpl) SetPongHandler(h func(string) error) {
+	c.conn.SetPongHandler(h)
+}