Merge branch 'master' into fluffy-evm-integration

.github/workflows/kurtosis.yml

@@ -83,7 +83,7 @@ jobs:
         cat kurtosis-network-params.yml | envsubst > assertoor.yaml
         sed -i "s/el_image: .*/el_image: localtestnet/" assertoor.yaml
 
-        kurtosis run github.com/ethpandaops/ethereum-package@4.3.0 --enclave assertoor-${{ github.run_id }} --args-file assertoor.yaml
+        kurtosis run github.com/ethpandaops/ethereum-package --enclave assertoor-${{ github.run_id }} --args-file assertoor.yaml
 
         enclave_dump=$(kurtosis enclave inspect assertoor-${{ github.run_id }})
 

.github/workflows/simulators.yml

@@ -43,7 +43,7 @@ jobs:
         run: |
           ncpu=$(nproc)
           make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci
-          make -j${ncpu} deps
+          make -j${ncpu} deps rocksdb
 
       - name: Run Simulators
         run: |
@@ -84,7 +84,7 @@ jobs:
         run: |
           ncpu=$(sysctl -n hw.ncpu)
           make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci
-          make -j${ncpu} deps
+          make -j${ncpu} deps rocksdb
 
       - name: Run Simulators
         run: |

Makefile

@@ -105,6 +105,7 @@ VERIF_PROXY_OUT_PATH ?= build/libverifproxy/
 	deps \
 	update \
 	nimbus \
+	nimbus_execution_client \
 	fluffy \
 	nimbus_verified_proxy \
 	libverifproxy \
@@ -157,7 +158,7 @@ GIT_SUBMODULE_UPDATE := git -c submodule."vendor/nimbus-eth2".update=none submod
 else # "variables.mk" was included. Business as usual until the end of this file.
 
 # default target, because it's the first one that doesn't start with '.'
-all: | $(TOOLS) nimbus
+all: | $(TOOLS) nimbus_execution_client
 
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
@@ -210,11 +211,12 @@ $(TOOLS): | build deps rocksdb
 		echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/$@ "$${TOOL_DIR}/$@.nim"
 
-# a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated
-execution_client_name = nimbus_execution_client
-nimbus: | build deps rocksdb
-	echo -e $(BUILD_MSG) "build/$(execution_client_name)" && \
-		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/$(execution_client_name) "nimbus/$(execution_client_name).nim"
+nimbus_execution_client: | build deps rocksdb
+	echo -e $(BUILD_MSG) "build/nimbus_execution_client" && \
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/nimbus_execution_client "nimbus/nimbus_execution_client.nim"
+
+nimbus: nimbus_execution_client
+	echo "The nimbus target is deprecated and will soon change meaning, use 'nimbus_execution_client' instead"
 
 # symlink
 nimbus.nims:
@@ -244,7 +246,7 @@ test: | build deps rocksdb
 	$(ENV_SCRIPT) nim test_rocksdb $(NIM_PARAMS) nimbus.nims
 	$(ENV_SCRIPT) nim test $(NIM_PARAMS) nimbus.nims
 
-test_import: nimbus
+test_import: nimbus_execution_client
 	$(ENV_SCRIPT) nim test_import $(NIM_PARAMS) nimbus.nims
 
 # builds and runs an EVM-related subset of the nimbus test suite
@@ -257,10 +259,10 @@ test-evm: | build deps rocksdb
 # deterministic order for debugging info sections - even with
 # "-frandom-seed=...". Striping the binaries should make them identical, though.
 test-reproducibility:
-	+ [ -e build/nimbus_execution_client ] || $(MAKE) V=0 nimbus; \
+	+ [ -e build/nimbus_execution_client ] || $(MAKE) V=0 nimbus_execution_client; \
 		MD5SUM1=$$($(MD5SUM) build/nimbus_execution_client | cut -d ' ' -f 1) && \
 		rm -rf nimcache/*/nimbus_execution_client && \
-		$(MAKE) V=0 nimbus && \
+		$(MAKE) V=0 nimbus_execution_client && \
 		MD5SUM2=$$($(MD5SUM) build/nimbus_execution_client | cut -d ' ' -f 1) && \
 		[ "$$MD5SUM1" = "$$MD5SUM2" ] && echo -e "\e[92mSuccess: identical binaries.\e[39m" || \
 			{ echo -e "\e[91mFailure: the binary changed between builds.\e[39m"; exit 1; }
@@ -320,17 +322,17 @@ utp-test: | build deps
 # Nimbus Verified Proxy related targets
 
 # Builds the nimbus_verified_proxy
-nimbus_verified_proxy: | build deps
+nimbus_verified_proxy: | build deps rocksdb
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim nimbus_verified_proxy $(NIM_PARAMS) nimbus.nims
 
 # builds and runs the nimbus_verified_proxy test suite
-nimbus-verified-proxy-test: | build deps
+nimbus-verified-proxy-test: | build deps rocksdb
 	$(ENV_SCRIPT) nim nimbus_verified_proxy_test $(NIM_PARAMS) nimbus.nims
 
 # Shared library for verified proxy
 
-libverifproxy: | build deps
+libverifproxy: | build deps rocksdb
 	+ echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim --version && \
 		$(ENV_SCRIPT) nim c --app:lib -d:"libp2p_pki_schemes=secp256k1" --noMain:on --threads:on --nimcache:nimcache/libverifproxy -o:$(VERIF_PROXY_OUT_PATH)/$@.$(VERIF_PROXY_SHAREDLIBEXT) $(NIM_PARAMS) nimbus_verified_proxy/libverifproxy/verifproxy.nim
@@ -359,7 +361,7 @@ txparse: | build deps
 
 # usual cleaning
 clean: | clean-common
-	rm -rf build/{nimbus,fluffy,libverifproxy,nimbus_verified_proxy,$(TOOLS_CSV),$(FLUFFY_TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_history_network_custom_chain_tests,test_portal_testnet,utp_test_app,utp_test,*.dSYM}
+	rm -rf build/{nimbus,nimbus_execution_client,fluffy,libverifproxy,nimbus_verified_proxy,$(TOOLS_CSV),$(FLUFFY_TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_history_network_custom_chain_tests,test_portal_testnet,utp_test_app,utp_test,*.dSYM}
 	rm -rf tools/t8n/{t8n,t8n_test}
 	rm -rf tools/evmstate/{evmstate,evmstate_test}
 ifneq ($(USE_LIBBACKTRACE), 0)

TracerTests.md

@@ -2,16 +2,16 @@ TracerTests
 ===
 ## TracerTests
 ```diff
-+ block46147.json                                                 OK
-+ block46400.json                                                 OK
-+ block46402.json                                                 OK
-+ block47205.json                                                 OK
-+ block48712.json                                                 OK
-+ block48915.json                                                 OK
-+ block49018.json                                                 OK
-+ block97.json                                                    OK
+  block46147.json                                                 Skip
+  block46400.json                                                 Skip
+  block46402.json                                                 Skip
+  block47205.json                                                 Skip
+  block48712.json                                                 Skip
+  block48915.json                                                 Skip
+  block49018.json                                                 Skip
+  block97.json                                                    Skip
 ```
-OK: 8/8 Fail: 0/8 Skip: 0/8
+OK: 0/8 Fail: 0/8 Skip: 8/8
 
 ---TOTAL---
-OK: 8/8 Fail: 0/8 Skip: 0/8
+OK: 0/8 Fail: 0/8 Skip: 8/8

fluffy/common/common_types.nim

@@ -36,6 +36,12 @@ func decodeRlp*(input: openArray[byte], T: type): Result[T, string] =
   except RlpError as e:
     err(e.msg)
 
+func decodeRlpOrRaise*(input: openArray[byte], T: type): T =
+  try:
+    rlp.decode(input, T)
+  except RlpError as e:
+    raiseAssert(e.msg)
+
 func decodeSsz*(input: openArray[byte], T: type): Result[T, string] =
   try:
     ok(SSZ.decode(input, T))

fluffy/conf.nim

@@ -99,7 +99,8 @@ type
 
     portalSubnetworks* {.
       desc: "Select which networks (Portal sub-protocols) to enable",
-      defaultValue: {PortalSubnetwork.history, PortalSubnetwork.state},
+      defaultValue:
+        {PortalSubnetwork.history, PortalSubnetwork.state, PortalSubnetwork.beacon},
       name: "portal-subnetworks"
     .}: set[PortalSubnetwork]
 
@@ -258,7 +259,14 @@ type
       desc: "The maximum number of nodes to send content to during gossip",
       defaultValue: defaultPortalProtocolConfig.maxGossipNodes,
       defaultValueDesc: $defaultMaxGossipNodesDesc,
-      name: "max-gossip-nodes"
+      name: "debug-max-gossip-nodes"
+    .}: int
+
+    maxConcurrentOffers* {.
+      hidden,
+      desc: "The maximum number of offers to send concurrently",
+      defaultValue: defaultPortalProtocolConfig.maxConcurrentOffers,
+      name: "debug-max-concurrent-offers"
     .}: int
 
     radiusConfig* {.
@@ -315,14 +323,14 @@ type
         "Size of the in memory local content cache. This is the max number " &
         "of content values that can be stored in the cache.",
       defaultValue: defaultPortalProtocolConfig.contentCacheSize,
-      name: "content-cache-size"
+      name: "debug-content-cache-size"
     .}: int
 
     disableContentCache* {.
       hidden,
       desc: "Disable the in memory local content cache",
       defaultValue: defaultPortalProtocolConfig.disableContentCache,
-      name: "disable-content-cache"
+      name: "debug-disable-content-cache"
     .}: bool
 
     disablePoke* {.

fluffy/database/content_db.nim

@@ -285,45 +285,21 @@ proc close*(db: ContentDB) =
   db.largestDistanceStmt.disposeSafe()
   discard db.kv.close()
 
-## Private KvStoreRef Calls
-
-proc get(kv: KvStoreRef, key: openArray[byte]): Opt[seq[byte]] =
-  var res: Opt[seq[byte]]
-  proc onData(data: openArray[byte]) =
-    res = Opt.some(@data)
-
-  discard kv.get(key, onData).expectDb()
-
-  return res
-
-proc getSszDecoded(kv: KvStoreRef, key: openArray[byte], T: type auto): Opt[T] =
-  let res = kv.get(key)
-  if res.isSome():
-    try:
-      Opt.some(SSZ.decode(res.get(), T))
-    except SerializationError:
-      raiseAssert("Stored data should always be serialized correctly")
-  else:
-    Opt.none(T)
-
 ## Private ContentDB calls
 
-proc get(db: ContentDB, key: openArray[byte]): Opt[seq[byte]] =
-  db.kv.get(key)
+template get(db: ContentDB, key: openArray[byte], onData: DataProc): bool =
+  db.kv.get(key, onData).expectDb()
 
-proc put(db: ContentDB, key, value: openArray[byte]) =
+template put(db: ContentDB, key, value: openArray[byte]) =
   db.kv.put(key, value).expectDb()
 
-proc contains(db: ContentDB, key: openArray[byte]): bool =
+template contains(db: ContentDB, key: openArray[byte]): bool =
   db.kv.contains(key).expectDb()
 
-proc del(db: ContentDB, key: openArray[byte]) =
+template del(db: ContentDB, key: openArray[byte]) =
   # TODO: Do we want to return the bool here too?
   discard db.kv.del(key).expectDb()
 
-proc getSszDecoded(db: ContentDB, key: openArray[byte], T: type auto): Opt[T] =
-  db.kv.getSszDecoded(key, T)
-
 ## Public ContentId based ContentDB calls
 
 # TODO: Could also decide to use the ContentKey SSZ bytestring, as this is what
@@ -334,9 +310,9 @@ proc getSszDecoded(db: ContentDB, key: openArray[byte], T: type auto): Opt[T] =
 # checked with the Radius/distance of the node anyhow. So lets see how we end up
 # using this mostly in the code.
 
-proc get*(db: ContentDB, key: ContentId): Opt[seq[byte]] =
+proc get*(db: ContentDB, key: ContentId, onData: DataProc): bool =
   # TODO: Here it is unfortunate that ContentId is a uint256 instead of Digest256.
-  db.get(key.toBytesBE())
+  db.get(key.toBytesBE(), onData)
 
 proc put*(db: ContentDB, key: ContentId, value: openArray[byte]) =
   db.put(key.toBytesBE(), value)
@@ -347,9 +323,6 @@ proc contains*(db: ContentDB, key: ContentId): bool =
 proc del*(db: ContentDB, key: ContentId) =
   db.del(key.toBytesBE())
 
-proc getSszDecoded*(db: ContentDB, key: ContentId, T: type auto): Opt[T] =
-  db.getSszDecoded(key.toBytesBE(), T)
-
 ## Pruning related calls
 
 proc deleteContentFraction*(
@@ -484,10 +457,15 @@ proc adjustRadius(
 proc createGetHandler*(db: ContentDB): DbGetHandler =
   return (
     proc(contentKey: ContentKeyByteList, contentId: ContentId): Opt[seq[byte]] =
-      let content = db.get(contentId).valueOr:
-        return Opt.none(seq[byte])
+      var res: seq[byte]
+
+      proc onData(data: openArray[byte]) =
+        res = @data
 
-      ok(content)
+      if db.get(contentId, onData):
+        Opt.some(res)
+      else:
+        Opt.none(seq[byte])
   )
 
 proc createStoreHandler*(db: ContentDB, cfg: RadiusConfig): DbStoreHandler =
@@ -520,6 +498,12 @@ proc createStoreHandler*(db: ContentDB, cfg: RadiusConfig): DbStoreHandler =
         db.put(contentId, content)
   )
 
+proc createContainsHandler*(db: ContentDB): DbContainsHandler =
+  return (
+    proc(contentKey: ContentKeyByteList, contentId: ContentId): bool =
+      db.contains(contentId)
+  )
+
 proc createRadiusHandler*(db: ContentDB): DbRadiusHandler =
   return (
     proc(): UInt256 {.raises: [], gcsafe.} =

fluffy/database/content_db_custom_sql_functions.nim

@@ -13,7 +13,7 @@ func xorDistance(a: openArray[byte], b: openArray[byte]): seq[byte] =
   doAssert(a.len == b.len)
 
   let length = a.len
-  var distance: seq[byte] = newSeq[byte](length)
+  var distance: seq[byte] = newSeqUninitialized[byte](length)
   for i in 0 ..< length:
     distance[i] = a[i] xor b[i]
 

fluffy/docs/the_fluffy_book/docs/index.md

@@ -18,11 +18,39 @@ To quickly get your Fluffy node up and running, follow the quickstart page:
   - [Quickstart for Windows users](./quick-start-windows.md)
   - [Quickstart for Docker users](./quick-start-docker.md)
 
-# Development status
-The Portal Network is a project still in research phase.
-This client is thus still experimental.
+## Development status
+The Portal Network is a project still in research phase. This client is thus still experimental.
 
-However, the Portal history, beacon and state sub-networks are already operational and can be tested on the public network or in a local testnet.
+The development of this client is on par with the latest Portal specifications and will continue to evolve according to the Portal specifications.
+
+The Portal history, beacon and state sub-networks are already operational on the public Portal mainnet.
+
+Fluffy is default ran on the [Portal mainnet](https://github.com/ethereum/portal-network-specs/blob/master/bootnodes.md#bootnodes-mainnet) but can also be run on a (local) testnet.
+
+### Supported sub-networks and content:
+
+- [History network](https://github.com/ethereum/portal-network-specs/blob/e8e428c55f34893becfe936fe323608e9937956e/history/history-network.md): headers, blocks, and receipts.
+    - Note: Canonical verification is currently only enabled for pre-merge blocks.
+- [State network](https://github.com/ethereum/portal-network-specs/blob/e8e428c55f34893becfe936fe323608e9937956e/state/state-network.md): accounts and contract storage.
+    - Note: The Portal mainnet does not yet hold all states, nor the recent state.
+- [Beacon network](https://github.com/ethereum/portal-network-specs/blob/e8e428c55f34893becfe936fe323608e9937956e/beacon-chain/beacon-network.md): consensus light client data and historical summaries.
+
+### Supported functionality:
+
+- [Portal JSON-RPC API](https://github.com/ethereum/portal-network-specs/tree/e8e428c55f34893becfe936fe323608e9937956e/jsonrpc)
+- [Consensus light client sync](https://github.com/ethereum/consensus-specs/blob/a09d0c321550c5411557674a981e2b444a1178c0/specs/altair/light-client/light-client.md) through content available on the Portal beacon network.
+- Partial support of [Execution JSON-RPC API](https://github.com/ethereum/execution-apis):
+    - web3_clientVersion
+    - eth_chainId
+    - eth_getBalance
+    - eth_getBlockByHash
+    - eth_getBlockByNumber
+    - eth_getBlockTransactionCountByHash
+    - eth_getCode
+    - eth_getLogs (partial support: queries by blockhash)
+    - eth_getProof
+    - eth_getStorageAt
+    - eth_getTransactionCount
 
 ## Get in touch
 

fluffy/eth_data/era1.nim

@@ -13,7 +13,7 @@ import
   stew/[endians2, io2, byteutils, arrayops],
   stint,
   snappy,
-  eth/common/[headers, blocks_rlp, receipts_rlp],
+  eth/common/[headers_rlp, blocks_rlp, receipts_rlp],
   beacon_chain/spec/beacon_time,
   ssz_serialization,
   ncli/e2store,
@@ -184,7 +184,7 @@ proc fromCompressedRlpBytes(bytes: openArray[byte], T: type): Result[T, string]
   try:
     ok(rlp.decode(decodeFramed(bytes, checkIntegrity = false), T))
   except RlpError as e:
-    err("Invalid Compressed RLP data" & e.msg)
+    err("Invalid compressed RLP data for " & $T & ": " & e.msg)
 
 proc init*(T: type Era1Group, f: IoHandle, startNumber: uint64): Result[T, string] =
   discard ?f.appendHeader(E2Version, 0)
@@ -498,7 +498,7 @@ iterator era1BlockHeaders*(f: Era1File): headers.Header =
 
   for blockNumber in startNumber .. endNumber:
     let header = f.getBlockHeader(blockNumber).valueOr:
-      raiseAssert("Failed to read block header")
+      raiseAssert("Failed to read block header: " & error)
     yield header
 
 iterator era1BlockTuples*(f: Era1File): BlockTuple =
@@ -508,5 +508,5 @@ iterator era1BlockTuples*(f: Era1File): BlockTuple =
 
   for blockNumber in startNumber .. endNumber:
     let blockTuple = f.getBlockTuple(blockNumber).valueOr:
-      raiseAssert("Failed to read block header")
+      raiseAssert("Failed to read block tuple: " & error)
     yield blockTuple