Merge branch 'dev' into pr3255

ethereum · Mar 10, 2023 · baaab93 · baaab93
2 parents c7ac9cc + 3a23049
commit baaab93
Show file tree

Hide file tree

Showing 27 changed files with 1,508 additions and 253 deletions.
diff --git a/Makefile b/Makefile
@@ -23,16 +23,18 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
 
-MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) \
-                 $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SPEC_DIR)/altair/**/*.md) \
-                 $(wildcard $(SPEC_DIR)/bellatrix/*.md) \
-                 $(wildcard $(SPEC_DIR)/capella/*.md) $(wildcard $(SPEC_DIR)/capella/**/*.md) \
-                 $(wildcard $(SPEC_DIR)/deneb/*.md) $(wildcard $(SPEC_DIR)/deneb/**/*.md) \
-                 $(wildcard $(SPEC_DIR)/_features/custody/*.md) \
-                 $(wildcard $(SPEC_DIR)/_features/das/*.md) \
-                 $(wildcard $(SPEC_DIR)/_features/sharding/*.md) \
+MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
+                 $(wildcard $(SPEC_DIR)/*/*/*.md) \
+                 $(wildcard $(SPEC_DIR)/_features/*/*.md) \
+                 $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
+ALL_EXECUTABLE_SPECS = phase0 altair bellatrix capella deneb
+# The parameters for commands. Use `foreach` to avoid listing specs again.
+COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
+PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), ./eth2spec/$S)
+MYPY_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), -p eth2spec.$S)
+
 COV_HTML_OUT=.htmlcov
 COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT)
 COV_INDEX_FILE=$(COV_HTML_OUT_DIR)/index.html
@@ -63,15 +65,14 @@ partial_clean:
 	rm -f .coverage
 	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache
-	rm -rf $(ETH2SPEC_MODULE_DIR)/phase0
-	rm -rf $(ETH2SPEC_MODULE_DIR)/altair
-	rm -rf $(ETH2SPEC_MODULE_DIR)/bellatrix
-	rm -rf $(ETH2SPEC_MODULE_DIR)/capella
-	rm -rf $(ETH2SPEC_MODULE_DIR)/deneb
 	rm -rf $(COV_HTML_OUT_DIR)
 	rm -rf $(TEST_REPORT_DIR)
 	rm -rf eth2spec.egg-info dist build
-	rm -rf build
+	rm -rf build;
+	@for spec_name in $(ALL_EXECUTABLE_SPECS) ; do \
+		echo $$spec_name; \
+		rm -rf $(ETH2SPEC_MODULE_DIR)/$$spec_name; \
+	done
 
 clean: partial_clean
 	rm -rf venv
@@ -105,21 +106,21 @@ install_test:
 # Testing against `minimal` or `mainnet` config by default
 test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.deneb.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -n 4 --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 # Testing against `minimal` or `mainnet` config by default
 find_test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.deneb.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -k=$(K) --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: pyspec
 	mkdir -p $(TEST_REPORT_DIR);
 ifdef fork
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n 16 --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
 else
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 16 --bls-type=milagro  --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n 16 --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
 endif
 
 
@@ -137,13 +138,11 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 codespell:
 	codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist
 
-# TODO: add future protocol upgrade patch packages to linting.
-# NOTE: we use `pylint` just for catching unused arguments in spec code
 lint: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
-	&& pylint --rcfile $(LINTER_CONFIG_FILE) ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix ./eth2spec/capella ./eth2spec/deneb \
-	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix -p eth2spec.capella -p eth2spec.deneb
+	&& pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \
+	&& mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE)
 
 lint_generators: pyspec
 	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \

diff --git a/setup.py b/setup.py
@@ -653,9 +653,9 @@ def preparations(cls):
     @classmethod
     def sundry_functions(cls) -> str:
         return super().sundry_functions() + '\n\n' + '''
-def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> PyUnion[BlobsSidecar, str]:
+def retrieve_blobs_and_proofs(beacon_block_root: Root) -> PyUnion[Tuple[Blob, KZGProof], Tuple[str, str]]:
     # pylint: disable=unused-argument
-    return "TEST"'''
+    return ("TEST", "TEST")'''
 
     @classmethod
     def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
@@ -1174,5 +1174,6 @@ def run(self):
         RUAMEL_YAML_VERSION,
         "lru-dict==1.1.8",
         MARKO_VERSION,
+        "py_arkworks_bls12381==0.3.4",
     ]
 )
diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md
@@ -11,6 +11,7 @@
 - [Introduction](#introduction)
 - [Custom types](#custom-types)
 - [Constants](#constants)
+  - [Domain types](#domain-types)
   - [Blob](#blob)
 - [Preset](#preset)
   - [Execution](#execution)
@@ -44,15 +45,22 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi
 | Name | SSZ equivalent | Description |
 | - | - | - |
 | `VersionedHash` | `Bytes32` | |
+| `BlobIndex` | `uint64` | |
 
 ## Constants
 
+### Domain types
+
+| Name | Value |
+| - | - |
+| `DOMAIN_BLOB_SIDECAR` | `DomainType('0x0B000000')` |
+
 ### Blob
 
 | Name | Value |
 | - | - |
 | `BLOB_TX_TYPE` | `uint8(0x05)` |
-| `VERSIONED_HASH_VERSION_KZG` | `Bytes1('0x01')` | 
+| `VERSIONED_HASH_VERSION_KZG` | `Bytes1('0x01')` |
 
 ## Preset
 
@@ -249,7 +257,7 @@ def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody) -> N
 
 *Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Deneb testing only.
 
-The `BeaconState` initialization is unchanged, except for the use of the updated `deneb.BeaconBlockBody` type 
+The `BeaconState` initialization is unchanged, except for the use of the updated `deneb.BeaconBlockBody` type
 when initializing the first body-root.
 
 ```python

diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md
@@ -7,9 +7,8 @@
 
 - [Introduction](#introduction)
 - [Containers](#containers)
-  - [`BlobsSidecar`](#blobssidecar)
 - [Helpers](#helpers)
-    - [`validate_blobs_sidecar`](#validate_blobs_sidecar)
+    - [`validate_blobs`](#validate_blobs)
     - [`is_data_available`](#is_data_available)
 - [Updated fork-choice handlers](#updated-fork-choice-handlers)
   - [`on_block`](#on_block)
@@ -23,63 +22,48 @@ This is the modification of the fork choice accompanying the Deneb upgrade.
 
 ## Containers
 
-### `BlobsSidecar`
-
-```python
-class BlobsSidecar(Container):
-    beacon_block_root: Root
-    beacon_block_slot: Slot
-    blobs: List[Blob, MAX_BLOBS_PER_BLOCK]
-    kzg_aggregated_proof: KZGProof
-```
-
 ## Helpers
 
-#### `validate_blobs_sidecar`
+#### `validate_blobs`
 
 ```python
-def validate_blobs_sidecar(slot: Slot,
-                           beacon_block_root: Root,
-                           expected_kzg_commitments: Sequence[KZGCommitment],
-                           blobs_sidecar: BlobsSidecar) -> None:
-    assert slot == blobs_sidecar.beacon_block_slot
-    assert beacon_block_root == blobs_sidecar.beacon_block_root
-    blobs = blobs_sidecar.blobs
-    # kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof
+def validate_blobs(expected_kzg_commitments: Sequence[KZGCommitment],
+                   blobs: Sequence[Blob],
+                   proofs: Sequence[KZGProof]) -> None:
     assert len(expected_kzg_commitments) == len(blobs)
+    assert len(blobs) == len(proofs)
 
-    # Disabled because not available before switch to single blob sidecars
-    # assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof)
+    assert verify_blob_kzg_proof_batch(blobs, expected_kzg_commitments, proofs)
 ```
 
 #### `is_data_available`
 
 The implementation of `is_data_available` will become more sophisticated during later scaling upgrades.
-Initially, verification requires every verifying actor to retrieve the matching `BlobsSidecar`,
-and validate the sidecar with `validate_blobs_sidecar`.
+Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `validate_blobs`.
 
-The block MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded. Blocks that have been previously validated as available SHOULD be considered available even if the associated `BlobsSidecar` has subsequently been pruned.
+The block MUST NOT be considered valid until all valid `Blob`s have been downloaded. Blocks that have been previously validated as available SHOULD be considered available even if the associated `Blob`s have subsequently been pruned.
 
 ```python
-def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool:
-    # `retrieve_blobs_sidecar` is implementation and context dependent, raises an exception if not available.
-    # Note: the p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS`
-    sidecar = retrieve_blobs_sidecar(slot, beacon_block_root)
-
-    # For testing, `retrieve_blobs_sidecar` returns "TEST".
-    # TODO: Remove it once we have a way to inject `BlobsSidecar` into tests.
-    if isinstance(sidecar, str):
+def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool:
+    # `retrieve_blobs_and_proofs` is implementation and context dependent
+    # It returns all the blobs for the given block root, and raises an exception if not available
+    # Note: the p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS`
+    blobs, proofs = retrieve_blobs_and_proofs(beacon_block_root)
+
+    # For testing, `retrieve_blobs_and_proofs` returns ("TEST", "TEST").
+    # TODO: Remove it once we have a way to inject `BlobSidecar` into tests.
+    if isinstance(blobs, str) or isinstance(proofs, str):
         return True
 
-    validate_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar)
+    validate_blobs(blob_kzg_commitments, blobs, proofs)
     return True
 ```
 
 ## Updated fork-choice handlers
 
 ### `on_block`
 
-*Note*: The only modification is the addition of the verification of transition block conditions.
+*Note*: The only modification is the addition of the blob data availability check.
 
 ```python
 def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
@@ -103,7 +87,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     # [New in Deneb]
     # Check if blob data is available
     # If not, this block MAY be queued and subsequently considered when blob data becomes available
-    assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments) 
+    assert is_data_available(hash_tree_root(block), block.body.blob_kzg_commitments)
 
     # Check the block is valid and compute the post-state
     state = pre_state.copy()

diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md
@@ -64,8 +64,6 @@ Note that for the pure Deneb networks, we don't apply `upgrade_to_deneb` since i
 
 ### Upgrading the state
 
-Since the `deneb.BeaconState` format is equal to the `capella.BeaconState` format, we only have to update `BeaconState.fork`.
-
 ```python
 def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState:
     epoch = capella.get_current_epoch(pre)
@@ -82,10 +80,10 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState:
         timestamp=pre.latest_execution_payload_header.timestamp,
         extra_data=pre.latest_execution_payload_header.extra_data,
         base_fee_per_gas=pre.latest_execution_payload_header.base_fee_per_gas,
-        excess_data_gas=uint256(0),  # [New in Deneb]
         block_hash=pre.latest_execution_payload_header.block_hash,
         transactions_root=pre.latest_execution_payload_header.transactions_root,
         withdrawals_root=pre.latest_execution_payload_header.withdrawals_root,
+        excess_data_gas=uint256(0),  # [New in Deneb]
     )
     post = BeaconState(
         # Versioning