diff --git a/node/src/command.rs b/node/src/command.rs index 8472f50cf..35125749b 100644 --- a/node/src/command.rs +++ b/node/src/command.rs @@ -115,7 +115,7 @@ pub fn run() -> Result<()> { ); } cmd.run::(config) diff --git a/node/src/service.rs b/node/src/service.rs index 2401676aa..ee2e75d20 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -52,7 +52,7 @@ pub struct ExecutorDispatch; impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { type ExtendHostFunctions = ( frame_benchmarking::benchmarking::HostFunctions, - frame_system::header_builder::hosted_header_builder::HostFunctions, + frame_system::native::hosted_header_builder::hosted_header_builder::HostFunctions, avail_base::mem_tmp_storage::hosted_mem_tmp_storage::HostFunctions, da_runtime::kate::native::hosted_kate::HostFunctions, ); diff --git a/pallets/dactr/src/benchmarking.rs b/pallets/dactr/src/benchmarking.rs index 32571371b..976ef0370 100644 --- a/pallets/dactr/src/benchmarking.rs +++ b/pallets/dactr/src/benchmarking.rs @@ -12,7 +12,9 @@ use frame_benchmarking::{ impl_benchmark_test_suite, v1::BenchmarkError, v2::*, whitelisted_caller, }; use frame_support::traits::Get; -use frame_system::{header_builder::hosted_header_builder, limits::BlockLength, RawOrigin}; +use frame_system::{ + limits::BlockLength, native::hosted_header_builder::hosted_header_builder, RawOrigin, +}; use scale_info::{StaticTypeInfo, TypeInfo}; use sp_core::H256; use sp_runtime::{ diff --git a/pallets/dactr/src/extensions/extensions_mock.rs b/pallets/dactr/src/extensions/extensions_mock.rs index 009724c6c..53e18bdb4 100644 --- a/pallets/dactr/src/extensions/extensions_mock.rs +++ b/pallets/dactr/src/extensions/extensions_mock.rs @@ -3,7 +3,7 @@ use frame_support::weights::WeightToFee; use frame_support::{derive_impl, weights::IdentityFee}; use frame_system::{ - header_builder::da::HeaderExtensionBuilder, mocking::MockUncheckedExtrinsic, + mocking::MockUncheckedExtrinsic, native::hosted_header_builder::da::HeaderExtensionBuilder, test_utils::TestRandomness, }; use pallet_transaction_payment::CurrencyAdapter; diff --git a/pallets/dactr/src/mock.rs b/pallets/dactr/src/mock.rs index 7ec844291..111994f0f 100644 --- a/pallets/dactr/src/mock.rs +++ b/pallets/dactr/src/mock.rs @@ -6,7 +6,7 @@ use frame_support::{ weights::{ConstantMultiplier, IdentityFee}, }; use frame_system::{ - header_builder::da::HeaderExtensionBuilder, mocking::MockUncheckedExtrinsic, + mocking::MockUncheckedExtrinsic, native::hosted_header_builder::da::HeaderExtensionBuilder, test_utils::TestRandomness, }; use pallet_transaction_payment::CurrencyAdapter; diff --git a/pallets/mandate/src/mock.rs b/pallets/mandate/src/mock.rs index 145799843..36a6b5e93 100644 --- a/pallets/mandate/src/mock.rs +++ b/pallets/mandate/src/mock.rs @@ -31,7 +31,8 @@ parameter_types! { impl frame_system::Config for Test { type Block = Block; type BlockHashCount = BlockHashCount; - type HeaderExtensionBuilder = frame_system::header_builder::da::HeaderExtensionBuilder; + type HeaderExtensionBuilder = + frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder; type OnSetCode = (); type PalletInfo = PalletInfo; type Randomness = frame_system::test_utils::TestRandomness; diff --git a/pallets/mocked_runtime/src/lib.rs b/pallets/mocked_runtime/src/lib.rs index d19d57579..6f490941c 100644 --- a/pallets/mocked_runtime/src/lib.rs +++ b/pallets/mocked_runtime/src/lib.rs @@ -139,7 +139,8 @@ impl frame_system::Config for Runtime { type DbWeight = (); type Hash = sp_core::H256; type Hashing = BlakeTwo256; - type HeaderExtensionBuilder = frame_system::header_builder::da::HeaderExtensionBuilder; + type HeaderExtensionBuilder = + frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder; type Nonce = u64; type Lookup = IdentityLookup; type MaxConsumers = ConstU32<16>; diff --git a/pallets/system/benches/bench.rs b/pallets/system/benches/bench.rs index 3c51d7757..16363793f 100644 --- a/pallets/system/benches/bench.rs +++ b/pallets/system/benches/bench.rs @@ -21,9 +21,9 @@ use frame_support::{ weights::Weight, }; use frame_system::{ - header_builder::da::HeaderExtensionBuilder, limits::{BlockLength, BlockWeights}, mocking::{MockDaBlock, MockUncheckedExtrinsic}, + native::hosted_header_builder::da::HeaderExtensionBuilder, test_utils::TestRandomness, }; use sp_core::H256; diff --git a/pallets/system/benchmarking/src/lib.rs b/pallets/system/benchmarking/src/lib.rs index 8e0294fc0..4bb5d3f5e 100644 --- a/pallets/system/benchmarking/src/lib.rs +++ b/pallets/system/benchmarking/src/lib.rs @@ -23,7 +23,9 @@ use codec::Encode; use frame_benchmarking::{impl_benchmark_test_suite, v2::*}; use frame_support::{dispatch::DispatchClass, storage, traits::Get}; -use frame_system::{header_builder::HeaderExtensionBuilder, Call, Pallet as System, RawOrigin}; +use frame_system::{ + native::hosted_header_builder::HeaderExtensionBuilder, Call, Pallet as System, RawOrigin, +}; use sp_core::storage::well_known_keys; use sp_runtime::traits::Hash; use sp_std::{prelude::*, vec}; diff --git a/pallets/system/benchmarking/src/mock.rs b/pallets/system/benchmarking/src/mock.rs index f13c89956..1fc9e3986 100644 --- a/pallets/system/benchmarking/src/mock.rs +++ b/pallets/system/benchmarking/src/mock.rs @@ -22,7 +22,7 @@ use codec::Encode; use frame_support::{derive_impl, traits::ConstU32}; use frame_system::{ - header_builder::da::HeaderExtensionBuilder, mocking::MockUncheckedExtrinsic, + mocking::MockUncheckedExtrinsic, native::hosted_header_builder::da::HeaderExtensionBuilder, test_utils::TestRandomness, }; use sp_runtime::BuildStorage; diff --git a/pallets/system/src/lib.rs b/pallets/system/src/lib.rs index b26f641df..fbf4fc4a8 100644 --- a/pallets/system/src/lib.rs +++ b/pallets/system/src/lib.rs @@ -146,8 +146,8 @@ use sp_std::{fmt::Debug, marker::PhantomData, prelude::*}; use sp_version::RuntimeVersion; use sp_weights::{RuntimeDbWeight, Weight}; -pub mod header_builder; -pub use header_builder::HeaderExtensionBuilder; +pub mod native; +pub use native::hosted_header_builder::HeaderExtensionBuilder; pub mod limits; #[cfg(any(feature = "std", test))] @@ -527,7 +527,7 @@ pub mod pallet { /// Header builder #[pallet::no_default] - type HeaderExtensionBuilder: header_builder::HeaderExtensionBuilder; + type HeaderExtensionBuilder: native::hosted_header_builder::HeaderExtensionBuilder; /// Source of random seeds. #[pallet::no_default] @@ -1862,7 +1862,7 @@ impl Pallet { let block_length = Self::block_length(); - let extension = header_builder::da::HeaderExtensionBuilder::::build( + let extension = native::hosted_header_builder::da::HeaderExtensionBuilder::::build( header_extension_builder_data.to_app_extrinsics(), header_extension_builder_data.data_root(), block_length, diff --git a/pallets/system/src/mock.rs b/pallets/system/src/mock.rs index b472b157a..a91392def 100644 --- a/pallets/system/src/mock.rs +++ b/pallets/system/src/mock.rs @@ -112,8 +112,8 @@ impl Config for Test { type OnSetCode = (); type MaxConsumers = ConstU32<16>; type AccountData = u32; - - type HeaderExtensionBuilder = frame_system::header_builder::da::HeaderExtensionBuilder; + type HeaderExtensionBuilder = + frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder; type Randomness = TestRandomness; type Header = Header; type HeaderExtensionDataFilter = (); diff --git a/pallets/system/src/mocking.rs b/pallets/system/src/mocking.rs index 8dfd5b842..8e802222d 100644 --- a/pallets/system/src/mocking.rs +++ b/pallets/system/src/mocking.rs @@ -30,7 +30,7 @@ use sp_runtime::{ transaction_validity::TransactionValidityError, }; -use crate::{header_builder::da::BlockNumber, Config}; +use crate::{native::hosted_header_builder::da::BlockNumber, Config}; #[derive(Clone, Copy, Default, Debug, Encode, Decode, PartialEq, Eq, TypeInfo)] pub struct DefaultGetAppId {} diff --git a/pallets/system/src/native/build_extension_v1.rs b/pallets/system/src/native/build_extension_v1.rs new file mode 100644 index 000000000..88b953702 --- /dev/null +++ b/pallets/system/src/native/build_extension_v1.rs @@ -0,0 +1,128 @@ +// !!!! +// If the logic is changed in this file it will break Turing. Do not change it. +// If the logic is changed in avail-core it will break Turing as well. Do no change it. +// !!!! +#![cfg(feature = "std")] + +use super::hosted_header_builder::MIN_WIDTH; +use crate::limits::BlockLength; +use avail_base::metrics::avail::{ + HeaderExtensionBuilderMetrics as Metrics, MetricObserver, ObserveKind, +}; +use avail_core::{ + app_extrinsic::AppExtrinsic, + header::{extension as he, HeaderExtension}, + kate_commitment as kc, HeaderVersion, +}; +use kate::{ + couscous::multiproof_params, + gridgen::{AsBytes, EvaluationGrid}, + pmp::m1_blst::M1NoPrecomp, + Seed, +}; +use sp_core::H256; +use sp_runtime::SaturatedConversion; +use std::{sync::OnceLock, vec::Vec}; + +static PMP: OnceLock = OnceLock::new(); + +fn build_grid( + submitted: Vec, + block_length: BlockLength, + seed: Seed, +) -> Result { + let _metric_observer = MetricObserver::new(ObserveKind::HEGrid); + + let grid = EvaluationGrid::from_extrinsics( + submitted, + MIN_WIDTH, + block_length.cols.0.saturated_into(), // even if we run on a u16 target this is fine + block_length.rows.0.saturated_into(), + seed, + ) + .map_err(|e| format!("Grid construction failed: {e:?}"))?; + + Ok(grid) +} + +fn build_commitment(grid: &EvaluationGrid) -> Result, String> { + let _metric_observer = MetricObserver::new(ObserveKind::HECommitment); + + // couscous has pp for degree upto 1024 + let pmp = PMP.get_or_init(multiproof_params); + + let poly_grid = grid + .make_polynomial_grid() + .map_err(|e| format!("Make polynomial grid failed: {e:?}"))?; + + let extended_grid = poly_grid + .extended_commitments(pmp, 2) + .map_err(|e| format!("Grid extension failed: {e:?}"))?; + + let mut commitment = Vec::new(); + for c in extended_grid.iter() { + match c.to_bytes() { + Ok(bytes) => commitment.extend(bytes), + Err(e) => return Err(format!("Commitment serialization failed: {:?}", e)), + } + } + + Ok(commitment) +} + +pub fn build_extension( + submitted: Vec, + data_root: H256, + block_length: BlockLength, + _block_number: u32, + seed: Seed, + version: HeaderVersion, +) -> HeaderExtension { + let _metric_observer = MetricObserver::new(ObserveKind::HETotalExecutionTime); + + // Build the grid + let maybe_grid = build_grid(submitted, block_length, seed); + + // We get the grid or return an empty header in case of an error + let grid = match maybe_grid { + Ok(res) => res, + Err(message) => { + log::error!("NODE_CRITICAL_ERROR_001 - A critical error has occured: {message:?}."); + log::error!("NODE_CRITICAL_ERROR_001 - If you see this, please warn Avail team and raise an issue."); + return HeaderExtension::get_faulty_header(data_root, version); + }, + }; + + let maybe_commitment = build_commitment(&grid); + + // We get the commitment or return an empty header in case of an error + let commitment = match maybe_commitment { + Ok(res) => res, + Err(message) => { + log::error!("NODE_CRITICAL_ERROR_002 - A critical error has occured: {message:?}."); + log::error!("NODE_CRITICAL_ERROR_002 - If you see this, please warn Avail team and raise an issue."); + return HeaderExtension::get_faulty_header(data_root, version); + }, + }; + + // Note that this uses the original dims, _not the extended ones_ + let rows = grid.dims().rows().get(); + let cols = grid.dims().cols().get(); + + // Grid Metrics + Metrics::observe_grid_rows(rows as f64); + Metrics::observe_grid_cols(cols as f64); + + let app_lookup = grid.lookup().clone(); + + match version { + HeaderVersion::V3 => { + let commitment = kc::v3::KateCommitment::new(rows, cols, data_root, commitment); + he::v3::HeaderExtension { + app_lookup, + commitment, + } + .into() + }, + } +} diff --git a/pallets/system/src/header_builder/version_select.rs b/pallets/system/src/native/build_extension_v2.rs similarity index 65% rename from pallets/system/src/header_builder/version_select.rs rename to pallets/system/src/native/build_extension_v2.rs index e57b67152..6c163ff92 100644 --- a/pallets/system/src/header_builder/version_select.rs +++ b/pallets/system/src/native/build_extension_v2.rs @@ -1,6 +1,11 @@ -use super::MIN_WIDTH; -use crate::limits::BlockLength; +// !!!! +// If the logic is changed in this file it will break Turing/Mainnet. Do not change it. +// If the logic is changed in avail-core it will break Turing/Mainnet as well. Do no change it. +// !!!! +#![cfg(feature = "std")] +use super::hosted_header_builder::MIN_WIDTH; +use crate::limits::BlockLength; use avail_base::metrics::avail::{ HeaderExtensionBuilderMetrics as Metrics, MetricObserver, ObserveKind, }; @@ -15,15 +20,13 @@ use kate::{ pmp::m1_blst::M1NoPrecomp, Seed, }; - use sp_core::H256; use sp_runtime::SaturatedConversion; use std::{sync::OnceLock, vec::Vec}; static PMP: OnceLock = OnceLock::new(); -#[cfg(feature = "std")] -pub fn build_grid( +fn build_grid( submitted: Vec, block_length: BlockLength, seed: Seed, @@ -42,8 +45,7 @@ pub fn build_grid( Ok(grid) } -#[cfg(feature = "std")] -pub fn build_commitment(grid: &EvaluationGrid) -> Result, String> { +fn build_commitment(grid: &EvaluationGrid) -> Result, String> { let _metric_observer = MetricObserver::new(ObserveKind::HECommitment); // couscous has pp for degree upto 1024 @@ -75,63 +77,6 @@ pub fn build_extension( _block_number: u32, seed: Seed, version: HeaderVersion, -) -> HeaderExtension { - let _metric_observer = MetricObserver::new(ObserveKind::HETotalExecutionTime); - - // Build the grid - let maybe_grid = build_grid(submitted, block_length, seed); - - // We get the grid or return an empty header in case of an error - let grid = match maybe_grid { - Ok(res) => res, - Err(message) => { - log::error!("NODE_CRITICAL_ERROR_001 - A critical error has occured: {message:?}."); - log::error!("NODE_CRITICAL_ERROR_001 - If you see this, please warn Avail team and raise an issue."); - return HeaderExtension::get_faulty_header(data_root, version); - }, - }; - - let maybe_commitment = build_commitment(&grid); - - // We get the commitment or return an empty header in case of an error - let commitment = match maybe_commitment { - Ok(res) => res, - Err(message) => { - log::error!("NODE_CRITICAL_ERROR_002 - A critical error has occured: {message:?}."); - log::error!("NODE_CRITICAL_ERROR_002 - If you see this, please warn Avail team and raise an issue."); - return HeaderExtension::get_faulty_header(data_root, version); - }, - }; - - // Note that this uses the original dims, _not the extended ones_ - let rows = grid.dims().rows().get(); - let cols = grid.dims().cols().get(); - - // Grid Metrics - Metrics::observe_grid_rows(rows as f64); - Metrics::observe_grid_cols(cols as f64); - - let app_lookup = grid.lookup().clone(); - - match version { - HeaderVersion::V3 => { - let commitment = kc::v3::KateCommitment::new(rows, cols, data_root, commitment); - he::v3::HeaderExtension { - app_lookup, - commitment, - } - .into() - }, - } -} - -pub fn build_extension_2( - submitted: Vec, - data_root: H256, - block_length: BlockLength, - _block_number: u32, - seed: Seed, - version: HeaderVersion, ) -> HeaderExtension { // Blocks with non-DA extrinsics will have empty commitments if submitted.is_empty() { diff --git a/pallets/system/src/header_builder.rs b/pallets/system/src/native/hosted_header_builder.rs similarity index 92% rename from pallets/system/src/header_builder.rs rename to pallets/system/src/native/hosted_header_builder.rs index e1aed73b1..2bd78549e 100644 --- a/pallets/system/src/header_builder.rs +++ b/pallets/system/src/native/hosted_header_builder.rs @@ -1,3 +1,7 @@ +// !!!! +// More info about how runtime interfaces work: (https://docs.rs/sp-runtime-interface/latest/sp_runtime_interface/attr.runtime_interface.html +// !!!! + use crate::{limits::BlockLength, Config, LOG_TARGET}; #[cfg(feature = "std")] use avail_core::HeaderVersion; @@ -13,9 +17,6 @@ use sp_runtime::traits::Hash; use sp_runtime_interface::runtime_interface; use sp_std::vec::Vec; -#[cfg(feature = "std")] -mod version_select; - pub const MIN_WIDTH: usize = 4; pub mod da { @@ -98,7 +99,7 @@ pub trait HostedHeaderBuilder { block_number: u32, seed: Seed, ) -> HeaderExtension { - version_select::build_extension( + crate::native::build_extension_v1::build_extension( submitted, data_root, block_length, @@ -118,7 +119,7 @@ pub trait HostedHeaderBuilder { block_number: u32, seed: Seed, ) -> HeaderExtension { - version_select::build_extension_2( + crate::native::build_extension_v2::build_extension( submitted, data_root, block_length, diff --git a/pallets/system/src/native/mod.rs b/pallets/system/src/native/mod.rs new file mode 100644 index 000000000..c21108c9d --- /dev/null +++ b/pallets/system/src/native/mod.rs @@ -0,0 +1,6 @@ +#[cfg(feature = "std")] +pub mod build_extension_v1; +#[cfg(feature = "std")] +pub mod build_extension_v2; + +pub mod hosted_header_builder; diff --git a/pallets/vector/src/mock.rs b/pallets/vector/src/mock.rs index 2e8d47458..6a1c09eaa 100644 --- a/pallets/vector/src/mock.rs +++ b/pallets/vector/src/mock.rs @@ -1,5 +1,5 @@ use frame_support::{derive_impl, parameter_types, traits::ConstU64, PalletId}; -use frame_system::{header_builder::da, test_utils::TestRandomness}; +use frame_system::{native::hosted_header_builder::da, test_utils::TestRandomness}; use hex_literal::hex; use primitive_types::H256; use sp_runtime::{ diff --git a/runtime/benches/header_kate_commitment.rs b/runtime/benches/header_kate_commitment.rs index ed2d983f9..b2c800b7d 100644 --- a/runtime/benches/header_kate_commitment.rs +++ b/runtime/benches/header_kate_commitment.rs @@ -2,7 +2,7 @@ use avail_core::{AppExtrinsic, BlockLengthColumns, BLOCK_CHUNK_SIZE, DA_DISPATCH use da_control::Config as DAConfig; use da_runtime::Runtime; use frame_support::traits::Get as _; -use frame_system::{header_builder::hosted_header_builder, limits::BlockLength}; +use frame_system::{limits::BlockLength, native::hosted_header_builder::hosted_header_builder}; use sp_core::H256; use sp_std::iter::repeat; diff --git a/runtime/src/apis.rs b/runtime/src/apis.rs index 9e3870fea..0033d64c8 100644 --- a/runtime/src/apis.rs +++ b/runtime/src/apis.rs @@ -13,9 +13,7 @@ use avail_core::{ OpaqueExtrinsic, }; -use frame_system::{ - header_builder::da::HeaderExtensionBuilder, limits::BlockLength, HeaderExtensionBuilder as _, -}; +use frame_system::limits::BlockLength; use frame_support::{ genesis_builder_helper::{build_config, create_default_config}, @@ -364,6 +362,9 @@ impl_runtime_apis! { block_length: BlockLength, block_number: u32, ) -> HeaderExtension { + use frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder; + use frame_system::HeaderExtensionBuilder as _; + let app_extrinsics = HeaderExtensionBuilderData::from_opaque_extrinsics::(block_number, &extrinsics).to_app_extrinsics(); HeaderExtensionBuilder::::build(app_extrinsics, data_root, block_length, block_number) } diff --git a/runtime/src/impls.rs b/runtime/src/impls.rs index 20db2aa7d..d459cc1eb 100644 --- a/runtime/src/impls.rs +++ b/runtime/src/impls.rs @@ -817,7 +817,8 @@ impl frame_system::Config for Runtime { /// The hashing algorithm used. type Hashing = BlakeTwo256; /// The header builder type. - type HeaderExtensionBuilder = frame_system::header_builder::da::HeaderExtensionBuilder; + type HeaderExtensionBuilder = + frame_system::native::hosted_header_builder::da::HeaderExtensionBuilder; /// The lookup mechanism to get account ID from whatever is passed in dispatchers. type Lookup = Indices; type MaxConsumers = constants::system::MaxConsumers; diff --git a/runtime/src/impls_tests.rs b/runtime/src/impls_tests.rs index c4916ac1a..76b37534d 100644 --- a/runtime/src/impls_tests.rs +++ b/runtime/src/impls_tests.rs @@ -426,7 +426,7 @@ mod tests { PalletId, }; use frame_system::{ - header_builder::da::HeaderExtensionBuilder, mocking::MockUncheckedExtrinsic, + mocking::MockUncheckedExtrinsic, native::hosted_header_builder::da::HeaderExtensionBuilder, test_utils::TestRandomness, }; use sp_runtime::{traits::IdentityLookup, BuildStorage, Perquintill}; diff --git a/runtime/src/kate/native.rs b/runtime/src/kate/native.rs index 037cec868..afe3e099c 100644 --- a/runtime/src/kate/native.rs +++ b/runtime/src/kate/native.rs @@ -1,7 +1,7 @@ use super::{Error, GDataProof, GProof, GRawScalar, GRow}; use avail_core::{AppExtrinsic, AppId, BlockLengthColumns, BlockLengthRows}; use core::num::NonZeroU16; -use frame_system::{header_builder::MIN_WIDTH, limits::BlockLength}; +use frame_system::{limits::BlockLength, native::hosted_header_builder::MIN_WIDTH}; use kate::Seed; #[cfg(feature = "std")] use kate::{