From 45fe2423dcdb035e9d25d306c51493c7d0f87326 Mon Sep 17 00:00:00 2001 From: Sebastian Kunert Date: Tue, 17 Dec 2024 21:44:01 +0100 Subject: [PATCH] BACKPORT-CONFLICT --- Cargo.lock | 93 ++++++++ cumulus/polkadot-omni-node/lib/Cargo.toml | 127 +++++++++++ .../lib/src/common/runtime.rs | 213 ++++++++++++++++++ prdoc/pr_6923.prdoc | 12 + 4 files changed, 445 insertions(+) create mode 100644 cumulus/polkadot-omni-node/lib/Cargo.toml create mode 100644 cumulus/polkadot-omni-node/lib/src/common/runtime.rs create mode 100644 prdoc/pr_6923.prdoc diff --git a/Cargo.lock b/Cargo.lock index d98e9e2fc95a..0715e6972678 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13871,6 +13871,99 @@ dependencies = [ ] [[package]] +<<<<<<< HEAD +======= +name = "polkadot-omni-node" +version = "0.1.0" +dependencies = [ + "color-eyre", + "polkadot-omni-node-lib", + "substrate-build-script-utils", +] + +[[package]] +name = "polkadot-omni-node-lib" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "async-trait", + "clap 4.5.13", + "color-print", + "cumulus-client-cli", + "cumulus-client-collator", + "cumulus-client-consensus-aura", + "cumulus-client-consensus-common", + "cumulus-client-consensus-proposer", + "cumulus-client-consensus-relay-chain", + "cumulus-client-parachain-inherent", + "cumulus-client-service", + "cumulus-primitives-aura 0.7.0", + "cumulus-primitives-core 0.7.0", + "cumulus-relay-chain-interface", + "cumulus-test-runtime", + "docify", + "frame-benchmarking 28.0.0", + "frame-benchmarking-cli", + "frame-support 28.0.0", + "frame-system-rpc-runtime-api 26.0.0", + "frame-try-runtime 0.34.0", + "futures", + "futures-timer", + "jsonrpsee", + "log", + "nix 0.29.0", + "pallet-transaction-payment 28.0.0", + "pallet-transaction-payment-rpc", + "pallet-transaction-payment-rpc-runtime-api 28.0.0", + "parachains-common 7.0.0", + "parity-scale-codec", + "polkadot-cli", + "polkadot-primitives 7.0.0", + "sc-basic-authorship", + "sc-chain-spec", + "sc-cli", + "sc-client-api", + "sc-client-db", + "sc-consensus", + "sc-consensus-manual-seal", + "sc-executor 0.32.0", + "sc-network", + "sc-rpc", + "sc-runtime-utilities", + "sc-service", + "sc-sysinfo", + "sc-telemetry", + "sc-tracing", + "sc-transaction-pool", + "scale-info", + "serde", + "serde_json", + "sp-api 26.0.0", + "sp-block-builder 26.0.0", + "sp-consensus", + "sp-consensus-aura 0.32.0", + "sp-core 28.0.0", + "sp-crypto-hashing 0.1.0", + "sp-genesis-builder 0.8.0", + "sp-inherents 26.0.0", + "sp-keystore 0.34.0", + "sp-runtime 31.0.1", + "sp-session 27.0.0", + "sp-storage 19.0.0", + "sp-timestamp 26.0.0", + "sp-transaction-pool 26.0.0", + "sp-version 29.0.0", + "sp-weights 27.0.0", + "substrate-frame-rpc-system", + "substrate-prometheus-endpoint", + "substrate-state-trie-migration-rpc", + "subxt-metadata", + "tokio", + "wait-timeout", +] + +[[package]] +>>>>>>> e6ddd392 (omni-node: Tolerate failing metadata check (#6923)) name = "polkadot-overseer" version = "18.0.0" dependencies = [ diff --git a/cumulus/polkadot-omni-node/lib/Cargo.toml b/cumulus/polkadot-omni-node/lib/Cargo.toml new file mode 100644 index 000000000000..b1937427be66 --- /dev/null +++ b/cumulus/polkadot-omni-node/lib/Cargo.toml @@ -0,0 +1,127 @@ +[package] +name = "polkadot-omni-node-lib" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +description = "Helper library that can be used to build a parachain node" +license = "Apache-2.0" +homepage.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +async-trait = { workspace = true } +clap = { features = ["derive"], workspace = true } +codec = { workspace = true, default-features = true } +color-print = { workspace = true } +futures = { workspace = true } +log = { workspace = true, default-features = true } +serde = { features = ["derive"], workspace = true, default-features = true } +serde_json = { workspace = true, default-features = true } +docify = { workspace = true } + +# Local +jsonrpsee = { features = ["server"], workspace = true } +parachains-common = { workspace = true, default-features = true } +scale-info = { workspace = true } +subxt-metadata = { workspace = true, default-features = true } + +# Substrate +frame-benchmarking = { optional = true, workspace = true, default-features = true } +frame-benchmarking-cli = { workspace = true, default-features = true } +sp-crypto-hashing = { workspace = true } +sp-runtime = { workspace = true } +sp-core = { workspace = true, default-features = true } +sp-session = { workspace = true, default-features = true } +frame-try-runtime = { optional = true, workspace = true, default-features = true } +sc-consensus = { workspace = true, default-features = true } +frame-support = { optional = true, workspace = true, default-features = true } +sc-cli = { workspace = true, default-features = true } +sc-client-api = { workspace = true, default-features = true } +sc-client-db = { workspace = true, default-features = true } +sc-executor = { workspace = true, default-features = true } +sc-service = { workspace = true, default-features = true } +sc-telemetry = { workspace = true, default-features = true } +sc-transaction-pool = { workspace = true, default-features = true } +sp-transaction-pool = { workspace = true, default-features = true } +sc-network = { workspace = true, default-features = true } +sc-basic-authorship = { workspace = true, default-features = true } +sp-timestamp = { workspace = true, default-features = true } +sp-genesis-builder = { workspace = true } +sp-block-builder = { workspace = true, default-features = true } +sp-keystore = { workspace = true, default-features = true } +sc-chain-spec = { workspace = true, default-features = true } +sc-rpc = { workspace = true, default-features = true } +sp-version = { workspace = true, default-features = true } +sp-weights = { workspace = true, default-features = true } +sc-tracing = { workspace = true, default-features = true } +sc-runtime-utilities = { workspace = true, default-features = true } +sp-storage = { workspace = true, default-features = true } +frame-system-rpc-runtime-api = { workspace = true, default-features = true } +pallet-transaction-payment = { workspace = true, default-features = true } +pallet-transaction-payment-rpc-runtime-api = { workspace = true, default-features = true } +sp-inherents = { workspace = true, default-features = true } +sp-api = { workspace = true, default-features = true } +sp-consensus = { workspace = true, default-features = true } +sp-consensus-aura = { workspace = true, default-features = true } +sc-consensus-manual-seal = { workspace = true, default-features = true } +sc-sysinfo = { workspace = true, default-features = true } +prometheus-endpoint = { workspace = true, default-features = true } +substrate-frame-rpc-system = { workspace = true, default-features = true } +pallet-transaction-payment-rpc = { workspace = true, default-features = true } +substrate-state-trie-migration-rpc = { workspace = true, default-features = true } + +# Polkadot +polkadot-cli = { workspace = true, default-features = true } +polkadot-primitives = { workspace = true, default-features = true } + +# Cumulus +cumulus-client-cli = { workspace = true, default-features = true } +cumulus-client-collator = { workspace = true, default-features = true } +cumulus-client-consensus-aura = { workspace = true, default-features = true } +cumulus-client-consensus-relay-chain = { workspace = true, default-features = true } +cumulus-client-consensus-common = { workspace = true, default-features = true } +cumulus-client-consensus-proposer = { workspace = true, default-features = true } +cumulus-client-parachain-inherent = { workspace = true, default-features = true } +cumulus-client-service = { workspace = true, default-features = true } +cumulus-primitives-aura = { workspace = true, default-features = true } +cumulus-primitives-core = { workspace = true, default-features = true } +cumulus-relay-chain-interface = { workspace = true, default-features = true } +futures-timer = "3.0.3" + +[dev-dependencies] +assert_cmd = { workspace = true } +nix = { features = ["signal"], workspace = true } +tokio = { version = "1.32.0", features = ["macros", "parking_lot", "time"] } +wait-timeout = { workspace = true } +cumulus-test-runtime = { workspace = true } + +[features] +default = [] +rococo-native = ["polkadot-cli/rococo-native"] +westend-native = ["polkadot-cli/westend-native"] +runtime-benchmarks = [ + "cumulus-primitives-core/runtime-benchmarks", + "frame-benchmarking-cli/runtime-benchmarks", + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "pallet-transaction-payment/runtime-benchmarks", + "parachains-common/runtime-benchmarks", + "polkadot-cli/runtime-benchmarks", + "polkadot-primitives/runtime-benchmarks", + "sc-client-db/runtime-benchmarks", + "sc-service/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] +try-runtime = [ + "frame-support/try-runtime", + "frame-try-runtime/try-runtime", + "pallet-transaction-payment/try-runtime", + "polkadot-cli/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/cumulus/polkadot-omni-node/lib/src/common/runtime.rs b/cumulus/polkadot-omni-node/lib/src/common/runtime.rs new file mode 100644 index 000000000000..fcc1d7f0643e --- /dev/null +++ b/cumulus/polkadot-omni-node/lib/src/common/runtime.rs @@ -0,0 +1,213 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +//! Runtime parameters. + +use codec::Decode; +use cumulus_client_service::ParachainHostFunctions; +use sc_chain_spec::ChainSpec; +use sc_executor::WasmExecutor; +use sc_runtime_utilities::fetch_latest_metadata_from_code_blob; +use scale_info::{form::PortableForm, TypeDef, TypeDefPrimitive}; +use std::fmt::Display; +use subxt_metadata::{Metadata, StorageEntryType}; + +/// Expected parachain system pallet runtime type name. +pub const DEFAULT_PARACHAIN_SYSTEM_PALLET_NAME: &str = "ParachainSystem"; +/// Expected frame system pallet runtime type name. +pub const DEFAULT_FRAME_SYSTEM_PALLET_NAME: &str = "System"; + +/// The Aura ID used by the Aura consensus +#[derive(PartialEq)] +pub enum AuraConsensusId { + /// Ed25519 + Ed25519, + /// Sr25519 + Sr25519, +} + +/// The choice of consensus for the parachain omni-node. +#[derive(PartialEq)] +pub enum Consensus { + /// Aura consensus. + Aura(AuraConsensusId), +} + +/// The choice of block number for the parachain omni-node. +#[derive(PartialEq, Debug)] +pub enum BlockNumber { + /// u32 + U32, + /// u64 + U64, +} + +impl Display for BlockNumber { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BlockNumber::U32 => write!(f, "u32"), + BlockNumber::U64 => write!(f, "u64"), + } + } +} + +impl Into for BlockNumber { + fn into(self) -> TypeDefPrimitive { + match self { + BlockNumber::U32 => TypeDefPrimitive::U32, + BlockNumber::U64 => TypeDefPrimitive::U64, + } + } +} + +impl BlockNumber { + fn from_type_def(type_def: &TypeDef) -> Option { + match type_def { + TypeDef::Primitive(TypeDefPrimitive::U32) => Some(BlockNumber::U32), + TypeDef::Primitive(TypeDefPrimitive::U64) => Some(BlockNumber::U64), + _ => None, + } + } +} + +/// Helper enum listing the supported Runtime types +#[derive(PartialEq)] +pub enum Runtime { + /// None of the system-chain runtimes, rather the node will act agnostic to the runtime ie. be + /// an omni-node, and simply run a node with the given consensus algorithm. + Omni(BlockNumber, Consensus), +} + +/// Helper trait used for extracting the Runtime variant from the chain spec ID. +pub trait RuntimeResolver { + /// Extract the Runtime variant from the chain spec ID. + fn runtime(&self, chain_spec: &dyn ChainSpec) -> sc_cli::Result; +} + +/// Default implementation for `RuntimeResolver` that just returns +/// `Runtime::Omni(BlockNumber::U32, Consensus::Aura(AuraConsensusId::Sr25519))`. +pub struct DefaultRuntimeResolver; + +impl RuntimeResolver for DefaultRuntimeResolver { + fn runtime(&self, chain_spec: &dyn ChainSpec) -> sc_cli::Result { + let Ok(metadata_inspector) = MetadataInspector::new(chain_spec) else { + log::info!("Unable to check metadata. Skipping metadata checks. Metadata checks are supported for metadata versions v14 and higher."); + return Ok(Runtime::Omni(BlockNumber::U32, Consensus::Aura(AuraConsensusId::Sr25519))) + }; + + let block_number = match metadata_inspector.block_number() { + Some(inner) => inner, + None => { + log::warn!( + r#"⚠️ There isn't a runtime type named `System`, corresponding to the `frame-system` + pallet (https://docs.rs/frame-system/latest/frame_system/). Please check Omni Node docs for runtime conventions: + https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/reference_docs/omni_node/index.html#runtime-conventions. + Note: We'll assume a block number size of `u32`."# + ); + BlockNumber::U32 + }, + }; + + if !metadata_inspector.pallet_exists(DEFAULT_PARACHAIN_SYSTEM_PALLET_NAME) { + log::warn!( + r#"⚠️ The parachain system pallet (https://docs.rs/crate/cumulus-pallet-parachain-system/latest) is + missing from the runtime’s metadata. Please check Omni Node docs for runtime conventions: + https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/reference_docs/omni_node/index.html#runtime-conventions."# + ); + } + + Ok(Runtime::Omni(block_number, Consensus::Aura(AuraConsensusId::Sr25519))) + } +} + +struct MetadataInspector(Metadata); + +impl MetadataInspector { + fn new(chain_spec: &dyn ChainSpec) -> Result { + MetadataInspector::fetch_metadata(chain_spec).map(MetadataInspector) + } + + fn pallet_exists(&self, name: &str) -> bool { + self.0.pallet_by_name(name).is_some() + } + + fn block_number(&self) -> Option { + let pallet_metadata = self.0.pallet_by_name(DEFAULT_FRAME_SYSTEM_PALLET_NAME); + pallet_metadata + .and_then(|inner| inner.storage()) + .and_then(|inner| inner.entry_by_name("Number")) + .and_then(|number_ty| match number_ty.entry_type() { + StorageEntryType::Plain(ty_id) => Some(ty_id), + _ => None, + }) + .and_then(|ty_id| self.0.types().resolve(*ty_id)) + .and_then(|portable_type| BlockNumber::from_type_def(&portable_type.type_def)) + } + + fn fetch_metadata(chain_spec: &dyn ChainSpec) -> Result { + let mut storage = chain_spec.build_storage()?; + let code_bytes = storage + .top + .remove(sp_storage::well_known_keys::CODE) + .ok_or("chain spec genesis does not contain code")?; + let opaque_metadata = fetch_latest_metadata_from_code_blob( + &WasmExecutor::::builder() + .with_allow_missing_host_functions(true) + .build(), + sp_runtime::Cow::Borrowed(code_bytes.as_slice()), + ) + .map_err(|err| err.to_string())?; + + Metadata::decode(&mut (*opaque_metadata).as_slice()).map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use crate::runtime::{ + BlockNumber, MetadataInspector, DEFAULT_FRAME_SYSTEM_PALLET_NAME, + DEFAULT_PARACHAIN_SYSTEM_PALLET_NAME, + }; + use codec::Decode; + use cumulus_client_service::ParachainHostFunctions; + use sc_executor::WasmExecutor; + use sc_runtime_utilities::fetch_latest_metadata_from_code_blob; + + fn cumulus_test_runtime_metadata() -> subxt_metadata::Metadata { + let opaque_metadata = fetch_latest_metadata_from_code_blob( + &WasmExecutor::::builder() + .with_allow_missing_host_functions(true) + .build(), + sp_runtime::Cow::Borrowed(cumulus_test_runtime::WASM_BINARY.unwrap()), + ) + .unwrap(); + + subxt_metadata::Metadata::decode(&mut (*opaque_metadata).as_slice()).unwrap() + } + + #[test] + fn test_pallet_exists() { + let metadata_inspector = MetadataInspector(cumulus_test_runtime_metadata()); + assert!(metadata_inspector.pallet_exists(DEFAULT_PARACHAIN_SYSTEM_PALLET_NAME)); + assert!(metadata_inspector.pallet_exists(DEFAULT_FRAME_SYSTEM_PALLET_NAME)); + } + + #[test] + fn test_runtime_block_number() { + let metadata_inspector = MetadataInspector(cumulus_test_runtime_metadata()); + assert_eq!(metadata_inspector.block_number().unwrap(), BlockNumber::U32); + } +} diff --git a/prdoc/pr_6923.prdoc b/prdoc/pr_6923.prdoc new file mode 100644 index 000000000000..5d88d7158e7f --- /dev/null +++ b/prdoc/pr_6923.prdoc @@ -0,0 +1,12 @@ +title: 'omni-node: Tolerate failing metadata check' +doc: +- audience: Node Operator + description: |- + #6450 introduced metadata checks. Supported are metadata v14 and higher. + + However, of course old chain-specs have a genesis code blob that might be on older version. This needs to be tolerated. We should just skip the checks in that case. + + Fixes #6921 +crates: +- name: polkadot-omni-node-lib + bump: patch