From 34c750aa9ee4aff7e7f603be411444d02508cd4b Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 6 Aug 2024 15:48:12 +0100 Subject: [PATCH 1/6] first step to remove cache --- .../src/spendbundle_validation.rs | 64 +++++++++++++------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/crates/chia-consensus/src/spendbundle_validation.rs b/crates/chia-consensus/src/spendbundle_validation.rs index 8058274be..1040827c5 100644 --- a/crates/chia-consensus/src/spendbundle_validation.rs +++ b/crates/chia-consensus/src/spendbundle_validation.rs @@ -9,9 +9,10 @@ use crate::gen::opcodes::{ use crate::gen::owned_conditions::OwnedSpendBundleConditions; use crate::gen::validation_error::ErrorCode; use crate::spendbundle_conditions::get_conditions_from_spendbundle; -use chia_bls::BlsCache; use chia_bls::PairingInfo; +use chia_bls::{aggregate_verify_gt, hash_to_g2, BlsCache}; use chia_protocol::SpendBundle; +use clvmr::sha2::Sha256; use clvmr::{ENABLE_BLS_OPS_OUTSIDE_GUARD, ENABLE_FIXED_DIV, LIMIT_HEAP}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -31,7 +32,8 @@ pub fn validate_clvm_and_signature( let sbc = get_conditions_from_spendbundle(&mut a, spend_bundle, max_cost, height, constants) .map_err(|e| e.1)?; let npcresult = OwnedSpendBundleConditions::from(&a, sbc); - let iter = npcresult.spends.iter().flat_map(|spend| { + let spends = npcresult.spends.clone(); + let iter = spends.iter().flat_map(|spend| { let condition_items_pairs = [ (AGG_SIG_PARENT, &spend.agg_sig_parent), (AGG_SIG_PUZZLE, &spend.agg_sig_puzzle), @@ -46,32 +48,54 @@ pub fn validate_clvm_and_signature( .flat_map(move |(condition, items)| { let spend_clone = spend.clone(); items.iter().map(move |(pk, msg)| { - ( - pk, - make_aggsig_final_message( - condition, - msg.as_slice(), - &spend_clone, - constants, - ), - ) + let mut aug_msg = pk.to_bytes().to_vec(); + let msg = make_aggsig_final_message( + condition, + msg.as_slice(), + &spend_clone, + constants, + ); + aug_msg.extend_from_slice(msg.as_ref()); + let aug_hash = hash_to_g2(&aug_msg); + let pairing = aug_hash.pair(pk); + + let mut hasher = Sha256::new(); + hasher.update(pk.to_bytes()); + hasher.update(msg.as_slice()); + let hash: [u8; 32] = hasher.finalize(); + (hash, pairing) }) }) }); - let unsafe_items = npcresult - .agg_sig_unsafe - .iter() - .map(|(pk, msg)| (pk, msg.as_slice().to_vec())); + let unsafes = npcresult.agg_sig_unsafe.clone(); + let unsafe_items = unsafes.iter().map(|(pk, msg)| { + let mut aug_msg = pk.to_bytes().to_vec(); + aug_msg.extend_from_slice(msg.as_ref()); + let aug_hash = hash_to_g2(&aug_msg); + let pairing = aug_hash.pair(pk); + + let mut hasher = Sha256::new(); + hasher.update(pk.to_bytes()); + hasher.update(msg.as_slice()); + let hash: [u8; 32] = hasher.finalize(); + (hash, pairing) + }); let iter = iter.chain(unsafe_items); + // Verify aggregated signature - let (result, added) = cache - .lock() - .unwrap() - .aggregate_verify(iter, &spend_bundle.aggregated_signature); + let result = aggregate_verify_gt( + &spend_bundle.aggregated_signature, + iter.clone().map(|tuple| tuple.1), + ); if !result { return Err(ErrorCode::BadAggregateSignature); } - Ok((npcresult, added, start_time.elapsed())) + Ok(( + npcresult, + iter.map(|tuple| (tuple.0, tuple.1.to_bytes().to_vec())) + .collect(), + start_time.elapsed(), + )) } pub fn get_flags_for_height_and_constants(height: u32, constants: &ConsensusConstants) -> u32 { From 4a039a5cf8e48f0d41d1301100ed0b507e37173c Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 6 Aug 2024 15:52:41 +0100 Subject: [PATCH 2/6] change api to not use cache --- crates/chia-consensus/src/spendbundle_validation.rs | 12 +----------- wheel/generate_type_stubs.py | 1 - wheel/python/chia_rs/chia_rs.pyi | 1 - wheel/src/api.rs | 3 --- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/crates/chia-consensus/src/spendbundle_validation.rs b/crates/chia-consensus/src/spendbundle_validation.rs index 1040827c5..e5bcdbec7 100644 --- a/crates/chia-consensus/src/spendbundle_validation.rs +++ b/crates/chia-consensus/src/spendbundle_validation.rs @@ -10,11 +10,10 @@ use crate::gen::owned_conditions::OwnedSpendBundleConditions; use crate::gen::validation_error::ErrorCode; use crate::spendbundle_conditions::get_conditions_from_spendbundle; use chia_bls::PairingInfo; -use chia_bls::{aggregate_verify_gt, hash_to_g2, BlsCache}; +use chia_bls::{aggregate_verify_gt, hash_to_g2}; use chia_protocol::SpendBundle; use clvmr::sha2::Sha256; use clvmr::{ENABLE_BLS_OPS_OUTSIDE_GUARD, ENABLE_FIXED_DIV, LIMIT_HEAP}; -use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; // currently in mempool_manager.py @@ -25,7 +24,6 @@ pub fn validate_clvm_and_signature( max_cost: u64, constants: &ConsensusConstants, height: u32, - cache: &Arc>, ) -> Result<(OwnedSpendBundleConditions, Vec, Duration), ErrorCode> { let start_time = Instant::now(); let mut a = make_allocator(LIMIT_HEAP); @@ -185,7 +183,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm, &TEST_CONSTANTS, 236, - &Arc::new(Mutex::new(BlsCache::default())), ) .expect("SpendBundle should be valid for this test"); } @@ -217,7 +214,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm, &TEST_CONSTANTS, 236, - &Arc::new(Mutex::new(BlsCache::default())), ) .expect("SpendBundle should be valid for this test"); } @@ -246,7 +242,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm / 2, // same as mempool_manager default &TEST_CONSTANTS, 236, - &Arc::new(Mutex::new(BlsCache::default())), ); assert!(matches!(result, Ok(..))); let result = validate_clvm_and_signature( @@ -254,7 +249,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm / 3, // lower than mempool_manager default &TEST_CONSTANTS, 236, - &Arc::new(Mutex::new(BlsCache::default())), ); assert!(matches!(result, Err(ErrorCode::CostExceeded))); } @@ -295,7 +289,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm, &TEST_CONSTANTS, 1, - &Arc::new(Mutex::new(BlsCache::default())), ) .expect("SpendBundle should be valid for this test"); } @@ -345,7 +338,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm, &TEST_CONSTANTS, TEST_CONSTANTS.hard_fork_height + 1, - &Arc::new(Mutex::new(BlsCache::default())), ) .expect("SpendBundle should be valid for this test"); } @@ -389,7 +381,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm, &TEST_CONSTANTS, TEST_CONSTANTS.hard_fork_height + 1, - &Arc::new(Mutex::new(BlsCache::default())), ) .expect("SpendBundle should be valid for this test"); } @@ -433,7 +424,6 @@ ff01\ TEST_CONSTANTS.max_block_cost_clvm, &TEST_CONSTANTS, TEST_CONSTANTS.hard_fork_height + 1, - &Arc::new(Mutex::new(BlsCache::default())), ) .expect("SpendBundle should be valid for this test"); } diff --git a/wheel/generate_type_stubs.py b/wheel/generate_type_stubs.py index 9da1c1cd0..35bd9c0e2 100644 --- a/wheel/generate_type_stubs.py +++ b/wheel/generate_type_stubs.py @@ -310,7 +310,6 @@ def validate_clvm_and_signature( max_cost: int, constants: ConsensusConstants, peak_height: int, - cache: Optional[BLSCache], ) -> Tuple[SpendBundleConditions, List[Tuple[SpendBundleConditions, List[Tuple[bytes32, bytes]]]], float]: ... def get_conditions_from_spendbundle( diff --git a/wheel/python/chia_rs/chia_rs.pyi b/wheel/python/chia_rs/chia_rs.pyi index 696ad8c69..15e35bf98 100644 --- a/wheel/python/chia_rs/chia_rs.pyi +++ b/wheel/python/chia_rs/chia_rs.pyi @@ -54,7 +54,6 @@ def validate_clvm_and_signature( max_cost: int, constants: ConsensusConstants, peak_height: int, - cache: Optional[BLSCache], ) -> Tuple[SpendBundleConditions, List[Tuple[SpendBundleConditions, List[Tuple[bytes32, bytes]]]], float]: ... def get_conditions_from_spendbundle( diff --git a/wheel/src/api.rs b/wheel/src/api.rs index cc016e0f1..940b0dd0d 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -368,15 +368,12 @@ pub fn py_validate_clvm_and_signature( max_cost: u64, constants: &ConsensusConstants, peak_height: u32, - cache: Option, ) -> PyResult<(OwnedSpendBundleConditions, Vec, f32)> { - let real_cache = cache.unwrap_or_default(); let (owned_conditions, additions, duration) = validate_clvm_and_signature( new_spend, max_cost, constants, peak_height, - &Arc::new(Mutex::new(real_cache)), // TODO: use cache properly ) .map_err(|e| { let error_code: u32 = e.into(); From 460f3e3f32c9dc44d2e93031e04a7326e39281b4 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 6 Aug 2024 15:57:17 +0100 Subject: [PATCH 3/6] fmt again --- wheel/src/api.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/wheel/src/api.rs b/wheel/src/api.rs index 940b0dd0d..7e8834b10 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -369,16 +369,11 @@ pub fn py_validate_clvm_and_signature( constants: &ConsensusConstants, peak_height: u32, ) -> PyResult<(OwnedSpendBundleConditions, Vec, f32)> { - let (owned_conditions, additions, duration) = validate_clvm_and_signature( - new_spend, - max_cost, - constants, - peak_height, - ) - .map_err(|e| { - let error_code: u32 = e.into(); - PyErr::new::(error_code) - })?; // cast validation error to int + let (owned_conditions, additions, duration) = + validate_clvm_and_signature(new_spend, max_cost, constants, peak_height).map_err(|e| { + let error_code: u32 = e.into(); + PyErr::new::(error_code) + })?; // cast validation error to int Ok((owned_conditions, additions, duration.as_secs_f32())) } From 722d553a45ab1d9cc7f7bb661cdbdcdc21f07488 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 6 Aug 2024 16:16:31 +0100 Subject: [PATCH 4/6] fix pytest and unused api import --- tests/test_blscache.py | 1 - wheel/src/api.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/test_blscache.py b/tests/test_blscache.py index 9f8472eb6..8f7851171 100644 --- a/tests/test_blscache.py +++ b/tests/test_blscache.py @@ -339,7 +339,6 @@ def test_validate_clvm_and_sig(): DEFAULT_CONSTANTS.MAX_BLOCK_COST_CLVM, DEFAULT_CONSTANTS, DEFAULT_CONSTANTS.HARD_FORK_HEIGHT + 1, - cache, ) assert sbc is not None diff --git a/wheel/src/api.rs b/wheel/src/api.rs index 7e8834b10..8a2c84dd2 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -53,7 +53,6 @@ use pyo3::types::PyList; use pyo3::types::PyTuple; use pyo3::wrap_pyfunction; use std::iter::zip; -use std::sync::{Arc, Mutex}; use crate::run_program::{run_chia_program, serialized_length}; From 5cbed30299f9510230bd837f17214cd7ebed4447 Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 6 Aug 2024 16:43:22 +0100 Subject: [PATCH 5/6] hash_pk_and_msg as a factored function --- .../src/spendbundle_validation.rs | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/crates/chia-consensus/src/spendbundle_validation.rs b/crates/chia-consensus/src/spendbundle_validation.rs index e5bcdbec7..d097ece44 100644 --- a/crates/chia-consensus/src/spendbundle_validation.rs +++ b/crates/chia-consensus/src/spendbundle_validation.rs @@ -57,11 +57,7 @@ pub fn validate_clvm_and_signature( let aug_hash = hash_to_g2(&aug_msg); let pairing = aug_hash.pair(pk); - let mut hasher = Sha256::new(); - hasher.update(pk.to_bytes()); - hasher.update(msg.as_slice()); - let hash: [u8; 32] = hasher.finalize(); - (hash, pairing) + (hash_pk_and_msg(&pk.to_bytes(), &msg), pairing) }) }) }); @@ -71,12 +67,7 @@ pub fn validate_clvm_and_signature( aug_msg.extend_from_slice(msg.as_ref()); let aug_hash = hash_to_g2(&aug_msg); let pairing = aug_hash.pair(pk); - - let mut hasher = Sha256::new(); - hasher.update(pk.to_bytes()); - hasher.update(msg.as_slice()); - let hash: [u8; 32] = hasher.finalize(); - (hash, pairing) + (hash_pk_and_msg(&pk.to_bytes(), msg), pairing) }); let iter = iter.chain(unsafe_items); @@ -96,6 +87,13 @@ pub fn validate_clvm_and_signature( )) } +fn hash_pk_and_msg(pk: &[u8], msg: &[u8]) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(pk); + hasher.update(msg); + hasher.finalize() +} + pub fn get_flags_for_height_and_constants(height: u32, constants: &ConsensusConstants) -> u32 { let mut flags: u32 = 0; From 5d0f4d8088daa0db2eb1da01d7fdccec2af7f43f Mon Sep 17 00:00:00 2001 From: Matthew Howard Date: Tue, 6 Aug 2024 17:45:58 +0100 Subject: [PATCH 6/6] move to for loop and derive Copy for GTElement --- crates/chia-bls/src/bls_cache.rs | 6 +- crates/chia-bls/src/gtelement.rs | 4 +- crates/chia-bls/src/signature.rs | 2 +- .../src/spendbundle_validation.rs | 70 +++++++++---------- wheel/src/api.rs | 4 ++ 5 files changed, 42 insertions(+), 44 deletions(-) diff --git a/crates/chia-bls/src/bls_cache.rs b/crates/chia-bls/src/bls_cache.rs index dfba45011..ca62aa4c6 100644 --- a/crates/chia-bls/src/bls_cache.rs +++ b/crates/chia-bls/src/bls_cache.rs @@ -65,8 +65,8 @@ impl BlsCache { let hash: [u8; 32] = hasher.finalize(); // If the pairing is in the cache, we don't need to recalculate it. - if let Some(pairing) = self.cache.get(&hash).cloned() { - return pairing; + if let Some(pairing) = self.cache.get(&hash) { + return *pairing; } // Otherwise, we need to calculate the pairing and add it to the cache. @@ -79,7 +79,7 @@ impl BlsCache { let hash: [u8; 32] = hasher.finalize(); let pairing = aug_hash.pair(pk.borrow()); - self.cache.put(hash, pairing.clone()); + self.cache.put(hash, pairing); added.push((hash, pairing.to_bytes().to_vec())); pairing }); diff --git a/crates/chia-bls/src/gtelement.rs b/crates/chia-bls/src/gtelement.rs index 5c42fbd1c..690db6988 100644 --- a/crates/chia-bls/src/gtelement.rs +++ b/crates/chia-bls/src/gtelement.rs @@ -13,7 +13,7 @@ use std::ops::{Mul, MulAssign}; pyo3::pyclass, derive(chia_py_streamable_macro::PyStreamable) )] -#[derive(Clone)] +#[derive(Clone, Copy)] pub struct GTElement(pub(crate) blst_fp12); impl GTElement { @@ -114,7 +114,7 @@ impl GTElement { #[must_use] pub fn __mul__(&self, rhs: &Self) -> Self { - let mut ret = self.clone(); + let mut ret = *self; ret *= rhs; ret } diff --git a/crates/chia-bls/src/signature.rs b/crates/chia-bls/src/signature.rs index 879200e98..a4e3870c8 100644 --- a/crates/chia-bls/src/signature.rs +++ b/crates/chia-bls/src/signature.rs @@ -438,7 +438,7 @@ where return *sig == Signature::default(); }; - let mut agg = agg.borrow().clone(); + let mut agg = *agg.borrow(); for gt in data { agg *= gt.borrow(); } diff --git a/crates/chia-consensus/src/spendbundle_validation.rs b/crates/chia-consensus/src/spendbundle_validation.rs index d097ece44..d31eb1e33 100644 --- a/crates/chia-consensus/src/spendbundle_validation.rs +++ b/crates/chia-consensus/src/spendbundle_validation.rs @@ -9,29 +9,35 @@ use crate::gen::opcodes::{ use crate::gen::owned_conditions::OwnedSpendBundleConditions; use crate::gen::validation_error::ErrorCode; use crate::spendbundle_conditions::get_conditions_from_spendbundle; -use chia_bls::PairingInfo; +use chia_bls::GTElement; use chia_bls::{aggregate_verify_gt, hash_to_g2}; use chia_protocol::SpendBundle; use clvmr::sha2::Sha256; use clvmr::{ENABLE_BLS_OPS_OUTSIDE_GUARD, ENABLE_FIXED_DIV, LIMIT_HEAP}; use std::time::{Duration, Instant}; +// type definition makes clippy happy +pub type ValidationPair = ([u8; 32], GTElement); + // currently in mempool_manager.py // called in threads from pre_validate_spend_bundle() -// returns (error, cached_results, new_cache_entries, duration) +// pybinding returns (error, cached_results, new_cache_entries, duration) pub fn validate_clvm_and_signature( spend_bundle: &SpendBundle, max_cost: u64, constants: &ConsensusConstants, height: u32, -) -> Result<(OwnedSpendBundleConditions, Vec, Duration), ErrorCode> { +) -> Result<(OwnedSpendBundleConditions, Vec, Duration), ErrorCode> { let start_time = Instant::now(); let mut a = make_allocator(LIMIT_HEAP); let sbc = get_conditions_from_spendbundle(&mut a, spend_bundle, max_cost, height, constants) .map_err(|e| e.1)?; let npcresult = OwnedSpendBundleConditions::from(&a, sbc); - let spends = npcresult.spends.clone(); - let iter = spends.iter().flat_map(|spend| { + + // Collect all pairs in a single vector to avoid multiple iterations + let mut pairs = Vec::new(); + + for spend in &npcresult.spends { let condition_items_pairs = [ (AGG_SIG_PARENT, &spend.agg_sig_parent), (AGG_SIG_PUZZLE, &spend.agg_sig_puzzle), @@ -41,50 +47,39 @@ pub fn validate_clvm_and_signature( (AGG_SIG_PARENT_PUZZLE, &spend.agg_sig_parent_puzzle), (AGG_SIG_ME, &spend.agg_sig_me), ]; - condition_items_pairs - .into_iter() - .flat_map(move |(condition, items)| { - let spend_clone = spend.clone(); - items.iter().map(move |(pk, msg)| { - let mut aug_msg = pk.to_bytes().to_vec(); - let msg = make_aggsig_final_message( - condition, - msg.as_slice(), - &spend_clone, - constants, - ); - aug_msg.extend_from_slice(msg.as_ref()); - let aug_hash = hash_to_g2(&aug_msg); - let pairing = aug_hash.pair(pk); - - (hash_pk_and_msg(&pk.to_bytes(), &msg), pairing) - }) - }) - }); - let unsafes = npcresult.agg_sig_unsafe.clone(); - let unsafe_items = unsafes.iter().map(|(pk, msg)| { + + for (condition, items) in condition_items_pairs { + for (pk, msg) in items { + let mut aug_msg = pk.to_bytes().to_vec(); + let msg = make_aggsig_final_message(condition, msg.as_slice(), spend, constants); + aug_msg.extend_from_slice(msg.as_ref()); + let aug_hash = hash_to_g2(&aug_msg); + let pairing = aug_hash.pair(pk); + pairs.push((hash_pk_and_msg(&pk.to_bytes(), &msg), pairing)); + } + } + } + + // Adding unsafe items + for (pk, msg) in &npcresult.agg_sig_unsafe { let mut aug_msg = pk.to_bytes().to_vec(); aug_msg.extend_from_slice(msg.as_ref()); let aug_hash = hash_to_g2(&aug_msg); let pairing = aug_hash.pair(pk); - (hash_pk_and_msg(&pk.to_bytes(), msg), pairing) - }); - let iter = iter.chain(unsafe_items); + pairs.push((hash_pk_and_msg(&pk.to_bytes(), msg), pairing)); + } // Verify aggregated signature let result = aggregate_verify_gt( &spend_bundle.aggregated_signature, - iter.clone().map(|tuple| tuple.1), + pairs.iter().map(|tuple| tuple.1), ); if !result { return Err(ErrorCode::BadAggregateSignature); } - Ok(( - npcresult, - iter.map(|tuple| (tuple.0, tuple.1.to_bytes().to_vec())) - .collect(), - start_time.elapsed(), - )) + + // Collect results + Ok((npcresult, pairs, start_time.elapsed())) } fn hash_pk_and_msg(pk: &[u8], msg: &[u8]) -> [u8; 32] { @@ -137,7 +132,6 @@ mod tests { use hex::FromHex; use hex_literal::hex; use rstest::rstest; - use std::sync::Arc; #[rstest] #[case(0, 0)] diff --git a/wheel/src/api.rs b/wheel/src/api.rs index 8a2c84dd2..486093a5a 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -373,6 +373,10 @@ pub fn py_validate_clvm_and_signature( let error_code: u32 = e.into(); PyErr::new::(error_code) })?; // cast validation error to int + let additions = additions + .into_iter() + .map(|tuple| (tuple.0, tuple.1.to_bytes().to_vec())) + .collect(); Ok((owned_conditions, additions, duration.as_secs_f32())) }