From ee1f6834ecb526ec06d9c1a698c16c809cd4813c Mon Sep 17 00:00:00 2001 From: Balaji Arun Date: Thu, 27 Jun 2024 13:21:18 -0700 Subject: [PATCH 001/469] [runson] update AMI with rustup fix (#13847) --- .github/runs-on.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 01760868f0e61..d0ef4c41ad481 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -2,4 +2,4 @@ images: aptos-ubuntu-x64: platform: "linux" arch: "x64" - ami: "ami-0d8c19bef4893b1ea" + ami: "ami-03b93e88e2d195770" From 50dba283aaf42cd257e5419d618f6e773739bfe0 Mon Sep 17 00:00:00 2001 From: igor-aptos <110557261+igor-aptos@users.noreply.github.com> Date: Thu, 27 Jun 2024 14:33:56 -0700 Subject: [PATCH 002/469] Add speed comparison flags to remote debugger (#13610) being able to use: --use-same-block-boundaries --concurrency-level=1 --concurrency-level=4 --concurrency-level=8 --concurrency-leve l=16 --concurrency-level=32 --concurrency-level=48 and compare speeds as well. --- Cargo.lock | 4 +- aptos-move/aptos-debugger/Cargo.toml | 2 + .../aptos-debugger/src/aptos_debugger.rs | 279 +++++++++++++++--- .../aptos-debugger/src/bcs_txn_decoder.rs | 6 +- aptos-move/aptos-debugger/src/common.rs | 4 +- .../src/execute_past_transactions.rs | 8 +- .../src/execute_pending_block.rs | 4 +- .../aptos-transaction-benchmarks/Cargo.toml | 2 - .../src/transaction_bench_state.rs | 4 +- .../src/transactions.rs | 13 +- .../aptos-validator-interface/src/lib.rs | 6 +- aptos-move/aptos-vm-logging/src/log_schema.rs | 7 + aptos-move/aptos-vm/src/aptos_vm.rs | 14 +- aptos-move/aptos-vm/src/block_executor/mod.rs | 33 ++- .../sharded_executor_service.rs | 2 +- aptos-move/block-executor/src/executor.rs | 9 +- .../block-executor/src/limit_processor.rs | 17 +- aptos-move/e2e-tests/src/executor.rs | 2 +- .../smoke-test/src/aptos/mint_transfer.rs | 2 +- types/src/state_store/mod.rs | 13 +- types/src/transaction/mod.rs | 11 + 21 files changed, 342 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f90fd7f8daca4..cb5bd9324e21f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2651,6 +2651,7 @@ name = "aptos-move-debugger" version = "0.1.0" dependencies = [ "anyhow", + "aptos-block-executor", "aptos-consensus", "aptos-crypto", "aptos-gas-profiling", @@ -2663,6 +2664,7 @@ dependencies = [ "aptos-vm-types", "bcs 0.1.4", "clap 4.4.14", + "itertools 0.12.1", "regex", "reqwest", "tokio", @@ -3991,9 +3993,7 @@ dependencies = [ "criterion", "criterion-cpu-time", "num_cpus", - "once_cell", "proptest", - "rayon", ] [[package]] diff --git a/aptos-move/aptos-debugger/Cargo.toml b/aptos-move/aptos-debugger/Cargo.toml index 208fabb573c27..896f868ddcfc3 100644 --- a/aptos-move/aptos-debugger/Cargo.toml +++ b/aptos-move/aptos-debugger/Cargo.toml @@ -14,6 +14,7 @@ rust-version = { workspace = true } [dependencies] anyhow = { workspace = true } +aptos-block-executor = { workspace = true } aptos-consensus = { workspace = true } aptos-crypto = { workspace = true } aptos-gas-profiling = { workspace = true } @@ -26,6 +27,7 @@ aptos-vm-logging = { workspace = true } aptos-vm-types = { workspace = true } bcs = { workspace = true } clap = { workspace = true } +itertools = { workspace = true } regex = { workspace = true } reqwest = { workspace = true } tokio = { workspace = true } diff --git a/aptos-move/aptos-debugger/src/aptos_debugger.rs b/aptos-move/aptos-debugger/src/aptos_debugger.rs index 1fc70796f6cbc..a587ba83479ce 100644 --- a/aptos-move/aptos-debugger/src/aptos_debugger.rs +++ b/aptos-move/aptos-debugger/src/aptos_debugger.rs @@ -2,24 +2,34 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, format_err, Result}; +use aptos_block_executor::txn_commit_hook::NoOpTransactionCommitHook; use aptos_gas_profiling::{GasProfiler, TransactionGasLog}; use aptos_rest_client::Client; use aptos_types::{ account_address::AccountAddress, + block_executor::config::{ + BlockExecutorConfig, BlockExecutorConfigFromOnchain, BlockExecutorLocalConfig, + }, state_store::TStateView, transaction::{ - signature_verified_transaction::SignatureVerifiedTransaction, SignedTransaction, - Transaction, TransactionInfo, TransactionOutput, TransactionPayload, Version, + signature_verified_transaction::SignatureVerifiedTransaction, BlockOutput, + SignedTransaction, Transaction, TransactionInfo, TransactionOutput, TransactionPayload, + Version, }, vm_status::VMStatus, }; use aptos_validator_interface::{ AptosValidatorInterface, DBDebuggerInterface, DebuggerStateView, RestDebuggerInterface, }; -use aptos_vm::{data_cache::AsMoveResolver, AptosVM, VMExecutor}; +use aptos_vm::{ + block_executor::{AptosTransactionOutput, BlockAptosVM}, + data_cache::AsMoveResolver, + AptosVM, +}; use aptos_vm_logging::log_schema::AdapterLogSchema; use aptos_vm_types::output::VMOutput; -use std::{path::Path, sync::Arc}; +use itertools::Itertools; +use std::{path::Path, sync::Arc, time::Instant}; pub struct AptosDebugger { debugger: Arc, @@ -45,31 +55,50 @@ impl AptosDebugger { version: Version, txns: Vec, repeat_execution_times: u64, + concurrency_levels: &[usize], ) -> Result> { let sig_verified_txns: Vec = txns.into_iter().map(|x| x.into()).collect::>(); let state_view = DebuggerStateView::new(self.debugger.clone(), version); - let result = AptosVM::execute_block_no_limit(&sig_verified_txns, &state_view) - .map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; + print_transaction_stats(&sig_verified_txns, version); - for i in 1..repeat_execution_times { - let repeat_result = AptosVM::execute_block_no_limit(&sig_verified_txns, &state_view) - .map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; - println!( - "Finished execution round {}/{} with {} transactions", - i, - repeat_execution_times, - sig_verified_txns.len() - ); - if !Self::ensure_output_matches(&repeat_result, &result, version) { - bail!( - "Execution result mismatched in round {}/{}", - i, - repeat_execution_times + let mut result = None; + + for concurrency_level in concurrency_levels { + for i in 0..repeat_execution_times { + let start_time = Instant::now(); + let cur_result = + execute_block_no_limit(&sig_verified_txns, &state_view, *concurrency_level) + .map_err(|err| format_err!("Unexpected VM Error: {:?}", err))?; + + println!( + "[{} txns from {}] Finished execution round {}/{} with concurrency_level={} in {}ms", + sig_verified_txns.len(), + version, + i + 1, + repeat_execution_times, + concurrency_level, + start_time.elapsed().as_millis(), ); + + match &result { + None => result = Some(cur_result), + Some(prev_result) => { + if !Self::ensure_output_matches(&cur_result, prev_result, version) { + bail!( + "Execution result mismatched in round {}/{}", + i, + repeat_execution_times + ); + } + }, + } } } + + let result = result.unwrap(); + assert_eq!(sig_verified_txns.len(), result.len()); Ok(result) } @@ -121,33 +150,39 @@ impl AptosDebugger { pub async fn execute_past_transactions( &self, - mut begin: Version, - mut limit: u64, + begin: Version, + limit: u64, + use_same_block_boundaries: bool, repeat_execution_times: u64, + concurrency_levels: &[usize], ) -> Result> { - let (mut txns, mut txn_infos) = self + let (txns, txn_infos) = self .debugger .get_committed_transactions(begin, limit) .await?; - let mut ret = vec![]; - while limit != 0 { - println!( - "Starting epoch execution at {:?}, {:?} transactions remaining", - begin, limit - ); - let mut epoch_result = self - .execute_transactions_by_epoch(begin, txns.clone(), repeat_execution_times) - .await?; - begin += epoch_result.len() as u64; - limit -= epoch_result.len() as u64; - txns = txns.split_off(epoch_result.len()); - let epoch_txn_infos = txn_infos.drain(0..epoch_result.len()).collect::>(); - Self::print_mismatches(&epoch_result, &epoch_txn_infos, begin); - - ret.append(&mut epoch_result); + if use_same_block_boundaries { + // when going block by block, no need to worry about epoch boundaries + // as new epoch is always a new block. + Ok(self + .execute_transactions_by_block( + begin, + txns.clone(), + repeat_execution_times, + concurrency_levels, + ) + .await?) + } else { + self.execute_transactions_by_epoch( + limit, + begin, + txns, + repeat_execution_times, + concurrency_levels, + txn_infos, + ) + .await } - Ok(ret) } fn print_mismatches( @@ -186,13 +221,19 @@ impl AptosDebugger { all_match } - pub async fn execute_transactions_by_epoch( + async fn execute_transactions_until_epoch_end( &self, begin: Version, txns: Vec, repeat_execution_times: u64, + concurrency_levels: &[usize], ) -> Result> { - let results = self.execute_transactions_at_version(begin, txns, repeat_execution_times)?; + let results = self.execute_transactions_at_version( + begin, + txns, + repeat_execution_times, + concurrency_levels, + )?; let mut ret = vec![]; let mut is_reconfig = false; @@ -208,6 +249,78 @@ impl AptosDebugger { Ok(ret) } + async fn execute_transactions_by_epoch( + &self, + mut limit: u64, + mut begin: u64, + mut txns: Vec, + repeat_execution_times: u64, + concurrency_levels: &[usize], + mut txn_infos: Vec, + ) -> Result> { + let mut ret = vec![]; + while limit != 0 { + println!( + "Starting epoch execution at {:?}, {:?} transactions remaining", + begin, limit + ); + + let mut epoch_result = self + .execute_transactions_until_epoch_end( + begin, + txns.clone(), + repeat_execution_times, + concurrency_levels, + ) + .await?; + begin += epoch_result.len() as u64; + limit -= epoch_result.len() as u64; + txns = txns.split_off(epoch_result.len()); + let epoch_txn_infos = txn_infos.drain(0..epoch_result.len()).collect::>(); + Self::print_mismatches(&epoch_result, &epoch_txn_infos, begin); + + ret.append(&mut epoch_result); + } + Ok(ret) + } + + async fn execute_transactions_by_block( + &self, + begin: Version, + txns: Vec, + repeat_execution_times: u64, + concurrency_levels: &[usize], + ) -> Result> { + let mut ret = vec![]; + let mut cur = vec![]; + let mut cur_version = begin; + for txn in txns { + if txn.is_block_start() && !cur.is_empty() { + let to_execute = std::mem::take(&mut cur); + let results = self.execute_transactions_at_version( + cur_version, + to_execute, + repeat_execution_times, + concurrency_levels, + )?; + cur_version += results.len() as u64; + ret.extend(results); + } + cur.push(txn); + } + if !cur.is_empty() { + let results = self.execute_transactions_at_version( + cur_version, + cur, + repeat_execution_times, + concurrency_levels, + )?; + ret.extend(results); + } + + Ok(ret) + } + pub async fn get_version_by_account_sequence( &self, account: AccountAddress, @@ -237,6 +350,67 @@ impl AptosDebugger { } } +fn print_transaction_stats(sig_verified_txns: &[SignatureVerifiedTransaction], version: u64) { + let transaction_types = sig_verified_txns + .iter() + .map(|txn| txn.expect_valid().type_name().to_string()) + // conflate same consecutive elements into one with count + .group_by(|k| k.clone()) + .into_iter() + .map(|(k, r)| { + let num = r.count(); + if num > 1 { + format!("{} {}s", num, k) + } else { + k + } + }) + .collect::>(); + let entry_functions = sig_verified_txns + .iter() + .filter_map(|txn| { + txn.expect_valid() + .try_as_signed_user_txn() + .map(|txn| match &txn.payload() { + TransactionPayload::EntryFunction(txn) => format!( + "entry: {:?}::{:?}", + txn.module().name.as_str(), + txn.function().as_str() + ), + TransactionPayload::Script(_) => "script".to_string(), + TransactionPayload::ModuleBundle(_) => panic!("deprecated module bundle"), + TransactionPayload::Multisig(_) => "multisig".to_string(), + }) + }) + // Count number of instances for each (irrsepsecitve of order) + .sorted() + .group_by(|k| k.clone()) + .into_iter() + .map(|(k, r)| (r.count(), k)) + .sorted_by_key(|(num, _k)| *num) + .rev() + .map(|(num, k)| { + if num > 1 { + format!("{} {}s", num, k) + } else { + k + } + }) + .collect::>(); + println!( + "[{} txns from {}] Transaction types: {:?}", + sig_verified_txns.len(), + version, + transaction_types + ); + println!( + "[{} txns from {}] Entry Functions {:?}", + sig_verified_txns.len(), + version, + entry_functions + ); +} + fn is_reconfiguration(vm_output: &TransactionOutput) -> bool { let new_epoch_event_key = aptos_types::on_chain_config::new_epoch_event_key(); vm_output @@ -244,3 +418,24 @@ fn is_reconfiguration(vm_output: &TransactionOutput) -> bool { .iter() .any(|event| event.event_key() == Some(&new_epoch_event_key)) } + +fn execute_block_no_limit( + sig_verified_txns: &[SignatureVerifiedTransaction], + state_view: &DebuggerStateView, + concurrency_level: usize, +) -> Result, VMStatus> { + BlockAptosVM::execute_block::<_, NoOpTransactionCommitHook>( + sig_verified_txns, + state_view, + BlockExecutorConfig { + local: BlockExecutorLocalConfig { + concurrency_level, + allow_fallback: true, + discard_failed_blocks: false, + }, + onchain: BlockExecutorConfigFromOnchain::new_no_block_limit(), + }, + None, + ) + .map(BlockOutput::into_transaction_outputs_forced) +} diff --git a/aptos-move/aptos-debugger/src/bcs_txn_decoder.rs b/aptos-move/aptos-debugger/src/bcs_txn_decoder.rs index b444aa2311ff1..204b04621f2b0 100644 --- a/aptos-move/aptos-debugger/src/bcs_txn_decoder.rs +++ b/aptos-move/aptos-debugger/src/bcs_txn_decoder.rs @@ -5,7 +5,6 @@ use crate::aptos_debugger::AptosDebugger; use anyhow::Result; use aptos_rest_client::Client; use aptos_types::transaction::SignedTransaction; -use aptos_vm::AptosVM; use clap::Parser; use regex::Regex; use std::io; @@ -65,14 +64,15 @@ impl Command { ); if self.execute { - AptosVM::set_concurrency_level_once(self.concurrency_level); println!(); println!("==============================="); println!("Transaction re-execution result"); println!("==============================="); println!( "{:#?}", - debugger.execute_past_transactions(version, 1, 1).await? + debugger + .execute_past_transactions(version, 1, false, 1, &[self.concurrency_level]) + .await? ); } diff --git a/aptos-move/aptos-debugger/src/common.rs b/aptos-move/aptos-debugger/src/common.rs index 6b250b8108b42..584a040fb59fa 100644 --- a/aptos-move/aptos-debugger/src/common.rs +++ b/aptos-move/aptos-debugger/src/common.rs @@ -27,8 +27,8 @@ pub struct Opts { #[clap(flatten)] pub(crate) target: Target, - #[clap(long, default_value_t = 1)] - pub(crate) concurrency_level: usize, + #[clap(long, num_args = 0..)] + pub(crate) concurrency_level: Vec, } #[derive(Parser)] diff --git a/aptos-move/aptos-debugger/src/execute_past_transactions.rs b/aptos-move/aptos-debugger/src/execute_past_transactions.rs index 9c949b2a877da..0403e3f51e6db 100644 --- a/aptos-move/aptos-debugger/src/execute_past_transactions.rs +++ b/aptos-move/aptos-debugger/src/execute_past_transactions.rs @@ -4,7 +4,6 @@ use crate::{aptos_debugger::AptosDebugger, common::Opts}; use anyhow::Result; use aptos_rest_client::Client; -use aptos_vm::AptosVM; use clap::Parser; use url::Url; @@ -24,12 +23,13 @@ pub struct Command { #[clap(long)] repeat_execution_times: Option, + + #[clap(long)] + use_same_block_boundaries: bool, } impl Command { pub async fn run(self) -> Result<()> { - AptosVM::set_concurrency_level_once(self.opts.concurrency_level); - let debugger = if let Some(rest_endpoint) = self.opts.target.rest_endpoint { AptosDebugger::rest_client(Client::new(Url::parse(&rest_endpoint)?))? } else if let Some(db_path) = self.opts.target.db_path { @@ -42,7 +42,9 @@ impl Command { .execute_past_transactions( self.begin_version, self.limit, + self.use_same_block_boundaries, self.repeat_execution_times.unwrap_or(1), + &self.opts.concurrency_level, ) .await?; diff --git a/aptos-move/aptos-debugger/src/execute_pending_block.rs b/aptos-move/aptos-debugger/src/execute_pending_block.rs index 1f7b6dabc9506..7235dff6e9b01 100644 --- a/aptos-move/aptos-debugger/src/execute_pending_block.rs +++ b/aptos-move/aptos-debugger/src/execute_pending_block.rs @@ -6,7 +6,6 @@ use anyhow::Result; use aptos_crypto::HashValue; use aptos_logger::info; use aptos_rest_client::Client; -use aptos_vm::AptosVM; use clap::Parser; use std::path::PathBuf; use url::Url; @@ -42,8 +41,6 @@ pub struct Command { impl Command { pub async fn run(self) -> Result<()> { - AptosVM::set_concurrency_level_once(self.opts.concurrency_level); - let debugger = if let Some(rest_endpoint) = self.opts.target.rest_endpoint { AptosDebugger::rest_client(Client::new(Url::parse(&rest_endpoint)?))? } else if let Some(db_path) = self.opts.target.db_path { @@ -91,6 +88,7 @@ impl Command { self.begin_version, block, self.repeat_execution_times.unwrap_or(1), + &self.opts.concurrency_level, )?; println!("{txn_outputs:#?}"); diff --git a/aptos-move/aptos-transaction-benchmarks/Cargo.toml b/aptos-move/aptos-transaction-benchmarks/Cargo.toml index 4bb90c07a41ae..3fe147d12d47a 100644 --- a/aptos-move/aptos-transaction-benchmarks/Cargo.toml +++ b/aptos-move/aptos-transaction-benchmarks/Cargo.toml @@ -29,9 +29,7 @@ clap = { workspace = true } criterion = { workspace = true, features = ["html_reports"] } criterion-cpu-time = { workspace = true } num_cpus = { workspace = true } -once_cell = { workspace = true } proptest = { workspace = true } -rayon = { workspace = true } [[bench]] name = "transaction_benches" diff --git a/aptos-move/aptos-transaction-benchmarks/src/transaction_bench_state.rs b/aptos-move/aptos-transaction-benchmarks/src/transaction_bench_state.rs index 5e728586e547b..d3a71a1d22862 100644 --- a/aptos-move/aptos-transaction-benchmarks/src/transaction_bench_state.rs +++ b/aptos-move/aptos-transaction-benchmarks/src/transaction_bench_state.rs @@ -1,7 +1,7 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use crate::{transactions, transactions::RAYON_EXEC_POOL}; +use crate::transactions; use aptos_bitvec::BitVec; use aptos_block_executor::txn_commit_hook::NoOpTransactionCommitHook; use aptos_block_partitioner::{ @@ -216,7 +216,6 @@ where _, NoOpTransactionCommitHook, >( - Arc::clone(&RAYON_EXEC_POOL), transactions, self.state_view.as_ref(), BlockExecutorConfig::new_maybe_block_limit(1, maybe_block_gas_limit), @@ -265,7 +264,6 @@ where _, NoOpTransactionCommitHook, >( - Arc::clone(&RAYON_EXEC_POOL), transactions, self.state_view.as_ref(), BlockExecutorConfig::new_maybe_block_limit( diff --git a/aptos-move/aptos-transaction-benchmarks/src/transactions.rs b/aptos-move/aptos-transaction-benchmarks/src/transactions.rs index 645bbe5475afa..d8f96d9d099a5 100644 --- a/aptos-move/aptos-transaction-benchmarks/src/transactions.rs +++ b/aptos-move/aptos-transaction-benchmarks/src/transactions.rs @@ -13,19 +13,8 @@ use aptos_language_e2e_tests::{ gas_costs::TXN_RESERVED, }; use criterion::{measurement::Measurement, BatchSize, Bencher}; -use once_cell::sync::Lazy; use proptest::strategy::Strategy; -use std::{net::SocketAddr, sync::Arc}; - -pub static RAYON_EXEC_POOL: Lazy> = Lazy::new(|| { - Arc::new( - rayon::ThreadPoolBuilder::new() - .num_threads(num_cpus::get()) - .thread_name(|index| format!("par_exec_{}", index)) - .build() - .unwrap(), - ) -}); +use std::net::SocketAddr; /// Benchmarking support for transactions. #[derive(Clone)] diff --git a/aptos-move/aptos-validator-interface/src/lib.rs b/aptos-move/aptos-validator-interface/src/lib.rs index 24e6945fd5963..9414fb2298476 100644 --- a/aptos-move/aptos-validator-interface/src/lib.rs +++ b/aptos-move/aptos-validator-interface/src/lib.rs @@ -12,7 +12,7 @@ use aptos_types::{ account_address::AccountAddress, state_store::{ state_key::StateKey, state_storage_usage::StateStorageUsage, state_value::StateValue, - Result as StateViewResult, TStateView, + Result as StateViewResult, StateViewId, TStateView, }, transaction::{Transaction, TransactionInfo, Version}, }; @@ -160,6 +160,10 @@ impl DebuggerStateView { impl TStateView for DebuggerStateView { type Key = StateKey; + fn id(&self) -> StateViewId { + StateViewId::Replay + } + fn get_state_value(&self, state_key: &StateKey) -> StateViewResult> { self.get_state_value_internal(state_key, self.version) .map_err(Into::into) diff --git a/aptos-move/aptos-vm-logging/src/log_schema.rs b/aptos-move/aptos-vm-logging/src/log_schema.rs index 383dda848a4ec..494ad5dce1175 100644 --- a/aptos-move/aptos-vm-logging/src/log_schema.rs +++ b/aptos-move/aptos-vm-logging/src/log_schema.rs @@ -52,6 +52,13 @@ impl AdapterLogSchema { base_version: Some(base_version), txn_idx, }, + StateViewId::Replay => Self { + name: LogEntry::Execution, + block_id: None, + first_version: None, + base_version: None, + txn_idx, + }, StateViewId::Miscellaneous => Self { name: LogEntry::Miscellaneous, block_id: None, diff --git a/aptos-move/aptos-vm/src/aptos_vm.rs b/aptos-move/aptos-vm/src/aptos_vm.rs index 2f7f0afd368cd..f36e90c624e3c 100644 --- a/aptos-move/aptos-vm/src/aptos_vm.rs +++ b/aptos-move/aptos-vm/src/aptos_vm.rs @@ -100,7 +100,7 @@ use move_vm_runtime::{ }; use move_vm_types::gas::{GasMeter, UnmeteredGasMeter}; use num_cpus; -use once_cell::sync::{Lazy, OnceCell}; +use once_cell::sync::OnceCell; use std::{ cmp::{max, min}, collections::{BTreeMap, BTreeSet}, @@ -114,17 +114,6 @@ static NUM_PROOF_READING_THREADS: OnceCell = OnceCell::new(); static DISCARD_FAILED_BLOCKS: OnceCell = OnceCell::new(); static PROCESSED_TRANSACTIONS_DETAILED_COUNTERS: OnceCell = OnceCell::new(); -// TODO: Don't expose this in AptosVM, and use only in BlockAptosVM! -pub static RAYON_EXEC_POOL: Lazy> = Lazy::new(|| { - Arc::new( - rayon::ThreadPoolBuilder::new() - .num_threads(num_cpus::get()) - .thread_name(|index| format!("par_exec-{}", index)) - .build() - .unwrap(), - ) -}); - macro_rules! deprecated_module_bundle { () => { VMStatus::error( @@ -2455,7 +2444,6 @@ impl VMExecutor for AptosVM { _, NoOpTransactionCommitHook, >( - Arc::clone(&RAYON_EXEC_POOL), transactions, state_view, BlockExecutorConfig { diff --git a/aptos-move/aptos-vm/src/block_executor/mod.rs b/aptos-move/aptos-vm/src/block_executor/mod.rs index 1aec0a6254cf7..e15e52871c4a5 100644 --- a/aptos-move/aptos-vm/src/block_executor/mod.rs +++ b/aptos-move/aptos-vm/src/block_executor/mod.rs @@ -40,13 +40,23 @@ use move_core_types::{ vm_status::{StatusCode, VMStatus}, }; use move_vm_types::delayed_values::delayed_field_id::DelayedFieldID; -use once_cell::sync::OnceCell; +use once_cell::sync::{Lazy, OnceCell}; use rayon::ThreadPool; use std::{ collections::{BTreeMap, HashSet}, sync::Arc, }; +pub static RAYON_EXEC_POOL: Lazy> = Lazy::new(|| { + Arc::new( + rayon::ThreadPoolBuilder::new() + .num_threads(num_cpus::get()) + .thread_name(|index| format!("par_exec-{}", index)) + .build() + .unwrap(), + ) +}); + /// Output type wrapper used by block executor. VM output is stored first, then /// transformed into TransactionOutput type that is returned. #[derive(Debug)] @@ -396,7 +406,7 @@ impl BlockExecutorTransactionOutput for AptosTransactionOutput { pub struct BlockAptosVM(); impl BlockAptosVM { - pub fn execute_block< + pub fn execute_block_on_thread_pool< S: StateView + Sync, L: TransactionCommitHook, >( @@ -455,4 +465,23 @@ impl BlockAptosVM { Err(BlockExecutionError::FatalVMError(err)) => Err(err), } } + + /// Uses shared thread pool to execute blocks. + pub fn execute_block< + S: StateView + Sync, + L: TransactionCommitHook, + >( + signature_verified_block: &[SignatureVerifiedTransaction], + state_view: &S, + config: BlockExecutorConfig, + transaction_commit_listener: Option, + ) -> Result, VMStatus> { + Self::execute_block_on_thread_pool::( + Arc::clone(&RAYON_EXEC_POOL), + signature_verified_block, + state_view, + config, + transaction_commit_listener, + ) + } } diff --git a/aptos-move/aptos-vm/src/sharded_block_executor/sharded_executor_service.rs b/aptos-move/aptos-vm/src/sharded_block_executor/sharded_executor_service.rs index a2d2e76e9e6e6..e968f2416d47c 100644 --- a/aptos-move/aptos-vm/src/sharded_block_executor/sharded_executor_service.rs +++ b/aptos-move/aptos-vm/src/sharded_block_executor/sharded_executor_service.rs @@ -135,7 +135,7 @@ impl ShardedExecutorService { ); }); s.spawn(move |_| { - let ret = BlockAptosVM::execute_block( + let ret = BlockAptosVM::execute_block_on_thread_pool( executor_thread_pool, &signature_verified_transactions, aggr_overridden_state_view.as_ref(), diff --git a/aptos-move/block-executor/src/executor.rs b/aptos-move/block-executor/src/executor.rs index 88bb0dd279dcd..cde73c85b7992 100644 --- a/aptos-move/block-executor/src/executor.rs +++ b/aptos-move/block-executor/src/executor.rs @@ -464,6 +464,7 @@ where shared_counter: &AtomicU32, executor: &E, block: &[T], + num_workers: usize, ) -> Result<(), PanicOr> { let mut block_limit_processor = shared_commit_state.acquire(); @@ -592,6 +593,7 @@ where block_limit_processor.finish_parallel_update_counters_and_log_info( txn_idx + 1, scheduler.num_txns(), + num_workers, ); // failpoint triggering error at the last committed transaction, @@ -756,6 +758,7 @@ where shared_counter: &AtomicU32, shared_commit_state: &ExplicitSyncWrapper>, final_results: &ExplicitSyncWrapper>, + num_workers: usize, ) -> Result<(), PanicOr> { // Make executor for each task. TODO: fast concurrent executor. let init_timer = VM_INIT_SECONDS.start_timer(); @@ -795,6 +798,7 @@ where shared_counter, &executor, block, + num_workers, )?; scheduler.queueing_commits_mark_done(); } @@ -883,7 +887,7 @@ where } let num_txns = signature_verified_block.len(); - let concurrency_level = self.config.local.concurrency_level.min(num_txns / 2).max(2); + let num_workers = self.config.local.concurrency_level.min(num_txns / 2).max(2); let shared_commit_state = ExplicitSyncWrapper::new(BlockGasLimitProcessor::new( self.config.onchain.block_gas_limit_type.clone(), @@ -906,7 +910,7 @@ where let timer = RAYON_EXECUTION_SECONDS.start_timer(); self.executor_thread_pool.scope(|s| { - for _ in 0..concurrency_level { + for _ in 0..num_workers { s.spawn(|_| { if let Err(err) = self.worker_loop( env, @@ -919,6 +923,7 @@ where &shared_counter, &shared_commit_state, &final_results, + num_workers, ) { // If there are multiple errors, they all get logged: // ModulePathReadWriteError and FatalVMError variant is logged at construction, diff --git a/aptos-move/block-executor/src/limit_processor.rs b/aptos-move/block-executor/src/limit_processor.rs index cbfc38d98a83b..b687da43e1e39 100644 --- a/aptos-move/block-executor/src/limit_processor.rs +++ b/aptos-move/block-executor/src/limit_processor.rs @@ -9,6 +9,7 @@ use aptos_types::{ transaction::{block_epilogue::BlockEndInfo, BlockExecutableTransaction as Transaction}, }; use claims::{assert_le, assert_none}; +use std::time::Instant; pub struct BlockGasLimitProcessor { block_gas_limit_type: BlockGasLimitType, @@ -18,6 +19,7 @@ pub struct BlockGasLimitProcessor { txn_fee_statements: Vec, txn_read_write_summaries: Vec>, module_rw_conflict: bool, + start_time: Instant, } impl BlockGasLimitProcessor { @@ -30,6 +32,7 @@ impl BlockGasLimitProcessor { txn_fee_statements: Vec::with_capacity(init_size), txn_read_write_summaries: Vec::with_capacity(init_size), module_rw_conflict: false, + start_time: Instant::now(), } } @@ -190,6 +193,7 @@ impl BlockGasLimitProcessor { is_parallel: bool, num_committed: u32, num_total: u32, + num_workers: usize, ) { let accumulated_effective_block_gas = self.get_effective_accumulated_block_gas(); let accumulated_approx_output_size = self.get_accumulated_approx_output_size(); @@ -216,11 +220,15 @@ impl BlockGasLimitProcessor { .block_gas_limit_type .block_output_limit() .map_or(false, |limit| accumulated_approx_output_size >= limit), + elapsed_ms = self.start_time.elapsed().as_millis(), + num_committed = num_committed, + num_total = num_total, + num_workers = num_workers, "[BlockSTM]: {} execution completed. {} out of {} txns committed", if is_parallel { - "Parallel" + format!("Parallel[{}]", num_workers) } else { - "Sequential" + "Sequential".to_string() }, num_committed, num_total, @@ -231,8 +239,9 @@ impl BlockGasLimitProcessor { &self, num_committed: u32, num_total: u32, + num_workers: usize, ) { - self.finish_update_counters_and_log_info(true, num_committed, num_total) + self.finish_update_counters_and_log_info(true, num_committed, num_total, num_workers) } pub(crate) fn finish_sequential_update_counters_and_log_info( @@ -240,7 +249,7 @@ impl BlockGasLimitProcessor { num_committed: u32, num_total: u32, ) { - self.finish_update_counters_and_log_info(false, num_committed, num_total) + self.finish_update_counters_and_log_info(false, num_committed, num_total, 1) } pub(crate) fn get_block_end_info(&self) -> BlockEndInfo { diff --git a/aptos-move/e2e-tests/src/executor.rs b/aptos-move/e2e-tests/src/executor.rs index ece654e35b19f..46a8cfc084112 100644 --- a/aptos-move/e2e-tests/src/executor.rs +++ b/aptos-move/e2e-tests/src/executor.rs @@ -533,7 +533,7 @@ impl FakeExecutor { }, onchain: onchain_config, }; - BlockAptosVM::execute_block::< + BlockAptosVM::execute_block_on_thread_pool::< _, NoOpTransactionCommitHook, >( diff --git a/testsuite/smoke-test/src/aptos/mint_transfer.rs b/testsuite/smoke-test/src/aptos/mint_transfer.rs index ae2e593c60600..8be365164a269 100644 --- a/testsuite/smoke-test/src/aptos/mint_transfer.rs +++ b/testsuite/smoke-test/src/aptos/mint_transfer.rs @@ -81,7 +81,7 @@ async fn test_mint_transfer() { .unwrap(); let output = debugger - .execute_past_transactions(txn_ver, 1, 1) + .execute_past_transactions(txn_ver, 1, false, 1, &[1]) .await .unwrap() .pop() diff --git a/types/src/state_store/mod.rs b/types/src/state_store/mod.rs index a0e34f75af336..f9632a48ff86c 100644 --- a/types/src/state_store/mod.rs +++ b/types/src/state_store/mod.rs @@ -60,13 +60,20 @@ impl> StateView for T {} #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum StateViewId { /// State-sync applying a chunk of transactions. - ChunkExecution { first_version: Version }, + ChunkExecution { + first_version: Version, + }, /// LEC applying a block. - BlockExecution { block_id: HashValue }, + BlockExecution { + block_id: HashValue, + }, /// VmValidator verifying incoming transaction. - TransactionValidation { base_version: Version }, + TransactionValidation { + base_version: Version, + }, /// For test, db-bootstrapper, etc. Usually not aimed to pass to VM. Miscellaneous, + Replay, } impl TStateView for R diff --git a/types/src/transaction/mod.rs b/types/src/transaction/mod.rs index 5e8314352a9e8..6813e98ceb469 100644 --- a/types/src/transaction/mod.rs +++ b/types/src/transaction/mod.rs @@ -2033,6 +2033,17 @@ impl Transaction { | Transaction::ValidatorTransaction(_) => false, } } + + pub fn is_block_start(&self) -> bool { + match self { + Transaction::BlockMetadata(_) | Transaction::BlockMetadataExt(_) => true, + Transaction::StateCheckpoint(_) + | Transaction::BlockEpilogue(_) + | Transaction::UserTransaction(_) + | Transaction::GenesisTransaction(_) + | Transaction::ValidatorTransaction(_) => false, + } + } } impl TryFrom for SignedTransaction { From 0d92186f4a5423c4c30398b660e8f2e5937d788b Mon Sep 17 00:00:00 2001 From: Satya Vusirikala Date: Thu, 27 Jun 2024 17:53:36 -0500 Subject: [PATCH 003/469] Calculating skipped txns in mempool accurately (#13831) * Avoid duplicate transactions in batch * Addressing PR comments --- mempool/src/core_mempool/mempool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempool/src/core_mempool/mempool.rs b/mempool/src/core_mempool/mempool.rs index c39b6f1bebc06..24569d0379140 100644 --- a/mempool/src/core_mempool/mempool.rs +++ b/mempool/src/core_mempool/mempool.rs @@ -352,7 +352,7 @@ impl Mempool { // check if we can now include some transactions // that were skipped before for given account let mut skipped_txn = (txn.address, tx_seq + 1); - while skipped.contains(&skipped_txn) { + while skipped.remove(&skipped_txn) { inserted.insert(skipped_txn); result.push(skipped_txn); if (result.len() as u64) == max_txns { From ba06d09f1408670a6c700481bdec17f0523f45a7 Mon Sep 17 00:00:00 2001 From: Balaji Arun Date: Thu, 27 Jun 2024 16:01:05 -0700 Subject: [PATCH 004/469] [consensus][buffer_manager_test][bugfix] count batches properly (#13839) --- consensus/src/pipeline/tests/buffer_manager_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/src/pipeline/tests/buffer_manager_tests.rs b/consensus/src/pipeline/tests/buffer_manager_tests.rs index 919c96dcbccb6..8e6aee8bbed18 100644 --- a/consensus/src/pipeline/tests/buffer_manager_tests.rs +++ b/consensus/src/pipeline/tests/buffer_manager_tests.rs @@ -255,8 +255,9 @@ async fn assert_results( batches: Vec>, result_rx: &mut Receiver, ) { + let total_batches = batches.iter().flatten().count(); let mut blocks: Vec = Vec::new(); - for _ in 0..batches.len() { + while blocks.len() < total_batches { let OrderedBlocks { ordered_blocks, .. } = result_rx.next().await.unwrap(); blocks.extend(ordered_blocks.into_iter()); } @@ -340,7 +341,6 @@ fn buffer_manager_happy_path_test() { }); } -#[ignore] // TODO: turn this test back on once the flakes have resolved. #[test] fn buffer_manager_sync_test() { // happy path From ad42e03f244bfd8ea676397954bd2c12abc20fdd Mon Sep 17 00:00:00 2001 From: Christian Theilemann Date: Thu, 27 Jun 2024 17:44:31 -0700 Subject: [PATCH 005/469] Attempt #2: switch high-perf-docker to runs-on.com runners (#13851) --- .github/actionlint.yaml | 2 +- .github/workflows/cli-e2e-tests.yaml | 2 +- .github/workflows/coverage-move-only.yaml | 2 +- .github/workflows/coverage.yaml | 4 ++-- .github/workflows/docker-build-rosetta.yaml | 2 +- .github/workflows/faucet-tests-main.yaml | 2 +- .github/workflows/faucet-tests-prod.yaml | 4 ++-- ...ges-with-undeclared-feature-dependencies.yaml | 2 +- .../indexer-grpc-integration-tests.yaml | 2 +- .../workflows/keyless-circuit-daily-test.yaml | 2 +- .github/workflows/lint-test.yaml | 16 ++++++++-------- .github/workflows/move-test-compiler-v2.yaml | 2 +- .../workflows/node-api-compatibility-tests.yaml | 2 +- .github/workflows/prover-daily-test.yaml | 2 +- .github/workflows/run-gas-calibration.yaml | 2 +- .github/workflows/rust-client-tests.yaml | 6 +++--- .github/workflows/ts-sdk-e2e-tests.yaml | 2 +- 17 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index a92d6c8ea3bcc..11d59e9073cef 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,4 +1,4 @@ self-hosted-runner: # Labels of self-hosted runners in array of string labels: - - high-perf-docker + - runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} diff --git a/.github/workflows/cli-e2e-tests.yaml b/.github/workflows/cli-e2e-tests.yaml index c068d31516f4d..70a1415ba9708 100644 --- a/.github/workflows/cli-e2e-tests.yaml +++ b/.github/workflows/cli-e2e-tests.yaml @@ -25,7 +25,7 @@ jobs: # we ensure that the Aptos CLI works with all 3 prod networks, at least # based on the tests in the test suite. run-cli-tests: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} permissions: contents: read id-token: write diff --git a/.github/workflows/coverage-move-only.yaml b/.github/workflows/coverage-move-only.yaml index 5f6ab8bd013ed..e0b35b9671762 100644 --- a/.github/workflows/coverage-move-only.yaml +++ b/.github/workflows/coverage-move-only.yaml @@ -30,7 +30,7 @@ concurrency: jobs: rust-move-unit-coverage: timeout-minutes: 60 - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 1f844330d3e30..c599f3ef023dd 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -27,7 +27,7 @@ jobs: (github.event_name == 'schedule' && github.ref_name == 'main') # Note the tests run slowly due to instrutmentation. It takes CI 10 hrs timeout-minutes: 720 - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 with: @@ -58,7 +58,7 @@ jobs: contains(github.event.pull_request.labels.*.name, 'CICD:run-coverage') || (github.event_name == 'schedule' && github.ref_name == 'main') timeout-minutes: 720 # incremented from 240 due to execution time limit hit in cron - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/docker-build-rosetta.yaml b/.github/workflows/docker-build-rosetta.yaml index 8d5b32b60c86c..6139c8d8ed194 100644 --- a/.github/workflows/docker-build-rosetta.yaml +++ b/.github/workflows/docker-build-rosetta.yaml @@ -17,7 +17,7 @@ permissions: jobs: build: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/faucet-tests-main.yaml b/.github/workflows/faucet-tests-main.yaml index 82f35a0699ba2..0ca51a2f84708 100644 --- a/.github/workflows/faucet-tests-main.yaml +++ b/.github/workflows/faucet-tests-main.yaml @@ -49,7 +49,7 @@ jobs: # be compatible in production. run-tests-main: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 if: ${{ !inputs.SKIP_JOB }} diff --git a/.github/workflows/faucet-tests-prod.yaml b/.github/workflows/faucet-tests-prod.yaml index 58b0a2dfee5cc..940ef883a1e20 100644 --- a/.github/workflows/faucet-tests-prod.yaml +++ b/.github/workflows/faucet-tests-prod.yaml @@ -37,7 +37,7 @@ jobs: run-tests-devnet: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') needs: [permission-check] - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main @@ -57,7 +57,7 @@ jobs: run-tests-testnet: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') needs: [permission-check] - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} permissions: contents: read id-token: write diff --git a/.github/workflows/find-packages-with-undeclared-feature-dependencies.yaml b/.github/workflows/find-packages-with-undeclared-feature-dependencies.yaml index 42ee5c8f3db93..45dfe907189b5 100644 --- a/.github/workflows/find-packages-with-undeclared-feature-dependencies.yaml +++ b/.github/workflows/find-packages-with-undeclared-feature-dependencies.yaml @@ -4,7 +4,7 @@ on: jobs: find-packages-with-undeclared-feature-dependencies: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main diff --git a/.github/workflows/indexer-grpc-integration-tests.yaml b/.github/workflows/indexer-grpc-integration-tests.yaml index 5d739a532a24f..4a1c2aa93b181 100644 --- a/.github/workflows/indexer-grpc-integration-tests.yaml +++ b/.github/workflows/indexer-grpc-integration-tests.yaml @@ -30,7 +30,7 @@ jobs: run-tests-local-testnet: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') needs: [permission-check] - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} env: # spin up the local testnet using the latest devnet image VALIDATOR_IMAGE_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}/validator diff --git a/.github/workflows/keyless-circuit-daily-test.yaml b/.github/workflows/keyless-circuit-daily-test.yaml index 1ebb0374dc587..8fecb6df2c516 100644 --- a/.github/workflows/keyless-circuit-daily-test.yaml +++ b/.github/workflows/keyless-circuit-daily-test.yaml @@ -20,7 +20,7 @@ concurrency: jobs: run-all-circuit-tests: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} timeout-minutes: 30 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 6782c2701e09e..17ab25e7326d2 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -56,7 +56,7 @@ jobs: # Run the crypto hasher domain separation checks rust-cryptohasher-domain-separation-check: needs: file_change_determinator - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') steps: - uses: actions/checkout@v4 @@ -69,7 +69,7 @@ jobs: # Run all rust lints. This is a PR required job. rust-lints: needs: file_change_determinator - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v4 if: needs.file_change_determinator.outputs.only_docs_changed != 'true' @@ -94,7 +94,7 @@ jobs: github.event.pull_request.auto_merge != null) || contains(github.event.pull_request.body, '#e2e' ) - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v4 if: needs.file_change_determinator.outputs.only_docs_changed != 'true' @@ -109,7 +109,7 @@ jobs: # Run only the targeted rust unit tests. This is a PR required job. rust-targeted-unit-tests: needs: file_change_determinator - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v4 with: @@ -128,7 +128,7 @@ jobs: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CICD:run-all-unit-tests') ) - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v4 - name: Run rust unit tests @@ -147,7 +147,7 @@ jobs: contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || github.event.pull_request.auto_merge != null ) - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v4 if: needs.file_change_determinator.outputs.only_docs_changed != 'true' @@ -162,7 +162,7 @@ jobs: # Run the consensus only unit tests rust-consensus-only-unit-test: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} if: contains(github.event.pull_request.labels.*.name, 'CICD:build-consensus-only-image') steps: - uses: actions/checkout@v4 @@ -178,7 +178,7 @@ jobs: # Run the consensus only smoke test rust-consensus-only-smoke-test: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} if: contains(github.event.pull_request.labels.*.name, 'CICD:build-consensus-only-image') steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/move-test-compiler-v2.yaml b/.github/workflows/move-test-compiler-v2.yaml index 1b7bf22e85fa1..449892ab72f59 100644 --- a/.github/workflows/move-test-compiler-v2.yaml +++ b/.github/workflows/move-test-compiler-v2.yaml @@ -26,7 +26,7 @@ concurrency: jobs: # Run Aptos Move Compiler v2 tests. This is a PR required job. rust-move-tests: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 - name: Run Aptos Move tests with compiler V2 diff --git a/.github/workflows/node-api-compatibility-tests.yaml b/.github/workflows/node-api-compatibility-tests.yaml index 4be84bf6f02fd..8feef3eff7533 100644 --- a/.github/workflows/node-api-compatibility-tests.yaml +++ b/.github/workflows/node-api-compatibility-tests.yaml @@ -43,7 +43,7 @@ jobs: # if there are any changes that would affect it within the PR / commit. If # everything is checked in, run tests, build the SDK, and upload it to npmjs. node-api-compatibility-tests: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} permissions: contents: read id-token: write diff --git a/.github/workflows/prover-daily-test.yaml b/.github/workflows/prover-daily-test.yaml index aecf35e45cf75..5ed59ec538937 100644 --- a/.github/workflows/prover-daily-test.yaml +++ b/.github/workflows/prover-daily-test.yaml @@ -22,7 +22,7 @@ concurrency: jobs: prover-inconsistency-test: - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} timeout-minutes: ${{ github.event_name == 'pull_request' && 10 || 480}} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-gas-calibration.yaml b/.github/workflows/run-gas-calibration.yaml index bf861c8ffd2aa..1f91062a4f633 100644 --- a/.github/workflows/run-gas-calibration.yaml +++ b/.github/workflows/run-gas-calibration.yaml @@ -25,7 +25,7 @@ concurrency: jobs: run-gas-calibration: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/rust-client-tests.yaml b/.github/workflows/rust-client-tests.yaml index 90a3f07adff39..0b5ce8e5a9c9a 100644 --- a/.github/workflows/rust-client-tests.yaml +++ b/.github/workflows/rust-client-tests.yaml @@ -31,7 +31,7 @@ jobs: run-tests-devnet: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') needs: [permission-check] - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main @@ -50,7 +50,7 @@ jobs: run-tests-testnet: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') needs: [permission-check] - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main @@ -69,7 +69,7 @@ jobs: run-tests-mainnet: if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests') needs: [permission-check] - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main diff --git a/.github/workflows/ts-sdk-e2e-tests.yaml b/.github/workflows/ts-sdk-e2e-tests.yaml index 942d07e24f66d..c906451abd78b 100644 --- a/.github/workflows/ts-sdk-e2e-tests.yaml +++ b/.github/workflows/ts-sdk-e2e-tests.yaml @@ -53,7 +53,7 @@ jobs: # Now that the latter runs against the local testnet too we make these land blocking. run-tests-main-branch: needs: [permission-check, file_change_determinator] - runs-on: high-perf-docker + runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }} steps: - uses: actions/checkout@v3 if: needs.file_change_determinator.outputs.only_docs_changed != 'true' From 3d4d746416708ff422781c46e3a01861e9708345 Mon Sep 17 00:00:00 2001 From: Zekun Wang <41706692+fEst1ck@users.noreply.github.com> Date: Thu, 27 Jun 2024 23:58:00 -0400 Subject: [PATCH 006/469] [Compiler-V2] Check Unused Assignments (#13543) * add unused assignment checker * update tests * add checker * update tests * format * add test * reformat err msg * refactor * update tests * update tests * update tests * update tests * format * add test * import more tests * tab to space * add test * remove wrong doc * add test * update tests --------- Co-authored-by: Zekun Wang --- .../move/move-compiler-v2/src/experiments.rs | 6 + third_party/move/move-compiler-v2/src/lib.rs | 9 +- .../move/move-compiler-v2/src/pipeline/mod.rs | 1 + .../src/pipeline/unused_assignment_checker.rs | 84 ++++++++ .../borrowed_from_one_path.exp | 44 ++++ .../tests/ability-transform/by_reference.exp | 198 ++++++++++++++++++ .../ability-transform/copy_ability_tuple.exp | 34 +++ .../ability-transform/dead_but_borrowed.exp | 17 ++ .../ability-transform/destroy_after_call.exp | 40 ++++ .../ability-transform/drop_after_loop.exp | 66 ++++++ .../ability-transform/drop_at_branch.exp | 23 ++ .../foreach_mut_expanded.exp | 81 +++++++ .../tests/ability-transform/mutate_return.exp | 26 +++ .../tests/ability-transform/mutate_vector.exp | 41 ++++ .../bytecode-generator/freeze_mut_ref.exp | 38 ++++ .../tests/bytecode-generator/wildcard7.exp | 8 + .../tests/copy-propagation/call_1.exp | 8 + .../tests/copy-propagation/immut_refs_2.exp | 8 + .../tests/copy-propagation/seq_kills_1.exp | 8 + .../tests/copy-propagation/seq_kills_2.exp | 8 + .../copy-propagation/straight_line_kills.exp | 8 + .../tests/file-format-generator/const.exp | 68 ++++++ .../tests/file-format-generator/const.opt.exp | 68 ++++++ .../tests/reference-safety/write_ref_dest.exp | 8 + .../write_ref_dest.no-opt.exp | 8 + .../bind_with_type_annot.exp | 20 ++ .../use_before_assign.exp | 8 + .../use_before_assign_while.exp | 8 + .../tests/simplifier/random.exp | 8 + .../move/move-compiler-v2/tests/testsuite.rs | 14 ++ .../conditional_loop_unreachable.exp | 14 ++ .../assignment_in_loops.move | 8 + .../unused-assignment/struct_assign_swap.exp | 13 ++ .../unused-assignment/struct_assign_swap.move | 27 +++ .../unused_assign_to_param.exp | 19 ++ .../unused_assign_to_param.move | 12 ++ .../unused_call_assign_shadow.exp | 57 +++++ .../unused_call_assign_shadow.move | 26 +++ .../unused-assignment/unused_in_pattern.exp | 81 +++++++ .../unused-assignment/unused_in_pattern.move | 40 ++++ .../v1-commands/mixed_lvalue.exp | 13 ++ .../v1-commands/mixed_lvalue.move | 17 ++ .../v1-liveness/unused_assignment.exp | 57 +++++ .../v1-liveness/unused_assignment.move | 45 ++++ .../v1-locals/assign_partial_resource.exp | 25 +++ .../v1-locals/assign_partial_resource.move | 34 +++ .../v1-locals/assign_resource.exp | 13 ++ .../v1-locals/assign_resource.move | 37 ++++ .../v1-locals/struct_use_before_assign.exp | 131 ++++++++++++ .../v1-locals/struct_use_before_assign.move | 33 +++ .../v1-locals/unused_copyable.exp | 27 +++ .../v1-locals/unused_copyable.move | 16 ++ .../v1-locals/unused_resource.exp | 45 ++++ .../v1-locals/unused_resource.move | 39 ++++ .../unused_resource_explicit_return.exp | 39 ++++ .../unused_resource_explicit_return.move | 40 ++++ .../move/move-compiler-v2/tests/v1.matched | 8 + .../move/move-compiler-v2/tests/v1.unmatched | 8 - .../tests/variable-coalescing/call_1.exp | 8 + .../tests/variable-coalescing/call_1.opt.exp | 8 + .../variable-coalescing/cant_coalesce_1.exp | 8 + .../cant_coalesce_1.opt.exp | 8 + .../cant_copy_propagate.exp | 14 ++ .../cant_copy_propagate.opt.exp | 14 ++ .../cyclic_assignment_without_use.exp | 8 + .../cyclic_assignment_without_use.opt.exp | 8 + .../variable-coalescing/dead_assignment_3.exp | 8 + .../dead_assignment_3.opt.exp | 8 + .../variable-coalescing/dead_assignment_4.exp | 20 ++ .../dead_assignment_4.opt.exp | 14 ++ .../variable-coalescing/immut_refs_2.exp | 8 + .../variable-coalescing/immut_refs_2.opt.exp | 8 + .../variable-coalescing/intermingled_1.exp | 8 + .../intermingled_1.opt.exp | 8 + .../variable-coalescing/intermingled_3.exp | 8 + .../intermingled_3.opt.exp | 8 + .../variable-coalescing/multi_assigns.exp | 8 + .../variable-coalescing/multi_assigns.opt.exp | 8 + .../non_overlapping_vars1.exp | 14 ++ .../non_overlapping_vars1.opt.exp | 14 ++ .../non_overlapping_vars_diff_type.exp | 14 ++ .../non_overlapping_vars_diff_type.opt.exp | 14 ++ .../variable-coalescing/reassigned_var.exp | 8 + .../reassigned_var.opt.exp | 8 + .../variable-coalescing/self_assigns.exp | 8 + .../variable-coalescing/self_assigns.opt.exp | 8 + .../tests/variable-coalescing/seq_kills_1.exp | 8 + .../variable-coalescing/seq_kills_1.opt.exp | 8 + .../tests/variable-coalescing/seq_kills_2.exp | 8 + .../variable-coalescing/seq_kills_2.opt.exp | 8 + .../straight_line_kills.exp | 8 + .../straight_line_kills.opt.exp | 8 + .../tests/variable-coalescing/unused_add.exp | 8 + .../tools/testdiff/src/main.rs | 3 + .../tests/inlining/shadowing_renamed.exp | 21 +- .../inlining/shadowing_renamed_param.exp | 21 +- .../tests/misc/struct_assign_swap.move | 4 +- ...th_side_effect_14.operator-eval-lang-2.exp | 30 +++ .../functional/ModifiesErrorTest.v2_exp | 60 ++++++ .../sources/functional/ModifiesTest.v2_exp | 11 + .../tests/sources/functional/bitset.v2_exp | 11 + .../functional/bitwise_features.v2_exp | 5 + .../sources/functional/inline-lambda.v2_exp | 12 ++ .../functional/inline_fun_simple.v2_exp | 24 +++ .../sources/functional/specs_in_fun.v2_exp | 24 +++ .../functional/specs_in_fun_ref.v2_exp | 29 +++ .../functional/verify_custom_table.v2_exp | 12 ++ .../tests/sources/regression/bug_828.v2_exp | 5 + 108 files changed, 2465 insertions(+), 31 deletions(-) create mode 100644 third_party/move/move-compiler-v2/src/pipeline/unused_assignment_checker.rs create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/assignment_in_loops.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.move create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.exp create mode 100644 third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.move create mode 100644 third_party/move/move-prover/tests/sources/functional/ModifiesTest.v2_exp create mode 100644 third_party/move/move-prover/tests/sources/functional/bitset.v2_exp create mode 100644 third_party/move/move-prover/tests/sources/functional/bitwise_features.v2_exp create mode 100644 third_party/move/move-prover/tests/sources/functional/specs_in_fun_ref.v2_exp create mode 100644 third_party/move/move-prover/tests/sources/regression/bug_828.v2_exp diff --git a/third_party/move/move-compiler-v2/src/experiments.rs b/third_party/move/move-compiler-v2/src/experiments.rs index 6a75cebb01639..7003c7ee9a19d 100644 --- a/third_party/move/move-compiler-v2/src/experiments.rs +++ b/third_party/move/move-compiler-v2/src/experiments.rs @@ -141,6 +141,11 @@ pub static EXPERIMENTS: Lazy> = Lazy::new(|| { description: "Whether to check for unused struct type parameters".to_string(), default: Inherited(Experiment::CHECKS.to_string()), }, + Experiment { + name: Experiment::UNUSED_ASSIGNMENT_CHECK.to_string(), + description: "Whether to check for unused assignments".to_string(), + default: Inherited(Experiment::CHECKS.to_string()), + }, Experiment { name: Experiment::VARIABLE_COALESCING.to_string(), description: "Whether to run variable coalescing".to_string(), @@ -214,6 +219,7 @@ impl Experiment { pub const SPEC_REWRITE: &'static str = "spec-rewrite"; pub const SPLIT_CRITICAL_EDGES: &'static str = "split-critical-edges"; pub const UNINITIALIZED_CHECK: &'static str = "uninitialized-check"; + pub const UNUSED_ASSIGNMENT_CHECK: &'static str = "unused-assignment-check"; pub const UNUSED_STRUCT_PARAMS_CHECK: &'static str = "unused-struct-params-check"; pub const USAGE_CHECK: &'static str = "usage-check"; pub const VARIABLE_COALESCING: &'static str = "variable-coalescing"; diff --git a/third_party/move/move-compiler-v2/src/lib.rs b/third_party/move/move-compiler-v2/src/lib.rs index 2a2ad7a772321..d46eab6916f7e 100644 --- a/third_party/move/move-compiler-v2/src/lib.rs +++ b/third_party/move/move-compiler-v2/src/lib.rs @@ -33,7 +33,9 @@ use crate::{ split_critical_edges_processor::SplitCriticalEdgesProcessor, uninitialized_use_checker::UninitializedUseChecker, unreachable_code_analysis::UnreachableCodeProcessor, - unreachable_code_remover::UnreachableCodeRemover, variable_coalescing::VariableCoalescing, + unreachable_code_remover::UnreachableCodeRemover, + unused_assignment_checker::UnusedAssignmentChecker, + variable_coalescing::VariableCoalescing, }, }; use anyhow::bail; @@ -391,6 +393,11 @@ pub fn bytecode_pipeline(env: &GlobalEnv) -> FunctionTargetPipeline { pipeline.add_processor(Box::new(UninitializedUseChecker { keep_annotations })); } + if options.experiment_on(Experiment::UNUSED_ASSIGNMENT_CHECK) { + pipeline.add_processor(Box::new(LiveVarAnalysisProcessor::new(false))); + pipeline.add_processor(Box::new(UnusedAssignmentChecker {})); + } + // Reference check is always run, but the processor decides internally // based on `Experiment::REFERENCE_SAFETY` whether to report errors. pipeline.add_processor(Box::new(LiveVarAnalysisProcessor::new(false))); diff --git a/third_party/move/move-compiler-v2/src/pipeline/mod.rs b/third_party/move/move-compiler-v2/src/pipeline/mod.rs index aad9578b3f430..ece4cbf80e4d8 100644 --- a/third_party/move/move-compiler-v2/src/pipeline/mod.rs +++ b/third_party/move/move-compiler-v2/src/pipeline/mod.rs @@ -23,6 +23,7 @@ pub mod split_critical_edges_processor; pub mod uninitialized_use_checker; pub mod unreachable_code_analysis; pub mod unreachable_code_remover; +pub mod unused_assignment_checker; pub mod variable_coalescing; pub mod visibility_checker; diff --git a/third_party/move/move-compiler-v2/src/pipeline/unused_assignment_checker.rs b/third_party/move/move-compiler-v2/src/pipeline/unused_assignment_checker.rs new file mode 100644 index 0000000000000..631ed8b56ac8d --- /dev/null +++ b/third_party/move/move-compiler-v2/src/pipeline/unused_assignment_checker.rs @@ -0,0 +1,84 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +//! Implements a pipeline that checks and gives warning on unused assignments. +//! Prerequisite: live variable annotation. + +use crate::pipeline::livevar_analysis_processor::LiveVarAnnotation; +use codespan_reporting::diagnostic::Severity; +use move_binary_format::file_format::CodeOffset; +use move_model::{ast::TempIndex, model::FunctionEnv}; +use move_stackless_bytecode::{ + function_target::{FunctionData, FunctionTarget}, + function_target_pipeline::{FunctionTargetProcessor, FunctionTargetsHolder}, + stackless_bytecode::{AttrId, Bytecode}, +}; + +pub struct UnusedAssignmentChecker {} + +impl UnusedAssignmentChecker { + /// Check if the assignment to `dst` at offset after the position given by `offset` and `after`. + fn check_unused_assignment( + target: &FunctionTarget, + id: AttrId, + offset: CodeOffset, + dst: TempIndex, + ) { + let data = target.data; + // only check for user defined variables + if let Some(dst_name) = data.local_names.get(&dst) { + let live_var_info = target + .get_annotations() + .get::() + .expect("live variable annotation") + .get_info_at(offset); + let live_after = &live_var_info.after; + let dst_name = dst_name.display(target.func_env.symbol_pool()).to_string(); + if !dst_name.starts_with('_') && live_after.get(&dst).is_none() { + let loc = target.get_bytecode_loc(id); + target + .global_env() + .diag( + Severity::Warning, + &loc, + &format!("Unused assignment to `{}`. Consider removing or prefixing with an underscore: `_{}`", dst_name, dst_name) + ); + } + } + } +} + +impl FunctionTargetProcessor for UnusedAssignmentChecker { + fn process( + &self, + _targets: &mut FunctionTargetsHolder, + func_env: &FunctionEnv, + data: FunctionData, + _scc_opt: Option<&[FunctionEnv]>, + ) -> FunctionData { + if func_env.is_native() { + return data; + } + let target = FunctionTarget::new(func_env, &data); + for (offset, bytecode) in data.code.iter().enumerate() { + let offset = offset as u16; + use Bytecode::*; + match bytecode { + Load(id, dst, _) | Assign(id, dst, _, _) => { + UnusedAssignmentChecker::check_unused_assignment(&target, *id, offset, *dst) + }, + Call(id, dsts, _, _, _) => { + for dst in dsts { + UnusedAssignmentChecker::check_unused_assignment(&target, *id, offset, *dst) + } + }, + _ => {}, + } + } + data + } + + fn name(&self) -> String { + "UnusedAssignmentChecker".to_string() + } +} diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/borrowed_from_one_path.exp b/third_party/move/move-compiler-v2/tests/ability-transform/borrowed_from_one_path.exp index bf7ce3ca16be6..ae4da36466455 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/borrowed_from_one_path.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/borrowed_from_one_path.exp @@ -71,6 +71,50 @@ fun m::f($t0: u8, $t1: &vector): u64 { 14: return $t2 } +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +fun m::f($t0: u8, $t1: &vector): u64 { + var $t2: u64 + var $t3: &vector + var $t4: bool + var $t5: u8 + var $t6: &m::R + var $t7: address + var $t8: &u64 + var $t9: u64 + # live vars: $t0, $t1 + 0: $t5 := 0 + # live vars: $t0, $t1, $t5 + 1: $t4 := ==($t0, $t5) + # live vars: $t1, $t4 + 2: if ($t4) goto 3 else goto 8 + # live vars: $t1 + 3: label L0 + # live vars: + 4: $t7 := 0x1 + # live vars: $t7 + 5: $t6 := borrow_global($t7) + # live vars: $t6 + 6: $t3 := borrow_field.data($t6) + # live vars: $t3 + 7: goto 10 + # live vars: $t1 + 8: label L1 + # live vars: $t1 + 9: $t3 := infer($t1) + # live vars: $t3 + 10: label L2 + # live vars: $t3 + 11: $t9 := 0 + # live vars: $t3, $t9 + 12: $t8 := vector::borrow($t3, $t9) + # live vars: $t8 + 13: $t2 := read_ref($t8) + # live vars: $t2 + 14: return $t2 +} + ============ after ReferenceSafetyProcessor: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/by_reference.exp b/third_party/move/move-compiler-v2/tests/ability-transform/by_reference.exp index 30f24ceb0bff8..b6044fa8bb79c 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/by_reference.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/by_reference.exp @@ -313,6 +313,204 @@ fun _0::check() { 80: return () } +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +fun _0::check() { + var $t0: bool + var $t1: u64 + var $t2: bool + var $t3: u64 + var $t4: bool + var $t5: u64 + var $t6: u64 + var $t7: u64 + var $t8: bool + var $t9: vector + var $t10: vector + var $t11: u64 + var $t12: &mut u64 + var $t13: u64 + var $t14: u64 + var $t15: &mut vector + var $t16: vector + var $t17: vector + var $t18: bool + var $t19: u64 + var $t20: u64 + var $t21: u64 + var $t22: bool + var $t23: vector + var $t24: vector + var $t25: u64 + var $t26: bool + var $t27: u64 + var $t28: bool + var $t29: u64 + # live vars: + 0: $t0 := true + # live vars: $t0 + 1: if ($t0) goto 2 else goto 4 + # live vars: + 2: label L0 + # live vars: + 3: goto 7 + # live vars: + 4: label L1 + # live vars: + 5: $t1 := 42 + # live vars: $t1 + 6: abort($t1) + # live vars: + 7: label L2 + # live vars: + 8: $t2 := true + # live vars: $t2 + 9: if ($t2) goto 10 else goto 12 + # live vars: + 10: label L3 + # live vars: + 11: goto 15 + # live vars: + 12: label L4 + # live vars: + 13: $t3 := 42 + # live vars: $t3 + 14: abort($t3) + # live vars: + 15: label L5 + # live vars: + 16: $t5 := 0 + # live vars: $t5 + 17: $t6 := 0 + # live vars: $t5, $t6 + 18: $t4 := ==($t5, $t6) + # live vars: $t4 + 19: if ($t4) goto 20 else goto 22 + # live vars: + 20: label L6 + # live vars: + 21: goto 25 + # live vars: + 22: label L7 + # live vars: + 23: $t7 := 42 + # live vars: $t7 + 24: abort($t7) + # live vars: + 25: label L8 + # live vars: + 26: $t9 := [104, 101, 108, 108, 111] + # live vars: $t9 + 27: $t10 := [104, 101, 108, 108, 111] + # live vars: $t9, $t10 + 28: $t8 := ==($t9, $t10) + # live vars: $t8 + 29: if ($t8) goto 30 else goto 32 + # live vars: + 30: label L9 + # live vars: + 31: goto 35 + # live vars: + 32: label L10 + # live vars: + 33: $t11 := 42 + # live vars: $t11 + 34: abort($t11) + # live vars: + 35: label L11 + # live vars: + 36: $t13 := 0 + # live vars: $t13 + 37: $t12 := borrow_local($t13) + # live vars: $t12 + 38: $t14 := 1 + # live vars: $t12, $t14 + 39: write_ref($t12, $t14) + # live vars: $t12 + 40: $t16 := [104, 101, 108, 108, 111] + # live vars: $t12, $t16 + 41: $t15 := borrow_local($t16) + # live vars: $t12, $t15 + 42: $t17 := [98, 121, 101] + # live vars: $t12, $t15, $t17 + 43: write_ref($t15, $t17) + # live vars: $t12, $t15 + 44: $t19 := read_ref($t12) + # live vars: $t15, $t19 + 45: $t20 := 1 + # live vars: $t15, $t19, $t20 + 46: $t18 := ==($t19, $t20) + # live vars: $t15, $t18 + 47: if ($t18) goto 48 else goto 50 + # live vars: $t15 + 48: label L12 + # live vars: $t15 + 49: goto 53 + # live vars: $t15 + 50: label L13 + # live vars: + 51: $t21 := 42 + # live vars: $t21 + 52: abort($t21) + # live vars: $t15 + 53: label L14 + # live vars: $t15 + 54: $t23 := read_ref($t15) + # live vars: $t23 + 55: $t24 := [98, 121, 101] + # live vars: $t23, $t24 + 56: $t22 := ==($t23, $t24) + # live vars: $t22 + 57: if ($t22) goto 58 else goto 60 + # live vars: + 58: label L15 + # live vars: + 59: goto 63 + # live vars: + 60: label L16 + # live vars: + 61: $t25 := 42 + # live vars: $t25 + 62: abort($t25) + # live vars: + 63: label L17 + # live vars: + 64: $t26 := true + # live vars: $t26 + 65: if ($t26) goto 66 else goto 68 + # live vars: + 66: label L18 + # live vars: + 67: goto 71 + # live vars: + 68: label L19 + # live vars: + 69: $t27 := 42 + # live vars: $t27 + 70: abort($t27) + # live vars: + 71: label L20 + # live vars: + 72: $t28 := true + # live vars: $t28 + 73: if ($t28) goto 74 else goto 76 + # live vars: + 74: label L21 + # live vars: + 75: goto 79 + # live vars: + 76: label L22 + # live vars: + 77: $t29 := 42 + # live vars: $t29 + 78: abort($t29) + # live vars: + 79: label L23 + # live vars: + 80: return () +} + ============ after ReferenceSafetyProcessor: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/copy_ability_tuple.exp b/third_party/move/move-compiler-v2/tests/ability-transform/copy_ability_tuple.exp index 63346bfd6f41b..739afae97ab0f 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/copy_ability_tuple.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/copy_ability_tuple.exp @@ -38,6 +38,40 @@ public fun M::f($t0: M::R): (M::R, u64) { } +[variant baseline] +public fun M::g($t0: &signer) { + var $t1: M::R + var $t2: u64 + var $t3: u64 + # live vars: $t0 + 0: $t2 := 1 + # live vars: $t0, $t2 + 1: $t1 := pack M::R($t2) + # live vars: $t0, $t1 + 2: $t3 := 3 + # live vars: $t0, $t1 + 3: ($t1, $t3) := M::f($t1) + # live vars: $t0, $t1 + 4: move_to($t0, $t1) + # live vars: + 5: return () +} + +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +public fun M::f($t0: M::R): (M::R, u64) { + var $t1: M::R + var $t2: u64 + # live vars: $t0 + 0: $t1 := infer($t0) + # live vars: $t1 + 1: $t2 := 0 + # live vars: $t1, $t2 + 2: return ($t1, $t2) +} + + [variant baseline] public fun M::g($t0: &signer) { var $t1: M::R diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/dead_but_borrowed.exp b/third_party/move/move-compiler-v2/tests/ability-transform/dead_but_borrowed.exp index 66cf31cd5bc5c..2f9cb7596ce67 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/dead_but_borrowed.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/dead_but_borrowed.exp @@ -28,6 +28,23 @@ fun explicate_drop::test0(): u8 { 3: return $t0 } +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +fun explicate_drop::test0(): u8 { + var $t0: u8 + var $t1: &u8 + var $t2: u8 + # live vars: + 0: $t2 := 42 + # live vars: $t2 + 1: $t1 := borrow_local($t2) + # live vars: $t1 + 2: $t0 := read_ref($t1) + # live vars: $t0 + 3: return $t0 +} + ============ after ReferenceSafetyProcessor: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/destroy_after_call.exp b/third_party/move/move-compiler-v2/tests/ability-transform/destroy_after_call.exp index e0c1d85b4cade..1a8be7b364eb7 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/destroy_after_call.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/destroy_after_call.exp @@ -34,6 +34,46 @@ fun m::f($t0: &mut u64): &mut u64 { } +[variant baseline] +fun m::g() { + var $t0: u64 + var $t1: &mut u64 + var $t2: &mut u64 + var $t3: &u64 + # live vars: + 0: $t0 := 22 + # live vars: $t0 + 1: $t1 := borrow_local($t0) + # live vars: $t0, $t1 + 2: $t2 := m::f($t1) + # live vars: $t0, $t2 + 3: $t1 := infer($t2) + # live vars: $t0 + 4: $t3 := borrow_local($t0) + # live vars: + 5: return () +} + + +Diagnostics: +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/ability-transform/destroy_after_call.move:8:9 + │ +8 │ r = f(r); + │ ^^^^^^^^ + +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +fun m::f($t0: &mut u64): &mut u64 { + var $t1: &mut u64 + # live vars: $t0 + 0: $t1 := infer($t0) + # live vars: $t1 + 1: return $t1 +} + + [variant baseline] fun m::g() { var $t0: u64 diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/drop_after_loop.exp b/third_party/move/move-compiler-v2/tests/ability-transform/drop_after_loop.exp index 257b1bae0d6cd..77edac67936f6 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/drop_after_loop.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/drop_after_loop.exp @@ -104,6 +104,72 @@ fun m::drop_after_loop() { 25: return () } +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +fun m::drop_after_loop() { + var $t0: u64 + var $t1: &mut u64 + var $t2: bool + var $t3: u64 + var $t4: bool + var $t5: bool + var $t6: u64 + var $t7: u64 + # live vars: + 0: $t0 := 1 + # live vars: $t0 + 1: $t1 := borrow_local($t0) + # live vars: $t0, $t1 + 2: $t2 := true + # live vars: $t0, $t1, $t2 + 3: label L0 + # live vars: $t0, $t1, $t2 + 4: if ($t2) goto 5 else goto 11 + # live vars: $t0, $t1 + 5: label L2 + # live vars: $t0, $t1 + 6: $t3 := 2 + # live vars: $t0, $t1, $t3 + 7: write_ref($t1, $t3) + # live vars: $t0, $t1 + 8: $t4 := false + # live vars: $t0, $t1, $t4 + 9: $t2 := infer($t4) + # live vars: $t0, $t1, $t2 + 10: goto 13 + # live vars: $t0, $t1 + 11: label L3 + # live vars: $t0 + 12: goto 15 + # live vars: $t0, $t1, $t2 + 13: label L4 + # live vars: $t0, $t1, $t2 + 14: goto 3 + # live vars: $t0 + 15: label L1 + # live vars: $t0 + 16: $t6 := 2 + # live vars: $t0, $t6 + 17: $t5 := ==($t0, $t6) + # live vars: $t5 + 18: if ($t5) goto 19 else goto 21 + # live vars: + 19: label L5 + # live vars: + 20: goto 24 + # live vars: + 21: label L6 + # live vars: + 22: $t7 := 0 + # live vars: $t7 + 23: abort($t7) + # live vars: + 24: label L7 + # live vars: + 25: return () +} + ============ after ReferenceSafetyProcessor: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/drop_at_branch.exp b/third_party/move/move-compiler-v2/tests/ability-transform/drop_at_branch.exp index 7b5153dffd6dd..baaa593bf4a38 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/drop_at_branch.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/drop_at_branch.exp @@ -36,6 +36,29 @@ fun explicate_drop::drop_at_branch($t0: bool): u8 { 7: return $t1 } +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +fun explicate_drop::drop_at_branch($t0: bool): u8 { + var $t1: u8 + # live vars: $t0 + 0: if ($t0) goto 1 else goto 4 + # live vars: + 1: label L0 + # live vars: + 2: $t1 := 1 + # live vars: $t1 + 3: goto 6 + # live vars: + 4: label L1 + # live vars: + 5: $t1 := 0 + # live vars: $t1 + 6: label L2 + # live vars: $t1 + 7: return $t1 +} + ============ after ReferenceSafetyProcessor: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/foreach_mut_expanded.exp b/third_party/move/move-compiler-v2/tests/ability-transform/foreach_mut_expanded.exp index f696ee52a0ffb..57cddffaf1f90 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/foreach_mut_expanded.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/foreach_mut_expanded.exp @@ -129,6 +129,87 @@ fun m::test_for_each_mut() { 30: return () } +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +fun m::test_for_each_mut() { + var $t0: vector + var $t1: u64 + var $t2: u64 + var $t3: &vector + var $t4: &mut vector + var $t5: bool + var $t6: &mut u64 + var $t7: u64 + var $t8: u64 + var $t9: u64 + var $t10: bool + var $t11: vector + var $t12: u64 + # live vars: + 0: $t0 := ["1", "2", "3"] + # live vars: $t0 + 1: $t1 := 0 + # live vars: $t0, $t1 + 2: $t3 := borrow_local($t0) + # live vars: $t0, $t1, $t3 + 3: $t2 := vector::length($t3) + # live vars: $t0, $t1, $t2 + 4: $t4 := borrow_local($t0) + # live vars: $t0, $t1, $t2, $t4 + 5: label L0 + # live vars: $t0, $t1, $t2, $t4 + 6: $t5 := <($t1, $t2) + # live vars: $t0, $t1, $t2, $t4, $t5 + 7: if ($t5) goto 8 else goto 16 + # live vars: $t0, $t1, $t2, $t4 + 8: label L2 + # live vars: $t0, $t1, $t2, $t4 + 9: $t6 := vector::borrow_mut($t4, $t1) + # live vars: $t0, $t1, $t2, $t4, $t6 + 10: $t7 := 2 + # live vars: $t0, $t1, $t2, $t4, $t6, $t7 + 11: write_ref($t6, $t7) + # live vars: $t0, $t1, $t2, $t4 + 12: $t9 := 1 + # live vars: $t0, $t1, $t2, $t4, $t9 + 13: $t8 := +($t1, $t9) + # live vars: $t0, $t2, $t4, $t8 + 14: $t1 := infer($t8) + # live vars: $t0, $t1, $t2, $t4 + 15: goto 18 + # live vars: $t0, $t1, $t2, $t4 + 16: label L3 + # live vars: $t0 + 17: goto 20 + # live vars: $t0, $t1, $t2, $t4 + 18: label L4 + # live vars: $t0, $t1, $t2, $t4 + 19: goto 5 + # live vars: $t0 + 20: label L1 + # live vars: $t0 + 21: $t11 := ["2", "3", "4"] + # live vars: $t0, $t11 + 22: $t10 := ==($t0, $t11) + # live vars: $t10 + 23: if ($t10) goto 24 else goto 26 + # live vars: + 24: label L5 + # live vars: + 25: goto 29 + # live vars: + 26: label L6 + # live vars: + 27: $t12 := 0 + # live vars: $t12 + 28: abort($t12) + # live vars: + 29: label L7 + # live vars: + 30: return () +} + ============ after ReferenceSafetyProcessor: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/mutate_return.exp b/third_party/move/move-compiler-v2/tests/ability-transform/mutate_return.exp index e83571eda35db..f7f2e990167a2 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/mutate_return.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/mutate_return.exp @@ -38,6 +38,32 @@ public fun m::singleton<#0>($t0: #0): vector<#0> { } +[variant baseline] +fun m::g<#0>($t0: &mut vector<#0>) { + # live vars: $t0 + 0: return () +} + +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +public fun m::singleton<#0>($t0: #0): vector<#0> { + var $t1: vector<#0> + var $t2: vector<#0> + var $t3: &mut vector<#0> + # live vars: $t0 + 0: $t2 := vector($t0) + # live vars: $t2 + 1: $t3 := borrow_local($t2) + # live vars: $t2, $t3 + 2: m::g<#0>($t3) + # live vars: $t2 + 3: $t1 := infer($t2) + # live vars: $t1 + 4: return $t1 +} + + [variant baseline] fun m::g<#0>($t0: &mut vector<#0>) { # live vars: $t0 diff --git a/third_party/move/move-compiler-v2/tests/ability-transform/mutate_vector.exp b/third_party/move/move-compiler-v2/tests/ability-transform/mutate_vector.exp index dcfea63e6ee36..c9b8e8c2e4574 100644 --- a/third_party/move/move-compiler-v2/tests/ability-transform/mutate_vector.exp +++ b/third_party/move/move-compiler-v2/tests/ability-transform/mutate_vector.exp @@ -57,6 +57,47 @@ public fun m::new_scalar_from_u8($t0: u8): m::Scalar { } +[variant baseline] +public fun m::scalar_zero(): m::Scalar { + var $t0: m::Scalar + var $t1: vector + # live vars: + 0: $t1 := [0] + # live vars: $t1 + 1: $t0 := pack m::Scalar($t1) + # live vars: $t0 + 2: return $t0 +} + +============ after LiveVarAnalysisProcessor: ================ + +[variant baseline] +public fun m::new_scalar_from_u8($t0: u8): m::Scalar { + var $t1: m::Scalar + var $t2: m::Scalar + var $t3: &mut u8 + var $t4: &mut vector + var $t5: &mut m::Scalar + var $t6: u64 + # live vars: $t0 + 0: $t2 := m::scalar_zero() + # live vars: $t0, $t2 + 1: $t5 := borrow_local($t2) + # live vars: $t0, $t2, $t5 + 2: $t4 := borrow_field.data($t5) + # live vars: $t0, $t2, $t4 + 3: $t6 := 0 + # live vars: $t0, $t2, $t4, $t6 + 4: $t3 := vector::borrow_mut($t4, $t6) + # live vars: $t0, $t2, $t3 + 5: write_ref($t3, $t0) + # live vars: $t2 + 6: $t1 := infer($t2) + # live vars: $t1 + 7: return $t1 +} + + [variant baseline] public fun m::scalar_zero(): m::Scalar { var $t0: m::Scalar diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/freeze_mut_ref.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/freeze_mut_ref.exp index ab3905586b4a9..faedb01487623 100644 --- a/third_party/move/move-compiler-v2/tests/bytecode-generator/freeze_mut_ref.exp +++ b/third_party/move/move-compiler-v2/tests/bytecode-generator/freeze_mut_ref.exp @@ -294,6 +294,44 @@ fun freeze_mut_ref::t8($t0: bool, $t1: &mut freeze_mut_ref::S, $t2: &freeze_mut_ } +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/bytecode-generator/freeze_mut_ref.move:58:9 + │ +58 │ (x, y) = (&mut 0, &mut 0); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/bytecode-generator/freeze_mut_ref.move:58:9 + │ +58 │ (x, y) = (&mut 0, &mut 0); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/bytecode-generator/freeze_mut_ref.move:66:24 + │ +66 │ let g = &mut ({x = x + 1; s}).f; + │ ^^^^^^^^^ + +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/bytecode-generator/freeze_mut_ref.move:69:20 + │ +69 │ *({*f = 0; z = y; g}) = 2; + │ ^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/bytecode-generator/freeze_mut_ref.move:74:19 + │ +74 │ if (cond) x = copy s else x = other; + │ ^^^^^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/bytecode-generator/freeze_mut_ref.move:74:35 + │ +74 │ if (cond) x = copy s else x = other; + │ ^^^^^^^^^ + + Diagnostics: error: mutable reference in local `s` requires exclusive access but is borrowed ┌─ tests/bytecode-generator/freeze_mut_ref.move:66:35 diff --git a/third_party/move/move-compiler-v2/tests/bytecode-generator/wildcard7.exp b/third_party/move/move-compiler-v2/tests/bytecode-generator/wildcard7.exp index c0d923b9231f5..86431acca3905 100644 --- a/third_party/move/move-compiler-v2/tests/bytecode-generator/wildcard7.exp +++ b/third_party/move/move-compiler-v2/tests/bytecode-generator/wildcard7.exp @@ -42,6 +42,14 @@ public fun m::test(): u8 { } +Diagnostics: +warning: Unused assignment to `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/bytecode-generator/wildcard7.move:6:13 + │ +6 │ let (_, q) = (x, z); + │ ^^^^^^ + + Diagnostics: error: cannot move local `x` since it is still in use ┌─ tests/bytecode-generator/wildcard7.move:5:17 diff --git a/third_party/move/move-compiler-v2/tests/copy-propagation/call_1.exp b/third_party/move/move-compiler-v2/tests/copy-propagation/call_1.exp index 1d261d0a9ea43..4f541ca8bee89 100644 --- a/third_party/move/move-compiler-v2/tests/copy-propagation/call_1.exp +++ b/third_party/move/move-compiler-v2/tests/copy-propagation/call_1.exp @@ -33,6 +33,14 @@ fun m::test($t0: u64): u64 { 6: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/copy-propagation/call_1.move:7:17 + │ +7 │ let a = p; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/copy-propagation/immut_refs_2.exp b/third_party/move/move-compiler-v2/tests/copy-propagation/immut_refs_2.exp index 44e212ca9cdf2..e8f24c1e067df 100644 --- a/third_party/move/move-compiler-v2/tests/copy-propagation/immut_refs_2.exp +++ b/third_party/move/move-compiler-v2/tests/copy-propagation/immut_refs_2.exp @@ -23,6 +23,14 @@ fun m::test($t0: u64): u64 { 5: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/copy-propagation/immut_refs_2.move:4:17 + │ +4 │ let a = &p; + │ ^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_1.exp b/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_1.exp index 55bc0e2c25d3d..5cafa1aff0973 100644 --- a/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_1.exp +++ b/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_1.exp @@ -18,6 +18,14 @@ fun m::test($t0: u64): bool { 7: return $t1 } + +Diagnostics: +warning: Unused assignment to `b`. Consider removing or prefixing with an underscore: `_b` + ┌─ tests/copy-propagation/seq_kills_1.move:7:9 + │ +7 │ b = p + 1; // kill b := a, which removes the whole copy chain + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_2.exp b/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_2.exp index 3680de01d0e30..c4103b2556510 100644 --- a/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_2.exp +++ b/third_party/move/move-compiler-v2/tests/copy-propagation/seq_kills_2.exp @@ -18,6 +18,14 @@ fun m::test($t0: u64): bool { 7: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/copy-propagation/seq_kills_2.move:7:9 + │ +7 │ a = p + 1; // kill b := a + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/copy-propagation/straight_line_kills.exp b/third_party/move/move-compiler-v2/tests/copy-propagation/straight_line_kills.exp index aeb8910c1d774..5b0207d4fbdac 100644 --- a/third_party/move/move-compiler-v2/tests/copy-propagation/straight_line_kills.exp +++ b/third_party/move/move-compiler-v2/tests/copy-propagation/straight_line_kills.exp @@ -16,6 +16,14 @@ fun m::copy_kill($t0: u64): u64 { 6: return $t1 } + +Diagnostics: +warning: Unused assignment to `p`. Consider removing or prefixing with an underscore: `_p` + ┌─ tests/copy-propagation/straight_line_kills.move:5:9 + │ +5 │ p = p + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/file-format-generator/const.exp b/third_party/move/move-compiler-v2/tests/file-format-generator/const.exp index 5be541cbc8b90..e3c70827b11a7 100644 --- a/third_party/move/move-compiler-v2/tests/file-format-generator/const.exp +++ b/third_party/move/move-compiler-v2/tests/file-format-generator/const.exp @@ -67,6 +67,74 @@ warning: Unused local variable `s`. Consider removing or prefixing with an under │ ^ +Diagnostics: +warning: Unused assignment to `const_true`. Consider removing or prefixing with an underscore: `_const_true` + ┌─ tests/file-format-generator/const.move:3:26 + │ +3 │ let const_true = u(true); + │ ^^^^^^^ + +warning: Unused assignment to `const_false`. Consider removing or prefixing with an underscore: `_const_false` + ┌─ tests/file-format-generator/const.move:4:27 + │ +4 │ let const_false = u(false); + │ ^^^^^^^^ + +warning: Unused assignment to `hex_u8`. Consider removing or prefixing with an underscore: `_hex_u8` + ┌─ tests/file-format-generator/const.move:5:26 + │ +5 │ let hex_u8: u8 = u(0x1); + │ ^^^^^^ + +warning: Unused assignment to `hex_u16`. Consider removing or prefixing with an underscore: `_hex_u16` + ┌─ tests/file-format-generator/const.move:6:28 + │ +6 │ let hex_u16: u16 = u(0x1BAE); + │ ^^^^^^^^^ + +warning: Unused assignment to `hex_u32`. Consider removing or prefixing with an underscore: `_hex_u32` + ┌─ tests/file-format-generator/const.move:7:28 + │ +7 │ let hex_u32: u32 = u(0xDEAD80); + │ ^^^^^^^^^^^ + +warning: Unused assignment to `hex_u64`. Consider removing or prefixing with an underscore: `_hex_u64` + ┌─ tests/file-format-generator/const.move:8:28 + │ +8 │ let hex_u64: u64 = u(0xCAFE); + │ ^^^^^^^^^ + +warning: Unused assignment to `hex_u128`. Consider removing or prefixing with an underscore: `_hex_u128` + ┌─ tests/file-format-generator/const.move:9:30 + │ +9 │ let hex_u128: u128 = u(0xDEADBEEF); + │ ^^^^^^^^^^^^^ + +warning: Unused assignment to `hex_u256`. Consider removing or prefixing with an underscore: `_hex_u256` + ┌─ tests/file-format-generator/const.move:10:30 + │ +10 │ let hex_u256: u256 = u(0x1123_456A_BCDE_F); + │ ^^^^^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/file-format-generator/const.move:11:17 + │ +11 │ let a = u(@0x42); + │ ^^^^^^^^ + +warning: Unused assignment to `vec`. Consider removing or prefixing with an underscore: `_vec` + ┌─ tests/file-format-generator/const.move:12:19 + │ +12 │ let vec = u(vector[1, 2, 3]); + │ ^^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `s`. Consider removing or prefixing with an underscore: `_s` + ┌─ tests/file-format-generator/const.move:13:17 + │ +13 │ let s = u(b"Hello!\n"); + │ ^^^^^^^^^^^^^^ + + ============ disassembled file-format ================== // Move bytecode v7 module 42.constant { diff --git a/third_party/move/move-compiler-v2/tests/file-format-generator/const.opt.exp b/third_party/move/move-compiler-v2/tests/file-format-generator/const.opt.exp index a198ea19fbd64..8c6bd9d68e95d 100644 --- a/third_party/move/move-compiler-v2/tests/file-format-generator/const.opt.exp +++ b/third_party/move/move-compiler-v2/tests/file-format-generator/const.opt.exp @@ -67,6 +67,74 @@ warning: Unused local variable `s`. Consider removing or prefixing with an under │ ^ +Diagnostics: +warning: Unused assignment to `const_true`. Consider removing or prefixing with an underscore: `_const_true` + ┌─ tests/file-format-generator/const.move:3:26 + │ +3 │ let const_true = u(true); + │ ^^^^^^^ + +warning: Unused assignment to `const_false`. Consider removing or prefixing with an underscore: `_const_false` + ┌─ tests/file-format-generator/const.move:4:27 + │ +4 │ let const_false = u(false); + │ ^^^^^^^^ + +warning: Unused assignment to `hex_u8`. Consider removing or prefixing with an underscore: `_hex_u8` + ┌─ tests/file-format-generator/const.move:5:26 + │ +5 │ let hex_u8: u8 = u(0x1); + │ ^^^^^^ + +warning: Unused assignment to `hex_u16`. Consider removing or prefixing with an underscore: `_hex_u16` + ┌─ tests/file-format-generator/const.move:6:28 + │ +6 │ let hex_u16: u16 = u(0x1BAE); + │ ^^^^^^^^^ + +warning: Unused assignment to `hex_u32`. Consider removing or prefixing with an underscore: `_hex_u32` + ┌─ tests/file-format-generator/const.move:7:28 + │ +7 │ let hex_u32: u32 = u(0xDEAD80); + │ ^^^^^^^^^^^ + +warning: Unused assignment to `hex_u64`. Consider removing or prefixing with an underscore: `_hex_u64` + ┌─ tests/file-format-generator/const.move:8:28 + │ +8 │ let hex_u64: u64 = u(0xCAFE); + │ ^^^^^^^^^ + +warning: Unused assignment to `hex_u128`. Consider removing or prefixing with an underscore: `_hex_u128` + ┌─ tests/file-format-generator/const.move:9:30 + │ +9 │ let hex_u128: u128 = u(0xDEADBEEF); + │ ^^^^^^^^^^^^^ + +warning: Unused assignment to `hex_u256`. Consider removing or prefixing with an underscore: `_hex_u256` + ┌─ tests/file-format-generator/const.move:10:30 + │ +10 │ let hex_u256: u256 = u(0x1123_456A_BCDE_F); + │ ^^^^^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/file-format-generator/const.move:11:17 + │ +11 │ let a = u(@0x42); + │ ^^^^^^^^ + +warning: Unused assignment to `vec`. Consider removing or prefixing with an underscore: `_vec` + ┌─ tests/file-format-generator/const.move:12:19 + │ +12 │ let vec = u(vector[1, 2, 3]); + │ ^^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `s`. Consider removing or prefixing with an underscore: `_s` + ┌─ tests/file-format-generator/const.move:13:17 + │ +13 │ let s = u(b"Hello!\n"); + │ ^^^^^^^^^^^^^^ + + ============ disassembled file-format ================== // Move bytecode v7 module 42.constant { diff --git a/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.exp b/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.exp index 90b32906711cc..ea5c005f76f31 100644 --- a/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.exp +++ b/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.exp @@ -1,2 +1,10 @@ +Diagnostics: +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/reference-safety/write_ref_dest.move:16:9 + │ +16 │ (y, _) = (*&y, 1); + │ ^^^^^^^^^^^^^^^^^ + + ============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.no-opt.exp b/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.no-opt.exp index 90b32906711cc..ea5c005f76f31 100644 --- a/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.no-opt.exp +++ b/third_party/move/move-compiler-v2/tests/reference-safety/write_ref_dest.no-opt.exp @@ -1,2 +1,10 @@ +Diagnostics: +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/reference-safety/write_ref_dest.move:16:9 + │ +16 │ (y, _) = (*&y, 1); + │ ^^^^^^^^^^^^^^^^^ + + ============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/simplifier-elimination/bind_with_type_annot.exp b/third_party/move/move-compiler-v2/tests/simplifier-elimination/bind_with_type_annot.exp index 0ddea02275090..8568e22ecebc0 100644 --- a/third_party/move/move-compiler-v2/tests/simplifier-elimination/bind_with_type_annot.exp +++ b/third_party/move/move-compiler-v2/tests/simplifier-elimination/bind_with_type_annot.exp @@ -38,4 +38,24 @@ module 0x8675309::M { } // end 0x8675309::M +Diagnostics: +warning: Unused assignment to `f`. Consider removing or prefixing with an underscore: `_f` + ┌─ tests/simplifier-elimination/bind_with_type_annot.move:7:20 + │ +7 │ let (x, b, R{f}): (u64, bool, R) = (0, false, R { f: 0 }); x; b; f; + │ ^^^^ + +warning: Unused assignment to `b`. Consider removing or prefixing with an underscore: `_b` + ┌─ tests/simplifier-elimination/bind_with_type_annot.move:7:13 + │ +7 │ let (x, b, R{f}): (u64, bool, R) = (0, false, R { f: 0 }); x; b; f; + │ ^^^^^^^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/simplifier-elimination/bind_with_type_annot.move:7:13 + │ +7 │ let (x, b, R{f}): (u64, bool, R) = (0, false, R { f: 0 }); x; b; f; + │ ^^^^^^^^^^^^ + + ============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign.exp b/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign.exp index ac63e9d86f699..f10a3cdb4ab96 100644 --- a/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign.exp +++ b/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign.exp @@ -26,3 +26,11 @@ error: use of unassigned local `x` │ 4 │ let y = x; │ ^ + + +Diagnostics: +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/simplifier-elimination/use_before_assign.move:4:13 + │ +4 │ let y = x; + │ ^ diff --git a/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign_while.exp b/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign_while.exp index 223526a98870c..3f769d77944f3 100644 --- a/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign_while.exp +++ b/third_party/move/move-compiler-v2/tests/simplifier-elimination/use_before_assign_while.exp @@ -105,6 +105,14 @@ error: use of unassigned local `x` │ ^^ +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/simplifier-elimination/use_before_assign_while.move:19:60 + │ +19 │ while (cond) { let y = &x; _ = move y; if (cond) { x = 0 }; break } + │ ^^^^^ + + Diagnostics: error: cannot move local `x` since it is still in use ┌─ tests/simplifier-elimination/use_before_assign_while.move:9:32 diff --git a/third_party/move/move-compiler-v2/tests/simplifier/random.exp b/third_party/move/move-compiler-v2/tests/simplifier/random.exp index c476659b15e23..0e9d3d0e91e5b 100644 --- a/third_party/move/move-compiler-v2/tests/simplifier/random.exp +++ b/third_party/move/move-compiler-v2/tests/simplifier/random.exp @@ -144,6 +144,14 @@ module 0x8675::M { } // end 0x8675::M +Diagnostics: +warning: Unused assignment to `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/simplifier/random.move:48:17 + │ +48 │ let q = v; + │ ^ + + Diagnostics: error: cannot mutably borrow since immutable references exist ┌─ tests/simplifier/random.move:56:17 diff --git a/third_party/move/move-compiler-v2/tests/testsuite.rs b/third_party/move/move-compiler-v2/tests/testsuite.rs index e0065c653efae..14d5be58213a7 100644 --- a/third_party/move/move-compiler-v2/tests/testsuite.rs +++ b/third_party/move/move-compiler-v2/tests/testsuite.rs @@ -145,6 +145,20 @@ const TEST_CONFIGS: Lazy> = Lazy::new(|| { dump_bytecode: DumpLevel::None, dump_bytecode_filter: None, }, + TestConfig { + name: "unused-assignment", + runner: |p| run_test(p, get_config_by_name("unused-assignment")), + include: vec!["/unused-assignment/"], + exclude: vec![], + exp_suffix: None, + options: opts + .clone() + .set_experiment(Experiment::UNUSED_ASSIGNMENT_CHECK, true), + stop_after: StopAfter::BytecodePipeline(Some("UnusedAssignmentChecker")), + dump_ast: DumpLevel::None, + dump_bytecode: DumpLevel::None, + dump_bytecode_filter: None, + }, // Tests for lambda lifting TestConfig { name: "lambda-lifting", diff --git a/third_party/move/move-compiler-v2/tests/unreachable-code-remover/conditional_loop_unreachable.exp b/third_party/move/move-compiler-v2/tests/unreachable-code-remover/conditional_loop_unreachable.exp index 189a01e97ca64..6dc6b6253bb77 100644 --- a/third_party/move/move-compiler-v2/tests/unreachable-code-remover/conditional_loop_unreachable.exp +++ b/third_party/move/move-compiler-v2/tests/unreachable-code-remover/conditional_loop_unreachable.exp @@ -37,6 +37,20 @@ fun m::test($t0: bool, $t1: bool) { 26: return () } + +Diagnostics: +warning: Unused assignment to `i`. Consider removing or prefixing with an underscore: `_i` + ┌─ tests/unreachable-code-remover/conditional_loop_unreachable.move:7:17 + │ +7 │ i = i + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `i`. Consider removing or prefixing with an underscore: `_i` + ┌─ tests/unreachable-code-remover/conditional_loop_unreachable.move:12:13 + │ +12 │ i = i + 1; + │ ^^^^^^^^^ + ============ after UnreachableCodeProcessor: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/assignment_in_loops.move b/third_party/move/move-compiler-v2/tests/unused-assignment/assignment_in_loops.move new file mode 100644 index 0000000000000..7c0c38cc7faa7 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/assignment_in_loops.move @@ -0,0 +1,8 @@ +module 0x42::test { + fun unused_assignment_in_loop() { + let x = 1; + for (i in 0..10) { + x = x + 1; + }; + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.exp new file mode 100644 index 0000000000000..ced32810aeec8 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.exp @@ -0,0 +1,13 @@ + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/struct_assign_swap.move:16:13 + │ +16 │ let S { f: x, g: y } = S { f: y, g: x }; + │ ^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/struct_assign_swap.move:16:13 + │ +16 │ let S { f: x, g: y } = S { f: y, g: x }; + │ ^^^^^^^^^^^^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.move b/third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.move new file mode 100644 index 0000000000000..dbc71d497f38e --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/struct_assign_swap.move @@ -0,0 +1,27 @@ +//# publish +module 0xc0ffee::m { + struct S { + f: u32, + g: u32, + } + + fun swap1(x: u32, y: u32): (u32, u32) { + let S { f: x, g: y } = S { f: y, g: x }; + (x, y) + } + + fun swap2(): (u32, u32) { + let x = 44; + let y = 55; + let S { f: x, g: y } = S { f: y, g: x }; + (x, y) + } + + fun test1(): (u32, u32) { + swap1(1, 2) + } +} + +//# run 0xc0ffee::m::test1 + +//# run 0xc0ffee::m::swap2 diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.exp new file mode 100644 index 0000000000000..32ccfbd544060 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.exp @@ -0,0 +1,19 @@ + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_assign_to_param.move:3:9 + │ +3 │ x = 42; + │ ^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_assign_to_param.move:8:13 + │ +8 │ x = 42; + │ ^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_assign_to_param.move:10:9 + │ +10 │ x = 42; + │ ^^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.move b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.move new file mode 100644 index 0000000000000..7557fef94a3fa --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_assign_to_param.move @@ -0,0 +1,12 @@ +module 0x42::test { + fun foo(x: u8) { + x = 42; + } + + fun bar(x: u8) { + if (x > 3) { + x = 42; + }; + x = 42; + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.exp new file mode 100644 index 0000000000000..6b733974dbdba --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.exp @@ -0,0 +1,57 @@ + +Diagnostics: +warning: Unused parameter `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:2:13 + │ +2 │ fun foo(a: u8, _b: u8) { + │ ^ + +warning: Unused local variable `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:5:13 + │ +5 │ let y = 0; + │ ^ + +warning: Unused local variable `w`. Consider removing or prefixing with an underscore: `_w` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:8:13 + │ +8 │ let w = bar(false); + │ ^ + +warning: Unused local variable `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:22:13 + │ +22 │ let x = 0; + │ ^ + +warning: Unused local variable `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:24:13 + │ +24 │ let x = 1; + │ ^ + + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:4:9 + │ +4 │ x = x + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `w`. Consider removing or prefixing with an underscore: `_w` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:8:17 + │ +8 │ let w = bar(false); + │ ^^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:14:13 + │ +14 │ y = y + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_call_assign_shadow.move:16:13 + │ +16 │ y = y + 2; + │ ^^^^^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.move b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.move new file mode 100644 index 0000000000000..70e9688d98c36 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_call_assign_shadow.move @@ -0,0 +1,26 @@ +module 0x42::test { + fun foo(a: u8, _b: u8) { + let x = 0; + x = x + 1; + let y = 0; + let _z = 42; + // unused call assignment + let w = bar(false); + } + + fun bar(x: bool): u8 { + let y = 0; + if (x) { + y = y + 1; + } else { + y = y + 2; + }; + 42 + } + + fun baz() { + let x = 0; + // shadowing + let x = 1; + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.exp new file mode 100644 index 0000000000000..b4f2ce138890c --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.exp @@ -0,0 +1,81 @@ + +Diagnostics: +warning: Unused local variable `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_in_pattern.move:20:11 + │ +20 │ let S { x, y } = s; + │ ^ + +warning: Unused local variable `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_in_pattern.move:20:14 + │ +20 │ let S { x, y } = s; + │ ^ + +warning: Unused local variable `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/unused_in_pattern.move:26:7 + │ +26 │ let z: S; + │ ^ + +warning: Unused local variable `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_in_pattern.move:33:18 + │ +33 │ let T { z: S { x, y } } = t; + │ ^ + +warning: Unused local variable `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_in_pattern.move:33:21 + │ +33 │ let T { z: S { x, y } } = t; + │ ^ + + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_in_pattern.move:15:3 + │ +15 │ S { x, y } = s; + │ ^^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_in_pattern.move:15:3 + │ +15 │ S { x, y } = s; + │ ^^^^^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_in_pattern.move:20:7 + │ +20 │ let S { x, y } = s; + │ ^^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_in_pattern.move:20:7 + │ +20 │ let S { x, y } = s; + │ ^^^^^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_in_pattern.move:28:10 + │ +28 │ T { z: S { x, y } } = t; + │ ^^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_in_pattern.move:28:10 + │ +28 │ T { z: S { x, y } } = t; + │ ^^^^^^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/unused_in_pattern.move:33:14 + │ +33 │ let T { z: S { x, y } } = t; + │ ^^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/unused-assignment/unused_in_pattern.move:33:14 + │ +33 │ let T { z: S { x, y } } = t; + │ ^^^^^^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.move b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.move new file mode 100644 index 0000000000000..921c5d0a1a95e --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/unused_in_pattern.move @@ -0,0 +1,40 @@ +module 0x42::test { + struct S { + x: u8, + y: bool, + } + + struct T { + z: S, + } + + fun unused_assign_in_pattern() { + let x; + let y; + let s = S { x: 42, y: true }; + S { x, y } = s; + } + + fun unused_decl_in_pattern() { + let s = S { x: 42, y: true }; + let S { x, y } = s; + } + + fun unused_assign_in_nested_pattern() { + let x; + let y; + let z: S; + let t = T { z: S { x: 42, y: true } }; + T { z: S { x, y } } = t; + } + + fun unused_decl_in_nested_pattern() { + let t = T { z: S { x: 42, y: true } }; + let T { z: S { x, y } } = t; + } + + fun unused_in_pattern_ok() { + let s = S { x: 42, y: true }; + let S { x: _x, y: _y }= s; + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.exp new file mode 100644 index 0000000000000..f13c591a2243b --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.exp @@ -0,0 +1,13 @@ + +Diagnostics: +error: invalid assignment + ┌─ tests/unused-assignment/v1-commands/mixed_lvalue.move:14:19 + │ +14 │ (_, _, _, s.f) = four(); + │ ^^^ Invalid assignment syntax. Expected: a local, a field write, or a deconstructing assignment + +error: invalid assignment + ┌─ tests/unused-assignment/v1-commands/mixed_lvalue.move:15:19 + │ +15 │ (_, _, _, *r_ref) = four(); + │ ^^^^^^ Invalid assignment syntax. Expected: a local, a field write, or a deconstructing assignment diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.move new file mode 100644 index 0000000000000..2858e5b6d2a34 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.move @@ -0,0 +1,17 @@ +module 0x8675309::A { + + struct S has drop { f: u64 } + + fun four(): (u64, u64, u64, u64) { + (0, 1, 2, 3) + } + + public fun mixed() { + let r = 0; + let r_ref = &mut r; + let s = S { f: 0 }; + + (_, _, _, s.f) = four(); + (_, _, _, *r_ref) = four(); + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.exp new file mode 100644 index 0000000000000..8106befe7dad4 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.exp @@ -0,0 +1,57 @@ + +Diagnostics: +warning: Unused local variable `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:3:13 + │ +3 │ let x = 0; + │ ^ + +warning: Unused local variable `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:13:17 + │ +13 │ let x = 0; + │ ^ + + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:7:17 + │ +7 │ let x = 0; + │ ^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:8:9 + │ +8 │ x = 0; + │ ^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:21:13 + │ +21 │ x = 0; + │ ^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:26:17 + │ +26 │ let x = 0; + │ ^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:28:13 + │ +28 │ x = 1; + │ ^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:30:13 + │ +30 │ x = 2; + │ ^^^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-liveness/unused_assignment.move:41:13 + │ +41 │ x = 1; + │ ^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.move new file mode 100644 index 0000000000000..6b0516e51f7f7 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.move @@ -0,0 +1,45 @@ +module 0x8675309::M { + fun t0() { + let x = 0; + } + + fun t1() { + let x = 0; + x = 0; + } + + fun t2(cond: bool) { + if (cond) { + let x = 0; + } + } + + fun t3(cond: bool) { + let x = 0; + x; + if (cond) { + x = 0; + } + } + + fun t4(cond: bool) { + let x = 0; + if (cond) { + x = 1; + } else { + x = 2; + } + } + + fun t5(cond: bool) { + let x; + while (cond) { + x = 0; + if (cond) { + x; + }; + x = 1; + } + } + +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.exp new file mode 100644 index 0000000000000..7a85759453cee --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.exp @@ -0,0 +1,25 @@ + +Diagnostics: +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/assign_partial_resource.move:6:21 + │ +6 │ if (cond) { r = R{}; }; + │ ^^^^^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/assign_partial_resource.move:13:29 + │ +13 │ if (cond) {} else { r = R{}; }; + │ ^^^^^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/assign_partial_resource.move:20:24 + │ +20 │ while (cond) { r = R{} }; + │ ^^^^^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/assign_partial_resource.move:27:16 + │ +27 │ loop { r = R{} } + │ ^^^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.move new file mode 100644 index 0000000000000..6b77947727f1e --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.move @@ -0,0 +1,34 @@ +module 0x8675309::M { + struct R {} + + fun t1(cond: bool) { + let r: R; + if (cond) { r = R{}; }; + r = R{}; + R{} = r; + } + + fun t2(cond: bool) { + let r: R; + if (cond) {} else { r = R{}; }; + r = R{}; + R{} = r; + } + + fun t3(cond: bool) { + let r: R; + while (cond) { r = R{} }; + r = R{}; + R{} = r; + } + + fun t4() { + let r: R; + loop { r = R{} } + } + + fun t5(cond: bool, x: T, y: T): (T, T) { + if (cond) { x = y }; + (x, y) + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.exp new file mode 100644 index 0000000000000..a0ba8ccfbfe60 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.exp @@ -0,0 +1,13 @@ + +Diagnostics: +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/assign_resource.move:5:17 + │ +5 │ let r = R{}; + │ ^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/assign_resource.move:29:17 + │ +29 │ let r = R{}; + │ ^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.move new file mode 100644 index 0000000000000..cd6ace6d63360 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.move @@ -0,0 +1,37 @@ +module 0x8675309::M { + struct R {} + + fun t0() { + let r = R{}; + r = R{}; + R{} = r; + } + + fun t1(cond: bool) { + let r = R{}; + if (cond) { r = R{}; }; + R{} = r; + } + + fun t2(cond: bool) { + let r = R{}; + if (cond) {} else { r = R{}; }; + R{} = r; + } + + fun t3(cond: bool) { + let r = R{}; + while (cond) { r = R{} }; + R{} = r; + } + + fun t4() { + let r = R{}; + loop { r = R{}; R {} = r } + } + + fun t5(x: T, y: T): T { + x = y; + x + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.exp new file mode 100644 index 0000000000000..e8dbffafd0ea6 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.exp @@ -0,0 +1,131 @@ + +Diagnostics: +warning: Unused local variable `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:8:13 + │ +8 │ let z = y; + │ ^ + +warning: Unused local variable `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:9:13 + │ +9 │ let q = x; + │ ^ + +warning: Unused local variable `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:17:13 + │ +17 │ let z = y; + │ ^ + +warning: Unused local variable `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:18:13 + │ +18 │ let q = x; + │ ^ + +warning: Unused local variable `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:24:13 + │ +24 │ let z = y; + │ ^ + +warning: Unused local variable `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:25:13 + │ +25 │ let q = x; + │ ^ + +warning: Unused local variable `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:30:13 + │ +30 │ let z = y; + │ ^ + +warning: Unused local variable `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:31:13 + │ +31 │ let q = x; + │ ^ + + +Diagnostics: +error: use of unassigned local `g` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:6:17 + │ +6 │ let r = R { f: 3, g }; + │ ^^^^^^^^^^^^^ + +error: use of unassigned local `y0` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:15:17 + │ +15 │ let r = R { f: x0, g: y0 }; + │ ^^^^^^^^^^^^^^^^^^ + +error: use of unassigned local `r` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:23:13 + │ +23 │ let R { f: x, g: y } = r; + │ ^^^^^^^^^^^^^^^^ + +error: use of unassigned local `y` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:30:17 + │ +30 │ let z = y; + │ ^ + +error: use of unassigned local `x` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:31:17 + │ +31 │ let q = x; + │ ^ + + +Diagnostics: +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:8:17 + │ +8 │ let z = y; + │ ^ + +warning: Unused assignment to `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:9:17 + │ +9 │ let q = x; + │ ^ + +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:17:17 + │ +17 │ let z = y; + │ ^ + +warning: Unused assignment to `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:18:17 + │ +18 │ let q = x; + │ ^ + +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:24:17 + │ +24 │ let z = y; + │ ^ + +warning: Unused assignment to `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:25:17 + │ +25 │ let q = x; + │ ^ + +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:30:17 + │ +30 │ let z = y; + │ ^ + +warning: Unused assignment to `q`. Consider removing or prefixing with an underscore: `_q` + ┌─ tests/unused-assignment/v1-locals/struct_use_before_assign.move:31:17 + │ +31 │ let q = x; + │ ^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.move new file mode 100644 index 0000000000000..8d8997c73852d --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.move @@ -0,0 +1,33 @@ +module 0x876543::M { + struct R { f: u64, g: u64 } + + fun main() { + let g: u64; + let r = R { f: 3, g }; + let R { f: x, g: y } = r; + let z = y; + let q = x; + } + + fun main2() { + let x0: u64 = 0; + let y0: u64; + let r = R { f: x0, g: y0 }; + let R { f: x, g: y } = r; + let z = y; + let q = x; + } + + fun main3() { + let r: R; + let R { f: x, g: y } = r; + let z = y; + let q = x; + } + + fun main4() { + let R { f: x, g: y }; + let z = y; + let q = x; + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.exp new file mode 100644 index 0000000000000..2a79d71797f43 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.exp @@ -0,0 +1,27 @@ + +Diagnostics: +warning: Unused parameter `i`. Consider removing or prefixing with an underscore: `_i` + ┌─ tests/unused-assignment/v1-locals/unused_copyable.move:5:12 + │ +5 │ fun t0(i: u64, s: S) { + │ ^ + +warning: Unused parameter `s`. Consider removing or prefixing with an underscore: `_s` + ┌─ tests/unused-assignment/v1-locals/unused_copyable.move:5:20 + │ +5 │ fun t0(i: u64, s: S) { + │ ^ + +warning: Unused local variable `s`. Consider removing or prefixing with an underscore: `_s` + ┌─ tests/unused-assignment/v1-locals/unused_copyable.move:9:13 + │ +9 │ let s = S{}; + │ ^ + + +Diagnostics: +warning: Unused assignment to `s`. Consider removing or prefixing with an underscore: `_s` + ┌─ tests/unused-assignment/v1-locals/unused_copyable.move:9:17 + │ +9 │ let s = S{}; + │ ^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.move new file mode 100644 index 0000000000000..db26cb54321f7 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.move @@ -0,0 +1,16 @@ +module 0x8675309::M { + struct S has drop {} + + // this produces unused parameter warnings for i and s, but not unused resource warnings + fun t0(i: u64, s: S) { + } + + fun t1() { + let s = S{}; + } + + fun t2() { + // prefixing an unused non-resource with _ suppresses the warning + let _s = S{}; + } +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.exp new file mode 100644 index 0000000000000..13d05060d21fd --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.exp @@ -0,0 +1,45 @@ + +Diagnostics: +warning: Unused local variable `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource.move:5:13 + │ +5 │ let r = R{}; + │ ^ + +warning: Unused local variable `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource.move:29:20 + │ +29 │ loop { let r = R{}; } + │ ^ + + +Diagnostics: +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource.move:5:17 + │ +5 │ let r = R{}; + │ ^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource.move:15:21 + │ +15 │ if (cond) { r = R{}; }; + │ ^^^^^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource.move:20:29 + │ +20 │ if (cond) {} else { r = R{}; }; + │ ^^^^^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource.move:25:24 + │ +25 │ while (cond) { r = R{} }; + │ ^^^^^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource.move:29:24 + │ +29 │ loop { let r = R{}; } + │ ^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.move new file mode 100644 index 0000000000000..4121a8c05ea11 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.move @@ -0,0 +1,39 @@ +module 0x8675309::M { + struct R {} + + fun t0() { + let r = R{}; + } + + fun t1() { + // prefixing an unused resource with _ does not work + let _r = R{}; + } + + fun t2(cond: bool) { + let r; + if (cond) { r = R{}; }; + } + + fun t3(cond: bool) { + let r; + if (cond) {} else { r = R{}; }; + } + + fun t4(cond: bool) { + let r; + while (cond) { r = R{} }; + } + + fun t5() { + loop { let r = R{}; } + } + + fun t6() { + let _ = &R{}; + } + + fun t7(_x: R) { + } + +} diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.exp b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.exp new file mode 100644 index 0000000000000..8d78b5a05aadb --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.exp @@ -0,0 +1,39 @@ + +Diagnostics: +warning: Unused local variable `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource_explicit_return.move:5:13 + │ +5 │ let r = R{}; + │ ^ + +warning: Unused local variable `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource_explicit_return.move:28:13 + │ +28 │ let r = R{}; + │ ^ + +warning: Unused local variable `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-locals/unused_resource_explicit_return.move:33:13 + │ +33 │ let x = &R{}; + │ ^ + + +Diagnostics: +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource_explicit_return.move:5:17 + │ +5 │ let r = R{}; + │ ^^^ + +warning: Unused assignment to `r`. Consider removing or prefixing with an underscore: `_r` + ┌─ tests/unused-assignment/v1-locals/unused_resource_explicit_return.move:28:17 + │ +28 │ let r = R{}; + │ ^^^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/unused-assignment/v1-locals/unused_resource_explicit_return.move:33:17 + │ +33 │ let x = &R{}; + │ ^^^^ diff --git a/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.move b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.move new file mode 100644 index 0000000000000..cb70af3240e97 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.move @@ -0,0 +1,40 @@ +module 0x8675309::M { + struct R {} + + fun t0() { + let r = R{}; + return () + } + + fun t1(cond: bool) { + let r = R {}; + if (cond) { return () }; + R {} = r; + } + + fun t2(cond: bool) { + let r = R{}; + if (cond) {} else { return () }; + R {} = r; + } + + fun t3(cond: bool) { + let r = R {}; + while (cond) { return () }; + R {} = r; + } + + fun t4() { + let r = R{}; + loop { return () } + } + + fun t5() { + let x = &R{}; + return () + } + + fun t6(_x: R) { + return () + } +} diff --git a/third_party/move/move-compiler-v2/tests/v1.matched b/third_party/move/move-compiler-v2/tests/v1.matched index 2d09eb45f2054..25a816e67c53f 100644 --- a/third_party/move/move-compiler-v2/tests/v1.matched +++ b/third_party/move/move-compiler-v2/tests/v1.matched @@ -138,6 +138,7 @@ move-compiler/tests/move_check/translated_ir_tests/move/commands/continue_outsid move-compiler/tests/move_check/translated_ir_tests/move/commands/else_assigns_if_doesnt.exp move-compiler-v2/tests/uninit-use-checker/v1-commands/else_assigns_if_doesnt.exp move-compiler/tests/move_check/translated_ir_tests/move/commands/if_assigns_else_doesnt.exp move-compiler-v2/tests/uninit-use-checker/v1-commands/if_assigns_else_doesnt.exp move-compiler/tests/move_check/translated_ir_tests/move/commands/if_assigns_no_else.exp move-compiler-v2/tests/uninit-use-checker/v1-commands/if_assigns_no_else.exp +move-compiler/tests/move_check/translated_ir_tests/move/commands/mixed_lvalue.exp move-compiler-v2/tests/unused-assignment/v1-commands/mixed_lvalue.exp move-compiler/tests/move_check/translated_ir_tests/move/commands/move_before_assign.exp move-compiler-v2/tests/uninit-use-checker/v1-commands/move_before_assign.exp move-compiler/tests/move_check/translated_ir_tests/move/commands/no_let_outside_if.exp move-compiler-v2/tests/checking/typing/v1-commands/no_let_outside_if.exp move-compiler/tests/move_check/translated_ir_tests/move/commands/pop_negative.exp move-compiler-v2/tests/checking/typing/v1-commands/pop_negative.exp @@ -205,9 +206,16 @@ move-compiler/tests/move_check/translated_ir_tests/move/generics/instantiation_l move-compiler/tests/move_check/translated_ir_tests/move/generics/instantiation_loops/two_loops.exp move-compiler-v2/tests/cyclic-instantiation-checker/v1-tests/two_loops.exp move-compiler/tests/move_check/liveness/mut_inline.exp move-compiler-v2/tests/live-var/mut_inline.exp move-compiler/tests/move_check/liveness/mut_ref.exp move-compiler-v2/tests/live-var/mut_ref.exp +move-compiler/tests/move_check/liveness/unused_assignment.exp move-compiler-v2/tests/unused-assignment/v1-liveness/unused_assignment.exp +move-compiler/tests/move_check/locals/assign_partial_resource.exp move-compiler-v2/tests/unused-assignment/v1-locals/assign_partial_resource.exp +move-compiler/tests/move_check/locals/assign_resource.exp move-compiler-v2/tests/unused-assignment/v1-locals/assign_resource.exp move-compiler/tests/move_check/locals/drop_conditional.exp move-compiler-v2/tests/ability-check/v1-locals/drop_conditional.exp move-compiler/tests/move_check/locals/eliminate_temps.exp move-compiler-v2/tests/reference-safety/v1-locals/eliminate_temps.exp move-compiler/tests/move_check/locals/reassign_parameter.exp move-compiler-v2/tests/ability-check/v1-locals/reassign_parameter.exp +move-compiler/tests/move_check/locals/struct_use_before_assign.exp move-compiler-v2/tests/unused-assignment/v1-locals/struct_use_before_assign.exp +move-compiler/tests/move_check/locals/unused_copyable.exp move-compiler-v2/tests/unused-assignment/v1-locals/unused_copyable.exp +move-compiler/tests/move_check/locals/unused_resource.exp move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource.exp +move-compiler/tests/move_check/locals/unused_resource_explicit_return.exp move-compiler-v2/tests/unused-assignment/v1-locals/unused_resource_explicit_return.exp move-compiler/tests/move_check/locals/use_before_assign_if.exp move-compiler-v2/tests/uninit-use-checker/v1-locals/use_before_assign_if.exp move-compiler/tests/move_check/locals/use_before_assign_if_else.exp move-compiler-v2/tests/uninit-use-checker/v1-locals/use_before_assign_if_else.exp move-compiler/tests/move_check/locals/use_before_assign_loop.exp move-compiler-v2/tests/uninit-use-checker/v1-locals/use_before_assign_loop.exp diff --git a/third_party/move/move-compiler-v2/tests/v1.unmatched b/third_party/move/move-compiler-v2/tests/v1.unmatched index f30791f7fad7d..f2ad6f0903cec 100644 --- a/third_party/move/move-compiler-v2/tests/v1.unmatched +++ b/third_party/move/move-compiler-v2/tests/v1.unmatched @@ -21,7 +21,6 @@ move-compiler/tests/move_check/commands/{ invalid_fallthrough2.move, invalid_fallthrough3.move, join_failure.move, - mixed_lvalue.move, while_move_local.move, while_move_local_2.move, } @@ -66,15 +65,8 @@ move-compiler/tests/move_check/liveness/{ move_in_infinite_loop_branched.move, trailing_semi.move, trailing_semi_loops.move, - unused_assignment.move, } move-compiler/tests/move_check/locals/{ - assign_partial_resource.move, - assign_resource.move, - struct_use_before_assign.move, - unused_copyable.move, - unused_resource.move, - unused_resource_explicit_return.move, use_after_move_if.move, use_after_move_if_else.move, use_after_move_loop.move, diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.exp index 8cb6654d0e563..3f4ff64c3e1c9 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.exp @@ -33,6 +33,14 @@ fun m::test($t0: u64): u64 { 6: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/call_1.move:7:17 + │ +7 │ let a = p; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.opt.exp index 8cb6654d0e563..3f4ff64c3e1c9 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/call_1.opt.exp @@ -33,6 +33,14 @@ fun m::test($t0: u64): u64 { 6: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/call_1.move:7:17 + │ +7 │ let a = p; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.exp index c15fb7fa2d8ce..c3193e31baca1 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.exp @@ -19,6 +19,14 @@ public fun m::test($t0: u64): u64 { 3: return $t1 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/cant_coalesce_1.move:3:17 + │ +3 │ let x = a + a; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.opt.exp index 0421981781a4e..fbea2c056a83b 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_coalesce_1.opt.exp @@ -17,6 +17,14 @@ public fun m::test($t0: u64): u64 { 2: return $t1 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/cant_coalesce_1.move:3:17 + │ +3 │ let x = a + a; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.exp index 714466fd830f4..bbe349810a53c 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.exp @@ -31,6 +31,20 @@ fun m::test($t0: u64, $t1: bool) { 13: return () } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/cant_copy_propagate.move:11:13 + │ +11 │ a = 99; + │ ^^^^^^ + +warning: Unused assignment to `c`. Consider removing or prefixing with an underscore: `_c` + ┌─ tests/variable-coalescing/cant_copy_propagate.move:13:13 + │ +13 │ c = c + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.opt.exp index 714466fd830f4..bbe349810a53c 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/cant_copy_propagate.opt.exp @@ -31,6 +31,20 @@ fun m::test($t0: u64, $t1: bool) { 13: return () } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/cant_copy_propagate.move:11:13 + │ +11 │ a = 99; + │ ^^^^^^ + +warning: Unused assignment to `c`. Consider removing or prefixing with an underscore: `_c` + ┌─ tests/variable-coalescing/cant_copy_propagate.move:13:13 + │ +13 │ c = c + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.exp index 1be5d8b6fc8ed..e6a15a35a4e75 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.exp @@ -12,6 +12,14 @@ public fun m::test($t0: u64) { 4: return () } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/cyclic_assignment_without_use.move:6:9 + │ +6 │ a = c; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.opt.exp index 1be5d8b6fc8ed..e6a15a35a4e75 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/cyclic_assignment_without_use.opt.exp @@ -12,6 +12,14 @@ public fun m::test($t0: u64) { 4: return () } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/cyclic_assignment_without_use.move:6:9 + │ +6 │ a = c; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.exp index 2ab2fd8b365f9..d59457e546efe 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.exp @@ -20,6 +20,14 @@ public fun m::test($t0: bool): u32 { 12: return $t1 } + +Diagnostics: +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/variable-coalescing/dead_assignment_3.move:10:13 + │ +10 │ y = y; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.opt.exp index 10501d30e9943..6aca9b7b84499 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_3.opt.exp @@ -18,6 +18,14 @@ public fun m::test($t0: bool): u32 { 11: return $t1 } + +Diagnostics: +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/variable-coalescing/dead_assignment_3.move:10:13 + │ +10 │ y = y; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.exp index 4dac314323337..970a6b84ab094 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.exp @@ -67,6 +67,26 @@ public fun m::test4($t0: u64): u64 { 2: return $t1 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/dead_assignment_4.move:3:17 + │ +3 │ let x = 1; + │ ^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/dead_assignment_4.move:9:17 + │ +9 │ let x = y; + │ ^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/dead_assignment_4.move:14:17 + │ +14 │ let x = y; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.opt.exp index 6a523fee558e2..c4b4326890f8a 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/dead_assignment_4.opt.exp @@ -61,6 +61,20 @@ public fun m::test4($t0: u64): u64 { 1: return $t1 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/dead_assignment_4.move:9:17 + │ +9 │ let x = y; + │ ^ + +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/dead_assignment_4.move:14:17 + │ +14 │ let x = y; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.exp index aef9456582cdc..c54a49ba6d3c9 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.exp @@ -23,6 +23,14 @@ fun m::test($t0: u64): u64 { 5: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/immut_refs_2.move:4:17 + │ +4 │ let a = &p; + │ ^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.opt.exp index aef9456582cdc..c54a49ba6d3c9 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/immut_refs_2.opt.exp @@ -23,6 +23,14 @@ fun m::test($t0: u64): u64 { 5: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/immut_refs_2.move:4:17 + │ +4 │ let a = &p; + │ ^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.exp index 8ad92bf681375..feff68c3be180 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.exp @@ -16,6 +16,14 @@ fun m::test(): u64 { 6: return $t0 } + +Diagnostics: +warning: Unused assignment to `t`. Consider removing or prefixing with an underscore: `_t` + ┌─ tests/variable-coalescing/intermingled_1.move:5:9 + │ +5 │ t = t + u; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.opt.exp index c8d8e9c9ea66b..1669a1b36c2a9 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_1.opt.exp @@ -14,6 +14,14 @@ fun m::test(): u64 { 5: return $t0 } + +Diagnostics: +warning: Unused assignment to `t`. Consider removing or prefixing with an underscore: `_t` + ┌─ tests/variable-coalescing/intermingled_1.move:5:9 + │ +5 │ t = t + u; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.exp index 5d50b3f39fdac..a8a2312b817af 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.exp @@ -18,6 +18,14 @@ fun m::test(): u64 { 7: return $t0 } + +Diagnostics: +warning: Unused assignment to `t`. Consider removing or prefixing with an underscore: `_t` + ┌─ tests/variable-coalescing/intermingled_3.move:5:9 + │ +5 │ t = t + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.opt.exp index dd35312daf6a4..4a5daedbda24a 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/intermingled_3.opt.exp @@ -14,6 +14,14 @@ fun m::test(): u64 { 5: return $t0 } + +Diagnostics: +warning: Unused assignment to `t`. Consider removing or prefixing with an underscore: `_t` + ┌─ tests/variable-coalescing/intermingled_3.move:5:9 + │ +5 │ t = t + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.exp index 487f77be960ef..23af347b74613 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.exp @@ -12,6 +12,14 @@ fun m::test(): u64 { 4: return $t0 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/multi_assigns.move:3:17 + │ +3 │ let x = 1; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.opt.exp index 487f77be960ef..23af347b74613 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/multi_assigns.opt.exp @@ -12,6 +12,14 @@ fun m::test(): u64 { 4: return $t0 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/multi_assigns.move:3:17 + │ +3 │ let x = 1; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.exp index 754941c708880..e15966fe20563 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.exp @@ -19,6 +19,20 @@ fun m::test() { 8: return () } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/non_overlapping_vars1.move:4:9 + │ +4 │ x = x + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/variable-coalescing/non_overlapping_vars1.move:6:9 + │ +6 │ y = y + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.opt.exp index 754941c708880..e15966fe20563 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars1.opt.exp @@ -19,6 +19,20 @@ fun m::test() { 8: return () } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/non_overlapping_vars1.move:4:9 + │ +4 │ x = x + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/variable-coalescing/non_overlapping_vars1.move:6:9 + │ +6 │ y = y + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.exp index b7f839ff0a194..ad657705a8322 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.exp @@ -19,6 +19,20 @@ fun m::test() { 8: return () } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/non_overlapping_vars_diff_type.move:4:9 + │ +4 │ x = x + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/variable-coalescing/non_overlapping_vars_diff_type.move:6:9 + │ +6 │ y = y + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.opt.exp index b7f839ff0a194..ad657705a8322 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/non_overlapping_vars_diff_type.opt.exp @@ -19,6 +19,20 @@ fun m::test() { 8: return () } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/non_overlapping_vars_diff_type.move:4:9 + │ +4 │ x = x + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/variable-coalescing/non_overlapping_vars_diff_type.move:6:9 + │ +6 │ y = y + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.exp index 5a165b49ca4ca..a7a192cb48679 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.exp @@ -14,6 +14,14 @@ fun m::test(): u64 { 5: return $t0 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/reassigned_var.move:3:17 + │ +3 │ let a = 1; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.opt.exp index 23428eb311af1..0bff15a1e37e6 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/reassigned_var.opt.exp @@ -14,6 +14,14 @@ fun m::test(): u64 { 5: return $t0 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/reassigned_var.move:3:17 + │ +3 │ let a = 1; + │ ^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.exp index 22650ed85bc01..f58fb946a815c 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.exp @@ -75,6 +75,14 @@ public fun m::test4($t0: u64): u64 { 17: return $t1 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/self_assigns.move:3:9 + │ +3 │ x = x; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.opt.exp index 22650ed85bc01..f58fb946a815c 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/self_assigns.opt.exp @@ -75,6 +75,14 @@ public fun m::test4($t0: u64): u64 { 17: return $t1 } + +Diagnostics: +warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` + ┌─ tests/variable-coalescing/self_assigns.move:3:9 + │ +3 │ x = x; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.exp index dc7553994cc1a..82b01b37c032f 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.exp @@ -18,6 +18,14 @@ fun m::test($t0: u64): bool { 7: return $t1 } + +Diagnostics: +warning: Unused assignment to `b`. Consider removing or prefixing with an underscore: `_b` + ┌─ tests/variable-coalescing/seq_kills_1.move:7:9 + │ +7 │ b = p + 1; // kill b := a, which removes the whole copy chain + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.opt.exp index dc7553994cc1a..82b01b37c032f 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_1.opt.exp @@ -18,6 +18,14 @@ fun m::test($t0: u64): bool { 7: return $t1 } + +Diagnostics: +warning: Unused assignment to `b`. Consider removing or prefixing with an underscore: `_b` + ┌─ tests/variable-coalescing/seq_kills_1.move:7:9 + │ +7 │ b = p + 1; // kill b := a, which removes the whole copy chain + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.exp index 3cbb873f10bd0..19a70afe528ad 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.exp @@ -18,6 +18,14 @@ fun m::test($t0: u64): bool { 7: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/seq_kills_2.move:7:9 + │ +7 │ a = p + 1; // kill b := a + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.opt.exp index 3cbb873f10bd0..19a70afe528ad 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/seq_kills_2.opt.exp @@ -18,6 +18,14 @@ fun m::test($t0: u64): bool { 7: return $t1 } + +Diagnostics: +warning: Unused assignment to `a`. Consider removing or prefixing with an underscore: `_a` + ┌─ tests/variable-coalescing/seq_kills_2.move:7:9 + │ +7 │ a = p + 1; // kill b := a + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.exp index 244a78cdaba55..84ea8b1abbcb6 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.exp @@ -16,6 +16,14 @@ fun m::copy_kill($t0: u64): u64 { 6: return $t1 } + +Diagnostics: +warning: Unused assignment to `p`. Consider removing or prefixing with an underscore: `_p` + ┌─ tests/variable-coalescing/straight_line_kills.move:5:9 + │ +5 │ p = p + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.opt.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.opt.exp index 244a78cdaba55..84ea8b1abbcb6 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.opt.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/straight_line_kills.opt.exp @@ -16,6 +16,14 @@ fun m::copy_kill($t0: u64): u64 { 6: return $t1 } + +Diagnostics: +warning: Unused assignment to `p`. Consider removing or prefixing with an underscore: `_p` + ┌─ tests/variable-coalescing/straight_line_kills.move:5:9 + │ +5 │ p = p + 1; + │ ^^^^^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tests/variable-coalescing/unused_add.exp b/third_party/move/move-compiler-v2/tests/variable-coalescing/unused_add.exp index e1e09de6b41e0..850ba7d9bb591 100644 --- a/third_party/move/move-compiler-v2/tests/variable-coalescing/unused_add.exp +++ b/third_party/move/move-compiler-v2/tests/variable-coalescing/unused_add.exp @@ -19,6 +19,14 @@ public fun m::test() { 3: return () } + +Diagnostics: +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/variable-coalescing/unused_add.move:5:17 + │ +5 │ let z = x + y; + │ ^^^^^ + ============ after DeadStoreElimination: ================ [variant baseline] diff --git a/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs b/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs index dd0e7a11037fa..cc33da8171255 100644 --- a/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs +++ b/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs @@ -100,6 +100,9 @@ static UNIT_PATH_REMAP: Lazy> = Lazy::new(|| { // Map file v2../unit_test/test/foo.move // to file v1../unit_test/foo.unit_test.move. ("unit_test/test", "unit_test.unit_test"), + ("unused-assignment/v1-locals", "locals"), + ("unused-assignment/v1-liveness", "liveness"), + ("unused-assignment/v1-commands", "commands"), ("checking-lang-v1/v1-typing", "typing"), ("ability-check/v1-typing", "typing"), ("ability-check/v1-signer", "signer"), diff --git a/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed.exp b/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed.exp index 3504b5c31c5bd..dc09697dd69eb 100644 --- a/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed.exp +++ b/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed.exp @@ -1,14 +1,15 @@ comparison between v1 and v2 failed: -- processed 2 tasks -- -- task 0 'publish'. lines 1-19: += processed 2 tasks += += task 0 'publish'. lines 1-19: - warning[W09003]: unused assignment - ┌─ TEMPFILE:8:13 -- │ -- 8 │ let x = 1; ++ warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` ++ ┌─ TEMPFILE:8:17 += │ += 8 │ let x = 1; - │ ^ Unused assignment or binding for local 'x'. Consider removing, replacing with '_', or prefixing with '_' (e.g., '_x') -- -- -- -+ processed 2 tasks -+ ++ │ ^ += += += diff --git a/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed_param.exp b/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed_param.exp index d14d8c03995e3..968bf96261b93 100644 --- a/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed_param.exp +++ b/third_party/move/move-compiler-v2/transactional-tests/tests/inlining/shadowing_renamed_param.exp @@ -1,14 +1,15 @@ comparison between v1 and v2 failed: -- processed 2 tasks -- -- task 0 'publish'. lines 1-46: += processed 2 tasks += += task 0 'publish'. lines 1-46: - warning[W09003]: unused assignment - ┌─ TEMPFILE:23:13 -- │ -- 23 │ let x = q; ++ warning: Unused assignment to `x`. Consider removing or prefixing with an underscore: `_x` ++ ┌─ TEMPFILE:23:17 += │ += 23 │ let x = q; - │ ^ Unused assignment or binding for local 'x'. Consider removing, replacing with '_', or prefixing with '_' (e.g., '_x') -- -- -- -+ processed 2 tasks -+ ++ │ ^ += += += diff --git a/third_party/move/move-compiler-v2/transactional-tests/tests/misc/struct_assign_swap.move b/third_party/move/move-compiler-v2/transactional-tests/tests/misc/struct_assign_swap.move index dbc71d497f38e..1f334c701d7d5 100644 --- a/third_party/move/move-compiler-v2/transactional-tests/tests/misc/struct_assign_swap.move +++ b/third_party/move/move-compiler-v2/transactional-tests/tests/misc/struct_assign_swap.move @@ -13,8 +13,8 @@ module 0xc0ffee::m { fun swap2(): (u32, u32) { let x = 44; let y = 55; - let S { f: x, g: y } = S { f: y, g: x }; - (x, y) + let S { f: _x, g: _y } = S { f: y, g: x }; + (_x, _y) } fun test1(): (u32, u32) { diff --git a/third_party/move/move-compiler-v2/transactional-tests/tests/operator_eval/op_with_side_effect_14.operator-eval-lang-2.exp b/third_party/move/move-compiler-v2/transactional-tests/tests/operator_eval/op_with_side_effect_14.operator-eval-lang-2.exp index 4a6daef3b154c..66a78666a3ae6 100644 --- a/third_party/move/move-compiler-v2/transactional-tests/tests/operator_eval/op_with_side_effect_14.operator-eval-lang-2.exp +++ b/third_party/move/move-compiler-v2/transactional-tests/tests/operator_eval/op_with_side_effect_14.operator-eval-lang-2.exp @@ -1,6 +1,36 @@ comparison between v1 and v2 failed: = processed 3 tasks = ++ task 0 'publish'. lines 1-26: ++ warning: Unused assignment to `by`. Consider removing or prefixing with an underscore: `_by` ++ ┌─ TEMPFILE:11:16 ++ │ ++ 11 │ inline fun inc_xx(self: &mut S, by: u64) { ++ │ ^^^^^^ ++ · ++ 20 │ {inc_xx(&mut s, 6); s.x} + {inc_xx(&mut s, 47); s.x} + {inc_xx(&mut s, 117); s.x} ++ │ ----------------- from a call inlined at this callsite ++ ++ warning: Unused assignment to `by`. Consider removing or prefixing with an underscore: `_by` ++ ┌─ TEMPFILE:11:16 ++ │ ++ 11 │ inline fun inc_xx(self: &mut S, by: u64) { ++ │ ^^^^^^ ++ · ++ 20 │ {inc_xx(&mut s, 6); s.x} + {inc_xx(&mut s, 47); s.x} + {inc_xx(&mut s, 117); s.x} ++ │ ------------------ from a call inlined at this callsite ++ ++ warning: Unused assignment to `by`. Consider removing or prefixing with an underscore: `_by` ++ ┌─ TEMPFILE:11:16 ++ │ ++ 11 │ inline fun inc_xx(self: &mut S, by: u64) { ++ │ ^^^^^^ ++ · ++ 20 │ {inc_xx(&mut s, 6); s.x} + {inc_xx(&mut s, 47); s.x} + {inc_xx(&mut s, 117); s.x} ++ │ ------------------- from a call inlined at this callsite ++ ++ ++ = task 1 'run'. lines 28-28: - return values: 513 + return values: 232 diff --git a/third_party/move/move-prover/tests/sources/functional/ModifiesErrorTest.v2_exp b/third_party/move/move-prover/tests/sources/functional/ModifiesErrorTest.v2_exp index 620e4ee6a4e64..a4c827857f788 100644 --- a/third_party/move/move-prover/tests/sources/functional/ModifiesErrorTest.v2_exp +++ b/third_party/move/move-prover/tests/sources/functional/ModifiesErrorTest.v2_exp @@ -1,4 +1,64 @@ Move prover returns: exiting with verification errors +warning: Unused assignment to `x0`. Consider removing or prefixing with an underscore: `_x0` + ┌─ tests/sources/functional/ModifiesErrorTest.move:37:18 + │ +37 │ let x0 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x1`. Consider removing or prefixing with an underscore: `_x1` + ┌─ tests/sources/functional/ModifiesErrorTest.move:40:18 + │ +40 │ let x1 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x0`. Consider removing or prefixing with an underscore: `_x0` + ┌─ tests/sources/functional/ModifiesErrorTest.move:51:18 + │ +51 │ let x0 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x1`. Consider removing or prefixing with an underscore: `_x1` + ┌─ tests/sources/functional/ModifiesErrorTest.move:53:18 + │ +53 │ let x1 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x0`. Consider removing or prefixing with an underscore: `_x0` + ┌─ tests/sources/functional/ModifiesErrorTest.move:64:18 + │ +64 │ let x0 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x1`. Consider removing or prefixing with an underscore: `_x1` + ┌─ tests/sources/functional/ModifiesErrorTest.move:66:18 + │ +66 │ let x1 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x0`. Consider removing or prefixing with an underscore: `_x0` + ┌─ tests/sources/functional/ModifiesErrorTest.move:78:18 + │ +78 │ let x0 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x1`. Consider removing or prefixing with an underscore: `_x1` + ┌─ tests/sources/functional/ModifiesErrorTest.move:80:18 + │ +80 │ let x1 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x0`. Consider removing or prefixing with an underscore: `_x0` + ┌─ tests/sources/functional/ModifiesErrorTest.move:91:18 + │ +91 │ let x0 = A::read_at(addr); + │ ^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x1`. Consider removing or prefixing with an underscore: `_x1` + ┌─ tests/sources/functional/ModifiesErrorTest.move:93:18 + │ +93 │ let x1 = A::read_at(addr); + │ ^^^^^^^^^^^^^^^^ + error: caller does not have permission to modify `B::T` at given address ┌─ tests/sources/functional/ModifiesErrorTest.move:38:17 │ diff --git a/third_party/move/move-prover/tests/sources/functional/ModifiesTest.v2_exp b/third_party/move/move-prover/tests/sources/functional/ModifiesTest.v2_exp new file mode 100644 index 0000000000000..66abe10c08bf4 --- /dev/null +++ b/third_party/move/move-prover/tests/sources/functional/ModifiesTest.v2_exp @@ -0,0 +1,11 @@ +warning: Unused assignment to `x0`. Consider removing or prefixing with an underscore: `_x0` + ┌─ tests/sources/functional/ModifiesTest.move:67:18 + │ +67 │ let x0 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `x1`. Consider removing or prefixing with an underscore: `_x1` + ┌─ tests/sources/functional/ModifiesTest.move:69:18 + │ +69 │ let x1 = A::read_at(addr2); + │ ^^^^^^^^^^^^^^^^^ diff --git a/third_party/move/move-prover/tests/sources/functional/bitset.v2_exp b/third_party/move/move-prover/tests/sources/functional/bitset.v2_exp new file mode 100644 index 0000000000000..e50ba75b20d2e --- /dev/null +++ b/third_party/move/move-prover/tests/sources/functional/bitset.v2_exp @@ -0,0 +1,11 @@ +warning: Unused assignment to `s2`. Consider removing or prefixing with an underscore: `_s2` + ┌─ tests/sources/functional/bitset.move:59:18 + │ +59 │ let s2 = intersect(s, s); + │ ^^^^^^^^^^^^^^^ + +warning: Unused assignment to `s3`. Consider removing or prefixing with an underscore: `_s3` + ┌─ tests/sources/functional/bitset.move:60:18 + │ +60 │ let s3 = union(s, s); + │ ^^^^^^^^^^^ diff --git a/third_party/move/move-prover/tests/sources/functional/bitwise_features.v2_exp b/third_party/move/move-prover/tests/sources/functional/bitwise_features.v2_exp new file mode 100644 index 0000000000000..7c523ceea6767 --- /dev/null +++ b/third_party/move/move-prover/tests/sources/functional/bitwise_features.v2_exp @@ -0,0 +1,5 @@ +warning: Unused assignment to `old_features`. Consider removing or prefixing with an underscore: `_old_features` + ┌─ tests/sources/functional/bitwise_features.move:50:28 + │ +50 │ let old_features = *features; // ghost var + │ ^^^^^^^^^ diff --git a/third_party/move/move-prover/tests/sources/functional/inline-lambda.v2_exp b/third_party/move/move-prover/tests/sources/functional/inline-lambda.v2_exp index 4d220149d9a54..a939ef7f0c9c6 100644 --- a/third_party/move/move-prover/tests/sources/functional/inline-lambda.v2_exp +++ b/third_party/move/move-prover/tests/sources/functional/inline-lambda.v2_exp @@ -1,4 +1,16 @@ Move prover returns: exiting with verification errors +warning: Unused assignment to `r1`. Consider removing or prefixing with an underscore: `_r1` + ┌─ tests/sources/functional/inline-lambda.move:11:31 + │ +11 │ let r1 = apply(0, |v| v >= 0); + │ ^^^^^^ + +warning: Unused assignment to `r2`. Consider removing or prefixing with an underscore: `_r2` + ┌─ tests/sources/functional/inline-lambda.move:16:31 + │ +16 │ let r2 = apply(0, |v| v != a1 + a2); + │ ^^^^^^^^^^^^ + error: unknown assertion failed ┌─ tests/sources/functional/inline-lambda.move:5:13 │ diff --git a/third_party/move/move-prover/tests/sources/functional/inline_fun_simple.v2_exp b/third_party/move/move-prover/tests/sources/functional/inline_fun_simple.v2_exp index 00f90006edacb..a71d8eee327fb 100644 --- a/third_party/move/move-prover/tests/sources/functional/inline_fun_simple.v2_exp +++ b/third_party/move/move-prover/tests/sources/functional/inline_fun_simple.v2_exp @@ -1,4 +1,28 @@ Move prover returns: exiting with verification errors +warning: Unused assignment to `r1`. Consider removing or prefixing with an underscore: `_r1` + ┌─ tests/sources/functional/inline_fun_simple.move:10:32 + │ +10 │ let r1 = apply(42, |v| v >= 1); + │ ^^^^^^ + +warning: Unused assignment to `r2`. Consider removing or prefixing with an underscore: `_r2` + ┌─ tests/sources/functional/inline_fun_simple.move:15:32 + │ +15 │ let r2 = apply(43, |v| v <= 2); + │ ^^^^^^ + +warning: Unused assignment to `r1`. Consider removing or prefixing with an underscore: `_r1` + ┌─ tests/sources/functional/inline_fun_simple.move:22:32 + │ +22 │ let r1 = apply(42, |v| v >= 1); + │ ^^^^^^ + +warning: Unused assignment to `r2`. Consider removing or prefixing with an underscore: `_r2` + ┌─ tests/sources/functional/inline_fun_simple.move:27:31 + │ +27 │ let r2 = apply(3, |v| v <= 2); + │ ^^^^^^ + error: unknown assertion failed ┌─ tests/sources/functional/inline_fun_simple.move:4:13 │ diff --git a/third_party/move/move-prover/tests/sources/functional/specs_in_fun.v2_exp b/third_party/move/move-prover/tests/sources/functional/specs_in_fun.v2_exp index 4dc5d49a730b3..a41ce4e659842 100644 --- a/third_party/move/move-prover/tests/sources/functional/specs_in_fun.v2_exp +++ b/third_party/move/move-prover/tests/sources/functional/specs_in_fun.v2_exp @@ -1,4 +1,28 @@ Move prover returns: exiting with verification errors +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/sources/functional/specs_in_fun.move:18:9 + │ +18 │ y = x + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/sources/functional/specs_in_fun.move:33:9 + │ +33 │ z = x + y; + │ ^^^^^^^^^ + +warning: Unused assignment to `y`. Consider removing or prefixing with an underscore: `_y` + ┌─ tests/sources/functional/specs_in_fun.move:51:9 + │ +51 │ y = x + 1; + │ ^^^^^^^^^ + +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/sources/functional/specs_in_fun.move:66:9 + │ +66 │ z = x + y; + │ ^^^^^^^^^ + error: unknown assertion failed ┌─ tests/sources/functional/specs_in_fun.move:45:13 │ diff --git a/third_party/move/move-prover/tests/sources/functional/specs_in_fun_ref.v2_exp b/third_party/move/move-prover/tests/sources/functional/specs_in_fun_ref.v2_exp new file mode 100644 index 0000000000000..a7c557355adc9 --- /dev/null +++ b/third_party/move/move-prover/tests/sources/functional/specs_in_fun_ref.v2_exp @@ -0,0 +1,29 @@ +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/sources/functional/specs_in_fun_ref.move:11:9 + │ +11 │ z = x + y; + │ ^^^^^^^^^ + +warning: Unused assignment to `z`. Consider removing or prefixing with an underscore: `_z` + ┌─ tests/sources/functional/specs_in_fun_ref.move:44:9 + │ +44 │ z = *x + *y; + │ ^^^^^^^^^^^ + +warning: Unused assignment to `vx`. Consider removing or prefixing with an underscore: `_vx` + ┌─ tests/sources/functional/specs_in_fun_ref.move:45:18 + │ +45 │ let vx = *x; + │ ^^ + +warning: Unused assignment to `vy`. Consider removing or prefixing with an underscore: `_vy` + ┌─ tests/sources/functional/specs_in_fun_ref.move:46:18 + │ +46 │ let vy = *y; + │ ^^ + +warning: Unused assignment to `fx`. Consider removing or prefixing with an underscore: `_fx` + ┌─ tests/sources/functional/specs_in_fun_ref.move:47:18 + │ +47 │ let fx = freeze(x); + │ ^^^^^^^^^ diff --git a/third_party/move/move-prover/tests/sources/functional/verify_custom_table.v2_exp b/third_party/move/move-prover/tests/sources/functional/verify_custom_table.v2_exp index 868c816071271..034af905fdcd6 100644 --- a/third_party/move/move-prover/tests/sources/functional/verify_custom_table.v2_exp +++ b/third_party/move/move-prover/tests/sources/functional/verify_custom_table.v2_exp @@ -1,4 +1,16 @@ Move prover returns: exiting with verification errors +warning: Unused assignment to `k`. Consider removing or prefixing with an underscore: `_k` + ┌─ tests/sources/functional/verify_custom_table.move:115:22 + │ +115 │ let (k, v) = table::remove_return_key(&mut t, 2); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: Unused assignment to `v`. Consider removing or prefixing with an underscore: `_v` + ┌─ tests/sources/functional/verify_custom_table.move:115:22 + │ +115 │ let (k, v) = table::remove_return_key(&mut t, 2); + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + error: post-condition does not hold ┌─ tests/sources/functional/verify_custom_table.move:76:9 │ diff --git a/third_party/move/move-prover/tests/sources/regression/bug_828.v2_exp b/third_party/move/move-prover/tests/sources/regression/bug_828.v2_exp new file mode 100644 index 0000000000000..1632a07cecacd --- /dev/null +++ b/third_party/move/move-prover/tests/sources/regression/bug_828.v2_exp @@ -0,0 +1,5 @@ +warning: Unused assignment to `old_i`. Consider removing or prefixing with an underscore: `_old_i` + ┌─ tests/sources/regression/bug_828.move:4:21 + │ +4 │ let old_i = i; + │ ^ From a0bfd5b97ef6a508c5c4a8f3252d2df304fa17d6 Mon Sep 17 00:00:00 2001 From: Zekun Li Date: Fri, 28 Jun 2024 16:28:16 -0700 Subject: [PATCH 007/469] [consensus] fix edge case of block retrieval --- consensus/src/block_storage/sync_manager.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/consensus/src/block_storage/sync_manager.rs b/consensus/src/block_storage/sync_manager.rs index ddc2120608828..d1e1842b28d52 100644 --- a/consensus/src/block_storage/sync_manager.rs +++ b/consensus/src/block_storage/sync_manager.rs @@ -23,7 +23,7 @@ use crate::{ persistent_liveness_storage::{LedgerRecoveryData, PersistentLivenessStorage, RecoveryData}, pipeline::execution_client::TExecutionClient, }; -use anyhow::{bail, Context}; +use anyhow::{anyhow, bail, Context}; use aptos_consensus_types::{ block::Block, block_retrieval::{ @@ -47,7 +47,7 @@ use futures::{stream::FuturesUnordered, FutureExt, StreamExt}; use futures_channel::oneshot; use rand::{prelude::*, Rng}; use std::{clone::Clone, cmp::min, sync::Arc, time::Duration}; -use tokio::time; +use tokio::{time, time::timeout}; #[derive(Debug, PartialEq, Eq)] /// Whether we need to do block retrieval if we want to insert a Quorum Cert. @@ -568,15 +568,14 @@ impl BlockRetriever { let author = self.network.author(); futures.push( async move { - let response = rx - .await - .map(|block| { - BlockRetrievalResponse::new( - BlockRetrievalStatus::SucceededWithTarget, - vec![block], - ) - }) - .map_err(|_| anyhow::anyhow!("self retrieval failed")); + let response = match timeout(rpc_timeout, rx).await { + Ok(Ok(block)) => Ok(BlockRetrievalResponse::new( + BlockRetrievalStatus::SucceededWithTarget, + vec![block], + )), + Ok(Err(_)) => Err(anyhow!("self retrieval cancelled")), + Err(_) => Err(anyhow!("self retrieval timeout")), + }; (author, response) } .boxed(), From cc8a87ea85ee31f8e481f60c9d699febe5665f3e Mon Sep 17 00:00:00 2001 From: "Brian R. Murphy" <132495859+brmataptos@users.noreply.github.com> Date: Sat, 29 Jun 2024 16:59:17 -0700 Subject: [PATCH 008/469] Honor `#[verify_only]` in move-compiler-v2. Add relevant tests from v1. Also add tests of `#[deprecated]`. (#13732) Fixes #13696: filter out `#[verify_only]` declarations during compilation Fixes #13737: when filtering functions due to _only attributes, also remove corresponding spec targets. * disable verify_only code by default in move-compiler-v2 and add plumbing and tests that it is set when appropriate * add tests of deprecated warnings to move-compiler-v2 * add tests for #13737 * Filter out spec blocks of filtered module members --------- Co-authored-by: Wolfgang Grieskamp --- third_party/move/move-compiler-v2/src/lib.rs | 3 + .../move/move-compiler-v2/src/options.rs | 66 +++++++- .../deprecated_constant_duplicated_struct.exp | 27 ++++ ...deprecated_constant_duplicated_struct.move | 44 ++++++ ...deprecated_constant_duplicated_struct2.exp | 145 ++++++++++++++++++ ...eprecated_constant_duplicated_struct2.move | 44 ++++++ .../deprecated/deprecated_field_type.exp | 31 ++++ .../deprecated/deprecated_field_type.move | 30 ++++ .../deprecated/deprecated_field_type2.exp | 13 ++ .../deprecated/deprecated_field_type2.move | 31 ++++ .../deprecated_placement_address.exp | 22 +++ .../deprecated_placement_address.move | 34 ++++ ...cated_placement_address_module_members.exp | 76 +++++++++ ...ated_placement_address_module_members.move | 42 +++++ .../deprecated_placement_members.exp | 40 +++++ .../deprecated_placement_members.move | 39 +++++ .../deprecated_placement_module.exp | 22 +++ .../deprecated_placement_module.move | 34 ++++ .../deprecated_placement_module2.exp | 49 ++++++ .../deprecated_placement_module2.move | 34 ++++ .../deprecated_placement_module_members.exp | 76 +++++++++ .../deprecated_placement_module_members.move | 41 +++++ .../tests/deprecated/public_script.exp | 13 ++ .../tests/deprecated/public_script.move | 6 + .../move/move-compiler-v2/tests/testsuite.rs | 51 +++++- .../unit_test/notest/function_with_spec.exp | 7 + .../unit_test/notest/function_with_spec.move | 22 +++ .../unit_test/test/function_with_spec.exp | 10 ++ .../unit_test/test/function_with_spec.move | 22 +++ .../move/move-compiler-v2/tests/v1.matched | 23 +++ .../move/move-compiler-v2/tests/v1.unmatched | 21 +-- .../cross_module_valid.verification | 0 .../double_annotation.verification | 0 .../noverify/cross_module_invalid.exp | 7 + .../noverify/cross_module_invalid.move | 26 ++++ .../noverify/cross_module_valid.exp | 2 + .../noverify/cross_module_valid.move | 24 +++ .../noverify/double_annotation.exp | 2 + .../noverify/double_annotation.move | 21 +++ .../noverify/function_with_spec.exp | 7 + .../noverify/function_with_spec.move | 19 +++ .../noverify/single_module_invalid.exp | 19 +++ .../noverify/single_module_invalid.move | 19 +++ .../noverify/single_module_valid.exp | 2 + .../noverify/single_module_valid.move | 16 ++ .../single_module_invalid.verification | 0 .../single_module_valid.verification | 0 .../verify/cross_module_valid.exp | 2 + .../verify/cross_module_valid.move | 24 +++ .../verification/verify/double_annotation.exp | 20 +++ .../verify/double_annotation.move | 21 +++ .../verify/function_with_spec.exp | 10 ++ .../verify/function_with_spec.move | 19 +++ .../verify/single_module_valid.exp | 2 + .../verify/single_module_valid.move | 16 ++ .../tools/testdiff/src/main.rs | 10 +- .../transactional-tests/tests/v1.matched | 5 +- .../move/move-compiler/src/parser/filter.rs | 43 +++++- .../move/move-compiler/src/shared/mod.rs | 7 + third_party/move/move-model/src/lib.rs | 6 + third_party/move/move-prover/src/lib.rs | 3 + .../compilation/std-lib-conflicts/Move.v2_exp | 53 +++++++ 62 files changed, 1492 insertions(+), 31 deletions(-) create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.move create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/public_script.exp create mode 100644 third_party/move/move-compiler-v2/tests/deprecated/public_script.move create mode 100644 third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.exp create mode 100644 third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.move create mode 100644 third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.exp create mode 100644 third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/cross_module_valid.verification create mode 100644 third_party/move/move-compiler-v2/tests/verification/double_annotation.verification create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/single_module_invalid.verification create mode 100644 third_party/move/move-compiler-v2/tests/verification/single_module_valid.verification create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.move create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.exp create mode 100644 third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.move create mode 100644 third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.v2_exp diff --git a/third_party/move/move-compiler-v2/src/lib.rs b/third_party/move/move-compiler-v2/src/lib.rs index d46eab6916f7e..2ee3067e492d7 100644 --- a/third_party/move/move-compiler-v2/src/lib.rs +++ b/third_party/move/move-compiler-v2/src/lib.rs @@ -185,7 +185,10 @@ pub fn run_checker(options: Options) -> anyhow::Result { &options.known_attributes }, options.language_version.unwrap_or_default(), + options.warn_deprecated, + options.warn_of_deprecation_use_in_aptos_libs, options.compile_test_code, + options.compile_verify_code, )?; // Store address aliases let map = addrs diff --git a/third_party/move/move-compiler-v2/src/options.rs b/third_party/move/move-compiler-v2/src/options.rs index bb8933ad163c9..60bc379f7211f 100644 --- a/third_party/move/move-compiler-v2/src/options.rs +++ b/third_party/move/move-compiler-v2/src/options.rs @@ -6,8 +6,14 @@ use crate::experiments::{DefaultValue, EXPERIMENTS}; use clap::Parser; use codespan_reporting::diagnostic::Severity; use itertools::Itertools; -use move_command_line_common::env::read_env_var; -use move_compiler::command_line as cli; +use move_command_line_common::env::{bool_to_str, read_env_var}; +use move_compiler::{ + command_line as cli, + shared::{ + move_compiler_warn_of_deprecation_use_env_var, + warn_of_deprecation_use_in_aptos_libs_env_var, + }, +}; use move_model::metadata::LanguageVersion; use once_cell::sync::Lazy; use std::{ @@ -25,29 +31,36 @@ pub struct Options { num_args = 0.. )] pub dependencies: Vec, + /// Named address mapping. #[clap( short, num_args = 0.. )] pub named_address_mapping: Vec, + /// Output directory. #[clap(short, long, default_value = "")] pub output_dir: String, + /// The language version to use. #[clap(long, value_parser = clap::value_parser!(LanguageVersion))] pub language_version: Option, + /// Do not complain about unknown attributes in Move code. #[clap(long, default_value = "false")] pub skip_attribute_checks: bool, + /// Known attributes for this dialect of move; if empty, assumes third-party Move. /// Only used if skip_attribute_checks is false. #[clap(skip)] pub known_attributes: BTreeSet, + /// Whether we generate code for tests. This specifically guarantees stable output /// for baseline testing. #[clap(long)] pub testing: bool, + /// Active experiments. Experiments alter default behavior of the compiler. /// See `Experiment` struct. #[clap(short)] @@ -56,31 +69,52 @@ pub struct Options { num_args = 0.. )] pub experiments: Vec, + /// A transient cache for memoization of experiment checks. #[clap(skip)] pub experiment_cache: RefCell>, + /// Sources to compile (positional arg, therefore last). /// Each source should be a path to either (1) a Move file or (2) a directory containing Move /// files, all to be compiled (e.g., not the root directory of a package---which contains /// Move.toml---but a specific subdirectorysuch as `sources`, `scripts`, and/or `tests`, /// depending on compilation mode). pub sources: Vec, + /// Dependencies to compile but not treat as a test/docgen/warning/prover target. /// Each source_dep should be a path to either (1) a Move file or (2) a directory containing /// Move files, all to be compiled (e.g., not the root directory of a package---which contains /// Move.toml---but a specific subdirectorysuch as `sources`). #[clap(skip)] pub sources_deps: Vec, + + #[clap(long = cli::MOVE_COMPILER_WARN_OF_DEPRECATION_USE_FLAG, + default_value=bool_to_str(move_compiler_warn_of_deprecation_use_env_var()))] + pub warn_deprecated: bool, + + /// Show warnings about use of deprecated usage in the Aptos libraries, + /// which we should generally not bother users with. + /// Note that current value of this constant is "Wdeprecation-aptos" + #[clap(long = cli::WARN_OF_DEPRECATION_USE_IN_APTOS_LIBS_FLAG, + default_value=bool_to_str(warn_of_deprecation_use_in_aptos_libs_env_var()))] + pub warn_of_deprecation_use_in_aptos_libs: bool, + /// Show warnings about unused functions, fields, constants, etc. /// Note that the current value of this constant is "Wunused" #[clap(long = cli::WARN_UNUSED_FLAG, default_value="false")] pub warn_unused: bool, + /// Whether to compile everything, including dependencies. #[clap(long)] pub whole_program: bool, + /// Whether to compile #[test] and #[test_only] code #[clap(skip)] pub compile_test_code: bool, + + /// Whether to compile #[verify_only] code + #[clap(skip)] + pub compile_verify_code: bool, } impl Default for Options { @@ -169,6 +203,34 @@ impl Options { ..self } } + + pub fn set_compile_verify_code(self, value: bool) -> Self { + Self { + compile_verify_code: value, + ..self + } + } + + pub fn set_warn_deprecated(self, value: bool) -> Self { + Self { + warn_deprecated: value, + ..self + } + } + + pub fn set_warn_of_deprecation_use_in_aptos_libs(self, value: bool) -> Self { + Self { + warn_of_deprecation_use_in_aptos_libs: value, + ..self + } + } + + pub fn set_warn_unused(self, value: bool) -> Self { + Self { + warn_unused: value, + ..self + } + } } /// Finds the experiment in the list of definitions. A definition diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.exp new file mode 100644 index 0000000000000..a30fe67df07d0 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.exp @@ -0,0 +1,27 @@ + +Diagnostics: +error: duplicate declaration, item, or annotation + ┌─ tests/deprecated/deprecated_constant_duplicated_struct.move:5:11 + │ +4 │ struct C { } + │ - Alias previously defined here +5 │ const C: u64 = 0; + │ ^ Duplicate module member or alias 'C'. Top level names in a namespace must be unique + +warning: unused alias + ┌─ tests/deprecated/deprecated_constant_duplicated_struct.move:12:15 + │ +12 │ use 0x42::mod1; + │ ^^^^ Unused 'use' of alias 'mod1'. Consider removing it + +warning: unused alias + ┌─ tests/deprecated/deprecated_constant_duplicated_struct.move:31:15 + │ +31 │ use 0x42::mod1; + │ ^^^^ Unused 'use' of alias 'mod1'. Consider removing it + +error: invalid name + ┌─ tests/deprecated/deprecated_constant_duplicated_struct.move:32:26 + │ +32 │ use 0x42::mod1::C as mod1; + │ ^^^^ Invalid constant alias name 'mod1'. Constant alias names must start with 'A'..'Z' diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.move new file mode 100644 index 0000000000000..e700dd74e7aec --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.move @@ -0,0 +1,44 @@ +address 0x42 { +module mod1 { + #[deprecated] + struct C { } + const C: u64 = 0; + public fun mod1() {} +} +} + +address 0x41 { +module N { + use 0x42::mod1; + use 0x42::mod1::C as D; + use 0x42::mod1::C as C; + use 0x42::mod1::mod1; + + fun f1(): 0x42::mod1::C { + mod1(); + C; + { + use 0x42::mod1::C; + C + }; + D + } +} +} + + +script { + use 0x42::mod1; + use 0x42::mod1::C as mod1; + use 0x42::mod1::C as C; + use 0x42::mod1::mod1; + + fun f1(): 0x42::mod1::C { + mod1(); + C; + { + use 0x42::mod1::C; + C + } + } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.exp new file mode 100644 index 0000000000000..9f0348c6e4d2e --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.exp @@ -0,0 +1,145 @@ + +Diagnostics: +error: duplicate declaration, item, or annotation + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:5:11 + │ +3 │ struct C { } + │ - Alias previously defined here +4 │ #[deprecated] +5 │ const C: u64 = 0; + │ ^ Duplicate module member or alias 'C'. Top level names in a namespace must be unique + +warning: unused alias + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:12:15 + │ +12 │ use 0x42::mod1; + │ ^^^^ Unused 'use' of alias 'mod1'. Consider removing it + +warning: Use of deprecated constant + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:13:21 + │ + 4 │ #[deprecated] + │ ---------- Constant 'C' in module '0x42::mod1' deprecated here + · +13 │ use 0x42::mod1::C as D; + │ ^ Use of deprecated constant 'C' from module '0x42::mod1' + +warning: Use of deprecated constant + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:14:21 + │ + 4 │ #[deprecated] + │ ---------- Constant 'C' in module '0x42::mod1' deprecated here + · +14 │ use 0x42::mod1::C as C; + │ ^ Use of deprecated constant 'C' from module '0x42::mod1' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:17:27 + │ + 4 │ #[deprecated] + │ ---------- Struct 'C' in module '0x42::mod1' deprecated here + · +17 │ fun f1(): 0x42::mod1::C { + │ ^ Use of deprecated struct 'C' from module '0x42::mod1' + +warning: Use of deprecated member + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:19:2 + │ + 4 │ #[deprecated] + │ ---------- Member 'C' in module '0x42::mod1' deprecated here + · +19 │ C; + │ ^ Use of deprecated member 'C' from module '0x42::mod1' + +warning: Use of deprecated constant + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:21:22 + │ + 4 │ #[deprecated] + │ ---------- Constant 'C' in module '0x42::mod1' deprecated here + · +21 │ use 0x42::mod1::C; + │ ^ Use of deprecated constant 'C' from module '0x42::mod1' + +warning: Use of deprecated member + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:22:6 + │ + 4 │ #[deprecated] + │ ---------- Member 'C' in module '0x42::mod1' deprecated here + · +22 │ C + │ ^ Use of deprecated member 'C' from module '0x42::mod1' + +warning: Use of deprecated member + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:24:2 + │ + 4 │ #[deprecated] + │ ---------- Member 'C' in module '0x42::mod1' deprecated here + · +24 │ D + │ ^ Use of deprecated member 'C' from module '0x42::mod1' + +warning: unused alias + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:31:15 + │ +31 │ use 0x42::mod1; + │ ^^^^ Unused 'use' of alias 'mod1'. Consider removing it + +warning: Use of deprecated constant + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:32:21 + │ + 4 │ #[deprecated] + │ ---------- Constant 'C' in module '0x42::mod1' deprecated here + · +32 │ use 0x42::mod1::C as mod1; + │ ^ Use of deprecated constant 'C' from module '0x42::mod1' + +error: invalid name + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:32:26 + │ +32 │ use 0x42::mod1::C as mod1; + │ ^^^^ Invalid constant alias name 'mod1'. Constant alias names must start with 'A'..'Z' + +warning: Use of deprecated constant + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:33:21 + │ + 4 │ #[deprecated] + │ ---------- Constant 'C' in module '0x42::mod1' deprecated here + · +33 │ use 0x42::mod1::C as C; + │ ^ Use of deprecated constant 'C' from module '0x42::mod1' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:36:27 + │ + 4 │ #[deprecated] + │ ---------- Struct 'C' in module '0x42::mod1' deprecated here + · +36 │ fun f1(): 0x42::mod1::C { + │ ^ Use of deprecated struct 'C' from module '0x42::mod1' + +warning: Use of deprecated member + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:38:2 + │ + 4 │ #[deprecated] + │ ---------- Member 'C' in module '0x42::mod1' deprecated here + · +38 │ C; + │ ^ Use of deprecated member 'C' from module '0x42::mod1' + +warning: Use of deprecated constant + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:40:22 + │ + 4 │ #[deprecated] + │ ---------- Constant 'C' in module '0x42::mod1' deprecated here + · +40 │ use 0x42::mod1::C; + │ ^ Use of deprecated constant 'C' from module '0x42::mod1' + +warning: Use of deprecated member + ┌─ tests/deprecated/deprecated_constant_duplicated_struct2.move:41:6 + │ + 4 │ #[deprecated] + │ ---------- Member 'C' in module '0x42::mod1' deprecated here + · +41 │ C + │ ^ Use of deprecated member 'C' from module '0x42::mod1' diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.move new file mode 100644 index 0000000000000..faf1c9ff4b518 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.move @@ -0,0 +1,44 @@ +address 0x42 { +module mod1 { + struct C { } + #[deprecated] + const C: u64 = 0; + public fun mod1() {} +} +} + +address 0x41 { +module N { + use 0x42::mod1; + use 0x42::mod1::C as D; + use 0x42::mod1::C as C; + use 0x42::mod1::mod1; + + fun f1(): 0x42::mod1::C { + mod1(); + C; + { + use 0x42::mod1::C; + C + }; + D + } +} +} + + +script { + use 0x42::mod1; + use 0x42::mod1::C as mod1; + use 0x42::mod1::C as C; + use 0x42::mod1::mod1; + + fun f1(): 0x42::mod1::C { + mod1(); + C; + { + use 0x42::mod1::C; + C + } + } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.exp new file mode 100644 index 0000000000000..9b2175d2d571a --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.exp @@ -0,0 +1,31 @@ + +Diagnostics: +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_field_type.move:17:22 + │ + 3 │ #[deprecated] + │ ---------- Struct 'EventHandle' in module '0x42::event' deprecated here + · +17 │ use 0x42::event::EventHandle; + │ ^^^^^^^^^^^ Use of deprecated struct 'EventHandle' from module '0x42::event' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_field_type.move:27:25 + │ + 3 │ #[deprecated] + │ ---------- Struct 'EventHandle' in module '0x42::event' deprecated here + · +27 │ deposit_events: EventHandle, + │ ^^^^^^^^^^^ Use of deprecated struct 'EventHandle' from module '0x42::event' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_field_type.move:28:26 + │ + 3 │ #[deprecated] + │ ---------- Struct 'EventHandle' in module '0x42::event' deprecated here + · +28 │ withdraw_events: EventHandle, + │ ^^^^^^^^^^^ Use of deprecated struct 'EventHandle' from module '0x42::event' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.move new file mode 100644 index 0000000000000..7b8603579e617 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type.move @@ -0,0 +1,30 @@ +module 0x42::event { + + #[deprecated] + /// A handle for an event such that: + /// 1. Other modules can emit events to this handle. + /// 2. Storage can use this handle to prove the total number of events that happened in the past. + struct EventHandle has store { + /// Total number of events emitted to this event stream. + counter: u64, + /// A globally unique ID for this event stream. + guid: u64, + } + +} + +module 0x41::coin { + use 0x42::event::EventHandle; + + struct Coin has store { } + struct CoinType has key {} + struct DepositEvent has drop, store {} + struct WithdrawEvent has drop, store {} + + struct CoinStore has key { + coin: Coin, + frozen: bool, + deposit_events: EventHandle, + withdraw_events: EventHandle, + } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.exp new file mode 100644 index 0000000000000..9fd7327c3f1ee --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.exp @@ -0,0 +1,13 @@ + +Diagnostics: +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_field_type2.move:29:38 + │ +22 │ #[deprecated] + │ ---------- Struct 'WithdrawEvent' in module '0x41::coin' deprecated here + · +29 │ withdraw_events: EventHandle, + │ ^^^^^^^^^^^^^ Use of deprecated struct 'WithdrawEvent' from module '0x41::coin' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.move new file mode 100644 index 0000000000000..f21a61fedec9a --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_field_type2.move @@ -0,0 +1,31 @@ +module 0x42::event { + + /// A handle for an event such that: + /// 1. Other modules can emit events to this handle. + /// 2. Storage can use this handle to prove the total number of events that happened in the past. + struct EventHandle has store { + /// Total number of events emitted to this event stream. + counter: u64, + /// A globally unique ID for this event stream. + guid: u64, + } + +} + +module 0x41::coin { + use 0x42::event::EventHandle; + + struct Coin has store { } + struct CoinType has key {} + struct DepositEvent has drop, store {} + + #[deprecated] + struct WithdrawEvent has drop, store {} + + struct CoinStore has key { + coin: Coin, + frozen: bool, + deposit_events: EventHandle, + withdraw_events: EventHandle, + } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.exp new file mode 100644 index 0000000000000..c492663d4fe27 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.exp @@ -0,0 +1,22 @@ + +Diagnostics: +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_address.move:25:9 + │ + 1 │ #[deprecated] + │ ---------- Module '0x42::M' deprecated here + · +25 │ use 0x42::M; + │ ^^^^^^^ Use of deprecated module '0x42::M' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_address.move:29:15 + │ + 1 │ #[deprecated] + │ ---------- Module '0x42::M' deprecated here + · +29 │ let foo: S = M::foo(); + │ ^ Use of deprecated module '0x42::M' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.move new file mode 100644 index 0000000000000..368cb7469e473 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address.move @@ -0,0 +1,34 @@ +#[deprecated] +address 0x42 { +module M { + use 0x41::N; + + struct S {} + + const C: u64 = 0; + + public fun foo(): N::S { let _foo = C + 3; N::bar() } + + spec foo {} +} +} + +module 0x41::N { + struct S has drop { } + + public fun bar(): S { S { } } + + public fun consume(_x: S) { } +} + +script { + use 0x42::M; + use 0x41::N::S; + + fun main() { + let foo: S = M::foo(); + 0x41::N::consume(foo); + } + + spec main { } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.exp new file mode 100644 index 0000000000000..9e1d702f7c4a8 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.exp @@ -0,0 +1,76 @@ + +Diagnostics: +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:33:9 + │ + 1 │ #[deprecated] // Think about using 0x43 instead + │ ---------- Module '0x42::M' deprecated here + · +33 │ use 0x42::M; + │ ^^^^^^^ Use of deprecated module '0x42::M' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:34:9 + │ +20 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + · +34 │ use 0x41::N::S; + │ ^^^^^^^ Use of deprecated module '0x41::N' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:34:18 + │ +22 │ #[deprecated] + │ ---------- Struct 'S' in module '0x41::N' deprecated here + · +34 │ use 0x41::N::S; + │ ^ Use of deprecated struct 'S' from module '0x41::N' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:37:11 + │ +22 │ #[deprecated] + │ ---------- Struct 'S' in module '0x41::N' deprecated here + · +37 │ let foo: S = M::foo(); + │ ^ Use of deprecated struct 'S' from module '0x41::N' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:37:15 + │ + 1 │ #[deprecated] // Think about using 0x43 instead + │ ---------- Module '0x42::M' deprecated here + · +37 │ let foo: S = M::foo(); + │ ^ Use of deprecated module '0x42::M' + +warning: Use of deprecated function + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:37:18 + │ +13 │ #[deprecated] + │ ---------- Function 'foo' in module '0x42::M' deprecated here + · +37 │ let foo: S = M::foo(); + │ ^^^ Use of deprecated function 'foo' from module '0x42::M' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:38:2 + │ +20 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + · +38 │ 0x41::N::consume(foo); + │ ^^^^^^^ Use of deprecated module '0x41::N' + +warning: Use of deprecated function + ┌─ tests/deprecated/deprecated_placement_address_module_members.move:38:11 + │ +28 │ #[deprecated] + │ ---------- Function 'consume' in module '0x41::N' deprecated here + · +38 │ 0x41::N::consume(foo); + │ ^^^^^^^ Use of deprecated function 'consume' from module '0x41::N' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.move new file mode 100644 index 0000000000000..361ac55d38042 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.move @@ -0,0 +1,42 @@ +#[deprecated] // Think about using 0x43 instead +address 0x42 { +#[deprecated] // Try module M2 +module M { + use 0x41::N; + + #[deprecated] + struct S {} + + #[deprecated] + const C: u64 = 0; + + #[deprecated] + public fun foo(): N::S { let _foo = C + 3; N::bar() } + + spec foo {} +} +} + +#[deprecated] +module 0x41::N { + #[deprecated] + struct S has drop { } + + #[deprecated] + public fun bar(): S { S { } } + + #[deprecated] + public fun consume(_x: S) { } +} + +script { + use 0x42::M; + use 0x41::N::S; + + fun main() { + let foo: S = M::foo(); + 0x41::N::consume(foo); + } + + spec main { } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.exp new file mode 100644 index 0000000000000..c5aa40b0e3e3f --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.exp @@ -0,0 +1,40 @@ + +Diagnostics: +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_placement_members.move:31:18 + │ +19 │ #[deprecated] + │ ---------- Struct 'S' in module '0x41::N' deprecated here + · +31 │ use 0x41::N::S; + │ ^ Use of deprecated struct 'S' from module '0x41::N' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_placement_members.move:34:11 + │ +19 │ #[deprecated] + │ ---------- Struct 'S' in module '0x41::N' deprecated here + · +34 │ let foo: S = M::foo(); + │ ^ Use of deprecated struct 'S' from module '0x41::N' + +warning: Use of deprecated function + ┌─ tests/deprecated/deprecated_placement_members.move:34:18 + │ +11 │ #[deprecated] + │ ---------- Function 'foo' in module '0x42::M' deprecated here + · +34 │ let foo: S = M::foo(); + │ ^^^ Use of deprecated function 'foo' from module '0x42::M' + +warning: Use of deprecated function + ┌─ tests/deprecated/deprecated_placement_members.move:35:11 + │ +25 │ #[deprecated] + │ ---------- Function 'consume' in module '0x41::N' deprecated here + · +35 │ 0x41::N::consume(foo); + │ ^^^^^^^ Use of deprecated function 'consume' from module '0x41::N' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.move new file mode 100644 index 0000000000000..c05b48447a29a --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_members.move @@ -0,0 +1,39 @@ +address 0x42 { +module M { + use 0x41::N; + + #[deprecated] + struct S {} + + #[deprecated] + const C: u64 = 0; + + #[deprecated] + public fun foo(): N::S { let _foo = C + 3; N::bar() } + + spec foo {} +} +} + +module 0x41::N { + #[deprecated] + struct S has drop { } + + #[deprecated] + public fun bar(): S { S { } } + + #[deprecated] + public fun consume(_x: S) { } +} + +script { + use 0x42::M; + use 0x41::N::S; + + fun main() { + let foo: S = M::foo(); + 0x41::N::consume(foo); + } + + spec main { } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.exp new file mode 100644 index 0000000000000..68a6f21363dd5 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.exp @@ -0,0 +1,22 @@ + +Diagnostics: +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module.move:25:9 + │ + 2 │ #[deprecated] + │ ---------- Module '0x42::M' deprecated here + · +25 │ use 0x42::M; + │ ^^^^^^^ Use of deprecated module '0x42::M' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module.move:29:15 + │ + 2 │ #[deprecated] + │ ---------- Module '0x42::M' deprecated here + · +29 │ let foo: S = M::foo(); + │ ^ Use of deprecated module '0x42::M' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.move new file mode 100644 index 0000000000000..7f2041ed6215a --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module.move @@ -0,0 +1,34 @@ +address 0x42 { +#[deprecated] +module M { + use 0x41::N; + + struct S {} + + const C: u64 = 0; + + public fun foo(): N::S { let _foo = C + 3; N::bar() } + + spec foo {} +} +} + +module 0x41::N { + struct S has drop { } + + public fun bar(): S { S { } } + + public fun consume(_x: S) { } +} + +script { + use 0x42::M; + use 0x41::N::S; + + fun main() { + let foo: S = M::foo(); + 0x41::N::consume(foo); + } + + spec main { } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.exp new file mode 100644 index 0000000000000..ab9447a567186 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.exp @@ -0,0 +1,49 @@ + +Diagnostics: +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module2.move:3:9 + │ + 3 │ use 0x41::N; + │ ^^^^^^^ Use of deprecated module '0x41::N' + · +15 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module2.move:9:23 + │ + 9 │ public fun foo(): N::S { let _foo = C + 3; N::bar() } + │ ^ Use of deprecated module '0x41::N' + · +15 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module2.move:9:48 + │ + 9 │ public fun foo(): N::S { let _foo = C + 3; N::bar() } + │ ^ Use of deprecated module '0x41::N' + · +15 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module2.move:26:9 + │ +15 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + · +26 │ use 0x41::N::S; + │ ^^^^^^^ Use of deprecated module '0x41::N' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module2.move:30:2 + │ +15 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + · +30 │ 0x41::N::consume(foo); + │ ^^^^^^^ Use of deprecated module '0x41::N' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.move new file mode 100644 index 0000000000000..a1a98631b86d8 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module2.move @@ -0,0 +1,34 @@ +address 0x42 { +module M { + use 0x41::N; + + struct S {} + + const C: u64 = 0; + + public fun foo(): N::S { let _foo = C + 3; N::bar() } + + spec foo {} +} +} + +#[deprecated] +module 0x41::N { + struct S has drop { } + + public fun bar(): S { S { } } + + public fun consume(_x: S) { } +} + +script { + use 0x42::M; + use 0x41::N::S; + + fun main() { + let foo: S = M::foo(); + 0x41::N::consume(foo); + } + + spec main { } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.exp b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.exp new file mode 100644 index 0000000000000..d8e3badc5994f --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.exp @@ -0,0 +1,76 @@ + +Diagnostics: +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module_members.move:32:9 + │ + 2 │ #[deprecated] + │ ---------- Module '0x42::M' deprecated here + · +32 │ use 0x42::M; + │ ^^^^^^^ Use of deprecated module '0x42::M' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module_members.move:33:9 + │ +19 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + · +33 │ use 0x41::N::S; + │ ^^^^^^^ Use of deprecated module '0x41::N' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_placement_module_members.move:33:18 + │ +21 │ #[deprecated] + │ ---------- Struct 'S' in module '0x41::N' deprecated here + · +33 │ use 0x41::N::S; + │ ^ Use of deprecated struct 'S' from module '0x41::N' + +warning: Use of deprecated struct + ┌─ tests/deprecated/deprecated_placement_module_members.move:36:11 + │ +21 │ #[deprecated] + │ ---------- Struct 'S' in module '0x41::N' deprecated here + · +36 │ let foo: S = M::foo(); + │ ^ Use of deprecated struct 'S' from module '0x41::N' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module_members.move:36:15 + │ + 2 │ #[deprecated] + │ ---------- Module '0x42::M' deprecated here + · +36 │ let foo: S = M::foo(); + │ ^ Use of deprecated module '0x42::M' + +warning: Use of deprecated function + ┌─ tests/deprecated/deprecated_placement_module_members.move:36:18 + │ +12 │ #[deprecated] + │ ---------- Function 'foo' in module '0x42::M' deprecated here + · +36 │ let foo: S = M::foo(); + │ ^^^ Use of deprecated function 'foo' from module '0x42::M' + +warning: Use of deprecated module + ┌─ tests/deprecated/deprecated_placement_module_members.move:37:2 + │ +19 │ #[deprecated] + │ ---------- Module '0x41::N' deprecated here + · +37 │ 0x41::N::consume(foo); + │ ^^^^^^^ Use of deprecated module '0x41::N' + +warning: Use of deprecated function + ┌─ tests/deprecated/deprecated_placement_module_members.move:37:11 + │ +27 │ #[deprecated] + │ ---------- Function 'consume' in module '0x41::N' deprecated here + · +37 │ 0x41::N::consume(foo); + │ ^^^^^^^ Use of deprecated function 'consume' from module '0x41::N' + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.move b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.move new file mode 100644 index 0000000000000..e3b71a074b341 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/deprecated_placement_module_members.move @@ -0,0 +1,41 @@ +address 0x42 { +#[deprecated] +module M { + use 0x41::N; + + #[deprecated] + struct S {} + + #[deprecated] + const C: u64 = 0; + + #[deprecated] + public fun foo(): N::S { let _foo = C + 3; N::bar() } + + spec foo {} +} +} + +#[deprecated] +module 0x41::N { + #[deprecated] + struct S has drop { } + + #[deprecated] + public fun bar(): S { S { } } + + #[deprecated] + public fun consume(_x: S) { } +} + +script { + use 0x42::M; + use 0x41::N::S; + + fun main() { + let foo: S = M::foo(); + 0x41::N::consume(foo); + } + + spec main { } +} diff --git a/third_party/move/move-compiler-v2/tests/deprecated/public_script.exp b/third_party/move/move-compiler-v2/tests/deprecated/public_script.exp new file mode 100644 index 0000000000000..1463c4ca97cdf --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/public_script.exp @@ -0,0 +1,13 @@ + +Diagnostics: +warning: DEPRECATED. will be removed + ┌─ tests/deprecated/public_script.move:2:5 + │ +2 │ public(script) fun main() { + │ ^^^^^^^^^^^^^^ 'public(script)' is deprecated in favor of the 'entry' modifier. Replace with 'public entry' + +error: address with no value + ┌─ tests/deprecated/public_script.move:4:30 + │ +4 │ let _addr:address = @Test; + │ ^^^^ address 'Test' is not assigned a value. Try assigning it a value when calling the compiler diff --git a/third_party/move/move-compiler-v2/tests/deprecated/public_script.move b/third_party/move/move-compiler-v2/tests/deprecated/public_script.move new file mode 100644 index 0000000000000..d225e0f525526 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/deprecated/public_script.move @@ -0,0 +1,6 @@ +module 0x1::Test { + public(script) fun main() { + // Previously, deprecation plus an error led to a compiler assert. Make sure that doesn't come back. + let _addr:address = @Test; + } +} diff --git a/third_party/move/move-compiler-v2/tests/testsuite.rs b/third_party/move/move-compiler-v2/tests/testsuite.rs index 14d5be58213a7..e940d1aa6ab4e 100644 --- a/third_party/move/move-compiler-v2/tests/testsuite.rs +++ b/third_party/move/move-compiler-v2/tests/testsuite.rs @@ -198,7 +198,7 @@ const TEST_CONFIGS: Lazy> = Lazy::new(|| { dump_bytecode: DumpLevel::None, // do not dump anything dump_bytecode_filter: None, }, - // Tests for inlining, simplifier, and folding + // Tests for front-end, diagnostics (inlining, simplifier, folding, etc.) TestConfig { name: "inlining-et-al", runner: |p| run_test(p, get_config_by_name("inlining-et-al")), @@ -226,6 +226,20 @@ const TEST_CONFIGS: Lazy> = Lazy::new(|| { dump_bytecode: DumpLevel::None, // do not dump anything dump_bytecode_filter: None, }, + // Tests for diagnostics, where dumping AST isn't useful. + TestConfig { + name: "diagnostics", + runner: |p| run_test(p, get_config_by_name("diagnostics")), + include: vec!["/deprecated/"], + exclude: vec![], + exp_suffix: None, + options: opts.clone().set_experiment(Experiment::AST_SIMPLIFY, true), + // Run the entire compiler pipeline to double-check the result + stop_after: StopAfter::FileFormat, + dump_ast: DumpLevel::None, + dump_bytecode: DumpLevel::None, // do not dump anything + dump_bytecode_filter: None, + }, // --- Tests for bytecode generation TestConfig { name: "bytecode-gen", @@ -539,6 +553,39 @@ const TEST_CONFIGS: Lazy> = Lazy::new(|| { dump_bytecode: DumpLevel::None, dump_bytecode_filter: None, }, + // Test for verify on and off + TestConfig { + name: "verification", + runner: |p| run_test(p, get_config_by_name("verification")), + include: vec!["/verification/verify/"], + exclude: vec![], + exp_suffix: None, + options: opts + .clone() + .set_experiment(Experiment::AST_SIMPLIFY, true) + .set_compile_test_code(true), + // Run the entire compiler pipeline to double-check the result + stop_after: StopAfter::FileFormat, + dump_ast: DumpLevel::None, + dump_bytecode: DumpLevel::None, + dump_bytecode_filter: None, + }, + TestConfig { + name: "verification-off", + runner: |p| run_test(p, get_config_by_name("verification-off")), + include: vec!["/verification/noverify/"], + exclude: vec![], + exp_suffix: None, + options: opts + .clone() + .set_experiment(Experiment::AST_SIMPLIFY, true) + .set_compile_test_code(false), + // Run the entire compiler pipeline to double-check the result + stop_after: StopAfter::FileFormat, + dump_ast: DumpLevel::None, + dump_bytecode: DumpLevel::None, + dump_bytecode_filter: None, + }, TestConfig { name: "skip-attribute-checks", runner: |p| run_test(p, get_config_by_name("skip-attribute-checks")), @@ -574,6 +621,8 @@ fn run_test(path: &Path, config: TestConfig) -> datatest_stable::Result<()> { let path_str = path.display().to_string(); let mut options = config.options.clone(); options.warn_unused = path_str.contains("/unused/"); + options.warn_deprecated = path_str.contains("/deprecated/"); + options.compile_verify_code = path_str.contains("/verification/verify/"); options.sources_deps = extract_test_directives(path, "// dep:")?; options.sources = vec![path_str.clone()]; options.dependencies = if extract_test_directives(path, "// no-stdlib")?.is_empty() { diff --git a/third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.exp b/third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.exp new file mode 100644 index 0000000000000..ed74cc91d766e --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.exp @@ -0,0 +1,7 @@ + +Diagnostics: +error: unresolved spec target + ┌─ tests/unit_test/notest/function_with_spec.move:18:10 + │ +18 │ spec baz { + │ ^^^ diff --git a/third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.move b/third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.move new file mode 100644 index 0000000000000..ed9d79304088e --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unit_test/notest/function_with_spec.move @@ -0,0 +1,22 @@ +// #[test_only] functions should be filtered out in non-test mode +address 0x1234 { +module M { + public fun foo() { } + + #[test] + public fun bar_test() { bar() } + + #[test_only] + public fun bar() { } + + // This should not cause an error in either test- nor non-test-mode. + spec bar { + aborts_if false; + } + + // This should always cause an error due to typo. + spec baz { + aborts_if false; + } +} +} diff --git a/third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.exp b/third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.exp new file mode 100644 index 0000000000000..fbcda8c812a5b --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.exp @@ -0,0 +1,10 @@ + +Diagnostics: +error: unresolved spec target + ┌─ tests/unit_test/test/function_with_spec.move:18:10 + │ +18 │ spec baz { + │ ^^^ + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.move b/third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.move new file mode 100644 index 0000000000000..ed9d79304088e --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/unit_test/test/function_with_spec.move @@ -0,0 +1,22 @@ +// #[test_only] functions should be filtered out in non-test mode +address 0x1234 { +module M { + public fun foo() { } + + #[test] + public fun bar_test() { bar() } + + #[test_only] + public fun bar() { } + + // This should not cause an error in either test- nor non-test-mode. + spec bar { + aborts_if false; + } + + // This should always cause an error due to typo. + spec baz { + aborts_if false; + } +} +} diff --git a/third_party/move/move-compiler-v2/tests/v1.matched b/third_party/move/move-compiler-v2/tests/v1.matched index 25a816e67c53f..56577566e080d 100644 --- a/third_party/move/move-compiler-v2/tests/v1.matched +++ b/third_party/move/move-compiler-v2/tests/v1.matched @@ -1,3 +1,6 @@ +WARNING: test `move-compiler-v2/tests/more-v1/verification/cross_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/cross_module_valid.move` share common key `verification/cross_module_valid.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/more-v1/verification/double_annotation.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/double_annotation.move` share common key `verification/double_annotation.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/more-v1/verification/single_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/single_module_valid.move` share common key `verification/single_module_valid.verification.move`, discarding former one move-compiler/tests/move_check/translated_ir_tests/move/borrow_tests/borrow_global_acquires_1.exp move-compiler-v2/tests/acquires-checker/v1-borrow-tests/borrow_global_acquires_1.exp move-compiler/tests/move_check/translated_ir_tests/move/borrow_tests/borrow_global_acquires_2.exp move-compiler-v2/tests/acquires-checker/v1-borrow-tests/borrow_global_acquires_2.exp move-compiler/tests/move_check/translated_ir_tests/move/borrow_tests/borrow_global_acquires_3.exp move-compiler-v2/tests/acquires-checker/v1-borrow-tests/borrow_global_acquires_3.exp @@ -152,6 +155,17 @@ move-compiler/tests/move_check/translated_ir_tests/move/commands/use_before_assi move-compiler/tests/move_check/control_flow/for_loop_empty_novar.exp move-compiler-v2/tests/checking/control_flow/for_loop_empty_novar.exp move-compiler/tests/move_check/control_flow/for_type_mismatch.exp move-compiler-v2/tests/checking/control_flow/for_type_mismatch.exp move-compiler/tests/move_check/control_flow/loop_after_loop.exp move-compiler-v2/tests/checking/control_flow/loop_after_loop.exp +move-compiler/tests/move_check/deprecated/deprecated_constant_duplicated_struct.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_constant_duplicated_struct.exp +move-compiler/tests/move_check/deprecated/deprecated_constant_duplicated_struct2.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_constant_duplicated_struct2.exp +move-compiler/tests/move_check/deprecated/deprecated_field_type.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_field_type.exp +move-compiler/tests/move_check/deprecated/deprecated_field_type2.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_field_type2.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_address.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_address.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_address_module_members.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_address_module_members.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_members.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_members.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_module.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_module.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_module2.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_module2.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_module_members.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_module_members.exp +move-compiler/tests/move_check/deprecated/public_script.exp move-compiler-v2/tests/more-v1/deprecated/public_script.exp move-compiler/tests/move_check/folding/empty_vectors.exp move-compiler-v2/tests/folding/empty_vectors.exp move-compiler/tests/move_check/folding/empty_vectors2.exp move-compiler-v2/tests/folding/empty_vectors2.exp move-compiler/tests/move_check/folding/non_constant_empty_vec.exp move-compiler-v2/tests/folding/non_constant_empty_vec.exp @@ -623,3 +637,12 @@ move-compiler/tests/move_check/unit_test/test_filter_function.exp move-compile move-compiler/tests/move_check/unit_test/test_filter_struct.exp move-compiler-v2/tests/unit_test/notest/test_filter_struct.exp move-compiler/tests/move_check/unit_test/valid_test_module.exp move-compiler-v2/tests/unit_test/notest/valid_test_module.exp move-compiler/tests/move_check/unit_test/valid_test_module.unit_test.exp move-compiler-v2/tests/unit_test/test/valid_test_module.exp +move-compiler/tests/move_check/verification/cross_module_invalid.exp move-compiler-v2/tests/more-v1/verification/noverify/cross_module_invalid.exp +move-compiler/tests/move_check/verification/cross_module_valid.exp move-compiler-v2/tests/more-v1/verification/noverify/cross_module_valid.exp +move-compiler/tests/move_check/verification/cross_module_valid.verification.exp move-compiler-v2/tests/more-v1/verification/verify/cross_module_valid.exp +move-compiler/tests/move_check/verification/double_annotation.exp move-compiler-v2/tests/more-v1/verification/noverify/double_annotation.exp +move-compiler/tests/move_check/verification/double_annotation.verification.exp move-compiler-v2/tests/more-v1/verification/verify/double_annotation.exp +move-compiler/tests/move_check/verification/single_module_invalid.exp move-compiler-v2/tests/more-v1/verification/noverify/single_module_invalid.exp +move-compiler/tests/move_check/verification/single_module_invalid.verification.exp move-compiler-v2/tests/more-v1/verification/single_module_invalid.verification.exp +move-compiler/tests/move_check/verification/single_module_valid.exp move-compiler-v2/tests/more-v1/verification/noverify/single_module_valid.exp +move-compiler/tests/move_check/verification/single_module_valid.verification.exp move-compiler-v2/tests/more-v1/verification/verify/single_module_valid.exp diff --git a/third_party/move/move-compiler-v2/tests/v1.unmatched b/third_party/move/move-compiler-v2/tests/v1.unmatched index f2ad6f0903cec..9ee038364c30c 100644 --- a/third_party/move/move-compiler-v2/tests/v1.unmatched +++ b/third_party/move/move-compiler-v2/tests/v1.unmatched @@ -1,3 +1,6 @@ +WARNING: test `move-compiler-v2/tests/more-v1/verification/cross_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/cross_module_valid.move` share common key `verification/cross_module_valid.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/more-v1/verification/double_annotation.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/double_annotation.move` share common key `verification/double_annotation.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/more-v1/verification/single_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/single_module_valid.move` share common key `verification/single_module_valid.verification.move`, discarding former one move-compiler/tests/move_check/borrow_tests/{ borrow_global_acquires_duplicate_annotation.move, eq_bad.move, @@ -31,18 +34,7 @@ move-compiler/tests/move_check/control_flow/{ } move-compiler/tests/move_check/deprecated/{ assert_function.move, - deprecated_constant_duplicated_struct.move, - deprecated_constant_duplicated_struct2.move, - deprecated_field_type.move, - deprecated_field_type2.move, - deprecated_placement_address.move, - deprecated_placement_address_module_members.move, deprecated_placement_basecase.move, - deprecated_placement_members.move, - deprecated_placement_module.move, - deprecated_placement_module2.move, - deprecated_placement_module_members.move, - public_script.move, } move-compiler/tests/move_check/examples/{ multi_pool_money_market_token.move, @@ -279,10 +271,3 @@ move-compiler/tests/move_check/typing/{ move-compiler/tests/move_check/v2-not-supported/{ no_receiver_calls.move, } -move-compiler/tests/move_check/verification/{ - cross_module_invalid.move, - cross_module_valid.move, - double_annotation.move, - single_module_invalid.move, - single_module_valid.move, -} diff --git a/third_party/move/move-compiler-v2/tests/verification/cross_module_valid.verification b/third_party/move/move-compiler-v2/tests/verification/cross_module_valid.verification new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/third_party/move/move-compiler-v2/tests/verification/double_annotation.verification b/third_party/move/move-compiler-v2/tests/verification/double_annotation.verification new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.exp b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.exp new file mode 100644 index 0000000000000..ca9e9067e0ff9 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.exp @@ -0,0 +1,7 @@ + +Diagnostics: +error: unbound module + ┌─ tests/verification/noverify/cross_module_invalid.move:23:9 + │ +23 │ A::build_foo() + │ ^ Unbound module or type alias 'A' diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.move b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.move new file mode 100644 index 0000000000000..9d78d065a3d6c --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_invalid.move @@ -0,0 +1,26 @@ +// check that `use`'s are filtered out correctly +address 0x1 { +module A { + struct Foo has drop {} + + public fun build_foo(): Foo { Foo {} } +} + +module B { + #[verify_only] + use 0x1::A::{Self, Foo}; + + #[verify_only] + fun x(_: Foo) { } + + #[verify_only] + fun tester() { + x(A::build_foo()) + } + + // this should fail + public fun bad(): Foo { + A::build_foo() + } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.exp b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.exp new file mode 100644 index 0000000000000..90b32906711cc --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.exp @@ -0,0 +1,2 @@ + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.move b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.move new file mode 100644 index 0000000000000..c141dc283e65f --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/cross_module_valid.move @@ -0,0 +1,24 @@ +// Check that verify_only filtering and calling is supported across modules and +// different types of module members +address 0x1 { +module A { + #[verify_only] + struct Foo has drop {} + + #[verify_only] + public fun build_foo(): Foo { Foo {} } +} + +module B { + #[verify_only] + use 0x1::A::{Self, Foo}; + + #[verify_only] + fun x(_: Foo) { } + + #[verify_only] + fun tester() { + x(A::build_foo()) + } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.exp b/third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.exp new file mode 100644 index 0000000000000..90b32906711cc --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.exp @@ -0,0 +1,2 @@ + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.move b/third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.move new file mode 100644 index 0000000000000..60f63be52a624 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/double_annotation.move @@ -0,0 +1,21 @@ +address 0x1 { +module M { + #[verify_only] + struct Foo {} + + // failure: double annotation + #[verify_only] + #[verify_only] + struct Bar {} + + public fun foo() { } + + #[verify_only] + public fun bar() { } + + // failure: double annotation + #[verify_only] + #[verify_only] + public fun d(_a: signer, _b: signer) { } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.exp b/third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.exp new file mode 100644 index 0000000000000..ea64cb0f09875 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.exp @@ -0,0 +1,7 @@ + +Diagnostics: +error: unresolved spec target + ┌─ tests/verification/noverify/function_with_spec.move:15:10 + │ +15 │ spec baz { + │ ^^^ diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.move b/third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.move new file mode 100644 index 0000000000000..4671968e37fd2 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/function_with_spec.move @@ -0,0 +1,19 @@ +// #[test_only] functions should be filtered out in non-test mode +address 0x1234 { +module M { + public fun foo() { } + + #[verify_only] + public fun bar() { } + + // This should not cause an error in either test- nor non-test-mode. + spec bar { + aborts_if false; + } + + // This should always cause an error due to typo. + spec baz { + aborts_if false; + } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.exp b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.exp new file mode 100644 index 0000000000000..232c3ee5c36c7 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.exp @@ -0,0 +1,19 @@ + +Diagnostics: +error: undeclared `0x1::M::Foo` + ┌─ tests/verification/noverify/single_module_invalid.move:8:23 + │ +8 │ public fun foo(): Foo { + │ ^^^ + +error: undeclared struct `M::Foo` + ┌─ tests/verification/noverify/single_module_invalid.move:9:9 + │ +9 │ Foo {} + │ ^^^ + +error: no function named `bar` found + ┌─ tests/verification/noverify/single_module_invalid.move:17:24 + │ +17 │ public fun baz() { bar() } + │ ^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.move b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.move new file mode 100644 index 0000000000000..06043755f6307 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_invalid.move @@ -0,0 +1,19 @@ +address 0x1 { +module M { + #[verify_only] + struct Foo {} + + // This should cause an unbound type error in non-verify mode + // as the Foo struct declaration was filtered out + public fun foo(): Foo { + Foo {} + } + + #[verify_only] + public fun bar() { } + + // This should cause an unbound function error in non-verify mode + // as `bar` was filtered out + public fun baz() { bar() } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.exp b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.exp new file mode 100644 index 0000000000000..90b32906711cc --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.exp @@ -0,0 +1,2 @@ + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.move b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.move new file mode 100644 index 0000000000000..a50d4ab3cf7da --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/noverify/single_module_valid.move @@ -0,0 +1,16 @@ +// Make sure that legal usage is allowed +module 0x1::M { + // verify-only struct + #[verify_only] + struct Foo {} + + public fun foo() { } + + // verify-only struct used in a verify-only function + #[verify_only] + public fun bar(): Foo { Foo{} } + + // verify-only function used in a verify-only function + #[verify_only] + public fun baz(): Foo { bar() } +} diff --git a/third_party/move/move-compiler-v2/tests/verification/single_module_invalid.verification b/third_party/move/move-compiler-v2/tests/verification/single_module_invalid.verification new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/third_party/move/move-compiler-v2/tests/verification/single_module_valid.verification b/third_party/move/move-compiler-v2/tests/verification/single_module_valid.verification new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.exp b/third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.exp new file mode 100644 index 0000000000000..90b32906711cc --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.exp @@ -0,0 +1,2 @@ + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.move b/third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.move new file mode 100644 index 0000000000000..c141dc283e65f --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/cross_module_valid.move @@ -0,0 +1,24 @@ +// Check that verify_only filtering and calling is supported across modules and +// different types of module members +address 0x1 { +module A { + #[verify_only] + struct Foo has drop {} + + #[verify_only] + public fun build_foo(): Foo { Foo {} } +} + +module B { + #[verify_only] + use 0x1::A::{Self, Foo}; + + #[verify_only] + fun x(_: Foo) { } + + #[verify_only] + fun tester() { + x(A::build_foo()) + } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.exp b/third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.exp new file mode 100644 index 0000000000000..fbc781f959cb2 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.exp @@ -0,0 +1,20 @@ + +Diagnostics: +error: duplicate declaration, item, or annotation + ┌─ tests/verification/verify/double_annotation.move:8:7 + │ +7 │ #[verify_only] + │ ----------- Attribute previously given here +8 │ #[verify_only] + │ ^^^^^^^^^^^ Duplicate attribute 'verify_only' attached to the same item + +error: duplicate declaration, item, or annotation + ┌─ tests/verification/verify/double_annotation.move:18:7 + │ +17 │ #[verify_only] + │ ----------- Attribute previously given here +18 │ #[verify_only] + │ ^^^^^^^^^^^ Duplicate attribute 'verify_only' attached to the same item + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.move b/third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.move new file mode 100644 index 0000000000000..60f63be52a624 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/double_annotation.move @@ -0,0 +1,21 @@ +address 0x1 { +module M { + #[verify_only] + struct Foo {} + + // failure: double annotation + #[verify_only] + #[verify_only] + struct Bar {} + + public fun foo() { } + + #[verify_only] + public fun bar() { } + + // failure: double annotation + #[verify_only] + #[verify_only] + public fun d(_a: signer, _b: signer) { } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.exp b/third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.exp new file mode 100644 index 0000000000000..47f7a09cccdad --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.exp @@ -0,0 +1,10 @@ + +Diagnostics: +error: unresolved spec target + ┌─ tests/verification/verify/function_with_spec.move:15:10 + │ +15 │ spec baz { + │ ^^^ + + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.move b/third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.move new file mode 100644 index 0000000000000..4671968e37fd2 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/function_with_spec.move @@ -0,0 +1,19 @@ +// #[test_only] functions should be filtered out in non-test mode +address 0x1234 { +module M { + public fun foo() { } + + #[verify_only] + public fun bar() { } + + // This should not cause an error in either test- nor non-test-mode. + spec bar { + aborts_if false; + } + + // This should always cause an error due to typo. + spec baz { + aborts_if false; + } +} +} diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.exp b/third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.exp new file mode 100644 index 0000000000000..90b32906711cc --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.exp @@ -0,0 +1,2 @@ + +============ bytecode verification succeeded ======== diff --git a/third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.move b/third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.move new file mode 100644 index 0000000000000..a50d4ab3cf7da --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/verification/verify/single_module_valid.move @@ -0,0 +1,16 @@ +// Make sure that legal usage is allowed +module 0x1::M { + // verify-only struct + #[verify_only] + struct Foo {} + + public fun foo() { } + + // verify-only struct used in a verify-only function + #[verify_only] + public fun bar(): Foo { Foo{} } + + // verify-only function used in a verify-only function + #[verify_only] + public fun baz(): Foo { bar() } +} diff --git a/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs b/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs index cc33da8171255..c3c7b93ebaf3b 100644 --- a/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs +++ b/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs @@ -97,8 +97,8 @@ static UNIT_PATH_REMAP: Lazy> = Lazy::new(|| { ("acquires-checker/v1-borrow-tests", "borrow_tests"), ("attributes", "parser"), ("unit_test/notest", "unit_test"), - // Map file v2../unit_test/test/foo.move - // to file v1../unit_test/foo.unit_test.move. + // Map file v2../unit_test/test/foo.exp + // to file v1../unit_test/foo.unit_test.exp. ("unit_test/test", "unit_test.unit_test"), ("unused-assignment/v1-locals", "locals"), ("unused-assignment/v1-liveness", "liveness"), @@ -108,6 +108,10 @@ static UNIT_PATH_REMAP: Lazy> = Lazy::new(|| { ("ability-check/v1-signer", "signer"), ("ability-check/v1-borrow-tests", "commands"), ("ability-check/v1-locals", "locals"), + ("verification/noverify", "verification"), + // Map file v2../verification/verify/foo.exp + // to file v1../verification/foo.verification.exp. + ("verification/verify", "verification.verification"), ] }); @@ -205,7 +209,7 @@ fn collect_tests(root: &str) -> anyhow::Result> { let path_str = path.display().to_string(); if path_str.ends_with(".move") { result.insert(path); - } else if path_str.ends_with(".unit_test") { + } else if path_str.ends_with(".unit_test") || path_str.ends_with(".verification") { result.insert(PathBuf::from(format!("{}.{}", path_str, "move"))); } } diff --git a/third_party/move/move-compiler-v2/transactional-tests/tests/v1.matched b/third_party/move/move-compiler-v2/transactional-tests/tests/v1.matched index fecee90c8f243..db793a356d40e 100644 --- a/third_party/move/move-compiler-v2/transactional-tests/tests/v1.matched +++ b/third_party/move/move-compiler-v2/transactional-tests/tests/v1.matched @@ -1,5 +1,7 @@ move-compiler/transactional-tests/tests/constants/by_reference.exp move-compiler-v2/transactional-tests/tests/constants/by_reference.exp move-compiler/transactional-tests/tests/constants/default_int_size.exp move-compiler-v2/transactional-tests/tests/constants/default_int_size.exp +move-compiler/transactional-tests/tests/constants/empty_vectors.exp move-compiler-v2/transactional-tests/tests/constants/empty_vectors.exp +move-compiler/transactional-tests/tests/constants/empty_vectors2.exp move-compiler-v2/transactional-tests/tests/constants/empty_vectors2.exp move-compiler/transactional-tests/tests/constants/folding_boolean.exp move-compiler-v2/transactional-tests/tests/constants/folding_boolean.exp move-compiler/transactional-tests/tests/constants/folding_complex.exp move-compiler-v2/transactional-tests/tests/constants/folding_complex.exp move-compiler/transactional-tests/tests/constants/folding_equality.exp move-compiler-v2/transactional-tests/tests/constants/folding_equality.exp @@ -8,8 +10,9 @@ move-compiler/transactional-tests/tests/constants/folding_values.exp move-comp move-compiler/transactional-tests/tests/constants/folding_vector.exp move-compiler-v2/transactional-tests/tests/constants/folding_vector.exp move-compiler/transactional-tests/tests/constants/folding_vector_large.exp move-compiler-v2/transactional-tests/tests/constants/folding_vector_large.exp move-compiler/transactional-tests/tests/constants/non_constant_empty_vec.exp move-compiler-v2/transactional-tests/tests/constants/non_constant_empty_vec.exp +move-compiler/transactional-tests/tests/constants/nonempty_vectors.exp move-compiler-v2/transactional-tests/tests/constants/nonempty_vectors.exp move-compiler/transactional-tests/tests/control_flow/assert_in_while.exp move-compiler-v2/transactional-tests/tests/control_flow/assert_in_while.exp -move-compiler/transactional-tests/tests/control_flow/binop_eval_order.exp move-compiler-v2/transactional-tests/tests/control_flow/binop_eval_order.exp +move-compiler/transactional-tests/tests/control_flow/binop_eval_order.exp move-compiler-v2/transactional-tests/tests/operator_eval/control_flow/binop_eval_order.exp move-compiler/transactional-tests/tests/control_flow/branch_assigns_then_moves_then_assigns.exp move-compiler-v2/transactional-tests/tests/control_flow/branch_assigns_then_moves_then_assigns.exp move-compiler/transactional-tests/tests/control_flow/break_accumulator.exp move-compiler-v2/transactional-tests/tests/control_flow/break_accumulator.exp move-compiler/transactional-tests/tests/control_flow/break_continue_for_loop.exp move-compiler-v2/transactional-tests/tests/control_flow/break_continue_for_loop.exp diff --git a/third_party/move/move-compiler/src/parser/filter.rs b/third_party/move/move-compiler/src/parser/filter.rs index c205a3ffa36ed..fe09adcb4af28 100644 --- a/third_party/move/move-compiler/src/parser/filter.rs +++ b/third_party/move/move-compiler/src/parser/filter.rs @@ -2,8 +2,13 @@ // Copyright (c) The Move Contributors // SPDX-License-Identifier: Apache-2.0 -use crate::parser::ast as P; +use crate::parser::{ + ast as P, + ast::{ModuleMember, SpecBlockTarget_}, +}; use move_ir_types::location::sp; +use move_symbol_pool::Symbol; +use std::collections::BTreeSet; /// A trait that decides whether to include a parsed element in the compilation pub trait FilterContext { @@ -56,8 +61,10 @@ pub trait FilterContext { &mut self, function_def: P::Function, is_source_def: bool, + filtered_members: &mut BTreeSet, ) -> Option { if self.should_remove_by_attributes(&function_def.attributes, is_source_def) { + filtered_members.insert(function_def.name.0.value); None } else { Some(function_def) @@ -68,8 +75,10 @@ pub trait FilterContext { &mut self, struct_def: P::StructDefinition, is_source_def: bool, + filtered_members: &mut BTreeSet, ) -> Option { if self.should_remove_by_attributes(&struct_def.attributes, is_source_def) { + filtered_members.insert(struct_def.name.0.value); None } else { Some(struct_def) @@ -233,7 +242,8 @@ fn filter_script( // This is a bit weird, if the only function in the script is filtered, we consider // the whole script is filtered as well - let new_function = context.filter_map_function(function, is_source_def)?; + let new_function = + context.filter_map_function(function, is_source_def, &mut BTreeSet::new())?; let new_uses = uses .into_iter() @@ -278,9 +288,29 @@ fn filter_module( members, } = module_def; + // Collected filtered members in this set (functions and structs) + let mut filtered_members = BTreeSet::new(); + let new_members: Vec<_> = members .into_iter() - .filter_map(|member| filter_module_member(context, member, is_source_def)) + .filter_map(|member| { + filter_module_member(context, member, is_source_def, &mut filtered_members) + }) + .collect(); + + // Now remove all spec blocks for members which are filtered out + let new_members_filtered_spec_blocks = new_members + .into_iter() + .filter(|mem| { + if let ModuleMember::Spec(spec) = mem { + if let SpecBlockTarget_::Member(name, _) = &spec.value.target.value { + if filtered_members.contains(&name.value) { + return false; + } + } + } + true + }) .collect(); Some(P::ModuleDefinition { @@ -289,7 +319,7 @@ fn filter_module( address, name, is_spec_module, - members: new_members, + members: new_members_filtered_spec_blocks, }) } @@ -297,15 +327,16 @@ fn filter_module_member( context: &mut T, module_member: P::ModuleMember, is_source_def: bool, + filtered_members: &mut BTreeSet, ) -> Option { use P::ModuleMember as PM; match module_member { PM::Function(func_def) => context - .filter_map_function(func_def, is_source_def) + .filter_map_function(func_def, is_source_def, filtered_members) .map(PM::Function), PM::Struct(struct_def) => context - .filter_map_struct(struct_def, is_source_def) + .filter_map_struct(struct_def, is_source_def, filtered_members) .map(PM::Struct), PM::Spec(sp!(spec_loc, spec)) => context .filter_map_spec(spec, is_source_def) diff --git a/third_party/move/move-compiler/src/shared/mod.rs b/third_party/move/move-compiler/src/shared/mod.rs index c14b9ce386ac9..2a5c50f9a6ada 100644 --- a/third_party/move/move-compiler/src/shared/mod.rs +++ b/third_party/move/move-compiler/src/shared/mod.rs @@ -481,6 +481,13 @@ impl Flags { } } + pub fn set_verify(self, value: bool) -> Self { + Self { + verify: value, + ..self + } + } + pub fn set_keep_testing_functions(self, value: bool) -> Self { Self { keep_testing_functions: value, diff --git a/third_party/move/move-model/src/lib.rs b/third_party/move/move-model/src/lib.rs index dcc5b81e66fe0..d6d795ca5f67d 100644 --- a/third_party/move/move-model/src/lib.rs +++ b/third_party/move/move-model/src/lib.rs @@ -90,7 +90,10 @@ pub fn run_model_builder_in_compiler_mode( skip_attribute_checks: bool, known_attributes: &BTreeSet, language_version: LanguageVersion, + warn_of_deprecation_use: bool, + warn_of_deprecation_use_in_aptos_libs: bool, compile_test_code: bool, + compile_verify_code: bool, ) -> anyhow::Result { let to_package_paths = |PackageInfo { sources, @@ -110,7 +113,10 @@ pub fn run_model_builder_in_compiler_mode( ..ModelBuilderOptions::default() }, Flags::model_compilation() + .set_warn_of_deprecation_use(warn_of_deprecation_use) + .set_warn_of_deprecation_use_in_aptos_libs(warn_of_deprecation_use_in_aptos_libs) .set_skip_attribute_checks(skip_attribute_checks) + .set_verify(compile_verify_code) .set_keep_testing_functions(compile_test_code) .set_lang_v2(language_version != LanguageVersion::V1) .set_compiler_v2(true), diff --git a/third_party/move/move-prover/src/lib.rs b/third_party/move/move-prover/src/lib.rs index 9cf607d374075..d5f4e9f308463 100644 --- a/third_party/move/move-prover/src/lib.rs +++ b/third_party/move/move-prover/src/lib.rs @@ -84,9 +84,12 @@ pub fn run_move_prover_v2( experiment_cache: Default::default(), sources: cloned_options.move_sources, sources_deps: vec![], + warn_deprecated: false, + warn_of_deprecation_use_in_aptos_libs: false, warn_unused: false, whole_program: false, compile_test_code: false, + compile_verify_code: true, }; let mut env = move_compiler_v2::run_move_compiler_for_analysis(error_writer, compiler_options)?; diff --git a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.v2_exp b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.v2_exp new file mode 100644 index 0000000000000..842d430239fea --- /dev/null +++ b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.v2_exp @@ -0,0 +1,53 @@ +CompiledPackageInfo { + package_name: "std-lib-conflicts", + address_alias_instantiation: { + "A": 0000000000000000000000000000000000000000000000000000000000000042, + "B": 0000000000000000000000000000000000000000000000000000000000000043, + "Extensions": 0000000000000000000000000000000000000000000000000000000000000001, + "aptos_framework": 0000000000000000000000000000000000000000000000000000000000000001, + "aptos_fungible_asset": 000000000000000000000000000000000000000000000000000000000000000a, + "aptos_std": 0000000000000000000000000000000000000000000000000000000000000001, + "aptos_token": 0000000000000000000000000000000000000000000000000000000000000003, + "core_resources": 000000000000000000000000000000000000000000000000000000000a550c18, + "std": 0000000000000000000000000000000000000000000000000000000000000001, + "vm": 0000000000000000000000000000000000000000000000000000000000000000, + "vm_reserved": 0000000000000000000000000000000000000000000000000000000000000000, + }, + source_digest: Some( + "ELIDED_FOR_TEST", + ), + build_flags: BuildConfig { + dev_mode: true, + test_mode: false, + override_std: None, + generate_docs: false, + generate_abis: false, + generate_move_model: false, + full_model_generation: false, + install_dir: Some( + "ELIDED_FOR_TEST", + ), + force_recompilation: false, + additional_named_addresses: {}, + architecture: None, + fetch_deps_only: false, + skip_fetch_latest_git_deps: false, + compiler_config: CompilerConfig { + bytecode_version: None, + known_attributes: { + "bytecode_instruction", + "deprecated", + "expected_failure", + "native_interface", + "test", + "test_only", + "verify_only", + }, + skip_attribute_checks: false, + compiler_version: Some( + V2_0, + ), + language_version: None, + }, + }, +} From 4ec648af65dbf7bb3dafb4cecd4886515abb5a44 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Tue, 18 Jun 2024 10:51:11 -0700 Subject: [PATCH 009/469] [internalindexer] add indexer db to state sync delete statevalueindex save progress save progress [storage] remove duplicated ensure and add checking for version --- Cargo.lock | 1 + api/src/context.rs | 143 ++++++++++++------ api/src/tests/accounts_test.rs | 43 +++++- api/src/tests/events_test.rs | 26 +++- api/src/tests/mod.rs | 9 +- api/src/tests/transactions_test.rs | 17 ++- api/test-context/Cargo.toml | 1 + api/test-context/src/test_context.rs | 24 ++- .../src/internal_indexer_db_service.rs | 45 +++++- storage/aptosdb/src/backup/restore_utils.rs | 6 +- .../src/db/include/aptosdb_internal.rs | 6 +- .../src/db/include/aptosdb_testonly.rs | 8 +- .../aptosdb/src/db/include/aptosdb_writer.rs | 1 - storage/aptosdb/src/db/test_helper.rs | 5 +- .../src/pruner/state_merkle_pruner/test.rs | 1 - storage/aptosdb/src/schema/mod.rs | 2 - .../src/schema/state_value_index/mod.rs | 67 -------- .../src/schema/state_value_index/test.rs | 18 --- storage/aptosdb/src/state_store/mod.rs | 32 +--- .../src/state_store/state_store_test.rs | 1 - storage/aptosdb/src/utils/iterators.rs | 92 ++--------- storage/db-tool/src/tests.rs | 1 - storage/indexer/src/db_indexer.rs | 7 +- storage/indexer/src/indexer_reader.rs | 35 ++++- testsuite/single_node_performance.py | 2 +- 25 files changed, 305 insertions(+), 288 deletions(-) delete mode 100644 storage/aptosdb/src/schema/state_value_index/mod.rs delete mode 100644 storage/aptosdb/src/schema/state_value_index/test.rs diff --git a/Cargo.lock b/Cargo.lock index cb5bd9324e21f..cea7d5bb6d59e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -458,6 +458,7 @@ dependencies = [ "aptos-executor-types", "aptos-framework", "aptos-genesis", + "aptos-indexer-grpc-table-info", "aptos-mempool", "aptos-mempool-notifications", "aptos-sdk", diff --git a/api/src/context.rs b/api/src/context.rs index a077845cb0b63..eced1d2c4a783 100644 --- a/api/src/context.rs +++ b/api/src/context.rs @@ -23,7 +23,7 @@ use aptos_logger::{error, info, Schema}; use aptos_mempool::{MempoolClientRequest, MempoolClientSender, SubmissionStatus}; use aptos_storage_interface::{ state_view::{DbStateView, DbStateViewAtVersion, LatestDbStateCheckpointView}, - DbReader, Order, MAX_REQUEST_LIMIT, + AptosDbError, DbReader, Order, MAX_REQUEST_LIMIT, }; use aptos_types::{ access_path::{AccessPath, Path}, @@ -350,19 +350,26 @@ impl Context { address: AccountAddress, version: u64, ) -> Result> { - let mut iter = self.db.get_prefixed_state_value_iterator( - &StateKeyPrefix::from(address), - None, - version, - )?; + let mut iter = if !db_sharding_enabled(&self.node_config) { + Box::new( + self.db + .get_prefixed_state_value_iterator( + &StateKeyPrefix::from(address), + None, + version, + )? + .map(|item| item.map_err(|err| anyhow!(err.to_string()))), + ) + } else { + self.indexer_reader + .as_ref() + .ok_or_else(|| format_err!("Indexer reader doesn't exist"))? + .get_prefixed_state_value_iterator(&StateKeyPrefix::from(address), None, version)? + }; let kvs = iter .by_ref() .take(MAX_REQUEST_LIMIT as usize) - .map(|res| match res { - Ok((k, v)) => Ok((k, v)), - Err(res) => Err(anyhow::Error::from(res)), - }) .collect::>()?; if iter.next().transpose()?.is_some() { bail!("Too many state items under account ({:?}).", address); @@ -377,11 +384,26 @@ impl Context { version: u64, limit: u64, ) -> Result<(Vec<(StructTag, Vec)>, Option)> { - let account_iter = self.db.get_prefixed_state_value_iterator( - &StateKeyPrefix::from(address), - prev_state_key, - version, - )?; + let account_iter = if !db_sharding_enabled(&self.node_config) { + Box::new( + self.db + .get_prefixed_state_value_iterator( + &StateKeyPrefix::from(address), + prev_state_key, + version, + )? + .map(|item| item.map_err(|err| anyhow!(err.to_string()))), + ) + } else { + self.indexer_reader + .as_ref() + .ok_or_else(|| format_err!("Indexer reader doesn't exist"))? + .get_prefixed_state_value_iterator( + &StateKeyPrefix::from(address), + prev_state_key, + version, + )? + }; // TODO: Consider rewriting this to consider resource groups: // * If a resource group is found, expand // * Return Option)>> @@ -408,7 +430,7 @@ impl Context { Some(Err(format_err!( "storage prefix scan return inconsistent key ({:?})", k ))) } }, - Err(e) => Some(Err(e.into())), + Err(e) => Some(Err(e)), }) .take(limit as usize + 1); let kvs = resource_iter @@ -453,11 +475,26 @@ impl Context { version: u64, limit: u64, ) -> Result<(Vec<(ModuleId, Vec)>, Option)> { - let account_iter = self.db.get_prefixed_state_value_iterator( - &StateKeyPrefix::from(address), - prev_state_key, - version, - )?; + let account_iter = if !db_sharding_enabled(&self.node_config) { + Box::new( + self.db + .get_prefixed_state_value_iterator( + &StateKeyPrefix::from(address), + prev_state_key, + version, + )? + .map(|item| item.map_err(|err| anyhow!(err.to_string()))), + ) + } else { + self.indexer_reader + .as_ref() + .ok_or_else(|| format_err!("Indexer reader doesn't exist"))? + .get_prefixed_state_value_iterator( + &StateKeyPrefix::from(address), + prev_state_key, + version, + )? + }; let mut module_iter = account_iter .filter_map(|res| match res { Ok((k, v)) => match k.inner() { @@ -473,7 +510,7 @@ impl Context { Some(Err(format_err!( "storage prefix scan return inconsistent key ({:?})", k ))) } }, - Err(e) => Some(Err(e.into())), + Err(e) => Some(Err(e)), }) .take(limit as usize + 1); let kvs = module_iter @@ -730,15 +767,31 @@ impl Context { .saturating_sub(limit as u64) }; - let txns = self - .db - .get_account_transactions( + let txns_res = if !db_sharding_enabled(&self.node_config) { + self.db.get_account_transactions( address, start_seq_number, limit as u64, true, ledger_version, ) + } else { + self.indexer_reader + .as_ref() + .ok_or(anyhow!("Indexer reader is None")) + .map_err(|err| { + E::internal_with_code(err, AptosErrorCode::InternalError, ledger_info) + })? + .get_account_transactions( + address, + start_seq_number, + limit as u64, + true, + ledger_version, + ) + .map_err(|e| AptosDbError::Other(e.to_string())) + }; + let txns = txns_res .context("Failed to retrieve account transactions") .map_err(|err| { E::internal_with_code(err, AptosErrorCode::InternalError, ledger_info) @@ -813,29 +866,21 @@ impl Context { limit: u16, ledger_version: u64, ) -> Result> { - if let Some(start) = start { - Ok(self.db.get_events( - event_key, - start, - Order::Ascending, - limit as u64, - ledger_version, - )?) + let (start, order) = if let Some(start) = start { + (start, Order::Ascending) } else { - Ok(self - .db - .get_events( - event_key, - u64::MAX, - Order::Descending, - limit as u64, - ledger_version, - ) - .map(|mut result| { - result.reverse(); - result - })?) - } + (u64::MAX, Order::Descending) + }; + let res = if !db_sharding_enabled(&self.node_config) { + self.db + .get_events(event_key, start, order, limit as u64, ledger_version)? + } else { + self.indexer_reader + .as_ref() + .ok_or(anyhow!("Internal indexer reader doesn't exist"))? + .get_events(event_key, start, order, limit as u64, ledger_version)? + }; + Ok(res) } fn next_bucket(&self, gas_unit_price: u64) -> u64 { @@ -1444,3 +1489,7 @@ impl FunctionStats { } } } + +fn db_sharding_enabled(node_config: &NodeConfig) -> bool { + node_config.storage.rocksdb_configs.enable_storage_sharding +} diff --git a/api/src/tests/accounts_test.rs b/api/src/tests/accounts_test.rs index da00aeff218e7..55b40ddef8967 100644 --- a/api/src/tests/accounts_test.rs +++ b/api/src/tests/accounts_test.rs @@ -3,7 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 use super::new_test_context; -use aptos_api_test_context::{current_function_name, find_value}; +use crate::tests::new_test_context_with_db_sharding_and_internal_indexer; +use aptos_api_test_context::{current_function_name, find_value, TestContext}; use aptos_api_types::{MoveModuleBytecode, MoveResource, MoveStructTag, StateKeyWrapper}; use aptos_cached_packages::aptos_stdlib; use serde_json::json; @@ -36,9 +37,21 @@ async fn test_get_account_resources_by_address_0x0() { async fn test_get_account_resources_by_valid_account_address() { let context = new_test_context(current_function_name!()); let addresses = vec!["0x1", "0x00000000000000000000000000000001"]; + let mut res = vec![]; for address in &addresses { - context.get(&account_resources(address)).await; + let resp = context.get(&account_resources(address)).await; + res.push(resp); } + + let shard_context = + new_test_context_with_db_sharding_and_internal_indexer(current_function_name!()); + let mut shard_res = vec![]; + for address in &addresses { + let resp = shard_context.get(&account_resources(address)).await; + shard_res.push(resp); + } + + assert_eq!(res, shard_res); } // Unstable due to framework changes @@ -96,9 +109,7 @@ async fn test_account_modules_structs() { context.check_golden_output(resp); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_get_account_resources_by_ledger_version() { - let mut context = new_test_context(current_function_name!()); +async fn test_account_resources_by_ledger_version_with_context(mut context: TestContext) { let account = context.gen_account(); let txn = context.create_user_account(&account).await; context.commit_block(&vec![txn.clone()]).await; @@ -125,6 +136,15 @@ async fn test_get_account_resources_by_ledger_version() { assert_eq!(root_account["data"]["sequence_number"], "0"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_get_account_resources_by_ledger_version() { + let context = new_test_context(current_function_name!()); + test_account_resources_by_ledger_version_with_context(context).await; + let shard_context = + new_test_context_with_db_sharding_and_internal_indexer(current_function_name!()); + test_account_resources_by_ledger_version_with_context(shard_context).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_account_resources_by_too_large_ledger_version() { let mut context = new_test_context(current_function_name!()); @@ -151,9 +171,7 @@ async fn test_get_account_resources_by_invalid_ledger_version() { context.check_golden_output(resp); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_get_account_modules_by_ledger_version() { - let mut context = new_test_context(current_function_name!()); +async fn test_get_account_modules_by_ledger_version_with_context(mut context: TestContext) { let payload = aptos_stdlib::publish_module_source("test_module", "module 0xa550c18::test_module {}"); @@ -178,6 +196,15 @@ async fn test_get_account_modules_by_ledger_version() { assert_eq!(modules, json!([])); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_get_account_modules_by_ledger_version() { + let context = new_test_context(current_function_name!()); + test_get_account_modules_by_ledger_version_with_context(context).await; + let shard_context = + new_test_context_with_db_sharding_and_internal_indexer(current_function_name!()); + test_get_account_modules_by_ledger_version_with_context(shard_context).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_core_account_data() { let mut context = new_test_context(current_function_name!()); diff --git a/api/src/tests/events_test.rs b/api/src/tests/events_test.rs index acf517b61a04c..1c1f69830cc06 100644 --- a/api/src/tests/events_test.rs +++ b/api/src/tests/events_test.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use super::new_test_context; +use crate::tests::new_test_context_with_db_sharding_and_internal_indexer; use aptos_api_test_context::{current_function_name, TestContext}; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use serde_json::json; @@ -35,7 +36,21 @@ async fn test_get_events_filter_by_start_sequence_number() { .as_str(), ) .await; - context.check_golden_output(resp); + context.check_golden_output(resp.clone()); + + // assert the same resp after db sharding migration with internal indexer turned on + let shard_context = + new_test_context_with_db_sharding_and_internal_indexer(current_function_name!()); + let new_resp = shard_context + .get( + format!( + "/accounts/{}/events/{}?start=1", + ACCOUNT_ADDRESS, CREATION_NUMBER + ) + .as_str(), + ) + .await; + assert_eq!(resp, new_resp); } // turn it back until we have multiple events in genesis @@ -84,7 +99,14 @@ async fn test_get_events_by_account_event_handle() { let resp = context .get("/accounts/0x1/events/0x1::reconfiguration::Configuration/events") .await; - context.check_golden_output(resp); + context.check_golden_output(resp.clone()); + + let shard_context = + new_test_context_with_db_sharding_and_internal_indexer(current_function_name!()); + let new_resp = shard_context + .get("/accounts/0x1/events/0x1::reconfiguration::Configuration/events") + .await; + assert_eq!(resp, new_resp); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/api/src/tests/mod.rs b/api/src/tests/mod.rs index e581acdbad02a..e7978f66a126e 100644 --- a/api/src/tests/mod.rs +++ b/api/src/tests/mod.rs @@ -22,7 +22,7 @@ mod view_function; mod webauthn_secp256r1_ecdsa; use aptos_api_test_context::{new_test_context as super_new_test_context, TestContext}; -use aptos_config::config::NodeConfig; +use aptos_config::config::{internal_indexer_db_config::InternalIndexerDBConfig, NodeConfig}; fn new_test_context(test_name: String) -> TestContext { new_test_context_with_config(test_name, NodeConfig::default()) @@ -31,3 +31,10 @@ fn new_test_context(test_name: String) -> TestContext { fn new_test_context_with_config(test_name: String, node_config: NodeConfig) -> TestContext { super_new_test_context(test_name, node_config, false) } + +fn new_test_context_with_db_sharding_and_internal_indexer(test_name: String) -> TestContext { + let mut node_config = NodeConfig::default(); + node_config.storage.rocksdb_configs.enable_storage_sharding = true; + node_config.indexer_db_config = InternalIndexerDBConfig::new(true, true, true, 10_000); + super_new_test_context(test_name, node_config, true) +} diff --git a/api/src/tests/transactions_test.rs b/api/src/tests/transactions_test.rs index 537d8ac34b01a..700b28babc7c6 100644 --- a/api/src/tests/transactions_test.rs +++ b/api/src/tests/transactions_test.rs @@ -3,7 +3,9 @@ // SPDX-License-Identifier: Apache-2.0 use super::new_test_context; -use crate::tests::new_test_context_with_config; +use crate::tests::{ + new_test_context_with_config, new_test_context_with_db_sharding_and_internal_indexer, +}; use aptos_api_test_context::{assert_json, current_function_name, pretty, TestContext}; use aptos_config::config::{GasEstimationStaticOverride, NodeConfig}; use aptos_crypto::{ @@ -750,9 +752,7 @@ async fn test_signing_message_with_payload( assert_eq!(ledger["ledger_version"].as_str().unwrap(), "3"); // metadata + user txn + state checkpoint } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_get_account_transactions() { - let mut context = new_test_context(current_function_name!()); +async fn test_account_transaction_with_context(mut context: TestContext) { let account = context.gen_account(); let txn = context.create_user_account(&account).await; context.commit_block(&vec![txn]).await; @@ -771,6 +771,15 @@ async fn test_get_account_transactions() { assert_json(txns, expected_txns); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_get_account_transactions() { + let context = new_test_context(current_function_name!()); + test_account_transaction_with_context(context).await; + let shard_context = + new_test_context_with_db_sharding_and_internal_indexer(current_function_name!()); + test_account_transaction_with_context(shard_context).await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_account_transactions_filter_transactions_by_start_sequence_number() { let mut context = new_test_context(current_function_name!()); diff --git a/api/test-context/Cargo.toml b/api/test-context/Cargo.toml index 9c4a9e61a8858..db9be88d6dee1 100644 --- a/api/test-context/Cargo.toml +++ b/api/test-context/Cargo.toml @@ -23,6 +23,7 @@ aptos-executor = { workspace = true } aptos-executor-types = { workspace = true } aptos-framework = { workspace = true } aptos-genesis = { workspace = true } +aptos-indexer-grpc-table-info = { workspace = true } aptos-mempool = { workspace = true, features = ["fuzzing"] } aptos-mempool-notifications = { workspace = true } aptos-sdk = { workspace = true } diff --git a/api/test-context/src/test_context.rs b/api/test-context/src/test_context.rs index c8786900d73d4..457f4a2c9deb2 100644 --- a/api/test-context/src/test_context.rs +++ b/api/test-context/src/test_context.rs @@ -20,6 +20,7 @@ use aptos_db::AptosDB; use aptos_executor::{block_executor::BlockExecutor, db_bootstrapper}; use aptos_executor_types::BlockExecutorTrait; use aptos_framework::BuiltPackage; +use aptos_indexer_grpc_table_info::internal_indexer_db_service::MockInternalIndexerDBService; use aptos_mempool::mocks::MockSharedMempool; use aptos_mempool_notifications::MempoolNotificationSender; use aptos_sdk::{ @@ -94,7 +95,7 @@ impl ApiSpecificConfig { pub fn new_test_context( test_name: String, - node_config: NodeConfig, + mut node_config: NodeConfig, use_db_with_indexer: bool, ) -> TestContext { // Speculative logging uses a global variable and when many instances use it together, they @@ -119,14 +120,23 @@ pub fn new_test_context( let validator_owner = validator_identity.account_address.unwrap(); let (db, db_rw) = if use_db_with_indexer { - DbReaderWriter::wrap(AptosDB::new_for_test_with_indexer(&tmp_dir)) + DbReaderWriter::wrap(AptosDB::new_for_test_with_indexer( + &tmp_dir, + node_config.storage.rocksdb_configs.enable_storage_sharding, + )) } else { DbReaderWriter::wrap( AptosDB::open( StorageDirPaths::from_path(&tmp_dir), false, /* readonly */ NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */ - RocksdbConfigs::default(), + RocksdbConfigs { + enable_storage_sharding: node_config + .storage + .rocksdb_configs + .enable_storage_sharding, + ..Default::default() + }, false, /* indexer */ BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, @@ -140,12 +150,18 @@ pub fn new_test_context( let mempool = MockSharedMempool::new_in_runtime(&db_rw, VMValidator::new(db.clone())); + node_config + .storage + .set_data_dir(tmp_dir.path().to_path_buf()); + let mock_indexer_service = + MockInternalIndexerDBService::new_for_test(db_rw.reader.clone(), &node_config); + let context = Context::new( ChainId::test(), db.clone(), mempool.ac_client.clone(), node_config.clone(), - None, /* table info reader */ + mock_indexer_service.get_indexer_reader(), ); // Configure the testing depending on which API version we're testing. diff --git a/ecosystem/indexer-grpc/indexer-grpc-table-info/src/internal_indexer_db_service.rs b/ecosystem/indexer-grpc/indexer-grpc-table-info/src/internal_indexer_db_service.rs index 544348b81ddfb..0f430543f8183 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-table-info/src/internal_indexer_db_service.rs +++ b/ecosystem/indexer-grpc/indexer-grpc-table-info/src/internal_indexer_db_service.rs @@ -2,11 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 use aptos_config::config::NodeConfig; -use aptos_db_indexer::{db_indexer::DBIndexer, db_ops::open_internal_indexer_db}; +use aptos_db_indexer::{ + db_indexer::DBIndexer, db_ops::open_internal_indexer_db, indexer_reader::IndexerReaders, +}; use aptos_indexer_grpc_utils::counters::{log_grpc_step, IndexerGrpcStep}; use aptos_schemadb::DB; use aptos_storage_interface::DbReader; +use aptos_types::indexer::indexer_db_reader::IndexerReader; use std::sync::Arc; +use tokio::runtime::Handle; const SERVICE_TYPE: &str = "internal_indexer_db_service"; const INTERNAL_INDEXER_DB: &str = "internal_indexer_db"; @@ -84,3 +88,42 @@ impl InternalIndexerDBService { } } } + +pub struct MockInternalIndexerDBService { + pub indexer_readers: Option, + pub _handle: Option, +} + +impl MockInternalIndexerDBService { + pub fn new_for_test(db_reader: Arc, node_config: &NodeConfig) -> Self { + if !node_config + .indexer_db_config + .is_internal_indexer_db_enabled() + { + return Self { + indexer_readers: None, + _handle: None, + }; + } + + let db = InternalIndexerDBService::get_indexer_db(node_config).unwrap(); + let handle = Handle::current(); + let mut internal_indexer_db_service = + InternalIndexerDBService::new(db_reader, node_config, db); + let db_indexer = internal_indexer_db_service.get_db_indexer(); + handle.spawn(async move { + internal_indexer_db_service.run().await; + }); + Self { + indexer_readers: IndexerReaders::new(None, Some(db_indexer)), + _handle: Some(handle), + } + } + + pub fn get_indexer_reader(&self) -> Option> { + if let Some(indexer_reader) = &self.indexer_readers { + return Some(Arc::new(indexer_reader.to_owned())); + } + None + } +} diff --git a/storage/aptosdb/src/backup/restore_utils.rs b/storage/aptosdb/src/backup/restore_utils.rs index 92a9ec132ef8f..b2da4a01adb4f 100644 --- a/storage/aptosdb/src/backup/restore_utils.rs +++ b/storage/aptosdb/src/backup/restore_utils.rs @@ -122,7 +122,7 @@ pub(crate) fn save_transactions( )>, kv_replay: bool, ) -> Result<()> { - if let Some((ledger_db_batch, state_kv_batches, state_kv_metadata_batch)) = existing_batch { + if let Some((ledger_db_batch, state_kv_batches, _state_kv_metadata_batch)) = existing_batch { save_transactions_impl( state_store, ledger_db, @@ -133,7 +133,6 @@ pub(crate) fn save_transactions( write_sets.as_ref(), ledger_db_batch, state_kv_batches, - state_kv_metadata_batch, kv_replay, )?; } else { @@ -150,7 +149,6 @@ pub(crate) fn save_transactions( write_sets.as_ref(), &mut ledger_db_batch, &mut sharded_kv_schema_batch, - &state_kv_metadata_batch, kv_replay, )?; // get the last version and commit to the state kv db @@ -193,7 +191,6 @@ pub(crate) fn save_transactions_impl( write_sets: &[WriteSet], ledger_db_batch: &mut LedgerDbSchemaBatches, state_kv_batches: &mut ShardedStateKvSchemaBatch, - state_kv_metadata_batch: &SchemaBatch, kv_replay: bool, ) -> Result<()> { for (idx, txn) in txns.iter().enumerate() { @@ -241,7 +238,6 @@ pub(crate) fn save_transactions_impl( first_version, &ledger_db_batch.ledger_metadata_db_batches, // used for storing the storage usage state_kv_batches, - state_kv_metadata_batch, state_store.state_kv_db.enabled_sharding(), )?; } diff --git a/storage/aptosdb/src/db/include/aptosdb_internal.rs b/storage/aptosdb/src/db/include/aptosdb_internal.rs index ee327fa3cec29..5dfc7cbf690d1 100644 --- a/storage/aptosdb/src/db/include/aptosdb_internal.rs +++ b/storage/aptosdb/src/db/include/aptosdb_internal.rs @@ -152,12 +152,16 @@ impl AptosDB { buffered_state_target_items: usize, max_num_nodes_per_lru_cache_shard: usize, enable_indexer: bool, + enable_sharding: bool, ) -> Self { Self::open( StorageDirPaths::from_path(db_root_path), readonly, NO_OP_STORAGE_PRUNER_CONFIG, /* pruner */ - RocksdbConfigs::default(), + RocksdbConfigs { + enable_storage_sharding: enable_sharding, + ..Default::default() + }, enable_indexer, buffered_state_target_items, max_num_nodes_per_lru_cache_shard, diff --git a/storage/aptosdb/src/db/include/aptosdb_testonly.rs b/storage/aptosdb/src/db/include/aptosdb_testonly.rs index cc49b179a619a..cfe5aaf7456d6 100644 --- a/storage/aptosdb/src/db/include/aptosdb_testonly.rs +++ b/storage/aptosdb/src/db/include/aptosdb_testonly.rs @@ -18,6 +18,7 @@ impl AptosDB { BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, false, /* indexer */ + false, ) } @@ -44,17 +45,18 @@ impl AptosDB { /// This opens db in non-readonly mode, without the pruner and cache. pub fn new_for_test_no_cache + Clone>(db_root_path: P) -> Self { - Self::new_without_pruner(db_root_path, false, BUFFERED_STATE_TARGET_ITEMS, 0, false) + Self::new_without_pruner(db_root_path, false, BUFFERED_STATE_TARGET_ITEMS, 0, false, false) } /// This opens db in non-readonly mode, without the pruner, and with the indexer - pub fn new_for_test_with_indexer + Clone>(db_root_path: P) -> Self { + pub fn new_for_test_with_indexer + Clone>(db_root_path: P, enable_sharding: bool) -> Self { Self::new_without_pruner( db_root_path, false, BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, true, /* indexer */ + enable_sharding, ) } @@ -69,6 +71,7 @@ impl AptosDB { buffered_state_target_items, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, false, /* indexer */ + false, ) } @@ -80,6 +83,7 @@ impl AptosDB { BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, false, /* indexer */ + false, ) } diff --git a/storage/aptosdb/src/db/include/aptosdb_writer.rs b/storage/aptosdb/src/db/include/aptosdb_writer.rs index e60bea6cce558..7005061e97e40 100644 --- a/storage/aptosdb/src/db/include/aptosdb_writer.rs +++ b/storage/aptosdb/src/db/include/aptosdb_writer.rs @@ -355,7 +355,6 @@ impl AptosDB { sharded_state_cache, &ledger_metadata_batch, &sharded_state_kv_batches, - &state_kv_metadata_batch, // Always put in state value index for now. // TODO(grao): remove after APIs migrated off the DB to the indexer. self.state_store.state_kv_db.enabled_sharding(), diff --git a/storage/aptosdb/src/db/test_helper.rs b/storage/aptosdb/src/db/test_helper.rs index 89a7ad311c3ff..570bdf8bd4433 100644 --- a/storage/aptosdb/src/db/test_helper.rs +++ b/storage/aptosdb/src/db/test_helper.rs @@ -85,7 +85,7 @@ pub(crate) fn update_store( .unwrap(); let ledger_batch = SchemaBatch::new(); let sharded_state_kv_batches = new_sharded_kv_schema_batch(); - let state_kv_metadata_batch = SchemaBatch::new(); + let schema_batch = SchemaBatch::new(); store .put_value_sets( vec![&sharded_value_state_set], @@ -94,7 +94,6 @@ pub(crate) fn update_store( None, &ledger_batch, &sharded_state_kv_batches, - &state_kv_metadata_batch, /*put_state_value_indices=*/ false, /*skip_usage=*/ false, /*last_checkpoint_index=*/ None, @@ -107,7 +106,7 @@ pub(crate) fn update_store( .unwrap(); store .state_kv_db - .commit(version, state_kv_metadata_batch, sharded_state_kv_batches) + .commit(version, schema_batch, sharded_state_kv_batches) .unwrap(); } root_hash diff --git a/storage/aptosdb/src/pruner/state_merkle_pruner/test.rs b/storage/aptosdb/src/pruner/state_merkle_pruner/test.rs index 47d152a45f15f..a660a6082975e 100644 --- a/storage/aptosdb/src/pruner/state_merkle_pruner/test.rs +++ b/storage/aptosdb/src/pruner/state_merkle_pruner/test.rs @@ -69,7 +69,6 @@ fn put_value_set( None, &ledger_batch, &sharded_state_kv_batches, - &state_kv_metadata_batch, enable_sharding, /*skip_usage=*/ false, /*last_checkpoint_index=*/ None, diff --git a/storage/aptosdb/src/schema/mod.rs b/storage/aptosdb/src/schema/mod.rs index caa858b754dd0..9ad14096e0daf 100644 --- a/storage/aptosdb/src/schema/mod.rs +++ b/storage/aptosdb/src/schema/mod.rs @@ -20,7 +20,6 @@ pub(crate) mod stale_state_value_index; pub(crate) mod stale_state_value_index_by_key_hash; pub(crate) mod state_value; pub(crate) mod state_value_by_key_hash; -pub(crate) mod state_value_index; pub(crate) mod transaction; pub(crate) mod transaction_accumulator; pub(crate) mod transaction_auxiliary_data; @@ -110,7 +109,6 @@ pub mod fuzzing { assert_no_panic_decoding::( data, ); - assert_no_panic_decoding::(data); assert_no_panic_decoding::(data); assert_no_panic_decoding::( data, diff --git a/storage/aptosdb/src/schema/state_value_index/mod.rs b/storage/aptosdb/src/schema/state_value_index/mod.rs deleted file mode 100644 index 494ca93d5286b..0000000000000 --- a/storage/aptosdb/src/schema/state_value_index/mod.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -// Historically we support an API that returns everything by key prefix (e.g. return all states -// under an account) operation. This was implemented by seeking the StateKv db by prefix directly. -// However in the new sharding world, account (or whatever prefix) is not a first class concept in -// the storage layer, and we will store data for an account in different shards, based on the hash -// of StateKey. Therefore the API cannot be supported by doing a single db seek. -// -// Our long term vision, is to move such support into indexer, before they are ready, we add this -// index for now to temporarily unblock the sharded db migration. - -use crate::schema::{ensure_slice_len_eq, ensure_slice_len_gt, STATE_VALUE_INDEX_CF_NAME}; -use anyhow::Result; -use aptos_schemadb::{ - define_schema, - schema::{KeyCodec, SeekKeyCodec, ValueCodec}, -}; -use aptos_types::{ - state_store::state_key::{prefix::StateKeyPrefix, StateKey}, - transaction::Version, -}; -use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; -use std::{io::Write, mem::size_of}; - -type Key = (StateKey, Version); - -define_schema!(StateValueIndexSchema, Key, (), STATE_VALUE_INDEX_CF_NAME); - -impl KeyCodec for Key { - fn encode_key(&self) -> Result> { - let mut encoded = vec![]; - encoded.write_all(self.0.encoded())?; - encoded.write_u64::(!self.1)?; - Ok(encoded) - } - - fn decode_key(data: &[u8]) -> Result { - const VERSION_SIZE: usize = size_of::(); - - ensure_slice_len_gt(data, VERSION_SIZE)?; - let state_key_len = data.len() - VERSION_SIZE; - let state_key: StateKey = StateKey::decode(&data[..state_key_len])?; - let version = !(&data[state_key_len..]).read_u64::()?; - Ok((state_key, version)) - } -} - -impl ValueCodec for () { - fn encode_value(&self) -> Result> { - Ok(Vec::new()) - } - - fn decode_value(data: &[u8]) -> Result { - ensure_slice_len_eq(data, 0)?; - Ok(()) - } -} - -impl SeekKeyCodec for &StateKeyPrefix { - fn encode_seek_key(&self) -> Result> { - self.encode() - } -} - -#[cfg(test)] -mod test; diff --git a/storage/aptosdb/src/schema/state_value_index/test.rs b/storage/aptosdb/src/schema/state_value_index/test.rs deleted file mode 100644 index c6d87014d0a7d..0000000000000 --- a/storage/aptosdb/src/schema/state_value_index/test.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright © Aptos Foundation -// SPDX-License-Identifier: Apache-2.0 - -use super::*; -use aptos_schemadb::{schema::fuzzing::assert_encode_decode, test_no_panic_decoding}; -use proptest::prelude::*; - -proptest! { - #[test] - fn test_encode_decode( - state_key in any::(), - version in any::(), - ) { - assert_encode_decode::(&(state_key, version), &()); - } -} - -test_no_panic_decoding!(StateValueIndexSchema); diff --git a/storage/aptosdb/src/state_store/mod.rs b/storage/aptosdb/src/state_store/mod.rs index cf598a8cbab16..e7e631040df66 100644 --- a/storage/aptosdb/src/state_store/mod.rs +++ b/storage/aptosdb/src/state_store/mod.rs @@ -17,7 +17,6 @@ use crate::{ stale_state_value_index_by_key_hash::StaleStateValueIndexByKeyHashSchema, state_value::StateValueSchema, state_value_by_key_hash::StateValueByKeyHashSchema, - state_value_index::StateValueIndexSchema, version_data::VersionDataSchema, }, state_kv_db::StateKvDb, @@ -574,12 +573,17 @@ impl StateStore { first_key_opt: Option<&StateKey>, desired_version: Version, ) -> Result { + ensure!( + !self.state_db.state_kv_db.enabled_sharding(), + "This API is not supported for sharded db." + ); + // this can only handle non-sharded db scenario. + // For sharded db, should look at API side using internal indexer to handle this request PrefixedStateValueIterator::new( &self.state_kv_db, key_prefix.clone(), first_key_opt.cloned(), desired_version, - self.state_kv_db.enabled_sharding(), ) } @@ -599,7 +603,6 @@ impl StateStore { first_version: Version, batch: &SchemaBatch, sharded_state_kv_batches: &ShardedStateKvSchemaBatch, - state_kv_metadata_batch: &SchemaBatch, enable_sharding: bool, ) -> Result<()> { let _timer = OTHER_TIMERS_SECONDS @@ -637,7 +640,6 @@ impl StateStore { value_state_sets.to_vec(), first_version, sharded_state_kv_batches, - state_kv_metadata_batch, enable_sharding, )?; @@ -653,7 +655,6 @@ impl StateStore { sharded_state_cache: Option<&ShardedStateCache>, ledger_batch: &SchemaBatch, sharded_state_kv_batches: &ShardedStateKvSchemaBatch, - state_kv_metadata_batch: &SchemaBatch, enable_sharding: bool, skip_usage: bool, last_checkpoint_index: Option, @@ -682,7 +683,6 @@ impl StateStore { value_state_sets, first_version, sharded_state_kv_batches, - state_kv_metadata_batch, enable_sharding, ) } @@ -692,7 +692,6 @@ impl StateStore { value_state_sets: Vec<&ShardedStateUpdates>, first_version: Version, sharded_state_kv_batches: &ShardedStateKvSchemaBatch, - state_kv_metadata_batch: &SchemaBatch, enable_sharding: bool, ) -> Result<()> { sharded_state_kv_batches @@ -718,23 +717,6 @@ impl StateStore { }) .collect::>() })?; - - // Eventually this index will move to indexer side. For now we temporarily write this into - // metadata db to unblock the sharded DB migration. - // TODO(grao): Remove when we are ready. - if enable_sharding { - value_state_sets - .par_iter() - .enumerate() - .try_for_each(|(i, updates)| { - let version = first_version + i as Version; - updates.iter().flatten().try_for_each(|(k, _)| { - state_kv_metadata_batch - .put::(&(k.clone(), version), &()) - }) - })?; - } - Ok(()) } @@ -1185,13 +1167,11 @@ impl StateValueWriter for StateStore { &DbMetadataKey::StateSnapshotRestoreProgress(version), &DbMetadataValue::StateSnapshotProgress(progress), )?; - self.shard_state_value_batch( &sharded_schema_batch, node_batch, self.state_kv_db.enabled_sharding(), )?; - self.state_kv_db .commit(version, batch, sharded_schema_batch) } diff --git a/storage/aptosdb/src/state_store/state_store_test.rs b/storage/aptosdb/src/state_store/state_store_test.rs index a212c252cbb6a..add8ba9f3bfb1 100644 --- a/storage/aptosdb/src/state_store/state_store_test.rs +++ b/storage/aptosdb/src/state_store/state_store_test.rs @@ -58,7 +58,6 @@ fn put_value_set( None, &ledger_batch, &sharded_state_kv_batches, - &state_kv_metadata_batch, /*put_state_value_indices=*/ false, /*skip_usage=*/ false, /*last_checkpoint_index=*/ None, diff --git a/storage/aptosdb/src/utils/iterators.rs b/storage/aptosdb/src/utils/iterators.rs index 089efef50196c..98906684f8ce0 100644 --- a/storage/aptosdb/src/utils/iterators.rs +++ b/storage/aptosdb/src/utils/iterators.rs @@ -2,14 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - schema::{ - event::EventSchema, ledger_info::LedgerInfoSchema, state_value::StateValueSchema, - state_value_by_key_hash::StateValueByKeyHashSchema, - state_value_index::StateValueIndexSchema, - }, + schema::{event::EventSchema, ledger_info::LedgerInfoSchema, state_value::StateValueSchema}, state_kv_db::StateKvDb, }; -use aptos_crypto::hash::CryptoHash; use aptos_schemadb::{iterator::SchemaIterator, ReadOptions}; use aptos_storage_interface::{db_ensure as ensure, AptosDbError, Result}; use aptos_types::{ @@ -101,14 +96,11 @@ where } pub struct PrefixedStateValueIterator<'a> { - db: &'a StateKvDb, kv_iter: Option>, - index_iter: Option>, key_prefix: StateKeyPrefix, prev_key: Option, desired_version: Version, is_finished: bool, - use_index: bool, } impl<'a> PrefixedStateValueIterator<'a> { @@ -117,7 +109,6 @@ impl<'a> PrefixedStateValueIterator<'a> { key_prefix: StateKeyPrefix, first_key: Option, desired_version: Version, - use_index: bool, ) -> Result { let mut read_opts = ReadOptions::default(); // Without this, iterators are not guaranteed a total order of all keys, but only keys for the same prefix. @@ -130,36 +121,20 @@ impl<'a> PrefixedStateValueIterator<'a> { // here will stick with prefix `aptos/abc` and return `None` or any arbitrary result after visited all the // keys starting with `aptos/abc`. read_opts.set_total_order_seek(true); - let (kv_iter, index_iter) = if use_index { - let mut index_iter = db - .metadata_db() - .iter_with_opts::(read_opts)?; - if let Some(first_key) = &first_key { - index_iter.seek(&(first_key.clone(), u64::MAX))?; - } else { - index_iter.seek(&&key_prefix)?; - }; - (None, Some(index_iter)) + let mut kv_iter = db + .metadata_db() + .iter_with_opts::(read_opts)?; + if let Some(first_key) = &first_key { + kv_iter.seek(&(first_key.clone(), u64::MAX))?; } else { - let mut kv_iter = db - .metadata_db() - .iter_with_opts::(read_opts)?; - if let Some(first_key) = &first_key { - kv_iter.seek(&(first_key.clone(), u64::MAX))?; - } else { - kv_iter.seek(&&key_prefix)?; - }; - (Some(kv_iter), None) + kv_iter.seek(&&key_prefix)?; }; Ok(Self { - db, - kv_iter, - index_iter, + kv_iter: Some(kv_iter), key_prefix, prev_key: None, desired_version, is_finished: false, - use_index, }) } @@ -195,62 +170,13 @@ impl<'a> PrefixedStateValueIterator<'a> { } Ok(None) } - - fn next_by_index(&mut self) -> Result> { - let iter = self.index_iter.as_mut().unwrap(); - if !self.is_finished { - while let Some(((state_key, version), _)) = iter.next().transpose()? { - // In case the previous seek() ends on the same key with version 0. - if Some(&state_key) == self.prev_key.as_ref() { - continue; - } - // Cursor is currently at the first available version of the state key. - // Check if the key_prefix is a valid prefix of the state_key we got from DB. - if !self.key_prefix.is_prefix(&state_key)? { - // No more keys matching the key_prefix, we can return the result. - self.is_finished = true; - break; - } - - if version > self.desired_version { - iter.seek(&(state_key.clone(), self.desired_version))?; - continue; - } - - self.prev_key = Some(state_key.clone()); - // Seek to the next key - this can be done by seeking to the current key with version 0 - iter.seek(&(state_key.clone(), 0))?; - - // The kv db is sharded with state_key_hash - if let Some(state_value) = self - .db - .db_shard(state_key.get_shard_id()) - .get::(&(state_key.hash(), version))? - .ok_or_else(|| { - AptosDbError::NotFound(format!( - "Key {state_key:?} is not found at version {version}.", - state_key = state_key, - version = version - )) - })? - { - return Ok(Some((state_key, state_value))); - } - } - } - Ok(None) - } } impl<'a> Iterator for PrefixedStateValueIterator<'a> { type Item = Result<(StateKey, StateValue)>; fn next(&mut self) -> Option { - if self.use_index { - self.next_by_index().transpose() - } else { - self.next_by_kv().transpose() - } + self.next_by_kv().transpose() } } diff --git a/storage/db-tool/src/tests.rs b/storage/db-tool/src/tests.rs index eb31df420ee0e..578110a408b93 100644 --- a/storage/db-tool/src/tests.rs +++ b/storage/db-tool/src/tests.rs @@ -474,7 +474,6 @@ mod dbtool_tests { StateKeyPrefix::new(AccessPath, b"".to_vec()), None, ver, - force_sharding, ) .unwrap(); let old_iter = db diff --git a/storage/indexer/src/db_indexer.rs b/storage/indexer/src/db_indexer.rs index 3ce157ddceae0..25f65aa181bd7 100644 --- a/storage/indexer/src/db_indexer.rs +++ b/storage/indexer/src/db_indexer.rs @@ -158,10 +158,14 @@ impl DBIndexer { fn get_num_of_transactions(&self, version: Version) -> Result { let highest_version = self.main_db_reader.get_synced_version()?; + if version > highest_version { + // In case main db is not synced yet or recreated + return Ok(0); + } // we want to include the last transaction since the iterator interface will is right exclusive. let num_of_transaction = min( (self.config.batch_size + 1) as u64, - highest_version - version + 1, + highest_version + 1 - version, ); Ok(num_of_transaction) } @@ -211,7 +215,6 @@ impl DBIndexer { } }); } - version += 1; Ok::<(), AptosDbError>(()) })?; diff --git a/storage/indexer/src/indexer_reader.rs b/storage/indexer/src/indexer_reader.rs index 21ee4f71ae18d..740fb439504ff 100644 --- a/storage/indexer/src/indexer_reader.rs +++ b/storage/indexer/src/indexer_reader.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{db_indexer::DBIndexer, db_v2::IndexerAsyncV2}; -use anyhow::anyhow; +use anyhow::{anyhow, ensure}; use aptos_types::{ account_address::AccountAddress, contract_event::EventWithVersion, @@ -17,6 +17,7 @@ use aptos_types::{ }; use std::sync::Arc; +#[derive(Clone)] pub struct IndexerReaders { table_info_reader: Option>, db_indexer_reader: Option>, @@ -56,6 +57,11 @@ impl IndexerReader for IndexerReaders { ) -> anyhow::Result> { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.event_enabled() { + let indexer_latest_version = db_indexer_reader.get_persisted_version()?; + ensure!( + indexer_latest_version >= ledger_version, + "Ledger version too new" + ); return Ok(db_indexer_reader.get_events( event_key, start, @@ -64,7 +70,7 @@ impl IndexerReader for IndexerReaders { ledger_version, )?); } else { - anyhow::bail!("Event index is not enabled") + anyhow::bail!("Internal event index is not enabled") } } anyhow::bail!("DB Indexer reader is not available") @@ -80,6 +86,11 @@ impl IndexerReader for IndexerReaders { ) -> anyhow::Result> { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.event_enabled() { + let indexer_latest_version = db_indexer_reader.get_persisted_version()?; + ensure!( + indexer_latest_version >= ledger_version, + "Ledger version too new" + ); return Ok(db_indexer_reader.get_events_by_event_key( event_key, start_seq_num, @@ -88,7 +99,7 @@ impl IndexerReader for IndexerReaders { ledger_version, )?); } else { - anyhow::bail!("Event index is not enabled") + anyhow::bail!("Internal event index is not enabled") } } anyhow::bail!("DB indexer reader is not available") @@ -104,6 +115,11 @@ impl IndexerReader for IndexerReaders { ) -> anyhow::Result { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.transaction_enabled() { + let indexer_latest_version = db_indexer_reader.get_persisted_version()?; + ensure!( + indexer_latest_version >= ledger_version, + "Ledger version too new" + ); return Ok(db_indexer_reader.get_account_transactions( address, start_seq_num, @@ -112,7 +128,7 @@ impl IndexerReader for IndexerReaders { ledger_version, )?); } else { - anyhow::bail!("Transaction by account index is not enabled") + anyhow::bail!("Interal transaction by account index is not enabled") } } anyhow::bail!("DB indexer reader is not available") @@ -122,13 +138,18 @@ impl IndexerReader for IndexerReaders { &self, key_prefix: &StateKeyPrefix, cursor: Option<&StateKey>, - version: Version, + ledger_version: Version, ) -> anyhow::Result> + '_>> { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.statekeys_enabled() { + let indexer_latest_version = db_indexer_reader.get_persisted_version()?; + ensure!( + indexer_latest_version >= ledger_version, + "Ledger version too new" + ); return Ok(Box::new( db_indexer_reader - .get_prefixed_state_value_iterator(key_prefix, cursor, version) + .get_prefixed_state_value_iterator(key_prefix, cursor, ledger_version) .map_err(|err| { anyhow!(format!( "failed to get prefixed state value iterator {}", @@ -140,7 +161,7 @@ impl IndexerReader for IndexerReaders { dyn Iterator>, >); } else { - anyhow::bail!("StateKeys index is not enabled") + anyhow::bail!("Internal statekeys index is not enabled") } } anyhow::bail!("DB indexer reader is not available") diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 325b9798de1b3..fc21c780ef641 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -89,7 +89,7 @@ class RunGroupConfig: RunGroupConfig(expected_tps=21300, key=RunGroupKey("no-op"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=11500, key=RunGroupKey("no-op", module_working_set_size=1000), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=12800, key=RunGroupKey("coin-transfer"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), - RunGroupConfig(expected_tps=36820, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), + RunGroupConfig(expected_tps=39700, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=9000, key=RunGroupKey("account-generation"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), RunGroupConfig(expected_tps=27873, key=RunGroupKey("account-generation", executor_type="native"), included_in=Flow.CONTINUOUS), RunGroupConfig(expected_tps=18600, key=RunGroupKey("account-resource32-b"), included_in=Flow.CONTINUOUS), From ab8304a4cfde6d3f40a9704ed7d909581edcfa6a Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Fri, 28 Jun 2024 10:54:47 -0700 Subject: [PATCH 010/469] [storage] check for deprecated API usage --- api/src/context.rs | 9 ++++++-- .../executor/tests/internal_indexer_test.rs | 10 ++++++--- .../aptosdb/src/db/include/aptosdb_reader.rs | 4 ++++ storage/aptosdb/src/state_store/mod.rs | 4 ---- storage/indexer/src/db_indexer.rs | 21 ++++++++++++++++-- storage/indexer/src/indexer_reader.rs | 22 +------------------ testsuite/single_node_performance.py | 2 +- 7 files changed, 39 insertions(+), 33 deletions(-) diff --git a/api/src/context.rs b/api/src/context.rs index eced1d2c4a783..c91a3cf42bc3f 100644 --- a/api/src/context.rs +++ b/api/src/context.rs @@ -871,7 +871,7 @@ impl Context { } else { (u64::MAX, Order::Descending) }; - let res = if !db_sharding_enabled(&self.node_config) { + let mut res = if !db_sharding_enabled(&self.node_config) { self.db .get_events(event_key, start, order, limit as u64, ledger_version)? } else { @@ -880,7 +880,12 @@ impl Context { .ok_or(anyhow!("Internal indexer reader doesn't exist"))? .get_events(event_key, start, order, limit as u64, ledger_version)? }; - Ok(res) + if order == Order::Descending { + res.reverse(); + Ok(res) + } else { + Ok(res) + } } fn next_bucket(&self, gas_unit_price: u64) -> u64 { diff --git a/execution/executor/tests/internal_indexer_test.rs b/execution/executor/tests/internal_indexer_test.rs index 7c909414dbb44..84b819f50f9c9 100644 --- a/execution/executor/tests/internal_indexer_test.rs +++ b/execution/executor/tests/internal_indexer_test.rs @@ -161,7 +161,7 @@ fn test_db_indexer_data() { assert_eq!(db_indexer.get_persisted_version().unwrap(), total_version); let txn_iter = db_indexer - .get_account_transaction_version_iter(core_account.address(), 0, 1000, 1000) + .get_account_transaction_version_iter(core_account.address(), 0, 1000, total_version) .unwrap(); let res: Vec<_> = txn_iter.collect(); @@ -174,7 +174,11 @@ fn test_db_indexer_data() { assert_eq!(res.len(), 27); let core_kv_iter = db_indexer - .get_prefixed_state_value_iterator(&StateKeyPrefix::from(core_account.address()), None, 12) + .get_prefixed_state_value_iterator( + &StateKeyPrefix::from(core_account.address()), + None, + total_version, + ) .unwrap(); let core_kv_res: Vec<_> = core_kv_iter.collect(); assert_eq!(core_kv_res.len(), 5); @@ -182,7 +186,7 @@ fn test_db_indexer_data() { .get_prefixed_state_value_iterator( &StateKeyPrefix::from(AccountAddress::from_hex_literal("0x1").unwrap()), None, - 12, + total_version, ) .unwrap(); let address_one_kv_res: Vec<_> = address_one_kv_iter.collect(); diff --git a/storage/aptosdb/src/db/include/aptosdb_reader.rs b/storage/aptosdb/src/db/include/aptosdb_reader.rs index ed21b05cea629..c0e1467ebf150 100644 --- a/storage/aptosdb/src/db/include/aptosdb_reader.rs +++ b/storage/aptosdb/src/db/include/aptosdb_reader.rs @@ -23,6 +23,7 @@ impl DbReader for AptosDB { version: Version, ) -> Result> + '_>> { gauged_api("get_prefixed_state_value_iterator", || { + ensure!(!self.state_kv_db.enabled_sharding(), "This API is not supported with sharded DB"); self.error_if_state_kv_pruned("StateValue", version)?; Ok(Box::new( @@ -64,6 +65,7 @@ impl DbReader for AptosDB { ledger_version: Version, ) -> Result> { gauged_api("get_account_transaction", || { + ensure!(!self.state_kv_db.enabled_sharding(), "This API is not supported with sharded DB"); self.transaction_store .get_account_transaction_version(address, seq_num, ledger_version)? .map(|txn_version| { @@ -82,6 +84,7 @@ impl DbReader for AptosDB { ledger_version: Version, ) -> Result { gauged_api("get_account_transactions", || { + ensure!(!self.state_kv_db.enabled_sharding(), "This API is not supported with sharded DB"); error_if_too_many_requested(limit, MAX_REQUEST_LIMIT)?; let txns_with_proofs = self @@ -852,6 +855,7 @@ impl AptosDB { limit: u64, ledger_version: Version, ) -> Result> { + ensure!(!self.state_kv_db.enabled_sharding(), "This API is deprecated for sharded DB"); error_if_too_many_requested(limit, MAX_REQUEST_LIMIT)?; let get_latest = order == Order::Descending && start_seq_num == u64::max_value(); diff --git a/storage/aptosdb/src/state_store/mod.rs b/storage/aptosdb/src/state_store/mod.rs index e7e631040df66..f935587d1a20e 100644 --- a/storage/aptosdb/src/state_store/mod.rs +++ b/storage/aptosdb/src/state_store/mod.rs @@ -573,10 +573,6 @@ impl StateStore { first_key_opt: Option<&StateKey>, desired_version: Version, ) -> Result { - ensure!( - !self.state_db.state_kv_db.enabled_sharding(), - "This API is not supported for sharded db." - ); // this can only handle non-sharded db scenario. // For sharded db, should look at API side using internal indexer to handle this request PrefixedStateValueIterator::new( diff --git a/storage/indexer/src/db_indexer.rs b/storage/indexer/src/db_indexer.rs index 25f65aa181bd7..94c29ca553e4a 100644 --- a/storage/indexer/src/db_indexer.rs +++ b/storage/indexer/src/db_indexer.rs @@ -111,6 +111,15 @@ impl DBIndexer { } } + pub fn ensure_cover_ledger_version(&self, ledger_version: Version) -> Result<()> { + let indexer_latest_version = self.get_persisted_version()?; + ensure!( + indexer_latest_version >= ledger_version, + "ledger version too new" + ); + Ok(()) + } + pub fn get_persisted_version(&self) -> Result { // read the latest key from the db self.db @@ -236,6 +245,7 @@ impl DBIndexer { num_versions: u64, ledger_version: Version, ) -> Result { + self.ensure_cover_ledger_version(ledger_version)?; let mut iter = self.db.iter::()?; iter.seek(&(address, min_seq_num))?; Ok(AccountTransactionVersionIter::new( @@ -253,6 +263,8 @@ impl DBIndexer { ledger_version: Version, event_key: &EventKey, ) -> Result> { + self.ensure_cover_ledger_version(ledger_version)?; + let mut iter = self.db.iter::()?; iter.seek_for_prev(&(*event_key, ledger_version, u64::max_value()))?; @@ -277,6 +289,7 @@ impl DBIndexer { u64, // index among events for the same transaction )>, > { + self.ensure_cover_ledger_version(ledger_version)?; let mut iter = self.db.iter::()?; iter.seek(&(*event_key, start_seq_num))?; @@ -332,6 +345,7 @@ impl DBIndexer { limit: u64, ledger_version: Version, ) -> Result> { + self.ensure_cover_ledger_version(ledger_version)?; self.get_events_by_event_key(event_key, start, order, limit, ledger_version) } @@ -343,6 +357,7 @@ impl DBIndexer { limit: u64, ledger_version: Version, ) -> Result> { + self.ensure_cover_ledger_version(ledger_version)?; error_if_too_many_requested(limit, MAX_REQUEST_LIMIT)?; let get_latest = order == Order::Descending && start_seq_num == u64::max_value(); @@ -410,6 +425,7 @@ impl DBIndexer { include_events: bool, ledger_version: Version, ) -> Result { + self.ensure_cover_ledger_version(ledger_version)?; error_if_too_many_requested(limit, MAX_REQUEST_LIMIT)?; let txns_with_proofs = self @@ -431,14 +447,15 @@ impl DBIndexer { &self, key_prefix: &StateKeyPrefix, cursor: Option<&StateKey>, - version: Version, + ledger_version: Version, ) -> Result> + '_> { + self.ensure_cover_ledger_version(ledger_version)?; PrefixedStateValueIterator::new( self.main_db_reader.clone(), self.db.as_ref(), key_prefix.clone(), cursor.cloned(), - version, + ledger_version, ) } } diff --git a/storage/indexer/src/indexer_reader.rs b/storage/indexer/src/indexer_reader.rs index 740fb439504ff..ad032ee88dc7d 100644 --- a/storage/indexer/src/indexer_reader.rs +++ b/storage/indexer/src/indexer_reader.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{db_indexer::DBIndexer, db_v2::IndexerAsyncV2}; -use anyhow::{anyhow, ensure}; +use anyhow::anyhow; use aptos_types::{ account_address::AccountAddress, contract_event::EventWithVersion, @@ -57,11 +57,6 @@ impl IndexerReader for IndexerReaders { ) -> anyhow::Result> { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.event_enabled() { - let indexer_latest_version = db_indexer_reader.get_persisted_version()?; - ensure!( - indexer_latest_version >= ledger_version, - "Ledger version too new" - ); return Ok(db_indexer_reader.get_events( event_key, start, @@ -86,11 +81,6 @@ impl IndexerReader for IndexerReaders { ) -> anyhow::Result> { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.event_enabled() { - let indexer_latest_version = db_indexer_reader.get_persisted_version()?; - ensure!( - indexer_latest_version >= ledger_version, - "Ledger version too new" - ); return Ok(db_indexer_reader.get_events_by_event_key( event_key, start_seq_num, @@ -115,11 +105,6 @@ impl IndexerReader for IndexerReaders { ) -> anyhow::Result { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.transaction_enabled() { - let indexer_latest_version = db_indexer_reader.get_persisted_version()?; - ensure!( - indexer_latest_version >= ledger_version, - "Ledger version too new" - ); return Ok(db_indexer_reader.get_account_transactions( address, start_seq_num, @@ -142,11 +127,6 @@ impl IndexerReader for IndexerReaders { ) -> anyhow::Result> + '_>> { if let Some(db_indexer_reader) = &self.db_indexer_reader { if db_indexer_reader.statekeys_enabled() { - let indexer_latest_version = db_indexer_reader.get_persisted_version()?; - ensure!( - indexer_latest_version >= ledger_version, - "Ledger version too new" - ); return Ok(Box::new( db_indexer_reader .get_prefixed_state_value_iterator(key_prefix, cursor, ledger_version) diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index fc21c780ef641..3675525c9dde9 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -89,7 +89,7 @@ class RunGroupConfig: RunGroupConfig(expected_tps=21300, key=RunGroupKey("no-op"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=11500, key=RunGroupKey("no-op", module_working_set_size=1000), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=12800, key=RunGroupKey("coin-transfer"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), - RunGroupConfig(expected_tps=39700, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), + RunGroupConfig(expected_tps=41479, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=9000, key=RunGroupKey("account-generation"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), RunGroupConfig(expected_tps=27873, key=RunGroupKey("account-generation", executor_type="native"), included_in=Flow.CONTINUOUS), RunGroupConfig(expected_tps=18600, key=RunGroupKey("account-resource32-b"), included_in=Flow.CONTINUOUS), From 224c771f321f75c5ac26e7ee3f7094a4b54d17cf Mon Sep 17 00:00:00 2001 From: George Mitenkov Date: Mon, 1 Jul 2024 01:17:37 +0100 Subject: [PATCH 011/469] [e2e-testing] Switch to AptosVM execution (#13843) Use AptosVM-based execution for e2e comparison testing --- Cargo.lock | 1 + .../src/data_state_view.rs | 2 +- .../src/execution.rs | 139 +++++++----------- aptos-move/aptos-vm-types/src/environment.rs | 10 -- aptos-move/e2e-tests/Cargo.toml | 3 + aptos-move/e2e-tests/src/data_store.rs | 44 ++++++ aptos-move/e2e-tests/src/executor.rs | 79 +--------- 7 files changed, 109 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cea7d5bb6d59e..ecdc4b0cb3946 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2495,6 +2495,7 @@ dependencies = [ "aptos-vm-types", "bcs 0.1.4", "bytes", + "claims", "goldenfile", "move-binary-format", "move-command-line-common", diff --git a/aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs b/aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs index f5c81a5dc56e3..be1938618686f 100644 --- a/aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs +++ b/aptos-move/aptos-e2e-comparison-testing/src/data_state_view.rs @@ -12,6 +12,7 @@ use aptos_types::{ use aptos_validator_interface::{AptosValidatorInterface, DebuggerStateView}; use std::{ collections::HashMap, + ops::DerefMut, sync::{Arc, Mutex}, }; @@ -20,7 +21,6 @@ pub struct DataStateView { code_data: Option, data_read_state_keys: Option>>>, } -use std::ops::DerefMut; impl DataStateView { pub fn new( diff --git a/aptos-move/aptos-e2e-comparison-testing/src/execution.rs b/aptos-move/aptos-e2e-comparison-testing/src/execution.rs index 9e85437fce474..27838857b47e7 100644 --- a/aptos-move/aptos-e2e-comparison-testing/src/execution.rs +++ b/aptos-move/aptos-e2e-comparison-testing/src/execution.rs @@ -12,20 +12,19 @@ use aptos_language_e2e_tests::{data_store::FakeDataStore, executor::FakeExecutor use aptos_types::{ contract_event::ContractEvent, on_chain_config::{FeatureFlag, Features, OnChainConfig}, - transaction::{Transaction, TransactionPayload, Version}, + transaction::{Transaction, Version}, vm_status::VMStatus, write_set::WriteSet, }; use aptos_validator_interface::AptosValidatorInterface; -use aptos_vm::data_cache::AsMoveResolver; use clap::ValueEnum; use itertools::Itertools; use move_core_types::{account_address::AccountAddress, language_storage::ModuleId}; use move_model::metadata::CompilerVersion; use std::{cmp, collections::HashMap, path::PathBuf, sync::Arc}; -fn load_packages_to_executor( - executor: &mut FakeExecutor, +fn add_packages_to_data_store( + data_store: &mut FakeDataStore, package_info: &PackageInfo, compiled_package_cache: &HashMap>>, ) { @@ -34,12 +33,12 @@ fn load_packages_to_executor( } let compiled_package = compiled_package_cache.get(package_info).unwrap(); for (module_id, module_blob) in compiled_package { - executor.add_module(module_id, module_blob.clone()); + data_store.add_module(module_id, module_blob.clone()); } } -fn load_aptos_packages_to_executor( - executor: &mut FakeExecutor, +fn add_aptos_packages_to_data_store( + data_store: &mut FakeDataStore, compiled_package_map: &HashMap>>, ) { for package in APTOS_PACKAGES { @@ -48,7 +47,7 @@ fn load_aptos_packages_to_executor( package_name: package.to_string(), upgrade_number: None, }; - load_packages_to_executor(executor, &package_info, compiled_package_map); + add_packages_to_data_store(data_store, &package_info, compiled_package_map); } } @@ -270,9 +269,8 @@ impl Execution { return compiled_result; } } - // read the state data; + // read the state data let state = data_manager.get_state(cur_version); - // execute and compare self.execute_and_compare( cur_version, state, @@ -301,7 +299,7 @@ impl Execution { package_cache_main = compiled_package_cache_v2; v2_flag = true; } - let res_main_opt = self.execute_code( + let res_main = self.execute_code( cur_version, state.clone(), &txn_idx.package_info, @@ -311,7 +309,7 @@ impl Execution { v2_flag, ); if self.execution_mode.is_compare() { - let res_other_opt = self.execute_code( + let res_other = self.execute_code( cur_version, state, &txn_idx.package_info, @@ -320,20 +318,21 @@ impl Execution { debugger.clone(), true, ); - self.print_mismatches(cur_version, &res_main_opt.unwrap(), &res_other_opt.unwrap()); + self.print_mismatches(cur_version, &res_main, &res_other); } else { - let res = res_main_opt.unwrap(); - if let Ok(res_ok) = res { - self.output_result_str(format!( - "version:{}\nwrite set:{:?}\n events:{:?}\n", - cur_version, res_ok.0, res_ok.1 - )); - } else { - self.output_result_str(format!( - "execution error {} at version: {}, error", - res.unwrap_err(), - cur_version - )); + match res_main { + Ok((write_set, events)) => { + self.output_result_str(format!( + "version:{}\nwrite set:{:?}\n events:{:?}\n", + cur_version, write_set, events + )); + }, + Err(vm_status) => { + self.output_result_str(format!( + "execution error {} at version: {}, error", + vm_status, cur_version + )); + }, } } } @@ -341,74 +340,44 @@ impl Execution { fn execute_code( &self, version: Version, - state: FakeDataStore, + mut state: FakeDataStore, package_info: &PackageInfo, txn: &Transaction, compiled_package_cache: &HashMap>>, debugger_opt: Option>, v2_flag: bool, - ) -> Option), VMStatus>> { - let executor = FakeExecutor::no_genesis(); - let mut executor = executor.set_not_parallel(); - *executor.data_store_mut() = state; - if let Transaction::UserTransaction(signed_trans) = txn { - let sender = signed_trans.sender(); - let payload = signed_trans.payload(); - if let TransactionPayload::EntryFunction(entry_function) = payload { - // Always load 0x1 modules - load_aptos_packages_to_executor(&mut executor, compiled_package_cache); - // Load modules - if package_info.is_compilable() { - load_packages_to_executor(&mut executor, package_info, compiled_package_cache); - } - let mut senders = vec![sender]; - senders.extend(signed_trans.authenticator().secondary_signer_addresses()); - let enable_v7 = |features: &mut Features| { - if v2_flag { - features.enable(FeatureFlag::VM_BINARY_FORMAT_V7); - } else { - features.enable(FeatureFlag::VM_BINARY_FORMAT_V6); - } - }; - if let Some(debugger) = debugger_opt { - let data_view = - DataStateView::new(debugger, version, executor.data_store().clone()); - let mut features = - Features::fetch_config(&data_view.as_move_resolver()).unwrap_or_default(); - enable_v7(&mut features); - return Some(executor.try_exec_entry_with_state_view( - senders, - entry_function, - &data_view.as_move_resolver(), - features, - )); - } else { - let mut features = - Features::fetch_config(&executor.data_store().clone().as_move_resolver()) - .unwrap_or_default(); - enable_v7(&mut features); - return Some(executor.try_exec_entry_with_state_view( - senders, - entry_function, - &executor.data_store().clone().as_move_resolver(), - features, - )); - } - } + ) -> Result<(WriteSet, Vec), VMStatus> { + // Always add Aptos (0x1) packages. + add_aptos_packages_to_data_store(&mut state, compiled_package_cache); + + // Add other modules. + if package_info.is_compilable() { + add_packages_to_data_store(&mut state, package_info, compiled_package_cache); } + + // Update features if needed to the correct binary format used by V2 compiler. + let mut features = Features::fetch_config(&state).unwrap_or_default(); + if v2_flag { + features.enable(FeatureFlag::VM_BINARY_FORMAT_V7); + } else { + features.enable(FeatureFlag::VM_BINARY_FORMAT_V6); + } + state.set_features(features); + + // We use executor only to get access to block executor and avoid some of + // the initializations, but ignore its internal state, i.e., FakeDataStore. + let executor = FakeExecutor::no_genesis(); + let txns = vec![txn.clone()]; + if let Some(debugger) = debugger_opt { - let data_view = DataStateView::new(debugger, version, executor.data_store().clone()); - Some( - executor - .execute_transaction_block_with_state_view([txn.clone()].to_vec(), &data_view) - .map(|res| res[0].clone().into()), - ) + let data_view = DataStateView::new(debugger, version, state); + executor + .execute_transaction_block_with_state_view(txns, &data_view) + .map(|mut res| res.pop().unwrap().into()) } else { - Some( - executor - .execute_transaction_block(vec![txn.clone()]) - .map(|res| res[0].clone().into()), - ) + executor + .execute_transaction_block_with_state_view(txns, &state) + .map(|mut res| res.pop().unwrap().into()) } } diff --git a/aptos-move/aptos-vm-types/src/environment.rs b/aptos-move/aptos-vm-types/src/environment.rs index af1e3b72fabab..5a936115ab0c5 100644 --- a/aptos-move/aptos-vm-types/src/environment.rs +++ b/aptos-move/aptos-vm-types/src/environment.rs @@ -89,16 +89,6 @@ impl Environment { )) } - pub fn with_features_for_testing(self, features: Features) -> Arc { - let ty_builder = aptos_default_ty_builder(&features); - Arc::new(Self::initialize( - features, - self.timed_features, - self.chain_id, - ty_builder, - )) - } - pub fn try_enable_delayed_field_optimization(mut self) -> Self { if self.features.is_aggregator_v2_delayed_fields_enabled() { self.vm_config.delayed_field_optimization_enabled = true; diff --git a/aptos-move/e2e-tests/Cargo.toml b/aptos-move/e2e-tests/Cargo.toml index 859110fee4302..d19a2b61d2a94 100644 --- a/aptos-move/e2e-tests/Cargo.toml +++ b/aptos-move/e2e-tests/Cargo.toml @@ -51,6 +51,9 @@ rand = { workspace = true } rayon = { workspace = true } serde = { workspace = true } +[dev-dependencies] +claims = { workspace = true } + [features] default = [] fuzzing = [] diff --git a/aptos-move/e2e-tests/src/data_store.rs b/aptos-move/e2e-tests/src/data_store.rs index 7113f7b704b4b..8013243ca36b2 100644 --- a/aptos-move/e2e-tests/src/data_store.rs +++ b/aptos-move/e2e-tests/src/data_store.rs @@ -7,6 +7,7 @@ use crate::account::AccountData; use aptos_types::{ account_config::CoinInfoResource, + on_chain_config::{Features, OnChainConfig}, state_store::{ errors::StateviewError, in_memory_state_view::InMemoryStateView, state_key::StateKey, state_storage_usage::StateStorageUsage, state_value::StateValue, TStateView, @@ -118,6 +119,14 @@ impl FakeDataStore { StateValue::new_legacy(blob.into()), ); } + + pub fn set_features(&mut self, features: Features) { + let bytes = bcs::to_bytes(&features).expect("Features should always be serializable"); + self.set( + StateKey::resource(Features::address(), &Features::struct_tag()).unwrap(), + StateValue::new_legacy(bytes.into()), + ); + } } // This is used by the `execute_block` API. @@ -140,3 +149,38 @@ impl TStateView for FakeDataStore { InMemoryStateView::new(self.state_data.clone()) } } + +#[cfg(test)] +mod test { + use super::*; + use aptos_types::on_chain_config::{FeatureFlag, Features}; + use claims::*; + + #[test] + fn test_features_can_be_set() { + let mut data_store = FakeDataStore::default(); + assert_none!(Features::fetch_config(&data_store)); + + data_store.set_features(Features::default()); + let features = assert_some!(Features::fetch_config(&data_store)); + assert_eq!(features, Features::default()) + } + + #[test] + fn test_features_can_be_reset() { + use claims::*; + + let mut data_store = FakeDataStore::default(); + data_store.add_write_set(GENESIS_CHANGE_SET_HEAD.write_set()); + + // Reset the feature. + let mut features = assert_some!(Features::fetch_config(&data_store)); + assert!(features.is_enabled(FeatureFlag::STORAGE_SLOT_METADATA)); + features.disable(FeatureFlag::STORAGE_SLOT_METADATA); + data_store.set_features(features.clone()); + + let reset_features = assert_some!(Features::fetch_config(&data_store)); + assert!(!reset_features.is_enabled(FeatureFlag::STORAGE_SLOT_METADATA)); + assert_eq!(reset_features, features) + } +} diff --git a/aptos-move/e2e-tests/src/executor.rs b/aptos-move/e2e-tests/src/executor.rs index 46a8cfc084112..c562fb933c6f5 100644 --- a/aptos-move/e2e-tests/src/executor.rs +++ b/aptos-move/e2e-tests/src/executor.rs @@ -34,15 +34,14 @@ use aptos_types::{ chain_id::ChainId, contract_event::ContractEvent, move_utils::MemberId, - on_chain_config::{AptosVersion, FeatureFlag, Features, OnChainConfig, ValidatorSet}, + on_chain_config::{AptosVersion, OnChainConfig, ValidatorSet}, state_store::{state_key::StateKey, state_value::StateValue, StateView, TStateView}, transaction::{ signature_verified_transaction::{ into_signature_verified_block, SignatureVerifiedTransaction, }, - BlockOutput, EntryFunction, ExecutionStatus, SignedTransaction, Transaction, - TransactionOutput, TransactionPayload, TransactionStatus, VMValidatorResult, - ViewFunctionOutput, + BlockOutput, ExecutionStatus, SignedTransaction, Transaction, TransactionOutput, + TransactionPayload, TransactionStatus, VMValidatorResult, ViewFunctionOutput, }, vm_status::VMStatus, write_set::WriteSet, @@ -50,9 +49,9 @@ use aptos_types::{ use aptos_vm::{ block_executor::{AptosTransactionOutput, BlockAptosVM}, data_cache::AsMoveResolver, - gas::{get_gas_parameters, make_prod_gas_meter}, - move_vm_ext::{AptosMoveResolver, MoveVmExt, SessionId}, - verifier, AptosVM, VMValidator, + gas::make_prod_gas_meter, + move_vm_ext::{MoveVmExt, SessionId}, + AptosVM, VMValidator, }; use aptos_vm_genesis::{generate_genesis_change_set_for_testing_with_count, GenesisOptions}; use aptos_vm_logging::log_schema::AdapterLogSchema; @@ -1096,72 +1095,6 @@ impl FakeExecutor { self.exec_module(&Self::module(module_name), function_name, type_params, args) } - pub fn try_exec_entry_with_state_view( - &mut self, - senders: Vec, - entry_fn: &EntryFunction, - state_view: &impl AptosMoveResolver, - features: Features, - ) -> Result<(WriteSet, Vec), VMStatus> { - let (gas_params, storage_gas_params, gas_feature_version) = - get_gas_parameters(&features, state_view); - - let are_struct_constructors_enabled = features.is_enabled(FeatureFlag::STRUCT_CONSTRUCTORS); - let env = self - .env - .as_ref() - .clone() - .with_features_for_testing(features); - - let vm = MoveVmExt::new( - LATEST_GAS_FEATURE_VERSION, - gas_params.as_ref(), - env, - state_view, - ); - - let mut session = vm.new_session(state_view, SessionId::void(), None); - let func = - session.load_function(entry_fn.module(), entry_fn.function(), entry_fn.ty_args())?; - let args = verifier::transaction_arg_validation::validate_combine_signer_and_txn_args( - &mut session, - senders, - entry_fn.args().to_vec(), - &func, - are_struct_constructors_enabled, - )?; - - let mut gas_meter = make_prod_gas_meter( - gas_feature_version, - gas_params.unwrap().clone().vm, - storage_gas_params.unwrap(), - false, - 10_000_000_000_000.into(), - ); - - let storage = TraversalStorage::new(); - session - .execute_entry_function( - func, - args, - &mut gas_meter, - &mut TraversalContext::new(&storage), - ) - .map_err(|e| e.into_vm_status())?; - - let mut change_set = session - .finish(&ChangeSetConfigs::unlimited_at_gas_feature_version( - LATEST_GAS_FEATURE_VERSION, - )) - .expect("Failed to generate txn effects"); - change_set.try_materialize_aggregator_v1_delta_set(state_view)?; - let (write_set, events) = change_set - .try_into_storage_change_set() - .expect("Failed to convert to ChangeSet") - .into_inner(); - Ok((write_set, events)) - } - pub fn try_exec( &mut self, module_name: &str, From 7a21c148c2668f3023238a6b7879216ea7f7f7ed Mon Sep 17 00:00:00 2001 From: Teng Zhang Date: Sun, 30 Jun 2024 17:42:30 -0700 Subject: [PATCH 012/469] [Compiler-v2] Raise error when entry and inline both appear in a function signature (#13840) * fix * refactor --- .../src/file_format_generator/module_generator.rs | 2 +- third_party/move/move-compiler-v2/src/lib.rs | 5 ++++- .../entry_inline_err_no_report.exp | 14 ++++++++++++++ .../entry_inline_err_no_report.move | 11 +++++++++++ .../tests/checking/typing/entry_inline_err.exp | 13 +++++++++++++ .../tests/checking/typing/entry_inline_err.move | 11 +++++++++++ .../move/move-model/src/builder/exp_builder.rs | 2 +- .../move/move-model/src/builder/module_builder.rs | 4 ++++ third_party/move/move-model/src/metadata.rs | 5 +++++ 9 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.exp create mode 100644 third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.move create mode 100644 third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.exp create mode 100644 third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.move diff --git a/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs b/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs index cc8648e06d356..d4ad151a05572 100644 --- a/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs +++ b/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs @@ -105,7 +105,7 @@ impl ModuleGenerator { ) -> (FF::CompiledModule, SourceMap, Option) { let options = module_env.env.get_extension::().expect("options"); let language_version = options.language_version.unwrap_or_default(); - let gen_access_specifiers = language_version >= LanguageVersion::V2_0 + let gen_access_specifiers = language_version.is_at_least(LanguageVersion::V2_0) && options.experiment_on(Experiment::GEN_ACCESS_SPECIFIERS); let compilation_metadata = CompilationMetadata::new(CompilerVersion::V2_0, language_version); diff --git a/third_party/move/move-compiler-v2/src/lib.rs b/third_party/move/move-compiler-v2/src/lib.rs index 2ee3067e492d7..f817957b6b274 100644 --- a/third_party/move/move-compiler-v2/src/lib.rs +++ b/third_party/move/move-compiler-v2/src/lib.rs @@ -310,7 +310,10 @@ pub fn check_and_rewrite_pipeline<'a, 'b>( ); } - let check_seqs_in_binops = options.language_version.unwrap_or_default() < LanguageVersion::V2_0 + let check_seqs_in_binops = !options + .language_version + .unwrap_or_default() + .is_at_least(LanguageVersion::V2_0) && options.experiment_on(Experiment::SEQS_IN_BINOPS_CHECK); if !for_v1_model && check_seqs_in_binops { diff --git a/third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.exp b/third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.exp new file mode 100644 index 0000000000000..65d74465d64d0 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.exp @@ -0,0 +1,14 @@ +// -- Model dump before bytecode pipeline +module 0x123::a { + friend entry fun a() { + Tuple() + } +} // end 0x123::a +module 0x123::b { + private entry fun a() { + Tuple() + } + private fun b() { + b::a() + } +} // end 0x123::b diff --git a/third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.move b/third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.move new file mode 100644 index 0000000000000..bf4a894878f11 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking-lang-v1/entry_inline_err_no_report.move @@ -0,0 +1,11 @@ +module 0x123::a{ + public(friend) entry inline fun a(){} +} + +module 0x123::b{ + entry inline fun a(){} + + fun b() { + a() + } +} diff --git a/third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.exp b/third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.exp new file mode 100644 index 0000000000000..dadf6a8162e05 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.exp @@ -0,0 +1,13 @@ + +Diagnostics: +error: An entry function cannot be inlined. + ┌─ tests/checking/typing/entry_inline_err.move:2:5 + │ +2 │ public(friend) entry inline fun a(){} + │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error: An entry function cannot be inlined. + ┌─ tests/checking/typing/entry_inline_err.move:6:5 + │ +6 │ entry inline fun a(){} + │ ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.move b/third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.move new file mode 100644 index 0000000000000..bf4a894878f11 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/typing/entry_inline_err.move @@ -0,0 +1,11 @@ +module 0x123::a{ + public(friend) entry inline fun a(){} +} + +module 0x123::b{ + entry inline fun a(){} + + fun b() { + a() + } +} diff --git a/third_party/move/move-model/src/builder/exp_builder.rs b/third_party/move/move-model/src/builder/exp_builder.rs index 336edb1370ff1..e9f859d19bb55 100644 --- a/third_party/move/move-model/src/builder/exp_builder.rs +++ b/third_party/move/move-model/src/builder/exp_builder.rs @@ -171,7 +171,7 @@ impl<'env, 'translator, 'module_translator> ExpTranslator<'env, 'translator, 'mo } pub fn check_language_version(&self, loc: &Loc, feature: &str, version_min: LanguageVersion) { - if self.env().language_version() < version_min { + if !self.env().language_version().is_at_least(version_min) { self.env().error( loc, &format!( diff --git a/third_party/move/move-model/src/builder/module_builder.rs b/third_party/move/move-model/src/builder/module_builder.rs index a934b63aa7924..889d424963aeb 100644 --- a/third_party/move/move-model/src/builder/module_builder.rs +++ b/third_party/move/move-model/src/builder/module_builder.rs @@ -32,6 +32,7 @@ use crate::{ }, symbol::{Symbol, SymbolPool}, ty::{Constraint, ConstraintContext, ErrorMessageContext, PrimitiveType, Type, BOOL_TYPE}, + LanguageVersion, }; use codespan_reporting::diagnostic::Severity; use itertools::Itertools; @@ -518,6 +519,9 @@ impl<'env, 'translator> ModuleBuilder<'env, 'translator> { let params = et.analyze_and_add_params(&def.signature.parameters, true); let result_type = et.translate_type(&def.signature.return_type); let kind = if def.entry.is_some() { + if et.env().language_version.is_at_least(LanguageVersion::V2_0) && def.inline { + et.error(&et.to_loc(&def.loc), "An entry function cannot be inlined."); + } FunctionKind::Entry } else if def.inline { FunctionKind::Inline diff --git a/third_party/move/move-model/src/metadata.rs b/third_party/move/move-model/src/metadata.rs index b8990fa10ba7b..4a5f62b61f2b9 100644 --- a/third_party/move/move-model/src/metadata.rs +++ b/third_party/move/move-model/src/metadata.rs @@ -215,6 +215,11 @@ impl LanguageVersion { LanguageVersion::V2_0 => true, } } + + /// Whether the language version is equal to greater than `ver` + pub fn is_at_least(&self, ver: LanguageVersion) -> bool { + *self >= ver + } } impl Display for LanguageVersion { From ffc940f233760aa7f39242ab760b48e1d9f18dc2 Mon Sep 17 00:00:00 2001 From: Teng Zhang Date: Sun, 30 Jun 2024 20:18:47 -0700 Subject: [PATCH 013/469] [Compiler-v2] Fix native struct (#13835) --- .../file_format_generator/module_generator.rs | 32 +- .../multi_pool_money_market_token.exp | 300 ++++++++++++++++++ .../multi_pool_money_market_token.move | 240 ++++++++++++++ .../v1-examples/simple_money_market_token.exp | 247 ++++++++++++++ .../simple_money_market_token.move | 204 ++++++++++++ .../move/move-compiler-v2/tests/v1.matched | 48 +-- .../move/move-compiler-v2/tests/v1.unmatched | 10 +- .../tools/testdiff/src/main.rs | 1 + .../move-model/src/builder/model_builder.rs | 3 + .../move-model/src/builder/module_builder.rs | 4 +- third_party/move/move-model/src/model.rs | 9 + 11 files changed, 1053 insertions(+), 45 deletions(-) create mode 100644 third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.exp create mode 100644 third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.move create mode 100644 third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.exp create mode 100644 third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.move diff --git a/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs b/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs index d4ad151a05572..cf1a3962b96cc 100644 --- a/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs +++ b/third_party/move/move-compiler-v2/src/file_format_generator/module_generator.rs @@ -217,20 +217,24 @@ impl ModuleGenerator { } let struct_handle = self.struct_index(ctx, loc, struct_env); let fields = struct_env.get_fields(); - let field_information = FF::StructFieldInformation::Declared( - fields - .map(|f| { - let field_loc = f.get_loc(); - self.source_map - .add_struct_field_mapping(def_idx, ctx.env.to_ir_loc(field_loc)) - .expect(SOURCE_MAP_OK); - let name = self.name_index(ctx, field_loc, f.get_name()); - let signature = - FF::TypeSignature(self.signature_token(ctx, loc, &f.get_type())); - FF::FieldDefinition { name, signature } - }) - .collect(), - ); + let field_information = if struct_env.is_native() { + FF::StructFieldInformation::Native + } else { + FF::StructFieldInformation::Declared( + fields + .map(|f| { + let field_loc = f.get_loc(); + self.source_map + .add_struct_field_mapping(def_idx, ctx.env.to_ir_loc(field_loc)) + .expect(SOURCE_MAP_OK); + let name = self.name_index(ctx, field_loc, f.get_name()); + let signature = + FF::TypeSignature(self.signature_token(ctx, loc, &f.get_type())); + FF::FieldDefinition { name, signature } + }) + .collect(), + ) + }; let def = FF::StructDefinition { struct_handle, field_information, diff --git a/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.exp b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.exp new file mode 100644 index 0000000000000..afd5f2f73d767 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.exp @@ -0,0 +1,300 @@ + +Diagnostics: +warning: unused type parameter + ┌─ tests/checking/typing/v1-examples/multi_pool_money_market_token.move:5:21 + │ +5 │ native struct T has copy, drop, store; + │ ^ + │ │ + │ Unused type parameter `K`. Consider declaring it as phantom + +warning: unused type parameter + ┌─ tests/checking/typing/v1-examples/multi_pool_money_market_token.move:5:24 + │ +5 │ native struct T has copy, drop, store; + │ ^ + │ │ + │ Unused type parameter `V`. Consider declaring it as phantom + +// -- Model dump before bytecode pipeline +module 0x2::Token { + struct Coin { + type: #0, + value: u64, + } + public fun create(type: #0,value: u64): Token::Coin<#0> { + pack Token::Coin(type, value) + } + public fun value(coin: &Token::Coin<#0>): u64 { + select Token::Coin.value<&Token::Coin>(coin) + } + public fun deposit(coin: &mut Token::Coin<#0>,check: Token::Coin<#0>) { + { + let Token::Coin{ type, value } = check; + if Eq(Borrow(Immutable)(select Token::Coin.type<&mut Token::Coin>(coin)), Borrow(Immutable)(type)) { + Tuple() + } else { + Abort(42) + }; + select Token::Coin.value<&mut Token::Coin>(coin) = Add(select Token::Coin.value<&mut Token::Coin>(coin), value); + Tuple() + } + } + public fun destroy_zero(coin: Token::Coin<#0>) { + { + let Token::Coin{ type: _, value } = coin; + if Eq(value, 0) { + Tuple() + } else { + Abort(11) + } + } + } + public fun join(xus: Token::Coin<#0>,coin2: Token::Coin<#0>): Token::Coin<#0> { + Token::deposit(Borrow(Mutable)(xus), coin2); + xus + } + public fun split(coin: Token::Coin<#0>,amount: u64): (Token::Coin<#0>, Token::Coin<#0>) { + { + let other: Token::Coin = Token::withdraw(Borrow(Mutable)(coin), amount); + Tuple(coin, other) + } + } + public fun withdraw(coin: &mut Token::Coin<#0>,amount: u64): Token::Coin<#0> { + if Ge(select Token::Coin.value<&mut Token::Coin>(coin), amount) { + Tuple() + } else { + Abort(10) + }; + select Token::Coin.value<&mut Token::Coin>(coin) = Sub(select Token::Coin.value<&mut Token::Coin>(coin), amount); + pack Token::Coin(Deref(Borrow(Immutable)(select Token::Coin.type<&mut Token::Coin>(coin))), amount) + } +} // end 0x2::Token +module 0x2::Map { + struct T { + } + public native fun empty(): Map::T<#0, #1>; + public native fun remove(m: &Map::T<#0, #1>,k: �): #1; + public native fun contains_key(m: &Map::T<#0, #1>,k: �): bool; + public native fun get(m: &Map::T<#0, #1>,k: �):  + public native fun get_mut(m: &mut Map::T<#0, #1>,k: �): &mut #1; + public native fun insert(m: &Map::T<#0, #1>,k: #0,v: #1); +} // end 0x2::Map +module 0x3::OneToOneMarket { + use std::signer; + use 0x2::Map; // resolved as: 0x2::Map + use 0x2::Token; // resolved as: 0x2::Token + struct BorrowRecord { + record: Map::T, + } + struct DepositRecord { + record: Map::T, + } + struct Pool { + coin: Token::Coin<#0>, + } + struct Price { + price: u64, + } + public fun borrow(account: &signer,pool_owner: address,amount: u64): Token::Coin<#1> + acquires OneToOneMarket::Price(*) + acquires OneToOneMarket::Pool(*) + acquires OneToOneMarket::DepositRecord(*) + acquires OneToOneMarket::BorrowRecord(*) + { + if Le(amount, OneToOneMarket::max_borrow_amount(account, pool_owner)) { + Tuple() + } else { + Abort(1025) + }; + OneToOneMarket::update_borrow_record(account, pool_owner, amount); + { + let pool: &mut OneToOneMarket::Pool = BorrowGlobal(Mutable)>(pool_owner); + Token::withdraw(Borrow(Mutable)(select OneToOneMarket::Pool.coin<&mut OneToOneMarket::Pool>(pool)), amount) + } + } + public fun deposit(account: &signer,pool_owner: address,coin: Token::Coin<#0>) + acquires OneToOneMarket::Pool(*) + acquires OneToOneMarket::DepositRecord(*) + { + { + let amount: u64 = Token::value(Borrow(Immutable)(coin)); + OneToOneMarket::update_deposit_record(account, pool_owner, amount); + { + let pool: &mut OneToOneMarket::Pool = BorrowGlobal(Mutable)>(pool_owner); + Token::deposit(Borrow(Mutable)(select OneToOneMarket::Pool.coin<&mut OneToOneMarket::Pool>(pool)), coin) + } + } + } + private fun accept(account: &signer,init: Token::Coin<#0>) { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + Tuple() + } else { + Abort(42) + }; + MoveTo>(account, pack OneToOneMarket::Pool(init)) + } + } + private fun borrowed_amount(account: &signer,pool_owner: address): u64 + acquires OneToOneMarket::BorrowRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + return 0 + } else { + Tuple() + }; + { + let record: &Map::T = Borrow(Immutable)(select OneToOneMarket::BorrowRecord.record<&OneToOneMarket::BorrowRecord>(BorrowGlobal(Immutable)>(sender))); + if Map::contains_key(record, Borrow(Immutable)(pool_owner)) { + Deref(Map::get(record, Borrow(Immutable)(pool_owner))) + } else { + 0 + } + } + } + } + private fun deposited_amount(account: &signer,pool_owner: address): u64 + acquires OneToOneMarket::DepositRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + return 0 + } else { + Tuple() + }; + { + let record: &Map::T = Borrow(Immutable)(select OneToOneMarket::DepositRecord.record<&OneToOneMarket::DepositRecord>(BorrowGlobal(Immutable)>(sender))); + if Map::contains_key(record, Borrow(Immutable)(pool_owner)) { + Deref(Map::get(record, Borrow(Immutable)(pool_owner))) + } else { + 0 + } + } + } + } + private fun max_borrow_amount(account: &signer,pool_owner: address): u64 + acquires OneToOneMarket::Price(*) + acquires OneToOneMarket::Pool(*) + acquires OneToOneMarket::DepositRecord(*) + acquires OneToOneMarket::BorrowRecord(*) + { + { + let input_deposited: u64 = OneToOneMarket::deposited_amount(account, pool_owner); + { + let output_deposited: u64 = OneToOneMarket::borrowed_amount(account, pool_owner); + { + let input_into_output: u64 = Mul(input_deposited, select OneToOneMarket::Price.price<&OneToOneMarket::Price>(BorrowGlobal(Immutable)>(pool_owner))); + { + let max_output: u64 = if Lt(input_into_output, output_deposited) { + 0 + } else { + Sub(input_into_output, output_deposited) + }; + { + let available_output: u64 = { + let pool: &OneToOneMarket::Pool = BorrowGlobal(Immutable)>(pool_owner); + Token::value(Borrow(Immutable)(select OneToOneMarket::Pool.coin<&OneToOneMarket::Pool>(pool))) + }; + if Lt(max_output, available_output) { + max_output + } else { + available_output + } + } + } + } + } + } + } + public fun register_price(account: &signer,initial_in: Token::Coin<#0>,initial_out: Token::Coin<#1>,price: u64) { + OneToOneMarket::accept(account, initial_in); + OneToOneMarket::accept(account, initial_out); + MoveTo>(account, pack OneToOneMarket::Price(price)) + } + private fun update_borrow_record(account: &signer,pool_owner: address,amount: u64) + acquires OneToOneMarket::BorrowRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + MoveTo>(account, pack OneToOneMarket::BorrowRecord(Map::empty())) + } else { + Tuple() + }; + { + let record: &mut Map::T = Borrow(Mutable)(select OneToOneMarket::BorrowRecord.record<&mut OneToOneMarket::BorrowRecord>(BorrowGlobal(Mutable)>(sender))); + if Map::contains_key(Freeze(false)(record), Borrow(Immutable)(pool_owner)) { + { + let old_amount: u64 = Map::remove(Freeze(false)(record), Borrow(Immutable)(pool_owner)); + amount: u64 = Add(amount, old_amount); + Tuple() + } + } else { + Tuple() + }; + Map::insert(Freeze(false)(record), pool_owner, amount) + } + } + } + private fun update_deposit_record(account: &signer,pool_owner: address,amount: u64) + acquires OneToOneMarket::DepositRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + MoveTo>(account, pack OneToOneMarket::DepositRecord(Map::empty())) + } else { + Tuple() + }; + { + let record: &mut Map::T = Borrow(Mutable)(select OneToOneMarket::DepositRecord.record<&mut OneToOneMarket::DepositRecord>(BorrowGlobal(Mutable)>(sender))); + if Map::contains_key(Freeze(false)(record), Borrow(Immutable)(pool_owner)) { + { + let old_amount: u64 = Map::remove(Freeze(false)(record), Borrow(Immutable)(pool_owner)); + amount: u64 = Add(amount, old_amount); + Tuple() + } + } else { + Tuple() + }; + Map::insert(Freeze(false)(record), pool_owner, amount) + } + } + } +} // end 0x3::OneToOneMarket +module 0x70dd::ToddNickels { + use 0x2::Token; // resolved as: 0x2::Token + use std::signer; + struct T { + dummy_field: bool, + } + struct Wallet { + nickels: Token::Coin, + } + public fun init(account: &signer) { + if Eq
(signer::address_of(account), 0x70dd) { + Tuple() + } else { + Abort(42) + }; + MoveTo(account, pack ToddNickels::Wallet(Token::create(pack ToddNickels::T(false), 0))) + } + public fun destroy(c: Token::Coin) + acquires ToddNickels::Wallet(*) + { + Token::deposit(Borrow(Mutable)(select ToddNickels::Wallet.nickels<&mut ToddNickels::Wallet>(BorrowGlobal(Mutable)(0x70dd))), c) + } + public fun mint(account: &signer): Token::Coin { + if Eq
(signer::address_of(account), 0x70dd) { + Tuple() + } else { + Abort(42) + }; + Token::create(pack ToddNickels::T(false), 5) + } +} // end 0x70dd::ToddNickels diff --git a/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.move b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.move new file mode 100644 index 0000000000000..a7cdc2f366054 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.move @@ -0,0 +1,240 @@ + +address 0x2 { + +module Map { + native struct T has copy, drop, store; + + native public fun empty(): T; + + native public fun get(m: &T, k: &K): &V; + native public fun get_mut(m: &mut T, k: &K): &mut V; + + native public fun contains_key(m: &T, k: &K): bool; + // throws on duplicate as I don't feel like mocking up Option + native public fun insert(m: &T, k: K, v: V); + // throws on miss as I don't feel like mocking up Option + native public fun remove(m: &T, k: &K): V; +} + +} + +address 0x2 { + +module Token { + + struct Coin has store { + type: AssetType, + value: u64, + } + + // control the minting/creation in the defining module of `ATy` + public fun create(type: ATy, value: u64): Coin { + Coin { type, value } + } + + public fun value(coin: &Coin): u64 { + coin.value + } + + public fun split(coin: Coin, amount: u64): (Coin, Coin) { + let other = withdraw(&mut coin, amount); + (coin, other) + } + + public fun withdraw(coin: &mut Coin, amount: u64): Coin { + assert!(coin.value >= amount, 10); + coin.value = coin.value - amount; + Coin { type: *&coin.type, value: amount } + } + + public fun join(xus: Coin, coin2: Coin): Coin { + deposit(&mut xus, coin2); + xus + } + + public fun deposit(coin: &mut Coin, check: Coin) { + let Coin { value, type } = check; + assert!(&coin.type == &type, 42); + coin.value = coin.value + value; + } + + public fun destroy_zero(coin: Coin) { + let Coin { value, type: _ } = coin; + assert!(value == 0, 11) + } + +} + +} + +address 0x3 { + +module OneToOneMarket { + use std::signer; + use 0x2::Map; + use 0x2::Token; + + struct Pool has key { + coin: Token::Coin, + } + + struct DepositRecord has key { + // pool owner => amount + record: Map::T + } + + struct BorrowRecord has key { + // pool owner => amount + record: Map::T + } + + struct Price has key { + price: u64, + } + + fun accept(account: &signer, init: Token::Coin) { + let sender = signer::address_of(account); + assert!(!exists>(sender), 42); + move_to(account, Pool { coin: init }) + } + + public fun register_price( + account: &signer, + initial_in: Token::Coin, + initial_out: Token::Coin, + price: u64 + ) { + accept(account, initial_in); + accept(account, initial_out); + move_to(account, Price { price }) + } + + public fun deposit(account: &signer, pool_owner: address, coin: Token::Coin) + acquires Pool, DepositRecord + { + let amount = Token::value(&coin); + + update_deposit_record(account, pool_owner, amount); + + let pool = borrow_global_mut>(pool_owner); + Token::deposit(&mut pool.coin, coin) + } + + public fun borrow( + account: &signer, + pool_owner: address, + amount: u64, + ): Token::Coin + acquires Price, Pool, DepositRecord, BorrowRecord + { + assert!(amount <= max_borrow_amount(account, pool_owner), 1025); + + update_borrow_record(account, pool_owner, amount); + + let pool = borrow_global_mut>(pool_owner); + Token::withdraw(&mut pool.coin, amount) + } + + fun max_borrow_amount(account: &signer, pool_owner: address): u64 + acquires Price, Pool, DepositRecord, BorrowRecord + { + let input_deposited = deposited_amount(account, pool_owner); + let output_deposited = borrowed_amount(account, pool_owner); + + let input_into_output = + input_deposited * borrow_global>(pool_owner).price; + let max_output = + if (input_into_output < output_deposited) 0 + else (input_into_output - output_deposited); + let available_output = { + let pool = borrow_global>(pool_owner); + Token::value(&pool.coin) + }; + if (max_output < available_output) max_output else available_output + + } + + fun update_deposit_record(account: &signer, pool_owner: address, amount: u64) + acquires DepositRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) { + move_to(account, DepositRecord { record: Map::empty() }) + }; + let record = &mut borrow_global_mut>(sender).record; + if (Map::contains_key(record, &pool_owner)) { + let old_amount = Map::remove(record, &pool_owner); + amount = amount + old_amount; + }; + Map::insert(record, pool_owner, amount) + } + + fun update_borrow_record(account: &signer, pool_owner: address, amount: u64) + acquires BorrowRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) { + move_to(account, BorrowRecord { record: Map::empty() }) + }; + let record = &mut borrow_global_mut>(sender).record; + if (Map::contains_key(record, &pool_owner)) { + let old_amount = Map::remove(record, &pool_owner); + amount = amount + old_amount; + }; + Map::insert(record, pool_owner, amount) + } + + fun deposited_amount(account: &signer, pool_owner: address): u64 + acquires DepositRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) return 0; + + let record = &borrow_global>(sender).record; + if (Map::contains_key(record, &pool_owner)) *Map::get(record, &pool_owner) + else 0 + } + + fun borrowed_amount(account: &signer, pool_owner: address): u64 + acquires BorrowRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) return 0; + + let record = &borrow_global>(sender).record; + if (Map::contains_key(record, &pool_owner)) *Map::get(record, &pool_owner) + else 0 + } +} + +} + +address 0x70DD { + +module ToddNickels { + use 0x2::Token; + use std::signer; + + struct T has copy, drop, store {} + + struct Wallet has key { + nickels: Token::Coin, + } + + public fun init(account: &signer) { + assert!(signer::address_of(account) == @0x70DD, 42); + move_to(account, Wallet { nickels: Token::create(T{}, 0) }) + } + + public fun mint(account: &signer): Token::Coin { + assert!(signer::address_of(account) == @0x70DD, 42); + Token::create(T{}, 5) + } + + public fun destroy(c: Token::Coin) acquires Wallet { + Token::deposit(&mut borrow_global_mut(@0x70DD).nickels, c) + } + +} + +} diff --git a/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.exp b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.exp new file mode 100644 index 0000000000000..b2f8b5a64657b --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.exp @@ -0,0 +1,247 @@ +// -- Model dump before bytecode pipeline +module 0x2::Token { + struct Coin { + type: #0, + value: u64, + } + public fun create(type: #0,value: u64): Token::Coin<#0> { + pack Token::Coin(type, value) + } + public fun value(coin: &Token::Coin<#0>): u64 { + select Token::Coin.value<&Token::Coin>(coin) + } + public fun deposit(coin: &mut Token::Coin<#0>,check: Token::Coin<#0>) { + { + let Token::Coin{ type, value } = check; + if Eq(Borrow(Immutable)(select Token::Coin.type<&mut Token::Coin>(coin)), Borrow(Immutable)(type)) { + Tuple() + } else { + Abort(42) + }; + select Token::Coin.value<&mut Token::Coin>(coin) = Add(select Token::Coin.value<&mut Token::Coin>(coin), value); + Tuple() + } + } + public fun destroy_zero(coin: Token::Coin<#0>) { + { + let Token::Coin{ type: _, value } = coin; + if Eq(value, 0) { + Tuple() + } else { + Abort(11) + } + } + } + public fun join(xus: Token::Coin<#0>,coin2: Token::Coin<#0>): Token::Coin<#0> { + Token::deposit(Borrow(Mutable)(xus), coin2); + xus + } + public fun split(coin: Token::Coin<#0>,amount: u64): (Token::Coin<#0>, Token::Coin<#0>) { + { + let other: Token::Coin = Token::withdraw(Borrow(Mutable)(coin), amount); + Tuple(coin, other) + } + } + public fun withdraw(coin: &mut Token::Coin<#0>,amount: u64): Token::Coin<#0> { + if Ge(select Token::Coin.value<&mut Token::Coin>(coin), amount) { + Tuple() + } else { + Abort(10) + }; + select Token::Coin.value<&mut Token::Coin>(coin) = Sub(select Token::Coin.value<&mut Token::Coin>(coin), amount); + pack Token::Coin(Deref(Borrow(Immutable)(select Token::Coin.type<&mut Token::Coin>(coin))), amount) + } +} // end 0x2::Token +module 0x70dd::ToddNickels { + use 0x2::Token; // resolved as: 0x2::Token + use std::signer; + struct T { + dummy_field: bool, + } + struct Wallet { + nickels: Token::Coin, + } + public fun init(account: &signer) { + if Eq
(signer::address_of(account), 0x70dd) { + Tuple() + } else { + Abort(42) + }; + MoveTo(account, pack ToddNickels::Wallet(Token::create(pack ToddNickels::T(false), 0))) + } + public fun destroy(c: Token::Coin) + acquires ToddNickels::Wallet(*) + { + Token::deposit(Borrow(Mutable)(select ToddNickels::Wallet.nickels<&mut ToddNickels::Wallet>(BorrowGlobal(Mutable)(0x70dd))), c) + } + public fun mint(account: &signer): Token::Coin { + if Eq
(signer::address_of(account), 0x70dd) { + Tuple() + } else { + Abort(42) + }; + Token::create(pack ToddNickels::T(false), 5) + } +} // end 0x70dd::ToddNickels +module 0xb055::OneToOneMarket { + use std::signer; + use 0x2::Token; // resolved as: 0x2::Token + struct BorrowRecord { + record: u64, + } + struct DepositRecord { + record: u64, + } + struct Pool { + coin: Token::Coin<#0>, + } + struct Price { + price: u64, + } + public fun borrow(account: &signer,amount: u64): Token::Coin<#1> + acquires OneToOneMarket::Price(*) + acquires OneToOneMarket::Pool(*) + acquires OneToOneMarket::DepositRecord(*) + acquires OneToOneMarket::BorrowRecord(*) + { + if Le(amount, OneToOneMarket::max_borrow_amount(account)) { + Tuple() + } else { + Abort(1025) + }; + OneToOneMarket::update_borrow_record(account, amount); + { + let pool: &mut OneToOneMarket::Pool = BorrowGlobal(Mutable)>(0xb055); + Token::withdraw(Borrow(Mutable)(select OneToOneMarket::Pool.coin<&mut OneToOneMarket::Pool>(pool)), amount) + } + } + public fun deposit(account: &signer,coin: Token::Coin<#0>) + acquires OneToOneMarket::Pool(*) + acquires OneToOneMarket::DepositRecord(*) + { + { + let amount: u64 = Token::value(Borrow(Immutable)(coin)); + OneToOneMarket::update_deposit_record(account, amount); + { + let pool: &mut OneToOneMarket::Pool = BorrowGlobal(Mutable)>(0xb055); + Token::deposit(Borrow(Mutable)(select OneToOneMarket::Pool.coin<&mut OneToOneMarket::Pool>(pool)), coin) + } + } + } + private fun accept(account: &signer,init: Token::Coin<#0>) { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + Tuple() + } else { + Abort(42) + }; + MoveTo>(account, pack OneToOneMarket::Pool(init)) + } + } + private fun borrowed_amount(account: &signer): u64 + acquires OneToOneMarket::BorrowRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + return 0 + } else { + Tuple() + }; + select OneToOneMarket::BorrowRecord.record<&OneToOneMarket::BorrowRecord>(BorrowGlobal(Immutable)>(sender)) + } + } + private fun deposited_amount(account: &signer): u64 + acquires OneToOneMarket::DepositRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + return 0 + } else { + Tuple() + }; + select OneToOneMarket::DepositRecord.record<&OneToOneMarket::DepositRecord>(BorrowGlobal(Immutable)>(sender)) + } + } + private fun max_borrow_amount(account: &signer): u64 + acquires OneToOneMarket::Price(*) + acquires OneToOneMarket::Pool(*) + acquires OneToOneMarket::DepositRecord(*) + acquires OneToOneMarket::BorrowRecord(*) + { + { + let input_deposited: u64 = OneToOneMarket::deposited_amount(account); + { + let output_deposited: u64 = OneToOneMarket::borrowed_amount(account); + { + let input_into_output: u64 = Mul(input_deposited, select OneToOneMarket::Price.price<&OneToOneMarket::Price>(BorrowGlobal(Immutable)>(0xb055))); + { + let max_output: u64 = if Lt(input_into_output, output_deposited) { + 0 + } else { + Sub(input_into_output, output_deposited) + }; + { + let available_output: u64 = { + let pool: &OneToOneMarket::Pool = BorrowGlobal(Immutable)>(0xb055); + Token::value(Borrow(Immutable)(select OneToOneMarket::Pool.coin<&OneToOneMarket::Pool>(pool))) + }; + if Lt(max_output, available_output) { + max_output + } else { + available_output + } + } + } + } + } + } + } + public fun register_price(account: &signer,initial_in: Token::Coin<#0>,initial_out: Token::Coin<#1>,price: u64) { + { + let sender: address = signer::address_of(account); + if Eq
(sender, 0xb055) { + Tuple() + } else { + Abort(42) + }; + OneToOneMarket::accept(account, initial_in); + OneToOneMarket::accept(account, initial_out); + MoveTo>(account, pack OneToOneMarket::Price(price)) + } + } + private fun update_borrow_record(account: &signer,amount: u64) + acquires OneToOneMarket::BorrowRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + MoveTo>(account, pack OneToOneMarket::BorrowRecord(0)) + } else { + Tuple() + }; + { + let record: &mut u64 = Borrow(Mutable)(select OneToOneMarket::BorrowRecord.record<&mut OneToOneMarket::BorrowRecord>(BorrowGlobal(Mutable)>(sender))); + record = Add(Deref(record), amount) + } + } + } + private fun update_deposit_record(account: &signer,amount: u64) + acquires OneToOneMarket::DepositRecord(*) + { + { + let sender: address = signer::address_of(account); + if Not(exists>(sender)) { + MoveTo>(account, pack OneToOneMarket::DepositRecord(0)) + } else { + Tuple() + }; + { + let record: &mut u64 = Borrow(Mutable)(select OneToOneMarket::DepositRecord.record<&mut OneToOneMarket::DepositRecord>(BorrowGlobal(Mutable)>(sender))); + record = Add(Deref(record), amount) + } + } + } +} // end 0xb055::OneToOneMarket diff --git a/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.move b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.move new file mode 100644 index 0000000000000..64e69446ed1f7 --- /dev/null +++ b/third_party/move/move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.move @@ -0,0 +1,204 @@ +address 0x2 { + +module Token { + + struct Coin has store { + type: AssetType, + value: u64, + } + + // control the minting/creation in the defining module of `ATy` + public fun create(type: ATy, value: u64): Coin { + Coin { type, value } + } + + public fun value(coin: &Coin): u64 { + coin.value + } + + public fun split(coin: Coin, amount: u64): (Coin, Coin) { + let other = withdraw(&mut coin, amount); + (coin, other) + } + + public fun withdraw(coin: &mut Coin, amount: u64): Coin { + assert!(coin.value >= amount, 10); + coin.value = coin.value - amount; + Coin { type: *&coin.type, value: amount } + } + + public fun join(xus: Coin, coin2: Coin): Coin { + deposit(&mut xus, coin2); + xus + } + + public fun deposit(coin: &mut Coin, check: Coin) { + let Coin { value, type } = check; + assert!(&coin.type == &type, 42); + coin.value = coin.value + value; + } + + public fun destroy_zero(coin: Coin) { + let Coin { value, type: _ } = coin; + assert!(value == 0, 11) + } + +} + +} + +address 0xB055 { + +module OneToOneMarket { + use std::signer; + use 0x2::Token; + + struct Pool has key { + coin: Token::Coin, + } + + struct DepositRecord has key { + record: u64, + } + + struct BorrowRecord has key { + record: u64, + } + + struct Price has key { + price: u64, + } + + fun accept(account: &signer, init: Token::Coin) { + let sender = signer::address_of(account); + assert!(!exists>(sender), 42); + move_to(account, Pool { coin: init }) + } + + public fun register_price( + account: &signer, + initial_in: Token::Coin, + initial_out: Token::Coin, + price: u64 + ) { + let sender = signer::address_of(account); + assert!(sender == @0xB055, 42); // assert sender is module writer + accept(account, initial_in); + accept(account, initial_out); + move_to(account, Price { price }) + } + + public fun deposit(account: &signer, coin: Token::Coin) + acquires Pool, DepositRecord + { + let amount = Token::value(&coin); + + update_deposit_record(account, amount); + + let pool = borrow_global_mut>(@0xB055); + Token::deposit(&mut pool.coin, coin) + } + + public fun borrow( + account: &signer, + amount: u64, + ): Token::Coin + acquires Price, Pool, DepositRecord, BorrowRecord + { + assert!(amount <= max_borrow_amount(account), 1025); + + update_borrow_record(account, amount); + + let pool = borrow_global_mut>(@0xB055); + Token::withdraw(&mut pool.coin, amount) + } + + fun max_borrow_amount(account: &signer): u64 + acquires Price, Pool, DepositRecord, BorrowRecord + { + let input_deposited = deposited_amount(account); + let output_deposited = borrowed_amount(account); + + let input_into_output = + input_deposited * borrow_global>(@0xB055).price; + let max_output = + if (input_into_output < output_deposited) 0 + else (input_into_output - output_deposited); + let available_output = { + let pool = borrow_global>(@0xB055); + Token::value(&pool.coin) + }; + if (max_output < available_output) max_output else available_output + + } + + fun update_deposit_record(account: &signer, amount: u64) + acquires DepositRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) { + move_to(account, DepositRecord { record: 0 }) + }; + let record = &mut borrow_global_mut>(sender).record; + *record = *record + amount + } + + fun update_borrow_record(account: &signer, amount: u64) + acquires BorrowRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) { + move_to(account, BorrowRecord { record: 0 }) + }; + let record = &mut borrow_global_mut>(sender).record; + *record = *record + amount + } + + fun deposited_amount(account: &signer): u64 + acquires DepositRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) return 0; + borrow_global>(sender).record + } + + fun borrowed_amount(account: &signer): u64 + acquires BorrowRecord + { + let sender = signer::address_of(account); + if (!exists>(sender)) return 0; + borrow_global>(sender).record + } +} + +} + +address 0x70DD { + +module ToddNickels { + use 0x2::Token; + use std::signer; + + struct T has copy, drop, store {} + + struct Wallet has key { + nickels: Token::Coin, + } + + public fun init(account: &signer) { + assert!(signer::address_of(account) == @0x70DD, 42); + move_to(account, Wallet { nickels: Token::create(T{}, 0) }) + } + + public fun mint(account: &signer): Token::Coin { + assert!(signer::address_of(account) == @0x70DD, 42); + Token::create(T{}, 5) + } + + public fun destroy(c: Token::Coin) acquires Wallet { + Token::deposit(&mut borrow_global_mut(@0x70DD).nickels, c) + } + +} + +} diff --git a/third_party/move/move-compiler-v2/tests/v1.matched b/third_party/move/move-compiler-v2/tests/v1.matched index 56577566e080d..4d58ac5dee0e0 100644 --- a/third_party/move/move-compiler-v2/tests/v1.matched +++ b/third_party/move/move-compiler-v2/tests/v1.matched @@ -1,6 +1,6 @@ -WARNING: test `move-compiler-v2/tests/more-v1/verification/cross_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/cross_module_valid.move` share common key `verification/cross_module_valid.verification.move`, discarding former one -WARNING: test `move-compiler-v2/tests/more-v1/verification/double_annotation.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/double_annotation.move` share common key `verification/double_annotation.verification.move`, discarding former one -WARNING: test `move-compiler-v2/tests/more-v1/verification/single_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/single_module_valid.move` share common key `verification/single_module_valid.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/verification/cross_module_valid.verification.move` and `move-compiler-v2/tests/verification/verify/cross_module_valid.move` share common key `verification/cross_module_valid.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/verification/double_annotation.verification.move` and `move-compiler-v2/tests/verification/verify/double_annotation.move` share common key `verification/double_annotation.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/verification/single_module_valid.verification.move` and `move-compiler-v2/tests/verification/verify/single_module_valid.move` share common key `verification/single_module_valid.verification.move`, discarding former one move-compiler/tests/move_check/translated_ir_tests/move/borrow_tests/borrow_global_acquires_1.exp move-compiler-v2/tests/acquires-checker/v1-borrow-tests/borrow_global_acquires_1.exp move-compiler/tests/move_check/translated_ir_tests/move/borrow_tests/borrow_global_acquires_2.exp move-compiler-v2/tests/acquires-checker/v1-borrow-tests/borrow_global_acquires_2.exp move-compiler/tests/move_check/translated_ir_tests/move/borrow_tests/borrow_global_acquires_3.exp move-compiler-v2/tests/acquires-checker/v1-borrow-tests/borrow_global_acquires_3.exp @@ -155,17 +155,19 @@ move-compiler/tests/move_check/translated_ir_tests/move/commands/use_before_assi move-compiler/tests/move_check/control_flow/for_loop_empty_novar.exp move-compiler-v2/tests/checking/control_flow/for_loop_empty_novar.exp move-compiler/tests/move_check/control_flow/for_type_mismatch.exp move-compiler-v2/tests/checking/control_flow/for_type_mismatch.exp move-compiler/tests/move_check/control_flow/loop_after_loop.exp move-compiler-v2/tests/checking/control_flow/loop_after_loop.exp -move-compiler/tests/move_check/deprecated/deprecated_constant_duplicated_struct.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_constant_duplicated_struct.exp -move-compiler/tests/move_check/deprecated/deprecated_constant_duplicated_struct2.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_constant_duplicated_struct2.exp -move-compiler/tests/move_check/deprecated/deprecated_field_type.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_field_type.exp -move-compiler/tests/move_check/deprecated/deprecated_field_type2.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_field_type2.exp -move-compiler/tests/move_check/deprecated/deprecated_placement_address.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_address.exp -move-compiler/tests/move_check/deprecated/deprecated_placement_address_module_members.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_address_module_members.exp -move-compiler/tests/move_check/deprecated/deprecated_placement_members.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_members.exp -move-compiler/tests/move_check/deprecated/deprecated_placement_module.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_module.exp -move-compiler/tests/move_check/deprecated/deprecated_placement_module2.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_module2.exp -move-compiler/tests/move_check/deprecated/deprecated_placement_module_members.exp move-compiler-v2/tests/more-v1/deprecated/deprecated_placement_module_members.exp -move-compiler/tests/move_check/deprecated/public_script.exp move-compiler-v2/tests/more-v1/deprecated/public_script.exp +move-compiler/tests/move_check/deprecated/deprecated_constant_duplicated_struct.exp move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct.exp +move-compiler/tests/move_check/deprecated/deprecated_constant_duplicated_struct2.exp move-compiler-v2/tests/deprecated/deprecated_constant_duplicated_struct2.exp +move-compiler/tests/move_check/deprecated/deprecated_field_type.exp move-compiler-v2/tests/deprecated/deprecated_field_type.exp +move-compiler/tests/move_check/deprecated/deprecated_field_type2.exp move-compiler-v2/tests/deprecated/deprecated_field_type2.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_address.exp move-compiler-v2/tests/deprecated/deprecated_placement_address.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_address_module_members.exp move-compiler-v2/tests/deprecated/deprecated_placement_address_module_members.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_members.exp move-compiler-v2/tests/deprecated/deprecated_placement_members.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_module.exp move-compiler-v2/tests/deprecated/deprecated_placement_module.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_module2.exp move-compiler-v2/tests/deprecated/deprecated_placement_module2.exp +move-compiler/tests/move_check/deprecated/deprecated_placement_module_members.exp move-compiler-v2/tests/deprecated/deprecated_placement_module_members.exp +move-compiler/tests/move_check/deprecated/public_script.exp move-compiler-v2/tests/deprecated/public_script.exp +move-compiler/tests/move_check/examples/multi_pool_money_market_token.exp move-compiler-v2/tests/checking/typing/v1-examples/multi_pool_money_market_token.exp +move-compiler/tests/move_check/examples/simple_money_market_token.exp move-compiler-v2/tests/checking/typing/v1-examples/simple_money_market_token.exp move-compiler/tests/move_check/folding/empty_vectors.exp move-compiler-v2/tests/folding/empty_vectors.exp move-compiler/tests/move_check/folding/empty_vectors2.exp move-compiler-v2/tests/folding/empty_vectors2.exp move-compiler/tests/move_check/folding/non_constant_empty_vec.exp move-compiler-v2/tests/folding/non_constant_empty_vec.exp @@ -637,12 +639,12 @@ move-compiler/tests/move_check/unit_test/test_filter_function.exp move-compile move-compiler/tests/move_check/unit_test/test_filter_struct.exp move-compiler-v2/tests/unit_test/notest/test_filter_struct.exp move-compiler/tests/move_check/unit_test/valid_test_module.exp move-compiler-v2/tests/unit_test/notest/valid_test_module.exp move-compiler/tests/move_check/unit_test/valid_test_module.unit_test.exp move-compiler-v2/tests/unit_test/test/valid_test_module.exp -move-compiler/tests/move_check/verification/cross_module_invalid.exp move-compiler-v2/tests/more-v1/verification/noverify/cross_module_invalid.exp -move-compiler/tests/move_check/verification/cross_module_valid.exp move-compiler-v2/tests/more-v1/verification/noverify/cross_module_valid.exp -move-compiler/tests/move_check/verification/cross_module_valid.verification.exp move-compiler-v2/tests/more-v1/verification/verify/cross_module_valid.exp -move-compiler/tests/move_check/verification/double_annotation.exp move-compiler-v2/tests/more-v1/verification/noverify/double_annotation.exp -move-compiler/tests/move_check/verification/double_annotation.verification.exp move-compiler-v2/tests/more-v1/verification/verify/double_annotation.exp -move-compiler/tests/move_check/verification/single_module_invalid.exp move-compiler-v2/tests/more-v1/verification/noverify/single_module_invalid.exp -move-compiler/tests/move_check/verification/single_module_invalid.verification.exp move-compiler-v2/tests/more-v1/verification/single_module_invalid.verification.exp -move-compiler/tests/move_check/verification/single_module_valid.exp move-compiler-v2/tests/more-v1/verification/noverify/single_module_valid.exp -move-compiler/tests/move_check/verification/single_module_valid.verification.exp move-compiler-v2/tests/more-v1/verification/verify/single_module_valid.exp +move-compiler/tests/move_check/verification/cross_module_invalid.exp move-compiler-v2/tests/verification/noverify/cross_module_invalid.exp +move-compiler/tests/move_check/verification/cross_module_valid.exp move-compiler-v2/tests/verification/noverify/cross_module_valid.exp +move-compiler/tests/move_check/verification/cross_module_valid.verification.exp move-compiler-v2/tests/verification/verify/cross_module_valid.exp +move-compiler/tests/move_check/verification/double_annotation.exp move-compiler-v2/tests/verification/noverify/double_annotation.exp +move-compiler/tests/move_check/verification/double_annotation.verification.exp move-compiler-v2/tests/verification/verify/double_annotation.exp +move-compiler/tests/move_check/verification/single_module_invalid.exp move-compiler-v2/tests/verification/noverify/single_module_invalid.exp +move-compiler/tests/move_check/verification/single_module_invalid.verification.exp move-compiler-v2/tests/verification/single_module_invalid.verification.exp +move-compiler/tests/move_check/verification/single_module_valid.exp move-compiler-v2/tests/verification/noverify/single_module_valid.exp +move-compiler/tests/move_check/verification/single_module_valid.verification.exp move-compiler-v2/tests/verification/verify/single_module_valid.exp diff --git a/third_party/move/move-compiler-v2/tests/v1.unmatched b/third_party/move/move-compiler-v2/tests/v1.unmatched index 9ee038364c30c..8e1885290e94b 100644 --- a/third_party/move/move-compiler-v2/tests/v1.unmatched +++ b/third_party/move/move-compiler-v2/tests/v1.unmatched @@ -1,6 +1,6 @@ -WARNING: test `move-compiler-v2/tests/more-v1/verification/cross_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/cross_module_valid.move` share common key `verification/cross_module_valid.verification.move`, discarding former one -WARNING: test `move-compiler-v2/tests/more-v1/verification/double_annotation.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/double_annotation.move` share common key `verification/double_annotation.verification.move`, discarding former one -WARNING: test `move-compiler-v2/tests/more-v1/verification/single_module_valid.verification.move` and `move-compiler-v2/tests/more-v1/verification/verify/single_module_valid.move` share common key `verification/single_module_valid.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/verification/cross_module_valid.verification.move` and `move-compiler-v2/tests/verification/verify/cross_module_valid.move` share common key `verification/cross_module_valid.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/verification/double_annotation.verification.move` and `move-compiler-v2/tests/verification/verify/double_annotation.move` share common key `verification/double_annotation.verification.move`, discarding former one +WARNING: test `move-compiler-v2/tests/verification/single_module_valid.verification.move` and `move-compiler-v2/tests/verification/verify/single_module_valid.move` share common key `verification/single_module_valid.verification.move`, discarding former one move-compiler/tests/move_check/borrow_tests/{ borrow_global_acquires_duplicate_annotation.move, eq_bad.move, @@ -36,10 +36,6 @@ move-compiler/tests/move_check/deprecated/{ assert_function.move, deprecated_placement_basecase.move, } -move-compiler/tests/move_check/examples/{ - multi_pool_money_market_token.move, - simple_money_market_token.move, -} move-compiler/tests/move_check/liveness/{ copy_after_move.move, dead_refs_branch.move, diff --git a/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs b/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs index c3c7b93ebaf3b..1384458cb3422 100644 --- a/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs +++ b/third_party/move/move-compiler-v2/tools/testdiff/src/main.rs @@ -83,6 +83,7 @@ static UNIT_PATH_REMAP: Lazy> = Lazy::new(|| { ("typing/v1-naming", "naming"), ("typing/v1-operators", "operators"), ("typing/v1-signer", "signer"), + ("typing/v1-examples", "examples"), ("live-var/v1-commands", "commands"), ("live-var", "liveness"), ("visibility-checker/v1-typing", "typing"), diff --git a/third_party/move/move-model/src/builder/model_builder.rs b/third_party/move/move-model/src/builder/model_builder.rs index 5c478ba909400..7ff3a4c42d868 100644 --- a/third_party/move/move-model/src/builder/model_builder.rs +++ b/third_party/move/move-model/src/builder/model_builder.rs @@ -127,6 +127,7 @@ pub(crate) struct StructEntry { /// Whether the struct is originally empty /// always false when it is enum pub is_empty_struct: bool, + pub is_native: bool, } #[derive(Debug, Clone)] @@ -354,6 +355,7 @@ impl<'env> ModelBuilder<'env> { abilities: AbilitySet, type_params: Vec, layout: StructLayout, + is_native: bool, ) { let entry = StructEntry { loc, @@ -365,6 +367,7 @@ impl<'env> ModelBuilder<'env> { layout, receiver_functions: BTreeMap::new(), is_empty_struct: false, + is_native, }; self.struct_table.insert(name.clone(), entry); self.reverse_struct_table diff --git a/third_party/move/move-model/src/builder/module_builder.rs b/third_party/move/move-model/src/builder/module_builder.rs index 889d424963aeb..50fd498330115 100644 --- a/third_party/move/move-model/src/builder/module_builder.rs +++ b/third_party/move/move-model/src/builder/module_builder.rs @@ -492,7 +492,8 @@ impl<'env, 'translator> ModuleBuilder<'env, 'translator> { struct_id, abilities, type_params, - StructLayout::None, // will be filled in during definition analysis + StructLayout::None, // will be filled in during definition analysis, + matches!(def.layout, EA::StructLayout::Native(_)), ); } @@ -3451,6 +3452,7 @@ impl<'env, 'translator> ModuleBuilder<'env, 'translator> { Some(variants) }, spec: RefCell::new(spec), + is_native: entry.is_native, }; struct_data.insert(StructId::new(name.symbol), data); } diff --git a/third_party/move/move-model/src/model.rs b/third_party/move/move-model/src/model.rs index eb485ab3cd5ee..cc37fa9de0705 100644 --- a/third_party/move/move-model/src/model.rs +++ b/third_party/move/move-model/src/model.rs @@ -1641,6 +1641,7 @@ impl GlobalEnv { field_data, variants: None, spec: RefCell::new(Spec::default()), + is_native: false, } } @@ -3256,6 +3257,9 @@ pub struct StructData { /// Associated specification. pub(crate) spec: RefCell, + + /// Whether this struct is native + pub is_native: bool, } #[derive(Debug)] @@ -3474,6 +3478,11 @@ impl<'env> StructEnv<'env> { }) } + /// Return true if it is a native struct + pub fn is_native(&self) -> bool { + self.data.is_native + } + /// Return the number of fields in the struct. Notice of the struct has variants, this /// includes the sum of all fields in all variants. pub fn get_field_count(&self) -> usize { From b47348ce3ece20e703e78c81dee73a8b9f9cbd5f Mon Sep 17 00:00:00 2001 From: Renee Tso <8248583+rtso@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:16:27 -0400 Subject: [PATCH 014/469] Fix entry function filtering (#13867) * Fix entry function filtering * Comments --- .../indexer-grpc-data-service/src/service.rs | 206 +++++++++++++++++- .../src/filters/user_transaction.rs | 4 +- 2 files changed, 207 insertions(+), 3 deletions(-) diff --git a/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs b/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs index 752c642f59f72..f2faf42408631 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs +++ b/ecosystem/indexer-grpc/indexer-grpc-data-service/src/service.rs @@ -1011,13 +1011,50 @@ fn strip_transactions( mod tests { use super::*; use aptos_protos::transaction::v1::{ - transaction::TxnData, Event, Signature, Transaction, TransactionInfo, TransactionPayload, + transaction::TxnData, transaction_payload::Payload, EntryFunctionId, EntryFunctionPayload, + Event, MoveModuleId, Signature, Transaction, TransactionInfo, TransactionPayload, UserTransaction, UserTransactionRequest, WriteSetChange, }; use aptos_transaction_filter::{ boolean_transaction_filter::APIFilter, filters::UserTransactionFilterBuilder, + EntryFunctionFilterBuilder, UserTransactionPayloadFilterBuilder, }; + fn create_test_transaction( + module_address: String, + module_name: String, + function_name: String, + ) -> Transaction { + Transaction { + version: 1, + txn_data: Some(TxnData::User(UserTransaction { + request: Some(UserTransactionRequest { + payload: Some(TransactionPayload { + r#type: 1, + payload: Some(Payload::EntryFunctionPayload(EntryFunctionPayload { + function: Some(EntryFunctionId { + module: Some(MoveModuleId { + address: module_address, + name: module_name, + }), + name: function_name, + }), + ..Default::default() + })), + }), + signature: Some(Signature::default()), + ..Default::default() + }), + events: vec![Event::default()], + })), + info: Some(TransactionInfo { + changes: vec![WriteSetChange::default()], + ..Default::default() + }), + ..Default::default() + } + } + #[test] fn test_ensure_sequential_transactions_merges_and_sorts() { let transactions1 = (1..5) @@ -1063,6 +1100,10 @@ mod tests { assert_eq!(transactions1.last().unwrap().version, 11); } + const MODULE_ADDRESS: &str = "0x1234"; + const MODULE_NAME: &str = "module"; + const FUNCTION_NAME: &str = "function"; + #[test] fn test_transactions_are_stripped_correctly_sender_addresses() { let sender_address = "0x1234".to_string(); @@ -1112,4 +1153,167 @@ mod tests { assert_eq!(user_transaction.events.len(), 0); assert_eq!(txn.info.as_ref().unwrap().changes.len(), 0); } + + #[test] + fn test_transactions_are_stripped_correctly_module_address() { + let txn = create_test_transaction( + MODULE_ADDRESS.to_string(), + MODULE_NAME.to_string(), + FUNCTION_NAME.to_string(), + ); + // Testing filter with only address set + let filter = BooleanTransactionFilter::new_or(vec![BooleanTransactionFilter::from( + APIFilter::UserTransactionFilter( + UserTransactionFilterBuilder::default() + .payload( + UserTransactionPayloadFilterBuilder::default() + .function( + EntryFunctionFilterBuilder::default() + .address(MODULE_ADDRESS.to_string()) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ), + )]); + + let (filtered_txns, num_stripped) = strip_transactions(vec![txn.clone()], &filter); + assert_eq!(num_stripped, 1); + assert_eq!(filtered_txns.len(), 1); + let txn = filtered_txns.first().unwrap(); + let user_transaction = match &txn.txn_data { + Some(TxnData::User(user_transaction)) => user_transaction, + _ => panic!("Expected user transaction"), + }; + assert_eq!(user_transaction.request.as_ref().unwrap().payload, None); + assert_eq!(user_transaction.request.as_ref().unwrap().signature, None); + assert_eq!(user_transaction.events.len(), 0); + assert_eq!(txn.info.as_ref().unwrap().changes.len(), 0); + } + + #[test] + fn test_transactions_are_stripped_correctly_module_name() { + let txn = create_test_transaction( + MODULE_ADDRESS.to_string(), + MODULE_NAME.to_string(), + FUNCTION_NAME.to_string(), + ); + // Testing filter with only module set + let filter = BooleanTransactionFilter::new_or(vec![BooleanTransactionFilter::from( + APIFilter::UserTransactionFilter( + UserTransactionFilterBuilder::default() + .payload( + UserTransactionPayloadFilterBuilder::default() + .function( + EntryFunctionFilterBuilder::default() + .module(MODULE_NAME.to_string()) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ), + )]); + + let (filtered_txns, num_stripped) = strip_transactions(vec![txn.clone()], &filter); + assert_eq!(num_stripped, 1); + assert_eq!(filtered_txns.len(), 1); + let txn = filtered_txns.first().unwrap(); + let user_transaction = match &txn.txn_data { + Some(TxnData::User(user_transaction)) => user_transaction, + _ => panic!("Expected user transaction"), + }; + assert_eq!(user_transaction.request.as_ref().unwrap().payload, None); + assert_eq!(user_transaction.request.as_ref().unwrap().signature, None); + assert_eq!(user_transaction.events.len(), 0); + assert_eq!(txn.info.as_ref().unwrap().changes.len(), 0); + } + + #[test] + fn test_transactions_are_stripped_correctly_function_name() { + let txn = create_test_transaction( + MODULE_ADDRESS.to_string(), + MODULE_NAME.to_string(), + FUNCTION_NAME.to_string(), + ); + // Testing filter with only function set + let filter = BooleanTransactionFilter::new_or(vec![BooleanTransactionFilter::from( + APIFilter::UserTransactionFilter( + UserTransactionFilterBuilder::default() + .payload( + UserTransactionPayloadFilterBuilder::default() + .function( + EntryFunctionFilterBuilder::default() + .function(FUNCTION_NAME.to_string()) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ), + )]); + + let (filtered_txns, num_stripped) = strip_transactions(vec![txn.clone()], &filter); + assert_eq!(num_stripped, 1); + assert_eq!(filtered_txns.len(), 1); + let txn = filtered_txns.first().unwrap(); + let user_transaction = match &txn.txn_data { + Some(TxnData::User(user_transaction)) => user_transaction, + _ => panic!("Expected user transaction"), + }; + assert_eq!(user_transaction.request.as_ref().unwrap().payload, None); + assert_eq!(user_transaction.request.as_ref().unwrap().signature, None); + assert_eq!(user_transaction.events.len(), 0); + assert_eq!(txn.info.as_ref().unwrap().changes.len(), 0); + } + #[test] + fn test_transactions_are_not_stripped() { + let txn = create_test_transaction( + MODULE_ADDRESS.to_string(), + MODULE_NAME.to_string(), + FUNCTION_NAME.to_string(), + ); + // Testing filter with wrong filter + let filter = BooleanTransactionFilter::new_or(vec![BooleanTransactionFilter::from( + APIFilter::UserTransactionFilter( + UserTransactionFilterBuilder::default() + .payload( + UserTransactionPayloadFilterBuilder::default() + .function( + EntryFunctionFilterBuilder::default() + .function("0xrandom".to_string()) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ) + .build() + .unwrap(), + ), + )]); + + let (filtered_txns, num_stripped) = strip_transactions(vec![txn.clone()], &filter); + assert_eq!(num_stripped, 0); + assert_eq!(filtered_txns.len(), 1); + let txn = filtered_txns.first().unwrap(); + let user_transaction = match &txn.txn_data { + Some(TxnData::User(user_transaction)) => user_transaction, + _ => panic!("Expected user transaction"), + }; + assert_ne!(user_transaction.request.as_ref().unwrap().payload, None); + assert_ne!(user_transaction.request.as_ref().unwrap().signature, None); + assert_ne!(user_transaction.events.len(), 0); + assert_ne!(txn.info.as_ref().unwrap().changes.len(), 0); + } } diff --git a/ecosystem/indexer-grpc/transaction-filter/src/filters/user_transaction.rs b/ecosystem/indexer-grpc/transaction-filter/src/filters/user_transaction.rs index 6c4207bc39691..7e95b790f03c3 100644 --- a/ecosystem/indexer-grpc/transaction-filter/src/filters/user_transaction.rs +++ b/ecosystem/indexer-grpc/transaction-filter/src/filters/user_transaction.rs @@ -112,14 +112,14 @@ impl Filterable for EntryFunctionFilter { #[inline] fn is_allowed(&self, module_id: &EntryFunctionId) -> bool { - if !self.module.is_allowed(&module_id.name) { + if !self.function.is_allowed(&module_id.name) { return false; } if self.address.is_some() || self.function.is_some() { if let Some(module) = &module_id.module.as_ref() { if !(self.address.is_allowed(&module.address) - && self.function.is_allowed(&module.name)) + && self.module.is_allowed(&module.name)) { return false; } From 1550c46323990a283081d225a3973d481e9b99a7 Mon Sep 17 00:00:00 2001 From: Rustie Lin Date: Mon, 1 Jul 2024 10:59:25 -0700 Subject: [PATCH 015/469] [dev_setup] retry flaky dotnet install (#13877) --- scripts/dev_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev_setup.sh b/scripts/dev_setup.sh index da6f12abe86da..a5f2d520a502a 100755 --- a/scripts/dev_setup.sh +++ b/scripts/dev_setup.sh @@ -543,7 +543,7 @@ function install_dotnet { # Below we need to (a) set TERM variable because the .net installer expects it and it is not set # in some environments (b) use bash not sh because the installer uses bash features. # NOTE: use wget to better follow the redirect - wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh + wget --tries 10 --retry-connrefused --waitretry=5 https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh chmod +x dotnet-install.sh ./dotnet-install.sh --channel $DOTNET_VERSION --install-dir "${DOTNET_INSTALL_DIR}" --version latest rm dotnet-install.sh From fae6899f166fb36af019f27d911a3941671e5cf3 Mon Sep 17 00:00:00 2001 From: Satya Vusirikala Date: Mon, 1 Jul 2024 13:46:54 -0500 Subject: [PATCH 016/469] Change proof queue data structure (#13866) --- consensus/src/quorum_store/utils.rs | 97 ++++++++++++++--------------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/consensus/src/quorum_store/utils.rs b/consensus/src/quorum_store/utils.rs index e4ee5357dde9d..f584c836d7c54 100644 --- a/consensus/src/quorum_store/utils.rs +++ b/consensus/src/quorum_store/utils.rs @@ -198,11 +198,10 @@ pub struct ProofQueue { author_to_batches: HashMap>, // ProofOfStore and insertion_time. None if committed batch_to_proof: HashMap>, - // Map of txn_summary = (sender, sequence number, hash) to all the batches that contain - // the transaction. This helps in counting the number of unique transactions in the pipeline. - txn_summary_to_batches: HashMap>, - // List of batches for which we received txn summaries from the batch coordinator - batches_with_txn_summary: HashSet, + // Number of batches in which the txn_summary = (sender, sequence number, hash) has been included + txn_summary_num_occurrences: HashMap, + // List of transaction summaries for each batch + batch_to_txn_summaries: HashMap>, // Expiration index expirations: TimeExpirations, latest_block_timestamp: u64, @@ -218,8 +217,8 @@ impl ProofQueue { my_peer_id, author_to_batches: HashMap::new(), batch_to_proof: HashMap::new(), - txn_summary_to_batches: HashMap::new(), - batches_with_txn_summary: HashSet::new(), + txn_summary_num_occurrences: HashMap::new(), + batch_to_txn_summaries: HashMap::new(), expirations: TimeExpirations::new(), latest_block_timestamp: 0, remaining_txns_with_duplicates: 0, @@ -250,22 +249,7 @@ impl ProofQueue { } fn remaining_txns_without_duplicates(&self) -> u64 { - // All the batch keys for which batch_to_proof is not None. This is the set of unexpired and uncommitted proofs. - let unexpired_batch_keys = self - .batch_to_proof - .iter() - .filter(|(_, proof)| proof.is_some()) - .map(|(batch_key, _)| batch_key) - .collect::>(); - let mut remaining_txns = self - .txn_summary_to_batches - .iter() - .filter(|(_, batches)| { - batches - .iter() - .any(|batch_key| unexpired_batch_keys.contains(batch_key)) - }) - .count() as u64; + let mut remaining_txns = self.txn_summary_num_occurrences.len() as u64; // If a batch_key is not in batches_with_txn_summary, it means we've received the proof but haven't receive the // transaction summary of the batch from batch coordinator. Add the number of txns in the batch to remaining_txns. @@ -273,7 +257,7 @@ impl ProofQueue { .batch_to_proof .iter() .filter_map(|(batch_key, proof)| { - if proof.is_some() && !self.batches_with_txn_summary.contains(batch_key) { + if proof.is_some() && !self.batch_to_txn_summaries.contains_key(batch_key) { Some(proof.as_ref().unwrap().0.num_txns()) } else { None @@ -322,13 +306,19 @@ impl ProofQueue { let start = Instant::now(); for (batch_info, txn_summaries) in batch_summaries { let batch_key = BatchKey::from_info(&batch_info); - for txn_summary in txn_summaries { - self.txn_summary_to_batches - .entry(txn_summary) - .or_default() - .insert(batch_key.clone()); + if self + .batch_to_txn_summaries + .insert(batch_key, txn_summaries.clone()) + .is_none() + { + for txn_summary in txn_summaries { + if let Some(count) = self.txn_summary_num_occurrences.get_mut(&txn_summary) { + *count += 1; + } else { + self.txn_summary_num_occurrences.insert(txn_summary, 1); + } + } } - self.batches_with_txn_summary.insert(batch_key); } counters::PROOF_QUEUE_ADD_BATCH_SUMMARIES_DURATION.observe_duration(start.elapsed()); } @@ -472,11 +462,17 @@ impl ProofQueue { num_expired_but_not_committed += 1; counters::GAP_BETWEEN_BATCH_EXPIRATION_AND_CURRENT_TIME_WHEN_COMMIT .observe((block_timestamp - batch.expiration()) as f64); - self.txn_summary_to_batches.retain(|_, batches| { - batches.remove(&key.batch_key); - !batches.is_empty() - }); - self.batches_with_txn_summary.remove(&key.batch_key); + if let Some(txn_summaries) = self.batch_to_txn_summaries.get(&key.batch_key) + { + for txn_summary in txn_summaries { + if let Some(count) = + self.txn_summary_num_occurrences.get_mut(txn_summary) + { + *count -= 1; + }; + } + } + self.batch_to_txn_summaries.remove(&key.batch_key); self.dec_remaining(&batch.author(), batch.num_txns()); } claims::assert_some!(self.batch_to_proof.remove(&key.batch_key)); @@ -486,6 +482,8 @@ impl ProofQueue { } } } + self.txn_summary_num_occurrences + .retain(|_, count| *count > 0); counters::PROOF_QUEUE_UPDATE_TIMESTAMP_DURATION.observe_duration(start.elapsed()); counters::NUM_PROOFS_EXPIRED_WHEN_COMMIT.inc_by(num_expired_but_not_committed); } @@ -501,20 +499,20 @@ impl ProofQueue { .observe(remaining_txns_without_duplicates as f64); //count the number of transactions with more than one batches counters::TXNS_WITH_DUPLICATE_BATCHES.set( - self.txn_summary_to_batches + self.txn_summary_num_occurrences .iter() - .filter(|(_, batches)| batches.len() > 1) + .filter(|(_, count)| **count > 1) .count() as i64, ); - counters::TXNS_IN_PROOF_QUEUE.set(self.txn_summary_to_batches.len() as i64); + counters::TXNS_IN_PROOF_QUEUE.set(self.txn_summary_num_occurrences.len() as i64); // count the number of batches with proofs but without txn summaries counters::PROOFS_WITHOUT_BATCH_DATA.set( self.batch_to_proof .iter() .map(|(batch_key, proof)| { - if proof.is_some() && !self.batches_with_txn_summary.contains(batch_key) { + if proof.is_some() && !self.batch_to_txn_summaries.contains_key(batch_key) { 1 } else { 0 @@ -546,18 +544,17 @@ impl ProofQueue { self.dec_remaining(&batch.author(), batch.num_txns()); } self.batch_to_proof.insert(batch_key.clone(), None); - self.batches_with_txn_summary.remove(&batch_key); - } - let batch_keys = batches - .iter() - .map(BatchKey::from_info) - .collect::>(); - self.txn_summary_to_batches.retain(|_, batches| { - for batch_key in &batch_keys { - batches.remove(batch_key); + if let Some(txn_summaries) = self.batch_to_txn_summaries.get(&batch_key) { + for txn_summary in txn_summaries { + if let Some(count) = self.txn_summary_num_occurrences.get_mut(txn_summary) { + *count -= 1; + }; + } } - !batches.is_empty() - }); + self.batch_to_txn_summaries.remove(&batch_key); + } + self.txn_summary_num_occurrences + .retain(|_, count| *count > 0); counters::PROOF_QUEUE_COMMIT_DURATION.observe_duration(start.elapsed()); } } From dcdb04ab3e0e7174d0bc5549b05a24921287a919 Mon Sep 17 00:00:00 2001 From: igor-aptos <110557261+igor-aptos@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:23:09 -0700 Subject: [PATCH 017/469] fix --skip-commit in executor benchmark (#13857) --- .../src/ledger_update_stage.rs | 58 +++++++++++-------- execution/executor-benchmark/src/lib.rs | 14 +++-- execution/executor-benchmark/src/pipeline.rs | 43 ++++++++------ testsuite/single_node_performance.py | 13 +++-- 4 files changed, 75 insertions(+), 53 deletions(-) diff --git a/execution/executor-benchmark/src/ledger_update_stage.rs b/execution/executor-benchmark/src/ledger_update_stage.rs index ec92f8287f5b1..fe9d528aaf046 100644 --- a/execution/executor-benchmark/src/ledger_update_stage.rs +++ b/execution/executor-benchmark/src/ledger_update_stage.rs @@ -8,10 +8,16 @@ use aptos_executor_types::BlockExecutorTrait; use aptos_types::transaction::Version; use std::sync::{mpsc, Arc}; +pub enum CommitProcessing { + SendToQueue(mpsc::SyncSender), + #[allow(dead_code)] + ExecuteInline, + Skip, +} + pub struct LedgerUpdateStage { executor: Arc>, - // If commit_sender is `None`, we will commit all the execution result immediately in this struct. - commit_sender: Option>, + commit_processing: CommitProcessing, version: Version, } @@ -21,13 +27,13 @@ where { pub fn new( executor: Arc>, - commit_sender: Option>, + commit_processing: CommitProcessing, version: Version, ) -> Self { Self { executor, version, - commit_sender, + commit_processing, } } @@ -50,26 +56,30 @@ where self.version += output.transactions_to_commit_len() as Version; - if let Some(commit_sender) = &self.commit_sender { - let msg = CommitBlockMessage { - block_id, - root_hash: output.root_hash(), - first_block_start_time, - current_block_start_time, - partition_time, - execution_time, - num_txns: output.transactions_to_commit_len(), - }; - commit_sender.send(msg).unwrap(); - } else { - let ledger_info_with_sigs = super::transaction_committer::gen_li_with_sigs( - block_id, - output.root_hash(), - self.version, - ); - self.executor - .commit_blocks(vec![block_id], ledger_info_with_sigs) - .unwrap(); + match &self.commit_processing { + CommitProcessing::SendToQueue(commit_sender) => { + let msg = CommitBlockMessage { + block_id, + root_hash: output.root_hash(), + first_block_start_time, + current_block_start_time, + partition_time, + execution_time, + num_txns: output.transactions_to_commit_len(), + }; + commit_sender.send(msg).unwrap(); + }, + CommitProcessing::ExecuteInline => { + let ledger_info_with_sigs = super::transaction_committer::gen_li_with_sigs( + block_id, + output.root_hash(), + self.version, + ); + self.executor + .commit_blocks(vec![block_id], ledger_info_with_sigs) + .unwrap(); + }, + CommitProcessing::Skip => {}, } } } diff --git a/execution/executor-benchmark/src/lib.rs b/execution/executor-benchmark/src/lib.rs index fe1cca4d748f6..c558a27205efc 100644 --- a/execution/executor-benchmark/src/lib.rs +++ b/execution/executor-benchmark/src/lib.rs @@ -233,17 +233,19 @@ pub fn run_benchmark( } ); - let num_txns = db.reader.get_synced_version().unwrap() - version - num_blocks_created as u64; - overall_measuring.print_end("Overall", num_txns); + if !pipeline_config.skip_commit { + let num_txns = + db.reader.get_synced_version().unwrap() - version - num_blocks_created as u64; + overall_measuring.print_end("Overall", num_txns); - if verify_sequence_numbers { - generator.verify_sequence_numbers(db.reader.clone()); + if verify_sequence_numbers { + generator.verify_sequence_numbers(db.reader.clone()); + } + log_total_supply(&db.reader); } // Assert there were no error log lines in the run. assert_eq!(0, aptos_logger::ERROR_LOG_COUNT.get()); - - log_total_supply(&db.reader); } fn init_workload( diff --git a/execution/executor-benchmark/src/pipeline.rs b/execution/executor-benchmark/src/pipeline.rs index 81db3c2e8a019..8d4efaf380d70 100644 --- a/execution/executor-benchmark/src/pipeline.rs +++ b/execution/executor-benchmark/src/pipeline.rs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - block_preparation::BlockPreparationStage, ledger_update_stage::LedgerUpdateStage, - metrics::NUM_TXNS, OverallMeasuring, TransactionCommitter, TransactionExecutor, + block_preparation::BlockPreparationStage, + ledger_update_stage::{CommitProcessing, LedgerUpdateStage}, + metrics::NUM_TXNS, + OverallMeasuring, TransactionCommitter, TransactionExecutor, }; use aptos_block_partitioner::v2::config::PartitionerV2Config; use aptos_crypto::HashValue; @@ -85,7 +87,7 @@ where ); let (commit_sender, commit_receiver) = mpsc::sync_channel::( - if config.split_stages || config.skip_commit { + if config.split_stages { (num_blocks.unwrap() + 1).max(3) } else { 3 @@ -99,7 +101,7 @@ where (None, None) }; - let (start_commit_tx, start_commit_rx) = if config.split_stages || config.skip_commit { + let (start_commit_tx, start_commit_rx) = if config.split_stages { let (start_commit_tx, start_commit_rx) = mpsc::sync_channel::<()>(1); (Some(start_commit_tx), Some(start_commit_rx)) } else { @@ -120,8 +122,13 @@ where config.allow_retries, ); + let commit_processing = if config.skip_commit { + CommitProcessing::Skip + } else { + CommitProcessing::SendToQueue(commit_sender) + }; let mut ledger_update_stage = - LedgerUpdateStage::new(executor_2, Some(commit_sender), version); + LedgerUpdateStage::new(executor_2, commit_processing, version); let (executable_block_sender, executable_block_receiver) = mpsc::sync_channel::(3); @@ -191,7 +198,9 @@ where ); } - overall_measuring.print_end("Overall execution", executed); + if num_blocks.is_some() { + overall_measuring.print_end("Overall execution", executed); + } start_commit_tx.map(|tx| tx.send(())); }) .expect("Failed to spawn transaction executor thread."); @@ -212,21 +221,19 @@ where .expect("Failed to spawn ledger update thread."); join_handles.push(ledger_update_thread); - let skip_commit = config.skip_commit; - - let commit_thread = std::thread::Builder::new() - .name("txn_committer".to_string()) - .spawn(move || { - start_commit_rx.map(|rx| rx.recv()); - info!("Starting commit thread"); - if !skip_commit { + if !config.skip_commit { + let commit_thread = std::thread::Builder::new() + .name("txn_committer".to_string()) + .spawn(move || { + start_commit_rx.map(|rx| rx.recv()); + info!("Starting commit thread"); let mut committer = TransactionCommitter::new(executor_3, version, commit_receiver); committer.run(); - } - }) - .expect("Failed to spawn transaction committer thread."); - join_handles.push(commit_thread); + }) + .expect("Failed to spawn transaction committer thread."); + join_handles.push(commit_thread); + } ( Self { diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 3675525c9dde9..8586b192a537a 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -193,7 +193,7 @@ class RunGroupConfig: ) if os.environ.get("DETAILED"): - EXECUTION_ONLY_NUMBER_OF_THREADS = [1, 2, 4, 8, 16, 32, 60] + EXECUTION_ONLY_NUMBER_OF_THREADS = [1, 2, 4, 8, 16, 32, 48, 60] else: EXECUTION_ONLY_NUMBER_OF_THREADS = [] @@ -407,9 +407,12 @@ def print_table( if single_field is not None: _, field_getter = single_field for num_threads in number_of_execution_threads: - row.append( - field_getter(result.number_of_threads_results[num_threads]) - ) + if num_threads in result.number_of_threads_results: + row.append( + field_getter(result.number_of_threads_results[num_threads]) + ) + else: + row.append("-") if single_field is not None: _, field_getter = single_field @@ -504,7 +507,7 @@ def print_table( number_of_threads_results = {} for execution_threads in EXECUTION_ONLY_NUMBER_OF_THREADS: - test_db_command = f"RUST_BACKTRACE=1 {BUILD_FOLDER}/aptos-executor-benchmark --execution-threads {execution_threads} {common_command_suffix} --skip-commit --blocks {NUM_BLOCKS_DETAILED}" + test_db_command = f"RUST_BACKTRACE=1 {BUILD_FOLDER}/aptos-executor-benchmark --execution-threads {execution_threads} --skip-commit {common_command_suffix} --blocks {NUM_BLOCKS_DETAILED}" output = execute_command(test_db_command) number_of_threads_results[execution_threads] = extract_run_results( From 2e0b82e8a0002e4bc877ba8f67745487574192dc Mon Sep 17 00:00:00 2001 From: Stelian Ionescu Date: Fri, 28 Jun 2024 16:00:05 -0400 Subject: [PATCH 018/469] Make healtch check failure clearer. --- api/src/basic.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/api/src/basic.rs b/api/src/basic.rs index 0dc9ff170243d..d6fed2dd70e3f 100644 --- a/api/src/basic.rs +++ b/api/src/basic.rs @@ -87,12 +87,10 @@ impl BasicApi { let ledger_info = api_spawn_blocking(move || context.get_latest_ledger_info()).await?; // If we have a duration, check that it's close to the current time, otherwise it's ok - if let Some(duration) = duration_secs.0 { - let timestamp = ledger_info.timestamp(); - - let timestamp = Duration::from_micros(timestamp); - let expectation = SystemTime::now() - .sub(Duration::from_secs(duration as u64)) + if let Some(max_skew) = duration_secs.0 { + let ledger_timestamp = Duration::from_micros(ledger_info.timestamp()); + let skew_threshold = SystemTime::now() + .sub(Duration::from_secs(max_skew as u64)) .duration_since(UNIX_EPOCH) .context("Failed to determine absolute unix time based on given duration") .map_err(|err| { @@ -103,9 +101,9 @@ impl BasicApi { ) })?; - if timestamp < expectation { + if ledger_timestamp < skew_threshold { return Err(HealthCheckError::service_unavailable_with_code( - "The latest ledger info timestamp is less than the expected timestamp", + format!("The latest ledger info timestamp is {:?}, which is beyond the allowed skew ({}s).", ledger_timestamp, max_skew), AptosErrorCode::HealthCheckFailed, &ledger_info, )); From 7777599e258ba5757f45362dee93a445e8c116ba Mon Sep 17 00:00:00 2001 From: Sherry Xiao Date: Mon, 1 Jul 2024 14:56:18 -0700 Subject: [PATCH 019/469] bump forge version for 1.16 (#13879) --- .github/workflows/docker-build-test.yaml | 4 ++-- .github/workflows/forge-stable.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build-test.yaml b/.github/workflows/docker-build-test.yaml index 2b52052781fad..bed1625f8697e 100644 --- a/.github/workflows/docker-build-test.yaml +++ b/.github/workflows/docker-build-test.yaml @@ -312,7 +312,7 @@ jobs: with: GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} FORGE_TEST_SUITE: compat - IMAGE_TAG: f648076a280621dbfd4e73b1ca83e3a3f52878ed #aptos-node-v1.15.0 + IMAGE_TAG: 1c2ee7082d6eff8c811ee25d6f5a7d00860a75d5 #aptos-node-v1.16.0 FORGE_RUNNER_DURATION_SECS: 300 COMMENT_HEADER: forge-compat FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} @@ -340,7 +340,7 @@ jobs: with: GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} FORGE_TEST_SUITE: framework_upgrade - IMAGE_TAG: f648076a280621dbfd4e73b1ca83e3a3f52878ed #aptos-node-v1.15.0 + IMAGE_TAG: 1c2ee7082d6eff8c811ee25d6f5a7d00860a75d5 #aptos-node-v1.16.0 FORGE_RUNNER_DURATION_SECS: 3600 COMMENT_HEADER: forge-framework-upgrade FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} diff --git a/.github/workflows/forge-stable.yaml b/.github/workflows/forge-stable.yaml index b11985e169f04..2315e959a68d7 100644 --- a/.github/workflows/forge-stable.yaml +++ b/.github/workflows/forge-stable.yaml @@ -128,7 +128,7 @@ jobs: uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main secrets: inherit with: - IMAGE_TAG: f648076a280621dbfd4e73b1ca83e3a3f52878ed #aptos-node-v1.15.0 + IMAGE_TAG: 1c2ee7082d6eff8c811ee25d6f5a7d00860a75d5 #aptos-node-v1.16.0 FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }} FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours FORGE_TEST_SUITE: framework_upgrade @@ -257,7 +257,7 @@ jobs: FORGE_RUNNER_DURATION_SECS: 300 # Run for 5 minutes # This will upgrade from testnet branch to the latest main FORGE_TEST_SUITE: compat - IMAGE_TAG: f648076a280621dbfd4e73b1ca83e3a3f52878ed #aptos-node-v1.15.0 + IMAGE_TAG: 1c2ee7082d6eff8c811ee25d6f5a7d00860a75d5 #aptos-node-v1.16.0 GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout POST_TO_SLACK: true From 1dfd8a85c412dd08d4a61afd9e72475f54319db4 Mon Sep 17 00:00:00 2001 From: Satya Vusirikala Date: Mon, 1 Jul 2024 19:33:44 -0500 Subject: [PATCH 020/469] Ignoring duplicate transactions when calculating block size (#13709) * Use proof queue asynchronously * Committing what I have * Sending AddBatches message * Calcuating the remaining txns * Calculate proof queue size correctly * Add a counter * Update pfn_const_tps test * Minor changes * Minor change * Add some coutners * Rust lint * Increasing quorum store backpressure limits * setting dynamic_min_txns_per_sec to 160 * Fixing the calculation * increase vfns to 7 * Fixing the typo in batch generator * Add increase fraction * Removing skipped transactions after inserting them * Add some counters * Update consensus pending duration counter * Add more counters * Increasing block size to 2500 * Update a counter * Increase block size limit * Resetting execution config params * Moving proof queue to utils.rs * Moving counters * Use transaction summary * intelligent pull proofs * Fix a bug in pull proofs * Fix the bug * Rest to full to false in every iteration * Addressing PR comments * Move backpressure_tx to proof queue * Add info statement * Change buckets * Add some info statements * Cleanup * Remove an unrelated change * Addressing PR comments * Addressing PR comments * Add some timer counters * Add more timer counters * Minor optimization * Proof queue to be part of proof manager * Move some code to a function * Minor fixes * Add max_unique_txns parameter * Use Lazy * Removing comments * Minor change * Minor change * Minor fix * Add unit test and address PR comments * Minor fix in proof manager * Use saturating_sub * Addressing PR comments --- config/src/config/consensus_config.rs | 17 ++-- .../consensus-types/src/request_response.rs | 9 +- consensus/src/dag/dag_driver.rs | 1 + consensus/src/epoch_manager.rs | 1 + consensus/src/liveness/proposal_generator.rs | 18 +++- .../src/liveness/proposal_generator_test.rs | 4 + consensus/src/payload_client/mixed.rs | 20 +++-- consensus/src/payload_client/mod.rs | 1 + consensus/src/payload_client/user/mod.rs | 4 + .../user/quorum_store_client.rs | 5 ++ consensus/src/quorum_store/counters.rs | 18 ++++ .../direct_mempool_quorum_store.rs | 5 +- consensus/src/quorum_store/proof_manager.rs | 22 +++-- .../tests/direct_mempool_quorum_store_test.rs | 1 + .../quorum_store/tests/proof_manager_test.rs | 1 + consensus/src/quorum_store/tests/utils.rs | 82 ++++++++++++++++++- consensus/src/quorum_store/utils.rs | 63 ++++++++++++-- consensus/src/round_manager_fuzzing.rs | 1 + consensus/src/round_manager_test.rs | 1 + .../src/test_utils/mock_payload_manager.rs | 1 + 20 files changed, 239 insertions(+), 36 deletions(-) diff --git a/config/src/config/consensus_config.rs b/config/src/config/consensus_config.rs index 501a80c73f852..31ec95921e32f 100644 --- a/config/src/config/consensus_config.rs +++ b/config/src/config/consensus_config.rs @@ -13,9 +13,14 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; // NOTE: when changing, make sure to update QuorumStoreBackPressureConfig::backlog_txn_limit_count as well. -const MAX_SENDING_BLOCK_TXNS: u64 = 1900; +const MAX_SENDING_BLOCK_UNIQUE_TXNS: u64 = 1900; pub(crate) static MAX_RECEIVING_BLOCK_TXNS: Lazy = - Lazy::new(|| 10000.max(2 * MAX_SENDING_BLOCK_TXNS)); + Lazy::new(|| 10000.max(2 * MAX_SENDING_BLOCK_UNIQUE_TXNS)); +// The receiving validator can accept upto 2k more transactions in the block than the max sending limit. +// The extra cushion of 2k transactions is added just in case we need to increase the max sending limit in the future. +static MAX_SENDING_BLOCK_TXNS: Lazy = + Lazy::new(|| MAX_SENDING_BLOCK_UNIQUE_TXNS.max(MAX_RECEIVING_BLOCK_TXNS.saturating_sub(2000))); + // stop reducing size at this point, so 1MB transactions can still go through const MIN_BLOCK_BYTES_OVERRIDE: u64 = 1024 * 1024 + BATCH_PADDING_BYTES as u64; @@ -25,6 +30,7 @@ pub struct ConsensusConfig { // length of inbound queue of messages pub max_network_channel_size: usize, pub max_sending_block_txns: u64, + pub max_sending_block_unique_txns: u64, pub max_sending_block_bytes: u64, pub max_sending_inline_txns: u64, pub max_sending_inline_bytes: u64, @@ -152,7 +158,8 @@ impl Default for ConsensusConfig { fn default() -> ConsensusConfig { ConsensusConfig { max_network_channel_size: 1024, - max_sending_block_txns: MAX_SENDING_BLOCK_TXNS, + max_sending_block_txns: *MAX_SENDING_BLOCK_TXNS, + max_sending_block_unique_txns: MAX_SENDING_BLOCK_UNIQUE_TXNS, max_sending_block_bytes: 3 * 1024 * 1024, // 3MB max_receiving_block_txns: *MAX_RECEIVING_BLOCK_TXNS, max_sending_inline_txns: 100, @@ -192,14 +199,14 @@ impl Default for ConsensusConfig { // pipeline once quorum on execution result among validators has been reached // (so-(badly)-called "commit certificate"), meaning 2f+1 validators have finished execution. back_pressure_pipeline_latency_limit_ms: 800, - max_sending_block_txns_override: MAX_SENDING_BLOCK_TXNS, + max_sending_block_txns_override: *MAX_SENDING_BLOCK_TXNS, max_sending_block_bytes_override: 5 * 1024 * 1024, backpressure_proposal_delay_ms: 100, max_txns_from_block_to_execute: None, }, PipelineBackpressureValues { back_pressure_pipeline_latency_limit_ms: 1100, - max_sending_block_txns_override: MAX_SENDING_BLOCK_TXNS, + max_sending_block_txns_override: *MAX_SENDING_BLOCK_TXNS, max_sending_block_bytes_override: 5 * 1024 * 1024, backpressure_proposal_delay_ms: 200, max_txns_from_block_to_execute: None, diff --git a/consensus/consensus-types/src/request_response.rs b/consensus/consensus-types/src/request_response.rs index a28082b6c0c7f..a65273b19626c 100644 --- a/consensus/consensus-types/src/request_response.rs +++ b/consensus/consensus-types/src/request_response.rs @@ -9,7 +9,9 @@ use std::{fmt, fmt::Formatter}; pub enum GetPayloadCommand { /// Request to pull block to submit to consensus. GetPayloadRequest( - // max block size + // max number of transactions in the block + u64, + // max number of unique transactions in the block u64, // max byte size u64, @@ -31,6 +33,7 @@ impl fmt::Display for GetPayloadCommand { match self { GetPayloadCommand::GetPayloadRequest( max_txns, + max_unique_txns, max_bytes, max_inline_txns, max_inline_bytes, @@ -40,8 +43,8 @@ impl fmt::Display for GetPayloadCommand { ) => { write!( f, - "GetPayloadRequest [max_txns: {}, max_bytes: {}, max_inline_txns: {}, max_inline_bytes:{}, return_non_full: {}, excluded: {}]", - max_txns, max_bytes, max_inline_txns, max_inline_bytes, return_non_full, excluded + "GetPayloadRequest [max_txns: {}, max_unique_txns: {}, max_bytes: {}, max_inline_txns: {}, max_inline_bytes:{}, return_non_full: {}, excluded: {}]", + max_txns, max_unique_txns, max_bytes, max_inline_txns, max_inline_bytes, return_non_full, excluded ) }, } diff --git a/consensus/src/dag/dag_driver.rs b/consensus/src/dag/dag_driver.rs index 2c787f5b1664b..f2fe23ea42524 100644 --- a/consensus/src/dag/dag_driver.rs +++ b/consensus/src/dag/dag_driver.rs @@ -257,6 +257,7 @@ impl DagDriver { .pull_payload( Duration::from_millis(self.payload_config.payload_pull_max_poll_time_ms), max_txns, + max_txns, max_size_bytes, // TODO: Set max_inline_items and max_inline_bytes correctly 100, diff --git a/consensus/src/epoch_manager.rs b/consensus/src/epoch_manager.rs index b9957ef1d2d75..0de227d9fafde 100644 --- a/consensus/src/epoch_manager.rs +++ b/consensus/src/epoch_manager.rs @@ -839,6 +839,7 @@ impl EpochManager

{ self.time_service.clone(), Duration::from_millis(self.config.quorum_store_poll_time_ms), self.config.max_sending_block_txns, + self.config.max_sending_block_unique_txns, self.config.max_sending_block_bytes, self.config.max_sending_inline_txns, self.config.max_sending_inline_bytes, diff --git a/consensus/src/liveness/proposal_generator.rs b/consensus/src/liveness/proposal_generator.rs index ea2145a62d93a..c412278506712 100644 --- a/consensus/src/liveness/proposal_generator.rs +++ b/consensus/src/liveness/proposal_generator.rs @@ -164,6 +164,8 @@ pub struct ProposalGenerator { quorum_store_poll_time: Duration, // Max number of transactions to be added to a proposed block. max_block_txns: u64, + // Max number of unique transactions to be added to a proposed block. + max_block_unique_txns: u64, // Max number of bytes to be added to a proposed block. max_block_bytes: u64, // Max number of inline transactions to be added to a proposed block. @@ -193,6 +195,7 @@ impl ProposalGenerator { time_service: Arc, quorum_store_poll_time: Duration, max_block_txns: u64, + max_block_unique_txns: u64, max_block_bytes: u64, max_inline_txns: u64, max_inline_bytes: u64, @@ -210,6 +213,7 @@ impl ProposalGenerator { time_service, quorum_store_poll_time, max_block_txns, + max_block_unique_txns, max_block_bytes, max_inline_txns, max_inline_bytes, @@ -312,9 +316,14 @@ impl ProposalGenerator { let voting_power_ratio = proposer_election.get_voting_power_participation_ratio(round); - let (max_block_txns, max_block_bytes, max_txns_from_block_to_execute, proposal_delay) = - self.calculate_max_block_sizes(voting_power_ratio, timestamp, round) - .await; + let ( + max_block_unique_txns, + max_block_bytes, + max_txns_from_block_to_execute, + proposal_delay, + ) = self + .calculate_max_block_sizes(voting_power_ratio, timestamp, round) + .await; PROPOSER_DELAY_PROPOSAL.set(proposal_delay.as_secs_f64()); if !proposal_delay.is_zero() { @@ -349,7 +358,8 @@ impl ProposalGenerator { .payload_client .pull_payload( self.quorum_store_poll_time.saturating_sub(proposal_delay), - max_block_txns, + self.max_block_txns, + max_block_unique_txns, max_block_bytes, // TODO: Set max_inline_txns and max_inline_bytes correctly self.max_inline_txns, diff --git a/consensus/src/liveness/proposal_generator_test.rs b/consensus/src/liveness/proposal_generator_test.rs index ab4102b7a39f3..1ee89e2e6b756 100644 --- a/consensus/src/liveness/proposal_generator_test.rs +++ b/consensus/src/liveness/proposal_generator_test.rs @@ -37,6 +37,7 @@ async fn test_proposal_generation_empty_tree() { Arc::new(SimulatedTimeService::new()), Duration::ZERO, 1, + 1, 10, 1, 10, @@ -81,6 +82,7 @@ async fn test_proposal_generation_parent() { Arc::new(SimulatedTimeService::new()), Duration::ZERO, 1, + 1, 1000, 1, 500, @@ -157,6 +159,7 @@ async fn test_old_proposal_generation() { Arc::new(SimulatedTimeService::new()), Duration::ZERO, 1, + 1, 1000, 1, 500, @@ -198,6 +201,7 @@ async fn test_correct_failed_authors() { Arc::new(SimulatedTimeService::new()), Duration::ZERO, 1, + 1, 1000, 1, 500, diff --git a/consensus/src/payload_client/mixed.rs b/consensus/src/payload_client/mixed.rs index 63cc7f00a62c3..72a458c5481d2 100644 --- a/consensus/src/payload_client/mixed.rs +++ b/consensus/src/payload_client/mixed.rs @@ -68,6 +68,7 @@ impl PayloadClient for MixedPayloadClient { &self, mut max_poll_time: Duration, mut max_items: u64, + mut max_unique_items: u64, mut max_bytes: u64, max_inline_items: u64, max_inline_bytes: u64, @@ -101,6 +102,7 @@ impl PayloadClient for MixedPayloadClient { debug!("num_validator_txns={}", validator_txns.len()); // Update constraints with validator txn pull results. max_items -= validator_txns.len() as u64; + max_unique_items -= validator_txns.len() as u64; max_bytes -= validator_txns .iter() .map(|txn| txn.size_in_bytes()) @@ -113,6 +115,7 @@ impl PayloadClient for MixedPayloadClient { .pull( max_poll_time, max_items, + max_unique_items, max_bytes, max_inline_items, max_inline_bytes, @@ -151,7 +154,8 @@ async fn mixed_payload_client_should_prioritize_validator_txns() { let (pulled_validator_txns, Payload::DirectMempool(pulled_user_txns)) = client .pull_payload( Duration::from_secs(1), // max_poll_time - 99, // max_items + 120, // max_items + 99, // max_unique_items 1048576, // size limit: 1MB 50, 500000, // inline limit: 500KB @@ -174,7 +178,8 @@ async fn mixed_payload_client_should_prioritize_validator_txns() { let (pulled_validator_txns, Payload::DirectMempool(pulled_user_txns)) = client .pull_payload( Duration::from_micros(500), // max_poll_time - 99, // max_items + 120, // max_items + 99, // max_unique_items 1048576, // size limit: 1MB 50, 500000, // inline limit: 500KB @@ -197,7 +202,8 @@ async fn mixed_payload_client_should_prioritize_validator_txns() { let (pulled_validator_txns, Payload::DirectMempool(pulled_user_txns)) = client .pull_payload( Duration::from_secs(1), // max_poll_time - 1, // max_items + 2, // max_items + 2, // max_unique_items 1048576, // size limit: 1MB 0, 0, // inline limit: 0 @@ -214,13 +220,14 @@ async fn mixed_payload_client_should_prioritize_validator_txns() { unreachable!() }; - assert_eq!(1, pulled_validator_txns.len()); + assert_eq!(2, pulled_validator_txns.len()); assert_eq!(0, pulled_user_txns.len()); let (pulled_validator_txns, Payload::DirectMempool(pulled_user_txns)) = client .pull_payload( Duration::from_secs(1), // max_poll_time - 99, // max_items + 120, // max_items + 99, // max_unique_items all_validator_txns[0].size_in_bytes() as u64, 50, all_validator_txns[0].size_in_bytes() as u64, @@ -261,7 +268,8 @@ async fn mixed_payload_client_should_respect_validator_txn_feature_flag() { let (pulled_validator_txns, Payload::DirectMempool(pulled_user_txns)) = client .pull_payload( Duration::from_millis(50), // max_poll_time - 99, // max_items + 120, // max_items + 99, // max_unique_items 1048576, // size limit: 1MB 50, 500000, // inline limit: 500KB diff --git a/consensus/src/payload_client/mod.rs b/consensus/src/payload_client/mod.rs index d37cbfbbdb5ac..61cc98a5ba036 100644 --- a/consensus/src/payload_client/mod.rs +++ b/consensus/src/payload_client/mod.rs @@ -18,6 +18,7 @@ pub trait PayloadClient: Send + Sync { &self, max_poll_time: Duration, max_items: u64, + max_unique_items: u64, max_bytes: u64, max_inline_items: u64, max_inline_bytes: u64, diff --git a/consensus/src/payload_client/user/mod.rs b/consensus/src/payload_client/user/mod.rs index 6a5cc6db745ff..a19f77e3dfc11 100644 --- a/consensus/src/payload_client/user/mod.rs +++ b/consensus/src/payload_client/user/mod.rs @@ -18,6 +18,7 @@ pub trait UserPayloadClient: Send + Sync { &self, max_poll_time: Duration, max_items: u64, + max_unique_items: u64, max_bytes: u64, max_inline_items: u64, max_inline_bytes: u64, @@ -49,6 +50,7 @@ impl UserPayloadClient for DummyClient { &self, max_poll_time: Duration, mut max_items: u64, + mut max_unique_items: u64, mut max_bytes: u64, _max_inline_items: u64, _max_inline_bytes: u64, @@ -63,6 +65,7 @@ impl UserPayloadClient for DummyClient { let mut txns = vec![]; while timer.elapsed() < max_poll_time && max_items >= 1 + && max_unique_items >= 1 && max_bytes >= 1 && nxt_txn_idx < self.txns.len() { @@ -73,6 +76,7 @@ impl UserPayloadClient for DummyClient { break; } max_items -= 1; + max_unique_items -= 1; max_bytes -= txn_size; nxt_txn_idx += 1; txns.push(txn); diff --git a/consensus/src/payload_client/user/quorum_store_client.rs b/consensus/src/payload_client/user/quorum_store_client.rs index c6f49a2f1e6c9..1cd3caefd50aa 100644 --- a/consensus/src/payload_client/user/quorum_store_client.rs +++ b/consensus/src/payload_client/user/quorum_store_client.rs @@ -46,6 +46,7 @@ impl QuorumStoreClient { async fn pull_internal( &self, max_items: u64, + max_unique_items: u64, max_bytes: u64, max_inline_items: u64, max_inline_bytes: u64, @@ -55,6 +56,7 @@ impl QuorumStoreClient { let (callback, callback_rcv) = oneshot::channel(); let req = GetPayloadCommand::GetPayloadRequest( max_items, + max_unique_items, max_bytes, max_inline_items, max_inline_bytes, @@ -88,6 +90,7 @@ impl UserPayloadClient for QuorumStoreClient { &self, max_poll_time: Duration, max_items: u64, + max_unique_items: u64, max_bytes: u64, max_inline_items: u64, max_inline_bytes: u64, @@ -117,6 +120,7 @@ impl UserPayloadClient for QuorumStoreClient { let payload = self .pull_internal( max_items, + max_unique_items, max_bytes, max_inline_items, max_inline_bytes, @@ -138,6 +142,7 @@ impl UserPayloadClient for QuorumStoreClient { max_poll_time_ms = max_poll_time.as_millis() as u64, payload_len = payload.len(), max_items = max_items, + max_unique_items = max_unique_items, max_bytes = max_bytes, max_inline_items = max_inline_items, max_inline_bytes = max_inline_bytes, diff --git a/consensus/src/quorum_store/counters.rs b/consensus/src/quorum_store/counters.rs index 29d893c4f43f8..6de4c5496fd2f 100644 --- a/consensus/src/quorum_store/counters.rs +++ b/consensus/src/quorum_store/counters.rs @@ -184,6 +184,24 @@ pub static BLOCK_SIZE_WHEN_PULL: Lazy = Lazy::new(|| { .unwrap() }); +pub static TOTAL_BLOCK_SIZE_WHEN_PULL: Lazy = Lazy::new(|| { + register_histogram!( + "quorum_store_total_block_size_when_pull", + "Histogram for the total size of transactions per block when pulled for consensus.", + BYTE_BUCKETS.clone(), + ) + .unwrap() +}); + +pub static KNOWN_DUPLICATE_TXNS_WHEN_PULL: Lazy = Lazy::new(|| { + register_histogram!( + "quorum_store_known_duplicate_txns_when_pull", + "Histogram for the number of known duplicate transactions in a block when pulled for consensus.", + TRANSACTION_COUNT_BUCKETS.clone(), + ) + .unwrap() +}); + pub static NUM_INLINE_BATCHES: Lazy = Lazy::new(|| { register_histogram!( "num_inline_batches_in_block_proposal", diff --git a/consensus/src/quorum_store/direct_mempool_quorum_store.rs b/consensus/src/quorum_store/direct_mempool_quorum_store.rs index 661ff46a1d4dd..8a6a61b10c066 100644 --- a/consensus/src/quorum_store/direct_mempool_quorum_store.rs +++ b/consensus/src/quorum_store/direct_mempool_quorum_store.rs @@ -138,7 +138,8 @@ impl DirectMempoolQuorumStore { async fn handle_consensus_request(&self, req: GetPayloadCommand) { match req { GetPayloadCommand::GetPayloadRequest( - max_txns, + _max_txns, + max_unique_txns, max_bytes, _max_inline_txns, _max_inline_bytes, @@ -147,7 +148,7 @@ impl DirectMempoolQuorumStore { callback, ) => { self.handle_block_request( - max_txns, + max_unique_txns, max_bytes, return_non_full, payload_filter, diff --git a/consensus/src/quorum_store/proof_manager.rs b/consensus/src/quorum_store/proof_manager.rs index 92db27fc43c05..89203f7e0a167 100644 --- a/consensus/src/quorum_store/proof_manager.rs +++ b/consensus/src/quorum_store/proof_manager.rs @@ -205,6 +205,7 @@ impl ProofManager { match msg { GetPayloadCommand::GetPayloadRequest( max_txns, + max_unique_txns, max_bytes, max_inline_txns, max_inline_bytes, @@ -220,21 +221,32 @@ impl ProofManager { PayloadFilter::InQuorumStore(proofs) => proofs, }; - let (proof_block, proof_queue_fully_utilized) = self - .proofs_for_consensus - .pull_proofs(&excluded_batches, max_txns, max_bytes, return_non_full); + let (proof_block, cur_unique_txns, proof_queue_fully_utilized) = + self.proofs_for_consensus.pull_proofs( + &excluded_batches, + max_txns, + max_unique_txns, + max_bytes, + return_non_full, + ); counters::NUM_BATCHES_WITHOUT_PROOF_OF_STORE.observe(self.batch_queue.len() as f64); counters::PROOF_QUEUE_FULLY_UTILIZED .observe(if proof_queue_fully_utilized { 1.0 } else { 0.0 }); let mut inline_block: Vec<(BatchInfo, Vec)> = vec![]; - let cur_txns: u64 = proof_block.iter().map(|p| p.num_txns()).sum(); + let cur_all_txns: u64 = proof_block.iter().map(|p| p.num_txns()).sum(); let cur_bytes: u64 = proof_block.iter().map(|p| p.num_bytes()).sum(); if self.allow_batches_without_pos_in_proposal && proof_queue_fully_utilized { inline_block = self.batch_queue.pull_batches( - min(max_txns - cur_txns, max_inline_txns), + min( + min( + max_txns.saturating_sub(cur_all_txns), + max_unique_txns.saturating_sub(cur_unique_txns), + ), + max_inline_txns, + ), min(max_bytes - cur_bytes, max_inline_bytes), excluded_batches .iter() diff --git a/consensus/src/quorum_store/tests/direct_mempool_quorum_store_test.rs b/consensus/src/quorum_store/tests/direct_mempool_quorum_store_test.rs index 03a5e903a554c..35fdc3c7ed97a 100644 --- a/consensus/src/quorum_store/tests/direct_mempool_quorum_store_test.rs +++ b/consensus/src/quorum_store/tests/direct_mempool_quorum_store_test.rs @@ -30,6 +30,7 @@ async fn test_block_request_no_txns() { let (consensus_callback, consensus_callback_rcv) = oneshot::channel(); consensus_to_quorum_store_sender .try_send(GetPayloadCommand::GetPayloadRequest( + 100, 100, 1000, 50, diff --git a/consensus/src/quorum_store/tests/proof_manager_test.rs b/consensus/src/quorum_store/tests/proof_manager_test.rs index 812a854f62d9c..c9065737fdb15 100644 --- a/consensus/src/quorum_store/tests/proof_manager_test.rs +++ b/consensus/src/quorum_store/tests/proof_manager_test.rs @@ -54,6 +54,7 @@ async fn get_proposal( let (callback_tx, callback_rx) = oneshot::channel(); let filter_set = HashSet::from_iter(filter.iter().cloned()); let req = GetPayloadCommand::GetPayloadRequest( + max_txns, max_txns, 1000000, max_txns / 2, diff --git a/consensus/src/quorum_store/tests/utils.rs b/consensus/src/quorum_store/tests/utils.rs index 882a40dfce0c9..e88fcb80f9707 100644 --- a/consensus/src/quorum_store/tests/utils.rs +++ b/consensus/src/quorum_store/tests/utils.rs @@ -9,6 +9,7 @@ use aptos_consensus_types::{ use aptos_crypto::HashValue; use aptos_types::{aggregate_signature::AggregateSignature, PeerId}; use maplit::hashset; +use std::collections::HashSet; /// Return a ProofOfStore with minimal fields used by ProofQueue tests. fn proof_of_store(author: PeerId, batch_id: BatchId, gas_bucket_start: u64) -> ProofOfStore { @@ -55,7 +56,7 @@ fn test_proof_queue_sorting() { } // Expect: [600, 300] - let (pulled, _) = proof_queue.pull_proofs(&hashset![], 2, 2, true); + let (pulled, num_unique_txns, _) = proof_queue.pull_proofs(&hashset![], 4, 2, 2, true); let mut count_author_0 = 0; let mut count_author_1 = 0; let mut prev: Option<&ProofOfStore> = None; @@ -74,9 +75,10 @@ fn test_proof_queue_sorting() { } assert_eq!(count_author_0, 1); assert_eq!(count_author_1, 1); + assert_eq!(num_unique_txns, 2); // Expect: [600, 500, 300, 100] - let (pulled, _) = proof_queue.pull_proofs(&hashset![], 4, 4, true); + let (pulled, num_unique_txns, _) = proof_queue.pull_proofs(&hashset![], 6, 4, 4, true); let mut count_author_0 = 0; let mut count_author_1 = 0; let mut prev: Option<&ProofOfStore> = None; @@ -93,6 +95,7 @@ fn test_proof_queue_sorting() { } prev = Some(batch); } + assert_eq!(num_unique_txns, 4); assert_eq!(count_author_0, 2); assert_eq!(count_author_1, 2); } @@ -159,3 +162,78 @@ fn test_proof_calculate_remaining_txns_and_proofs() { assert_eq!(proof_queue.remaining_txns_and_proofs(), (6, 8)); } + +#[test] +fn test_proof_pull_proofs_with_duplicates() { + let my_peer_id = PeerId::random(); + let mut proof_queue = ProofQueue::new(my_peer_id); + + let txns = vec![ + TransactionSummary::new(PeerId::ONE, 0, HashValue::zero()), + TransactionSummary::new(PeerId::ONE, 1, HashValue::zero()), + TransactionSummary::new(PeerId::ONE, 2, HashValue::zero()), + TransactionSummary::new(PeerId::ONE, 3, HashValue::zero()), + ]; + + let author_0 = PeerId::random(); + let author_1 = PeerId::random(); + + let author_0_batches = vec![ + proof_of_store(author_0, BatchId::new_for_test(0), 100), + proof_of_store(author_0, BatchId::new_for_test(1), 200), + proof_of_store(author_0, BatchId::new_for_test(2), 50), + proof_of_store(author_0, BatchId::new_for_test(3), 300), + ]; + let info_0 = author_0_batches[0].info().clone(); + proof_queue.add_batch_summaries(vec![(author_0_batches[0].info().clone(), vec![txns[0]])]); + proof_queue.add_batch_summaries(vec![(author_0_batches[1].info().clone(), vec![txns[1]])]); + proof_queue.add_batch_summaries(vec![(author_0_batches[2].info().clone(), vec![txns[2]])]); + proof_queue.add_batch_summaries(vec![(author_0_batches[3].info().clone(), vec![txns[0]])]); + + for batch in author_0_batches { + proof_queue.push(batch); + } + + let author_1_batches = vec![ + proof_of_store(author_1, BatchId::new_for_test(4), 500), + proof_of_store(author_1, BatchId::new_for_test(5), 400), + proof_of_store(author_1, BatchId::new_for_test(6), 600), + proof_of_store(author_1, BatchId::new_for_test(7), 50), + ]; + proof_queue.add_batch_summaries(vec![(author_1_batches[0].info().clone(), vec![txns[1]])]); + proof_queue.add_batch_summaries(vec![(author_1_batches[1].info().clone(), vec![txns[2]])]); + proof_queue.add_batch_summaries(vec![(author_1_batches[2].info().clone(), vec![txns[3]])]); + proof_queue.add_batch_summaries(vec![(author_1_batches[3].info().clone(), vec![txns[0]])]); + + for batch in author_1_batches { + proof_queue.push(batch); + } + assert_eq!(proof_queue.remaining_txns_and_proofs(), (4, 8)); + + let result = proof_queue.pull_proofs(&hashset![], 8, 4, 3000, true); + assert!(result.0.len() >= 4); + assert!(result.0.len() <= 8); + let mut pulled_txns = HashSet::new(); + for proof in result.0 { + match proof.batch_id() { + BatchId { id: 0, nonce: 0 } => pulled_txns.insert(0), + BatchId { id: 1, nonce: 0 } => pulled_txns.insert(1), + BatchId { id: 2, nonce: 0 } => pulled_txns.insert(2), + BatchId { id: 3, nonce: 0 } => pulled_txns.insert(0), + BatchId { id: 4, nonce: 0 } => pulled_txns.insert(1), + BatchId { id: 5, nonce: 0 } => pulled_txns.insert(2), + BatchId { id: 6, nonce: 0 } => pulled_txns.insert(3), + BatchId { id: 7, nonce: 0 } => pulled_txns.insert(0), + _ => panic!("Unexpected batch id"), + }; + } + assert!(pulled_txns.len() == 4); + assert!(result.1 == 4); + assert!( + proof_queue + .pull_proofs(&hashset![info_0], 8, 4, 400, true) + .0 + .len() + == 7 + ); +} diff --git a/consensus/src/quorum_store/utils.rs b/consensus/src/quorum_store/utils.rs index f584c836d7c54..433dd9846d89f 100644 --- a/consensus/src/quorum_store/utils.rs +++ b/consensus/src/quorum_store/utils.rs @@ -364,18 +364,32 @@ impl ProofQueue { // The flag in the second return argument is true iff the entire proof queue is fully utilized // when pulling the proofs. If any proof from proof queue cannot be included due to size limits, // this flag is set false. + // Returns the proofs, the number of unique transactions in the proofs, and a flag indicating + // whether the proof queue is fully utilized. pub(crate) fn pull_proofs( &mut self, excluded_batches: &HashSet, max_txns: u64, + max_unique_txns: u64, max_bytes: u64, return_non_full: bool, - ) -> (Vec, bool) { + ) -> (Vec, u64, bool) { let mut ret = vec![]; let mut cur_bytes = 0; - let mut cur_txns = 0; + let mut cur_unique_txns = 0; + let mut cur_all_txns = 0; let mut excluded_txns = 0; let mut full = false; + // Set of all the excluded transactions and all the transactions included in the result + let mut filtered_txns = HashSet::new(); + for batch_info in excluded_batches { + let batch_key = BatchKey::from_info(batch_info); + if let Some(txn_summaries) = self.batch_to_txn_summaries.get(&batch_key) { + for txn_summary in txn_summaries { + filtered_txns.insert(*txn_summary); + } + } + } let mut iters = vec![]; for (_, batches) in self.author_to_batches.iter() { @@ -391,19 +405,46 @@ impl ProofQueue { } else if let Some(Some((proof, insertion_time))) = self.batch_to_proof.get(&sort_key.batch_key) { + // Calculate the number of unique transactions if this batch is included in the result + let unique_txns = if let Some(txn_summaries) = + self.batch_to_txn_summaries.get(&sort_key.batch_key) + { + cur_unique_txns + + txn_summaries + .iter() + .filter(|txn_summary| !filtered_txns.contains(txn_summary)) + .count() as u64 + } else { + cur_unique_txns + batch.num_txns() + }; if cur_bytes + batch.num_bytes() > max_bytes - || cur_txns + batch.num_txns() > max_txns + || unique_txns > max_unique_txns + || cur_all_txns + batch.num_txns() > max_txns { // Exceeded the limit for requested bytes or number of transactions. full = true; return false; } cur_bytes += batch.num_bytes(); - cur_txns += batch.num_txns(); + cur_all_txns += batch.num_txns(); + // Add this batch to filtered_txns and calculate the number of + // unique transactions added in the result so far. + cur_unique_txns += self + .batch_to_txn_summaries + .get(&sort_key.batch_key) + .map_or(batch.num_txns(), |summaries| { + summaries + .iter() + .filter(|summary| filtered_txns.insert(**summary)) + .count() as u64 + }); let bucket = proof.gas_bucket_start(); ret.push(proof.clone()); counters::pos_to_pull(bucket, insertion_time.elapsed().as_secs_f64()); - if cur_bytes == max_bytes || cur_txns == max_txns { + if cur_bytes == max_bytes + || cur_all_txns == max_txns + || cur_unique_txns == max_unique_txns + { full = true; return false; } @@ -417,7 +458,8 @@ impl ProofQueue { info!( // before non full check byte_size = cur_bytes, - block_size = cur_txns, + block_total_txns = cur_all_txns, + block_unique_txns = cur_unique_txns, batch_count = ret.len(), full = full, return_non_full = return_non_full, @@ -425,7 +467,10 @@ impl ProofQueue { ); if full || return_non_full { - counters::BLOCK_SIZE_WHEN_PULL.observe(cur_txns as f64); + counters::BLOCK_SIZE_WHEN_PULL.observe(cur_unique_txns as f64); + counters::TOTAL_BLOCK_SIZE_WHEN_PULL.observe(cur_all_txns as f64); + counters::KNOWN_DUPLICATE_TXNS_WHEN_PULL + .observe((cur_all_txns.saturating_sub(cur_unique_txns)) as f64); counters::BLOCK_BYTES_WHEN_PULL.observe(cur_bytes as f64); counters::PROOF_SIZE_WHEN_PULL.observe(ret.len() as f64); counters::EXCLUDED_TXNS_WHEN_PULL.observe(excluded_txns as f64); @@ -433,9 +478,9 @@ impl ProofQueue { self.log_remaining_data_after_pull(excluded_batches, &ret); // Stable sort, so the order of proofs within an author will not change. ret.sort_by_key(|proof| Reverse(proof.gas_bucket_start())); - (ret, !full) + (ret, cur_unique_txns, !full) } else { - (Vec::new(), !full) + (Vec::new(), 0, !full) } } diff --git a/consensus/src/round_manager_fuzzing.rs b/consensus/src/round_manager_fuzzing.rs index 8c58935500ffc..a41b966a48ff9 100644 --- a/consensus/src/round_manager_fuzzing.rs +++ b/consensus/src/round_manager_fuzzing.rs @@ -183,6 +183,7 @@ fn create_node_for_fuzzing() -> RoundManager { time_service, Duration::ZERO, 1, + 1, 1024, 1, 1024, diff --git a/consensus/src/round_manager_test.rs b/consensus/src/round_manager_test.rs index 5b4861c4b5d80..1502922cfed15 100644 --- a/consensus/src/round_manager_test.rs +++ b/consensus/src/round_manager_test.rs @@ -303,6 +303,7 @@ impl NodeSetup { Arc::new(MockPayloadManager::new(None)), time_service.clone(), Duration::ZERO, + 20, 10, 1000, 5, diff --git a/consensus/src/test_utils/mock_payload_manager.rs b/consensus/src/test_utils/mock_payload_manager.rs index d28337e51ebfe..e855347ec4ebb 100644 --- a/consensus/src/test_utils/mock_payload_manager.rs +++ b/consensus/src/test_utils/mock_payload_manager.rs @@ -58,6 +58,7 @@ impl PayloadClient for MockPayloadManager { &self, _max_poll_time: Duration, _max_size: u64, + _max_unique_size: u64, _max_bytes: u64, _max_inline_size: u64, _max_inline_bytes: u64, From 56f4002feb85a30e01b6c578d8d6e28e4bf064fe Mon Sep 17 00:00:00 2001 From: George Mitenkov Date: Tue, 2 Jul 2024 08:32:42 +0100 Subject: [PATCH 021/469] [aptos-vm] Remove writeset txn generator (#13870) --- Cargo.lock | 13 -- Cargo.toml | 2 - .../writeset-transaction-generator/Cargo.toml | 22 --- .../writeset-transaction-generator/src/lib.rs | 7 - .../src/writeset_builder.rs | 135 ------------------ 5 files changed, 179 deletions(-) delete mode 100644 aptos-move/writeset-transaction-generator/Cargo.toml delete mode 100644 aptos-move/writeset-transaction-generator/src/lib.rs delete mode 100644 aptos-move/writeset-transaction-generator/src/writeset_builder.rs diff --git a/Cargo.lock b/Cargo.lock index ecdc4b0cb3946..138e7e2207d95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4426,19 +4426,6 @@ dependencies = [ "warp", ] -[[package]] -name = "aptos-writeset-generator" -version = "0.1.0" -dependencies = [ - "anyhow", - "aptos-crypto", - "aptos-types", - "aptos-vm", - "move-core-types", - "move-vm-runtime", - "move-vm-types", -] - [[package]] name = "arbitrary" version = "1.3.2" diff --git a/Cargo.toml b/Cargo.toml index 78792c45d7105..1f154a5183cf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,6 @@ members = [ "aptos-move/mvhashmap", "aptos-move/package-builder", "aptos-move/vm-genesis", - "aptos-move/writeset-transaction-generator", "aptos-node", "aptos-utils", "config", @@ -454,7 +453,6 @@ aptos-vm-genesis = { path = "aptos-move/vm-genesis" } aptos-vm-types = { path = "aptos-move/aptos-vm-types" } aptos-vm-validator = { path = "vm-validator" } aptos-warp-webserver = { path = "crates/aptos-warp-webserver" } -aptos-writeset-generator = { path = "aptos-move/writeset-transaction-generator" } aptos-cargo-cli = { path = "devtools/aptos-cargo-cli" } # External crate dependencies. diff --git a/aptos-move/writeset-transaction-generator/Cargo.toml b/aptos-move/writeset-transaction-generator/Cargo.toml deleted file mode 100644 index b35659a241013..0000000000000 --- a/aptos-move/writeset-transaction-generator/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "aptos-writeset-generator" -description = "Generating writesets used for incident management" -version = "0.1.0" - -# Workspace inherited keys -authors = { workspace = true } -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -publish = { workspace = true } -repository = { workspace = true } -rust-version = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -aptos-crypto = { workspace = true } -aptos-types = { workspace = true } -aptos-vm = { workspace = true } -move-core-types = { workspace = true } -move-vm-runtime = { workspace = true } -move-vm-types = { workspace = true } diff --git a/aptos-move/writeset-transaction-generator/src/lib.rs b/aptos-move/writeset-transaction-generator/src/lib.rs deleted file mode 100644 index e69d8c54014bb..0000000000000 --- a/aptos-move/writeset-transaction-generator/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright © Aptos Foundation -// Parts of the project are originally copyright © Meta Platforms, Inc. -// SPDX-License-Identifier: Apache-2.0 - -mod writeset_builder; - -pub use writeset_builder::{build_changeset, GenesisSession}; diff --git a/aptos-move/writeset-transaction-generator/src/writeset_builder.rs b/aptos-move/writeset-transaction-generator/src/writeset_builder.rs deleted file mode 100644 index 8d3640a8547a3..0000000000000 --- a/aptos-move/writeset-transaction-generator/src/writeset_builder.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright © Aptos Foundation -// Parts of the project are originally copyright © Meta Platforms, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::format_err; -use aptos_crypto::HashValue; -use aptos_types::{ - account_address::AccountAddress, - account_config::{self, aptos_test_root_address}, - chain_id::ChainId, - state_store::StateView, - transaction::{ChangeSet, Script, Version}, -}; -use aptos_vm::{ - data_cache::AsMoveResolver, - move_vm_ext::{GenesisMoveVM, SessionExt}, -}; -use move_core_types::{ - identifier::Identifier, - language_storage::{ModuleId, TypeTag}, - transaction_argument::convert_txn_args, - value::{serialize_values, MoveValue}, -}; -use move_vm_runtime::module_traversal::{TraversalContext, TraversalStorage}; -use move_vm_types::gas::UnmeteredGasMeter; - -pub struct GenesisSession<'r, 'l>(SessionExt<'r, 'l>); - -impl<'r, 'l> GenesisSession<'r, 'l> { - pub fn exec_func( - &mut self, - module_name: &str, - function_name: &str, - ty_args: Vec, - args: Vec>, - ) { - let traversal_storage = TraversalStorage::new(); - self.0 - .execute_function_bypass_visibility( - &ModuleId::new( - account_config::CORE_CODE_ADDRESS, - Identifier::new(module_name).unwrap(), - ), - &Identifier::new(function_name).unwrap(), - ty_args, - args, - &mut UnmeteredGasMeter, - &mut TraversalContext::new(&traversal_storage), - ) - .unwrap_or_else(|e| { - panic!( - "Error calling {}.{}: {}", - module_name, - function_name, - e.into_vm_status() - ) - }); - } - - pub fn exec_script(&mut self, sender: AccountAddress, script: &Script) { - let mut temp = vec![sender.to_vec()]; - temp.extend(convert_txn_args(script.args())); - let traversal_storage = TraversalStorage::new(); - self.0 - .execute_script( - script.code().to_vec(), - script.ty_args().to_vec(), - temp, - &mut UnmeteredGasMeter, - &mut TraversalContext::new(&traversal_storage), - ) - .unwrap() - } - - fn disable_reconfiguration(&mut self) { - self.exec_func( - "Reconfiguration", - "disable_reconfiguration", - vec![], - serialize_values(&vec![MoveValue::Signer(aptos_test_root_address())]), - ) - } - - fn enable_reconfiguration(&mut self) { - self.exec_func( - "Reconfiguration", - "enable_reconfiguration", - vec![], - serialize_values(&vec![MoveValue::Signer(aptos_test_root_address())]), - ) - } - - pub fn set_aptos_version(&mut self, version: Version) { - self.exec_func( - "AptosVersion", - "set_version", - vec![], - serialize_values(&vec![ - MoveValue::Signer(aptos_test_root_address()), - MoveValue::U64(version), - ]), - ) - } -} - -pub fn build_changeset( - state_view: &S, - procedure: F, - chain_id: ChainId, - genesis_id: HashValue, -) -> ChangeSet -where - F: FnOnce(&mut GenesisSession), -{ - let vm = GenesisMoveVM::new(chain_id); - - let change_set = { - let resolver = state_view.as_move_resolver(); - let mut session = GenesisSession(vm.new_genesis_session(&resolver, genesis_id)); - session.disable_reconfiguration(); - procedure(&mut session); - session.enable_reconfiguration(); - session - .0 - .finish(&vm.genesis_change_set_configs()) - .map_err(|err| format_err!("Unexpected VM Error: {:?}", err)) - .unwrap() - }; - - // Genesis never produces the delta change set. - assert!(change_set.aggregator_v1_delta_set().is_empty()); - change_set - .try_into_storage_change_set() - .expect("Conversion from VMChangeSet into ChangeSet should always succeed") -} From 1e3f52778d587934f9f77996e1f4af7f08c780ae Mon Sep 17 00:00:00 2001 From: George Mitenkov Date: Tue, 2 Jul 2024 10:13:22 +0100 Subject: [PATCH 022/469] Type tag construction (#13884) --- aptos-move/aptos-gas-schedule/src/ver.rs | 9 ++++++- aptos-move/aptos-vm-types/src/environment.rs | 6 ++--- aptos-move/aptos-vm/src/move_vm_ext/vm.rs | 27 +++++++------------ .../move/move-vm/runtime/src/config.rs | 2 ++ .../move/move-vm/runtime/src/loader/mod.rs | 9 ++++++- types/src/vm/configs.rs | 7 +++-- 6 files changed, 34 insertions(+), 26 deletions(-) diff --git a/aptos-move/aptos-gas-schedule/src/ver.rs b/aptos-move/aptos-gas-schedule/src/ver.rs index 713f79709a6e1..975ffe7663169 100644 --- a/aptos-move/aptos-gas-schedule/src/ver.rs +++ b/aptos-move/aptos-gas-schedule/src/ver.rs @@ -8,6 +8,8 @@ /// - Changing how gas is calculated in any way /// /// Change log: +/// - V21 +/// - Fix type to type tag conversion in MoveVM /// - V20 /// - Limits for bounding MoveVM type sizes /// - V19 @@ -64,7 +66,7 @@ /// global operations. /// - V1 /// - TBA -pub const LATEST_GAS_FEATURE_VERSION: u64 = 20; +pub const LATEST_GAS_FEATURE_VERSION: u64 = gas_feature_versions::RELEASE_V1_16; pub mod gas_feature_versions { pub const RELEASE_V1_8: u64 = 11; @@ -76,4 +78,9 @@ pub mod gas_feature_versions { pub const RELEASE_V1_13: u64 = 18; pub const RELEASE_V1_14: u64 = 19; pub const RELEASE_V1_15: u64 = 20; + pub const RELEASE_V1_16: u64 = 21; + pub const RELEASE_V1_17: u64 = 22; + pub const RELEASE_V1_18: u64 = 23; + pub const RELEASE_V1_19: u64 = 24; + pub const RELEASE_V1_20: u64 = 25; } diff --git a/aptos-move/aptos-vm-types/src/environment.rs b/aptos-move/aptos-vm-types/src/environment.rs index 5a936115ab0c5..1b97652f05d2d 100644 --- a/aptos-move/aptos-vm-types/src/environment.rs +++ b/aptos-move/aptos-vm-types/src/environment.rs @@ -122,14 +122,12 @@ impl Environment { chain_id: ChainId, ty_builder: TypeBuilder, ) -> Self { - // By default, do not use delayed field optimization. Instead, clients should enable it - // manually where applicable. - let delayed_field_optimization_enabled = false; + let pseudo_meter_vector_ty_to_ty_tag_construction = true; let vm_config = aptos_prod_vm_config( &features, &timed_features, - delayed_field_optimization_enabled, + pseudo_meter_vector_ty_to_ty_tag_construction, ty_builder, ); diff --git a/aptos-move/aptos-vm/src/move_vm_ext/vm.rs b/aptos-move/aptos-vm/src/move_vm_ext/vm.rs index 2a68662478123..6f70bbe796167 100644 --- a/aptos-move/aptos-vm/src/move_vm_ext/vm.rs +++ b/aptos-move/aptos-vm/src/move_vm_ext/vm.rs @@ -8,7 +8,8 @@ use crate::{ use aptos_crypto::HashValue; use aptos_gas_algebra::DynamicExpression; use aptos_gas_schedule::{ - AptosGasParameters, MiscGasParameters, NativeGasParameters, LATEST_GAS_FEATURE_VERSION, + gas_feature_versions::RELEASE_V1_16, AptosGasParameters, MiscGasParameters, + NativeGasParameters, LATEST_GAS_FEATURE_VERSION, }; use aptos_native_interface::SafeNativeBuilder; use aptos_types::{ @@ -21,7 +22,7 @@ use aptos_vm_types::{ environment::{aptos_default_ty_builder, aptos_prod_ty_builder, Environment}, storage::change_set_configs::ChangeSetConfigs, }; -use move_vm_runtime::{config::VMConfig, move_vm::MoveVM}; +use move_vm_runtime::move_vm::MoveVM; use std::{ops::Deref, sync::Arc}; /// MoveVM wrapper which is used to run genesis initializations. Designed as a @@ -39,13 +40,11 @@ impl GenesisMoveVM { let features = Features::default(); let timed_features = TimedFeaturesBuilder::enable_all().build(); - // Genesis runs sessions, where there is no concept of block execution. - // Hence, delayed fields are not enabled. - let delayed_field_optimization_enabled = false; + let pseudo_meter_vector_ty_to_ty_tag_construction = true; let vm_config = aptos_prod_vm_config( &features, &timed_features, - delayed_field_optimization_enabled, + pseudo_meter_vector_ty_to_ty_tag_construction, aptos_default_ty_builder(&features), ); @@ -142,18 +141,10 @@ impl MoveVmExt { ); // TODO(George): Move gas configs to environment to avoid this clone! - let vm_config = VMConfig { - verifier_config: env.vm_config().verifier_config.clone(), - deserializer_config: env.vm_config().deserializer_config.clone(), - paranoid_type_checks: env.vm_config().paranoid_type_checks, - check_invariant_in_swap_loc: env.vm_config().check_invariant_in_swap_loc, - max_value_nest_depth: env.vm_config().max_value_nest_depth, - type_max_cost: env.vm_config().type_max_cost, - type_base_cost: env.vm_config().type_base_cost, - type_byte_cost: env.vm_config().type_byte_cost, - delayed_field_optimization_enabled: env.vm_config().delayed_field_optimization_enabled, - ty_builder, - }; + let mut vm_config = env.vm_config().clone(); + vm_config.pseudo_meter_vector_ty_to_ty_tag_construction = + gas_feature_version >= RELEASE_V1_16; + vm_config.ty_builder = ty_builder; Self { inner: WarmVmCache::get_warm_vm( diff --git a/third_party/move/move-vm/runtime/src/config.rs b/third_party/move/move-vm/runtime/src/config.rs index 98cc0d619ebc9..7bb93a5a1ffdd 100644 --- a/third_party/move/move-vm/runtime/src/config.rs +++ b/third_party/move/move-vm/runtime/src/config.rs @@ -22,6 +22,7 @@ pub struct VMConfig { pub type_max_cost: u64, pub type_base_cost: u64, pub type_byte_cost: u64, + pub pseudo_meter_vector_ty_to_ty_tag_construction: bool, pub delayed_field_optimization_enabled: bool, pub ty_builder: TypeBuilder, } @@ -37,6 +38,7 @@ impl Default for VMConfig { type_max_cost: 0, type_base_cost: 0, type_byte_cost: 0, + pseudo_meter_vector_ty_to_ty_tag_construction: true, delayed_field_optimization_enabled: false, ty_builder: TypeBuilder::Legacy, } diff --git a/third_party/move/move-vm/runtime/src/loader/mod.rs b/third_party/move/move-vm/runtime/src/loader/mod.rs index 1a4cd9c2982b3..4c849c9af4b29 100644 --- a/third_party/move/move-vm/runtime/src/loader/mod.rs +++ b/third_party/move/move-vm/runtime/src/loader/mod.rs @@ -1647,7 +1647,14 @@ impl Loader { Type::U256 => TypeTag::U256, Type::Address => TypeTag::Address, Type::Signer => TypeTag::Signer, - Type::Vector(ty) => TypeTag::Vector(Box::new(self.type_to_type_tag(ty)?)), + Type::Vector(ty) => { + let el_ty_tag = if self.vm_config.pseudo_meter_vector_ty_to_ty_tag_construction { + self.type_to_type_tag_impl(ty, gas_context)? + } else { + self.type_to_type_tag(ty)? + }; + TypeTag::Vector(Box::new(el_ty_tag)) + }, Type::Struct { idx, .. } => TypeTag::Struct(Box::new(self.struct_name_to_type_tag( *idx, &[], diff --git a/types/src/vm/configs.rs b/types/src/vm/configs.rs index 76463a1cafe10..c48be4c7e8f75 100644 --- a/types/src/vm/configs.rs +++ b/types/src/vm/configs.rs @@ -69,7 +69,7 @@ pub fn aptos_prod_verifier_config(features: &Features) -> VerifierConfig { pub fn aptos_prod_vm_config( features: &Features, timed_features: &TimedFeatures, - delayed_field_optimization_enabled: bool, + pseudo_meter_vector_ty_to_ty_tag_construction: bool, ty_builder: TypeBuilder, ) -> VMConfig { let check_invariant_in_swap_loc = @@ -98,7 +98,10 @@ pub fn aptos_prod_vm_config( type_max_cost, type_base_cost, type_byte_cost, - delayed_field_optimization_enabled, + pseudo_meter_vector_ty_to_ty_tag_construction, + // By default, do not use delayed field optimization. Instead, clients should enable it + // manually where applicable. + delayed_field_optimization_enabled: false, ty_builder, } } From 03f4cf3e7a94d4f351325f61fafa834add07ae72 Mon Sep 17 00:00:00 2001 From: Greg Nazario Date: Fri, 28 Jun 2024 15:02:47 -0700 Subject: [PATCH 023/469] [cli] Add network to config --- crates/aptos/src/common/init.rs | 3 +++ crates/aptos/src/common/types.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/crates/aptos/src/common/init.rs b/crates/aptos/src/common/init.rs index 904ca225706ec..df07f1c8a6abd 100644 --- a/crates/aptos/src/common/init.rs +++ b/crates/aptos/src/common/init.rs @@ -122,6 +122,9 @@ impl CliCommand<()> for InitTool { } }; + // Ensure the config contains the network used + profile_config.network = Some(network); + // Ensure that there is at least a REST URL set for the network match network { Network::Mainnet => { diff --git a/crates/aptos/src/common/types.rs b/crates/aptos/src/common/types.rs index 496bc523dae72..3874153cf5970 100644 --- a/crates/aptos/src/common/types.rs +++ b/crates/aptos/src/common/types.rs @@ -232,6 +232,7 @@ pub const CONFIG_FOLDER: &str = ".aptos"; /// An individual profile #[derive(Debug, Default, Serialize, Deserialize)] pub struct ProfileConfig { + /// Name of network being used, if setup from aptos init #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, /// Private key for commands. From 35202bdb453cac6bb0108f9697e451e4d3c627cf Mon Sep 17 00:00:00 2001 From: Greg Nazario Date: Fri, 28 Jun 2024 15:25:22 -0700 Subject: [PATCH 024/469] [cli] Add helpful explorer links to CLI output --- crates/aptos/CHANGELOG.md | 2 ++ crates/aptos/src/common/init.rs | 31 +++++++++++++++++-- crates/aptos/src/common/types.rs | 52 +++++++++++++++++++------------- crates/aptos/src/common/utils.rs | 37 +++++++++++++++++++++-- 4 files changed, 95 insertions(+), 27 deletions(-) diff --git a/crates/aptos/CHANGELOG.md b/crates/aptos/CHANGELOG.md index c5afa9e567e26..1e1bbd63fab1b 100644 --- a/crates/aptos/CHANGELOG.md +++ b/crates/aptos/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to the Aptos CLI will be captured in this file. This project ## Unreleased - Add balance command to easily get account balances for APT currently +- Add network to config file +- Add explorer links to initialized accounts, and transaction submissions ## [3.4.1] - 2024/05/31 - Upgraded indexer processors for localnet from ca60e51b53c3be6f9517de7c73d4711e9c1f7236 to 5244b84fa5ed872e5280dc8df032d744d62ad29d. Upgraded Hasura metadata accordingly. diff --git a/crates/aptos/src/common/init.rs b/crates/aptos/src/common/init.rs index df07f1c8a6abd..46a5a7da8f6ad 100644 --- a/crates/aptos/src/common/init.rs +++ b/crates/aptos/src/common/init.rs @@ -9,7 +9,7 @@ use crate::{ ConfigSearchMode, EncodingOptions, HardwareWalletOptions, PrivateKeyInputOptions, ProfileConfig, ProfileOptions, PromptOptions, RngArgs, DEFAULT_PROFILE, }, - utils::{fund_account, prompt_yes_with_override, read_line}, + utils::{explorer_account_link, fund_account, prompt_yes_with_override, read_line}, }, }; use aptos_crypto::{ed25519::Ed25519PrivateKey, PrivateKey, ValidCryptoMaterialStringExt}; @@ -22,7 +22,11 @@ use async_trait::async_trait; use clap::Parser; use reqwest::Url; use serde::{Deserialize, Serialize}; -use std::{collections::BTreeMap, str::FromStr}; +use std::{ + collections::BTreeMap, + fmt::{Display, Formatter}, + str::FromStr, +}; /// 1 APT (might not actually get that much, depending on the faucet) const NUM_DEFAULT_OCTAS: u64 = 100000000; @@ -340,7 +344,16 @@ impl CliCommand<()> for InitTool { .expect("Must have profiles, as created above") .insert(profile_name.to_string(), profile_config); config.save()?; - eprintln!("\n---\nAptos CLI is now set up for account {} as profile {}! Run `aptos --help` for more information about commands", address, self.profile_options.profile_name().unwrap_or(DEFAULT_PROFILE)); + let profile_name = self + .profile_options + .profile_name() + .unwrap_or(DEFAULT_PROFILE); + eprintln!( + "\n---\nAptos CLI is now set up for account {} as profile {}!\n See the account here: {}\n Run `aptos --help` for more information about commands", + address, + profile_name, + explorer_account_link(address, Some(network)) + ); Ok(()) } } @@ -434,6 +447,18 @@ pub enum Network { Custom, } +impl Display for Network { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", match self { + Network::Mainnet => "mainnet", + Network::Testnet => "testnet", + Network::Devnet => "devnet", + Network::Local => "local", + Network::Custom => "custom", + }) + } +} + impl FromStr for Network { type Err = CliError; diff --git a/crates/aptos/src/common/types.rs b/crates/aptos/src/common/types.rs index 3874153cf5970..9db5e99f90ff2 100644 --- a/crates/aptos/src/common/types.rs +++ b/crates/aptos/src/common/types.rs @@ -1,7 +1,7 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use super::utils::fund_account; +use super::utils::{explorer_transaction_link, fund_account}; use crate::{ common::{ init::Network, @@ -1717,25 +1717,19 @@ impl TransactionOptions { adjusted_max_gas }; - // Sign and submit transaction + // Build a transaction let transaction_factory = TransactionFactory::new(chain_id) .with_gas_unit_price(gas_unit_price) .with_max_gas_amount(max_gas) .with_transaction_expiration_time(self.gas_options.expiration_secs); - match self.get_transaction_account_type() { + // Sign it with the appropriate signer + let transaction = match self.get_transaction_account_type() { Ok(AccountType::Local) => { let (private_key, _) = self.get_key_and_address()?; let sender_account = &mut LocalAccount::new(sender_address, private_key, sequence_number); - let transaction = sender_account - .sign_with_transaction_builder(transaction_factory.payload(payload)); - let response = client - .submit_and_wait(&transaction) - .await - .map_err(|err| CliError::ApiError(err.to_string()))?; - - Ok(response.into_inner()) + sender_account.sign_with_transaction_builder(transaction_factory.payload(payload)) }, Ok(AccountType::HardwareWallet) => { let sender_account = &mut HardwareWalletAccount::new( @@ -1748,17 +1742,33 @@ impl TransactionOptions { HardwareWalletType::Ledger, sequence_number, ); - let transaction = sender_account - .sign_with_transaction_builder(transaction_factory.payload(payload))?; - let response = client - .submit_and_wait(&transaction) - .await - .map_err(|err| CliError::ApiError(err.to_string()))?; - - Ok(response.into_inner()) + sender_account + .sign_with_transaction_builder(transaction_factory.payload(payload))? }, - Err(err) => Err(err), - } + Err(err) => return Err(err), + }; + + // Submit the transaction, printing out a useful transaction link + client + .submit_bcs(&transaction) + .await + .map_err(|err| CliError::ApiError(err.to_string()))?; + let transaction_hash = transaction.clone().committed_hash(); + let network = self + .profile_options + .profile() + .ok() + .and_then(|profile| profile.network); + eprintln!( + "Transaction submitted: {}", + explorer_transaction_link(transaction_hash, network) + ); + let response = client + .wait_for_signed_transaction(&transaction) + .await + .map_err(|err| CliError::ApiError(err.to_string()))?; + + Ok(response.into_inner()) } /// Simulates a transaction locally, using the debugger to fetch required data from remote. diff --git a/crates/aptos/src/common/utils.rs b/crates/aptos/src/common/utils.rs index 9fcd7bc507c08..fcdaf59d4267f 100644 --- a/crates/aptos/src/common/utils.rs +++ b/crates/aptos/src/common/utils.rs @@ -2,9 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - common::types::{ - account_address_from_public_key, CliError, CliTypedResult, PromptOptions, - TransactionOptions, TransactionSummary, + common::{ + init::Network, + types::{ + account_address_from_public_key, CliError, CliTypedResult, PromptOptions, + TransactionOptions, TransactionSummary, + }, }, config::GlobalConfig, CliResult, @@ -563,3 +566,31 @@ pub fn view_json_option_str(option_ref: &serde_json::Value) -> CliTypedResult) -> String { + // For now, default to what the browser is already on, though the link could be wrong + if let Some(network) = network { + format!( + "https://explorer.aptoslabs.com/account/{}?network={}", + hash, network + ) + } else { + format!("https://explorer.aptoslabs.com/account/{}", hash) + } +} + +pub fn explorer_transaction_link( + hash: aptos_crypto::HashValue, + network: Option, +) -> String { + // For now, default to what the browser is already on, though the link could be wrong + if let Some(network) = network { + format!( + "https://explorer.aptoslabs.com/txn/{}?network={}", + hash.to_hex_literal(), + network + ) + } else { + format!("https://explorer.aptoslabs.com/txn/{}", hash) + } +} From 075f18995962fb5ce319dbbeb5ffacfd86cce51f Mon Sep 17 00:00:00 2001 From: igor-aptos <110557261+igor-aptos@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:59:50 -0700 Subject: [PATCH 025/469] Txn emitter changes for it to be used for bulk emitter script (#13248) --- crates/transaction-emitter-lib/src/args.rs | 2 +- crates/transaction-emitter-lib/src/cluster.rs | 23 +- .../src/emitter/account_minter.rs | 248 +++++++++++++----- .../src/emitter/mod.rs | 227 ++++++++-------- .../src/emitter/transaction_executor.rs | 21 +- .../transaction-emitter-lib/src/wrappers.rs | 53 ++-- sdk/src/transaction_builder.rs | 4 + 7 files changed, 342 insertions(+), 236 deletions(-) diff --git a/crates/transaction-emitter-lib/src/args.rs b/crates/transaction-emitter-lib/src/args.rs index 45fe819576cbb..99ff392327ac5 100644 --- a/crates/transaction-emitter-lib/src/args.rs +++ b/crates/transaction-emitter-lib/src/args.rs @@ -194,7 +194,7 @@ pub struct EmitArgs { #[clap(long, default_value = "false")] /// Skip minting account during initialization - pub skip_minting_accounts: bool, + pub skip_funding_accounts: bool, #[clap(long)] pub latency_polling_interval_s: Option, diff --git a/crates/transaction-emitter-lib/src/cluster.rs b/crates/transaction-emitter-lib/src/cluster.rs index b0cd4e73a4be5..a1584b3ef992f 100644 --- a/crates/transaction-emitter-lib/src/cluster.rs +++ b/crates/transaction-emitter-lib/src/cluster.rs @@ -1,7 +1,7 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use crate::{emitter::query_sequence_number, instance::Instance, ClusterArgs}; +use crate::{emitter::load_specific_account, instance::Instance, ClusterArgs}; use anyhow::{anyhow, bail, format_err, Result}; use aptos_crypto::{ ed25519::{Ed25519PrivateKey, Ed25519PublicKey}, @@ -9,9 +9,7 @@ use aptos_crypto::{ }; use aptos_logger::{info, warn}; use aptos_rest_client::{Client as RestClient, State}; -use aptos_sdk::types::{ - account_config::aptos_test_root_address, chain_id::ChainId, AccountKey, LocalAccount, -}; +use aptos_sdk::types::{chain_id::ChainId, AccountKey, LocalAccount}; use futures::{stream::FuturesUnordered, StreamExt}; use rand::seq::SliceRandom; use std::{convert::TryFrom, time::Instant}; @@ -200,22 +198,7 @@ impl Cluster { } pub async fn load_coin_source_account(&self, client: &RestClient) -> Result { - let account_key = self.account_key(); - let address = if self.coin_source_is_root { - aptos_test_root_address() - } else { - account_key.authentication_key().account_address() - }; - - let sequence_number = query_sequence_number(client, address).await.map_err(|e| { - format_err!( - "query_sequence_number on {:?} for account {} failed: {:?}", - client, - address, - e - ) - })?; - Ok(LocalAccount::new(address, account_key, sequence_number)) + load_specific_account(self.account_key(), self.coin_source_is_root, client).await } pub fn random_instance(&self) -> Instance { diff --git a/crates/transaction-emitter-lib/src/emitter/account_minter.rs b/crates/transaction-emitter-lib/src/emitter/account_minter.rs index 6b41ddb4cb7cd..30d591d3936b4 100644 --- a/crates/transaction-emitter-lib/src/emitter/account_minter.rs +++ b/crates/transaction-emitter-lib/src/emitter/account_minter.rs @@ -1,8 +1,13 @@ // Copyright © Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use crate::{emitter::local_account_generator::LocalAccountGenerator, EmitJobRequest}; +use super::{ + local_account_generator::LocalAccountGenerator, parse_seed, + transaction_executor::RestApiReliableTransactionSubmitter, +}; +use crate::EmitJobRequest; use anyhow::{anyhow, bail, format_err, Context, Result}; +use aptos_config::config::DEFAULT_MAX_SUBMIT_TRANSACTION_BATCH_SIZE; use aptos_crypto::{ ed25519::{Ed25519PrivateKey, Ed25519PublicKey}, encoding_type::EncodingType, @@ -16,14 +21,11 @@ use aptos_sdk::{ }, }; use aptos_transaction_generator_lib::{ - CounterState, ReliableTransactionSubmitter, RootAccountHandle, SEND_AMOUNT, + CounterState, ReliableTransactionSubmitter, RootAccountHandle, }; use aptos_types::account_address::AccountAddress; -use core::{ - cmp::min, - result::Result::{Err, Ok}, -}; -use futures::StreamExt; +use core::result::Result::{Err, Ok}; +use futures::{future::try_join_all, StreamExt}; use rand::{rngs::StdRng, Rng, SeedableRng}; use std::{ path::Path, @@ -34,7 +36,8 @@ use std::{ pub struct SourceAccountManager<'t> { pub source_account: Arc, pub txn_executor: &'t dyn ReliableTransactionSubmitter, - pub req: &'t EmitJobRequest, + pub mint_to_root: bool, + pub prompt_before_spending: bool, pub txn_factory: TransactionFactory, } @@ -60,7 +63,7 @@ impl<'t> SourceAccountManager<'t> { .txn_executor .get_account_balance(self.source_account_address()) .await?; - Ok(if self.req.mint_to_root { + Ok(if self.mint_to_root { // We have a root account, so amount of funds minted is not a problem // We can have multiple txn emitter running simultaneously, each coming to this check at the same time. // So they might all pass the check, but not be able to consume funds they need. So we check more conservatively @@ -107,7 +110,7 @@ impl<'t> SourceAccountManager<'t> { )); } - if self.req.prompt_before_spending { + if self.prompt_before_spending { if !prompt_yes(&format!( "plan will consume in total {} balance for {}, are you sure you want to proceed", amount, @@ -156,7 +159,7 @@ impl<'t> SourceAccountManager<'t> { pub struct AccountMinter<'t> { txn_factory: TransactionFactory, - rng: StdRng, + account_rng: StdRng, source_account: &'t SourceAccountManager<'t>, } @@ -164,39 +167,12 @@ impl<'t> AccountMinter<'t> { pub fn new( source_account: &'t SourceAccountManager<'t>, txn_factory: TransactionFactory, - rng: StdRng, + account_rng: StdRng, ) -> Self { Self { source_account, txn_factory, - rng, - } - } - - pub fn get_needed_balance_per_account(&self, req: &EmitJobRequest, num_accounts: usize) -> u64 { - if let Some(val) = req.coins_per_account_override { - info!(" with {} balance each because of override", val); - val - } else { - // round up: - let txnx_per_account = - (req.expected_max_txns + num_accounts as u64 - 1) / num_accounts as u64; - let min_balance = req.max_gas_per_txn * req.gas_price; - let coins_per_account = txnx_per_account - .checked_mul(SEND_AMOUNT + req.get_expected_gas_per_txn() * req.gas_price) - .unwrap() - .checked_add(min_balance) - .unwrap(); // extra coins for secure to pay none zero gas price - - info!( - " with {} balance each because of expecting {} txns per account, with {} gas at {} gas price per txn, and min balance {}", - coins_per_account, - txnx_per_account, - req.get_expected_gas_per_txn(), - req.gas_price, - min_balance, - ); - coins_per_account + account_rng, } } @@ -254,21 +230,22 @@ impl<'t> AccountMinter<'t> { pub async fn create_and_fund_accounts( &mut self, txn_executor: &dyn ReliableTransactionSubmitter, - req: &EmitJobRequest, account_generator: Box, - max_submit_batch_size: usize, local_accounts: Vec>, + coins_per_account: u64, + max_submit_batch_size: usize, + mint_to_root: bool, + create_secondary_source_account: bool, ) -> Result<()> { let num_accounts = local_accounts.len(); info!( - "Account creation plan created for {} accounts and {} txns:", - num_accounts, req.expected_max_txns, + "Account creation plan created for {} accounts and {} coins per account", + num_accounts, coins_per_account, ); let expected_num_seed_accounts = (num_accounts / 50).clamp(1, (num_accounts as f32).sqrt() as usize + 1); - let coins_per_account = self.get_needed_balance_per_account(req, num_accounts); let expected_children_per_seed_account = (num_accounts + expected_num_seed_accounts - 1) / expected_num_seed_accounts; @@ -280,7 +257,7 @@ impl<'t> AccountMinter<'t> { self.txn_factory.get_gas_unit_price(), ); let coins_for_source = Self::funds_needed_for_multi_transfer( - if req.mint_to_root { "root" } else { "source" }, + if mint_to_root { "root" } else { "source" }, expected_num_seed_accounts as u64, coins_per_seed_account, self.txn_factory.get_max_gas_amount(), @@ -293,8 +270,8 @@ impl<'t> AccountMinter<'t> { .await? { // recheck value makes sense for auto-approval. - let max_allowed = (3 * req.expected_max_txns as u128) - .checked_mul((req.get_expected_gas_per_txn() * req.gas_price).into()) + let max_allowed = (3 * coins_per_account as u128) + .checked_mul(num_accounts as u128) .unwrap(); assert!(coins_for_source as u128 <= max_allowed, "Overhead too large to consume funds without approval - estimated total coins needed for load test ({}) are larger than expected_max_txns * expected_gas_per_txn, multiplied by 3 to account for rounding up and overheads ({})", @@ -303,7 +280,7 @@ impl<'t> AccountMinter<'t> { ); } - let new_source_account = if !req.coordination_delay_between_instances.is_zero() { + let new_source_account = if create_secondary_source_account { Some( self.create_new_source_account(txn_executor, coins_for_source) .await?, @@ -410,20 +387,18 @@ impl<'t> AccountMinter<'t> { "Creating and funding seeds accounts (txn {} gas price)", self.txn_factory.get_gas_unit_price() ); - let mut i = 0; - let mut seed_accounts = vec![]; let source_account = match new_source_account { None => self.source_account.get_root_account().clone(), Some(param_account) => Arc::new(param_account), }; - while i < seed_account_num { - let batch_size = min(max_submit_batch_size, seed_account_num - i); - let mut rng = StdRng::from_rng(self.rng()).unwrap(); - let mut batch = account_generator - .gen_local_accounts(txn_executor, batch_size, &mut rng) - .await?; + + let seed_accounts = account_generator + .gen_local_accounts(txn_executor, seed_account_num, self.account_rng()) + .await?; + + for chunk in seed_accounts.chunks(max_submit_batch_size) { let txn_factory = &self.txn_factory; - let create_requests: Vec<_> = batch + let create_requests: Vec<_> = chunk .iter() .map(|account| { create_and_fund_account_request( @@ -437,9 +412,6 @@ impl<'t> AccountMinter<'t> { txn_executor .execute_transactions_with_counter(&create_requests, counters) .await?; - - i += batch_size; - seed_accounts.append(&mut batch); } Ok(seed_accounts) @@ -483,7 +455,7 @@ impl<'t> AccountMinter<'t> { root_account.set_sequence_number(new_sequence_number); } - let new_source_account = LocalAccount::generate(self.rng()); + let new_source_account = LocalAccount::generate(self.account_rng()); let txn = create_and_fund_account_request( root_account.clone(), coins_for_source, @@ -516,8 +488,8 @@ impl<'t> AccountMinter<'t> { bail!("Couldn't create new source account"); } - pub fn rng(&mut self) -> &mut StdRng { - &mut self.rng + pub fn account_rng(&mut self) -> &mut StdRng { + &mut self.account_rng } } @@ -593,3 +565,151 @@ pub fn prompt_yes(prompt: &str) -> bool { } result.unwrap() } + +pub struct BulkAccountCreationConfig { + max_submit_batch_size: usize, + skip_funding_accounts: bool, + seed: Option<[u8; 32]>, + mint_to_root: bool, + prompt_before_spending: bool, + create_secondary_source_account: bool, + expected_gas_per_transfer: u64, + expected_gas_per_account_create: u64, +} + +impl BulkAccountCreationConfig { + pub fn new( + max_submit_batch_size: usize, + skip_funding_accounts: bool, + seed: Option<&str>, + mint_to_root: bool, + prompt_before_spending: bool, + create_secondary_source_account: bool, + expected_gas_per_transfer: u64, + expected_gas_per_account_create: u64, + ) -> Self { + Self { + max_submit_batch_size, + skip_funding_accounts, + seed: seed.map(parse_seed), + mint_to_root, + prompt_before_spending, + create_secondary_source_account, + expected_gas_per_transfer, + expected_gas_per_account_create, + } + } +} + +impl From<&EmitJobRequest> for BulkAccountCreationConfig { + fn from(req: &EmitJobRequest) -> Self { + Self { + max_submit_batch_size: DEFAULT_MAX_SUBMIT_TRANSACTION_BATCH_SIZE, + skip_funding_accounts: req.skip_funding_accounts, + seed: req.account_minter_seed, + mint_to_root: req.mint_to_root, + prompt_before_spending: req.prompt_before_spending, + create_secondary_source_account: !req.coordination_delay_between_instances.is_zero(), + expected_gas_per_transfer: req.get_expected_gas_per_transfer(), + expected_gas_per_account_create: req.get_expected_gas_per_account_create(), + } + } +} + +pub async fn bulk_create_accounts( + coin_source_account: Arc, + txn_executor: &RestApiReliableTransactionSubmitter, + txn_factory: &TransactionFactory, + account_generator: Box, + config: BulkAccountCreationConfig, + num_accounts: usize, + coins_per_account: u64, +) -> Result> { + let source_account_manager = SourceAccountManager { + source_account: coin_source_account, + txn_executor, + mint_to_root: config.mint_to_root, + prompt_before_spending: config.prompt_before_spending, + txn_factory: txn_factory.clone(), + }; + + let seed = config.seed.unwrap_or_else(|| { + let mut rng = StdRng::from_entropy(); + rng.gen() + }); + info!( + "AccountMinter Seed (reuse accounts by passing into --account-minter-seed): {:?}", + seed + ); + + let accounts = account_generator + .gen_local_accounts(txn_executor, num_accounts, &mut StdRng::from_seed(seed)) + .await?; + info!( + "Generated and fetched re-usable accounts for seed {:?}", + seed + ); + + let all_accounts_already_exist = accounts.iter().all(|account| account.sequence_number() > 0); + let send_money_gas = if all_accounts_already_exist { + config.expected_gas_per_transfer + } else { + config.expected_gas_per_account_create + }; + + let mut account_minter = AccountMinter::new( + &source_account_manager, + txn_factory.clone().with_max_gas_amount(send_money_gas), + // Wrap seed once, to not have conflicts between worker and seed accounts. + // We also don't want to continue from the same rng, as number of accounts will affect + // seed accounts. + StdRng::from_seed(StdRng::from_seed(seed).gen()), + ); + + if !config.skip_funding_accounts { + let accounts: Vec<_> = accounts.into_iter().map(Arc::new).collect(); + account_minter + .create_and_fund_accounts( + txn_executor, + account_generator, + accounts.clone(), + coins_per_account, + config.max_submit_batch_size, + config.mint_to_root, + config.create_secondary_source_account, + ) + .await?; + let accounts: Vec<_> = accounts + .into_iter() + .map(|a| Arc::try_unwrap(a).unwrap()) + .collect(); + info!("Accounts created and funded"); + Ok(accounts) + } else { + info!( + "Account reuse plan created for {} accounts and min balance {}", + accounts.len(), + coins_per_account, + ); + + let balance_futures = accounts + .iter() + .map(|account| txn_executor.get_account_balance(account.address())); + let balances: Vec<_> = try_join_all(balance_futures).await?; + accounts + .iter() + .zip(balances) + .for_each(|(account, balance)| { + assert!( + balance >= coins_per_account, + "Account {} has balance {} < needed_min_balance {}", + account.address(), + balance, + coins_per_account + ); + }); + + info!("Skipping funding accounts"); + Ok(accounts) + } +} diff --git a/crates/transaction-emitter-lib/src/emitter/mod.rs b/crates/transaction-emitter-lib/src/emitter/mod.rs index 4b31491d81b42..06302259b27c8 100644 --- a/crates/transaction-emitter-lib/src/emitter/mod.rs +++ b/crates/transaction-emitter-lib/src/emitter/mod.rs @@ -8,8 +8,8 @@ pub mod submission_worker; pub mod transaction_executor; use crate::emitter::{ - account_minter::{AccountMinter, SourceAccountManager}, - local_account_generator::{create_account_generator, LocalAccountGenerator}, + account_minter::{bulk_create_accounts, SourceAccountManager}, + local_account_generator::create_account_generator, stats::{DynamicStatsTracking, TxnStats}, submission_worker::SubmissionWorker, transaction_executor::RestApiReliableTransactionSubmitter, @@ -22,11 +22,12 @@ use aptos_rest_client::{aptos_api_types::AptosErrorCode, error::RestError, Clien use aptos_sdk::{ move_types::account_address::AccountAddress, transaction_builder::{aptos_stdlib, TransactionFactory}, - types::{transaction::SignedTransaction, LocalAccount}, + types::{transaction::SignedTransaction, AccountKey, LocalAccount}, }; use aptos_transaction_generator_lib::{ - create_txn_generator_creator, AccountType, ReliableTransactionSubmitter, TransactionType, + create_txn_generator_creator, AccountType, TransactionType, SEND_AMOUNT, }; +use aptos_types::account_config::aptos_test_root_address; use futures::future::{try_join_all, FutureExt}; use once_cell::sync::Lazy; use rand::{rngs::StdRng, seq::IteratorRandom, Rng}; @@ -46,11 +47,11 @@ use tokio::{runtime::Handle, task::JoinHandle, time}; // Max is 100k TPS for 3 hours const MAX_TXNS: u64 = 1_000_000_000; -const MAX_RETRIES: usize = 12; - // TODO Transfer cost increases during Coin => FA migration, we can reduce back later. -const EXPECTED_GAS_PER_TRANSFER: u64 = 10; -const EXPECTED_GAS_PER_ACCOUNT_CREATE: u64 = 2000 + 8; +pub const EXPECTED_GAS_PER_TRANSFER: u64 = 10; +pub const EXPECTED_GAS_PER_ACCOUNT_CREATE: u64 = 2000 + 8; + +const MAX_RETRIES: usize = 12; // This retry policy is used for important client calls necessary for setting // up the test (e.g. account creation) and collecting its results (e.g. checking @@ -148,6 +149,7 @@ pub struct EmitJobRequest { transaction_mix_per_phase: Vec>, account_type: AccountType, + max_gas_per_txn: u64, init_max_gas_per_txn: Option, @@ -163,7 +165,7 @@ pub struct EmitJobRequest { init_gas_price_multiplier: u64, mint_to_root: bool, - skip_minting_accounts: bool, + skip_funding_accounts: bool, txn_expiration_time_secs: u64, init_expiration_multiplier: f64, @@ -193,7 +195,7 @@ impl Default for EmitJobRequest { init_max_gas_per_txn: None, init_gas_price_multiplier: 2, mint_to_root: false, - skip_minting_accounts: false, + skip_funding_accounts: false, txn_expiration_time_secs: 60, init_expiration_multiplier: 3.0, init_retry_interval: Duration::from_secs(10), @@ -336,8 +338,8 @@ impl EmitJobRequest { self } - pub fn skip_minting_accounts(mut self) -> Self { - self.skip_minting_accounts = true; + pub fn skip_funding_accounts(mut self) -> Self { + self.skip_funding_accounts = true; self } @@ -464,7 +466,7 @@ impl EmitJobRequest { }; info!( - " Transaction emitter targeting {} TPS, expecting {} TPS", + " Transaction emitter targetting {} TPS, expecting {} TPS", tps, num_accounts * transactions_per_account / wait_seconds as usize ); @@ -703,20 +705,21 @@ impl TxnEmitter { .with_transaction_expiration_time(init_expiration_time); let init_retries: usize = usize::try_from(init_expiration_time / req.init_retry_interval.as_secs()).unwrap(); - let seed = req.account_minter_seed.unwrap_or_else(|| self.rng.gen()); let account_generator = create_account_generator(req.account_type); - let mut all_accounts = create_accounts( + let mut all_accounts = bulk_create_accounts( root_account.clone(), + &RestApiReliableTransactionSubmitter::new( + req.rest_clients.clone(), + init_retries, + req.init_retry_interval, + ), &init_txn_factory, account_generator, - &req, - mode_params.max_submit_batch_size, - req.skip_minting_accounts, - seed, + (&req).into(), num_accounts, - init_retries, + get_needed_balance_per_account_from_req(&req, num_accounts), ) .await?; @@ -724,16 +727,17 @@ impl TxnEmitter { let stats = Arc::new(DynamicStatsTracking::new(stats_tracking_phases)); let tokio_handle = Handle::current(); - let txn_executor = RestApiReliableTransactionSubmitter { - rest_clients: req.rest_clients.clone(), - max_retries: init_retries, - retry_after: req.init_retry_interval, - }; + let txn_executor = RestApiReliableTransactionSubmitter::new( + req.rest_clients.clone(), + init_retries, + req.init_retry_interval, + ); let source_account_manager = SourceAccountManager { source_account: root_account.clone(), txn_executor: &txn_executor, - req: &req, txn_factory: init_txn_factory.clone(), + mint_to_root: req.mint_to_root, + prompt_before_spending: req.prompt_before_spending, }; let (txn_generator_creator, _, _) = create_txn_generator_creator( &req.transaction_mix_per_phase, @@ -994,8 +998,9 @@ pub async fn query_sequence_numbers<'a, I>( where I: Iterator, { - let futures = addresses - .map(|address| RETRY_POLICY.retry(move || get_account_if_exists(client, *address))); + let futures = addresses.map(|address| { + RETRY_POLICY.retry(move || get_account_address_and_seq_num(client, *address)) + }); let (seq_nums, timestamps): (Vec<_>, Vec<_>) = try_join_all(futures) .await @@ -1008,14 +1013,23 @@ where Ok((seq_nums, timestamps.into_iter().min().unwrap())) } -async fn get_account_if_exists( +async fn get_account_address_and_seq_num( client: &RestClient, address: AccountAddress, ) -> Result<((AccountAddress, u64), u64)> { + get_account_seq_num(client, address) + .await + .map(|(seq_num, ts)| ((address, seq_num), ts)) +} + +pub async fn get_account_seq_num( + client: &RestClient, + address: AccountAddress, +) -> Result<(u64, u64)> { let result = client.get_account_bcs(address).await; match &result { Ok(resp) => Ok(( - (address, resp.inner().sequence_number()), + resp.inner().sequence_number(), Duration::from_micros(resp.state().timestamp_usecs).as_secs(), )), Err(e) => { @@ -1023,7 +1037,7 @@ async fn get_account_if_exists( if let RestError::Api(api_error) = e { if let AptosErrorCode::AccountNotFound = api_error.error.error_code { return Ok(( - (address, 0), + 0, Duration::from_micros(api_error.state.as_ref().unwrap().timestamp_usecs) .as_secs(), )); @@ -1035,6 +1049,28 @@ async fn get_account_if_exists( } } +pub async fn load_specific_account( + account_key: AccountKey, + is_root: bool, + client: &RestClient, +) -> Result { + let address = if is_root { + aptos_test_root_address() + } else { + account_key.authentication_key().account_address() + }; + + let sequence_number = query_sequence_number(client, address).await.map_err(|e| { + format_err!( + "query_sequence_number on {:?} for account {} failed: {:?}", + client, + address, + e + ) + })?; + Ok(LocalAccount::new(address, account_key, sequence_number)) +} + pub fn gen_transfer_txn_request( sender: &mut LocalAccount, receiver: &AccountAddress, @@ -1060,103 +1096,50 @@ pub fn parse_seed(seed_string: &str) -> [u8; 32] { .expect("failed to convert to array") } -pub async fn create_accounts( - root_account: Arc, - txn_factory: &TransactionFactory, - account_generator: Box, - req: &EmitJobRequest, - max_submit_batch_size: usize, - skip_minting_accounts: bool, - seed: [u8; 32], +pub fn get_needed_balance_per_account( + num_workload_transactions: u64, + gas_per_workload_transaction: u64, + octas_per_workload_transaction: u64, num_accounts: usize, - retries: usize, -) -> Result> { - info!( - "Using reliable/retriable init transaction executor with {} retries, every {}s", - retries, - req.init_retry_interval.as_secs_f32() - ); + gas_price: u64, + max_gas_per_txn: u64, +) -> u64 { + // round up: + let txnx_per_account = + (num_workload_transactions + num_accounts as u64 - 1) / num_accounts as u64; + let coins_per_account = txnx_per_account + .checked_mul(octas_per_workload_transaction + gas_per_workload_transaction * gas_price) + .unwrap() + .checked_add(max_gas_per_txn * gas_price) + .unwrap(); info!( - "AccountMinter Seed (reuse accounts by passing into --account-minter-seed): {:?}", - seed - ); - let txn_executor = RestApiReliableTransactionSubmitter { - rest_clients: req.rest_clients.clone(), - max_retries: retries, - retry_after: req.init_retry_interval, - }; - let source_account_manager = SourceAccountManager { - source_account: root_account, - txn_executor: &txn_executor, - req, - txn_factory: txn_factory.clone(), - }; - - let mut rng = StdRng::from_seed(seed); - - let accounts = account_generator - .gen_local_accounts(&txn_executor, num_accounts, &mut rng) - .await?; - - info!("Generated re-usable accounts for seed {:?}", seed); - - let all_accounts_already_exist = accounts.iter().all(|account| account.sequence_number() > 0); - let send_money_gas = if all_accounts_already_exist { - req.get_expected_gas_per_transfer() - } else { - req.get_expected_gas_per_account_create() - }; - - let mut account_minter = AccountMinter::new( - &source_account_manager, - txn_factory.clone().with_max_gas_amount(send_money_gas), - StdRng::from_seed(seed), + "Needed {} balance for each account because of expecting {} txns per account with {} gas and {} octas, with leaving {} gas for max_txn_gas, all at {} gas price", + coins_per_account, + txnx_per_account, + gas_per_workload_transaction, + octas_per_workload_transaction, + max_gas_per_txn, + gas_price, ); + coins_per_account +} - if !skip_minting_accounts { - let accounts: Vec<_> = accounts.into_iter().map(Arc::new).collect(); - account_minter - .create_and_fund_accounts( - &txn_executor, - req, - account_generator, - max_submit_batch_size, - accounts.clone(), - ) - .await?; - let accounts: Vec<_> = accounts - .into_iter() - .map(|a| Arc::try_unwrap(a).unwrap()) - .collect(); - info!("Accounts created and funded"); - Ok(accounts) - } else { +pub fn get_needed_balance_per_account_from_req(req: &EmitJobRequest, num_accounts: usize) -> u64 { + if let Some(val) = req.coins_per_account_override { info!( - "Account reuse plan created for {} accounts and {} txns:", - accounts.len(), - req.expected_max_txns, + "Needed {} balance for each account because of override", + val ); - - let needed_min_balance = account_minter.get_needed_balance_per_account(req, accounts.len()); - let balance_futures = accounts - .iter() - .map(|account| txn_executor.get_account_balance(account.address())); - let balances: Vec<_> = try_join_all(balance_futures).await?; - accounts - .iter() - .zip(balances) - .for_each(|(account, balance)| { - assert!( - balance >= needed_min_balance, - "Account {} has balance {} < needed_min_balance {}", - account.address(), - balance, - needed_min_balance - ); - }); - - info!("Skipping minting accounts"); - Ok(accounts) + val + } else { + get_needed_balance_per_account( + req.expected_max_txns, + req.get_expected_gas_per_txn(), + SEND_AMOUNT, + num_accounts, + req.gas_price, + req.max_gas_per_txn, + ) } } diff --git a/crates/transaction-emitter-lib/src/emitter/transaction_executor.rs b/crates/transaction-emitter-lib/src/emitter/transaction_executor.rs index 9db067bbe6e0f..4a2cae20c5f13 100644 --- a/crates/transaction-emitter-lib/src/emitter/transaction_executor.rs +++ b/crates/transaction-emitter-lib/src/emitter/transaction_executor.rs @@ -3,7 +3,7 @@ use super::RETRY_POLICY; use anyhow::{Context, Result}; -use aptos_logger::{debug, sample, sample::SampleRate, warn}; +use aptos_logger::{debug, info, sample, sample::SampleRate, warn}; use aptos_rest_client::{aptos_api_types::AptosErrorCode, error::RestError, Client as RestClient}; use aptos_sdk::{ move_types::account_address::AccountAddress, types::transaction::SignedTransaction, @@ -19,12 +19,25 @@ use std::{ // Reliable/retrying transaction executor, used for initializing pub struct RestApiReliableTransactionSubmitter { - pub rest_clients: Vec, - pub max_retries: usize, - pub retry_after: Duration, + rest_clients: Vec, + max_retries: usize, + retry_after: Duration, } impl RestApiReliableTransactionSubmitter { + pub fn new(rest_clients: Vec, max_retries: usize, retry_after: Duration) -> Self { + info!( + "Using reliable/retriable init transaction executor with {} retries, every {}s", + max_retries, + retry_after.as_secs_f32() + ); + Self { + rest_clients, + max_retries, + retry_after, + } + } + fn random_rest_client(&self) -> &RestClient { let mut rng = thread_rng(); self.rest_clients.choose(&mut rng).unwrap() diff --git a/crates/transaction-emitter-lib/src/wrappers.rs b/crates/transaction-emitter-lib/src/wrappers.rs index 813c4a1ae07dc..4314371229bbb 100644 --- a/crates/transaction-emitter-lib/src/wrappers.rs +++ b/crates/transaction-emitter-lib/src/wrappers.rs @@ -5,18 +5,19 @@ use crate::{ args::{ClusterArgs, EmitArgs}, cluster::Cluster, emitter::{ - create_accounts, local_account_generator::PrivateKeyAccountGenerator, parse_seed, - stats::TxnStats, EmitJobMode, EmitJobRequest, NumAccountsMode, TxnEmitter, + account_minter::bulk_create_accounts, get_needed_balance_per_account_from_req, + local_account_generator::PrivateKeyAccountGenerator, stats::TxnStats, + transaction_executor::RestApiReliableTransactionSubmitter, EmitJobMode, EmitJobRequest, + NumAccountsMode, TxnEmitter, }, instance::Instance, CreateAccountsArgs, }; use anyhow::{bail, Context, Result}; -use aptos_config::config::DEFAULT_MAX_SUBMIT_TRANSACTION_BATCH_SIZE; use aptos_logger::{error, info}; use aptos_sdk::transaction_builder::TransactionFactory; use aptos_transaction_generator_lib::{args::TransactionTypeArg, WorkflowProgress}; -use rand::{rngs::StdRng, Rng, SeedableRng}; +use rand::{rngs::StdRng, SeedableRng}; use std::{ sync::Arc, time::{Duration, Instant}, @@ -156,8 +157,8 @@ pub async fn emit_transactions_with_cluster( .latency_polling_interval(Duration::from_secs_f32(latency_polling_interval_s)); } - if args.skip_minting_accounts { - emit_job_request = emit_job_request.skip_minting_accounts(); + if args.skip_funding_accounts { + emit_job_request = emit_job_request.skip_funding_accounts(); } let coin_source_account = std::sync::Arc::new(coin_source_account); @@ -185,30 +186,32 @@ pub async fn create_accounts_command( let txn_factory = TransactionFactory::new(cluster.chain_id) .with_transaction_expiration_time(60) .with_max_gas_amount(create_accounts_args.max_gas_per_txn); - let emit_job_request = - EmitJobRequest::new(cluster.all_instances().map(Instance::rest_client).collect()) - .init_gas_price_multiplier(1) - .expected_gas_per_txn(create_accounts_args.max_gas_per_txn) - .max_gas_per_txn(create_accounts_args.max_gas_per_txn) - .coins_per_account_override(0) - .expected_max_txns(0) - .prompt_before_spending(); - let seed = match &create_accounts_args.account_minter_seed { - Some(str) => parse_seed(str), - None => StdRng::from_entropy().gen(), - }; - - create_accounts( + let rest_clients = cluster + .all_instances() + .map(Instance::rest_client) + .collect::>(); + let mut emit_job_request = EmitJobRequest::new(rest_clients.clone()) + .init_gas_price_multiplier(1) + .expected_gas_per_txn(create_accounts_args.max_gas_per_txn) + .max_gas_per_txn(create_accounts_args.max_gas_per_txn) + .coins_per_account_override(0) + .expected_max_txns(0) + .prompt_before_spending(); + + if let Some(seed) = &create_accounts_args.account_minter_seed { + emit_job_request = emit_job_request.account_minter_seed(seed); + } + + bulk_create_accounts( coin_source_account, + &RestApiReliableTransactionSubmitter::new(rest_clients, 6, Duration::from_secs(10)), &txn_factory, Box::new(PrivateKeyAccountGenerator), - &emit_job_request, - DEFAULT_MAX_SUBMIT_TRANSACTION_BATCH_SIZE, - false, - seed, + (&emit_job_request).into(), create_accounts_args.count, - 4, + get_needed_balance_per_account_from_req(&emit_job_request, create_accounts_args.count), ) .await?; + Ok(()) } diff --git a/sdk/src/transaction_builder.rs b/sdk/src/transaction_builder.rs index bb28f20c1c95e..9f5e958fef9b8 100644 --- a/sdk/src/transaction_builder.rs +++ b/sdk/src/transaction_builder.rs @@ -137,6 +137,10 @@ impl TransactionFactory { self.transaction_expiration_time } + pub fn get_chain_id(&self) -> ChainId { + self.chain_id + } + pub fn payload(&self, payload: TransactionPayload) -> TransactionBuilder { self.transaction_builder(payload) } From 436c931296f6a824cc24f22e446a8159e0c4f29c Mon Sep 17 00:00:00 2001 From: Alan Luong Date: Tue, 2 Jul 2024 16:02:47 -0400 Subject: [PATCH 026/469] update testnet ranges for replay-verify (#13852) --- testsuite/module_verify.py | 2 +- testsuite/replay_verify.py | 46 +++++++++++++++++---------------- testsuite/verify_core/common.py | 4 ++- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/testsuite/module_verify.py b/testsuite/module_verify.py index 8adcc55590c50..14f53f75b845d 100755 --- a/testsuite/module_verify.py +++ b/testsuite/module_verify.py @@ -7,7 +7,7 @@ import shutil import subprocess -from verify_core.common import query_backup_latest_version, clear_artifacts +from verify_core.common import warm_cache_and_get_latest_backup_version, clear_artifacts def main(): diff --git a/testsuite/replay_verify.py b/testsuite/replay_verify.py index 67ab4895a9b12..e685c3e2c82fb 100755 --- a/testsuite/replay_verify.py +++ b/testsuite/replay_verify.py @@ -12,29 +12,32 @@ from collections import deque -from verify_core.common import clear_artifacts, query_backup_latest_version +from verify_core.common import ( + clear_artifacts, + warm_cache_and_get_latest_backup_version, +) TESTNET_RANGES = [ - (962_000_000, 978_000_000), - (978_000_000, 994_000_000), - (994_000_000, 1_009_000_000), - (1_009_000_000, 1_025_000_000), - (1_025_000_000, 1_041_000_000), - (1_041_000_000, 1_057_000_000), - (1_057_000_000, 1_073_000_000), - (1_073_000_000, 1_088_000_000), - (1_088_000_000, 1_104_000_000), - (1_104_000_000, 1_120_000_000), - (1_120_000_000, 1_136_000_000), - (1_136_000_000, 1_151_000_000), - (1_151_000_000, 1_167_000_000), - (1_167_000_000, 1_183_000_000), - (1_183_000_000, 1_199_000_000), - (1_199_000_000, 1_215_000_000), - (1_215_000_000, 1_230_000_000), - (1_230_000_000, 1_246_000_000), - (1_246_000_000, 1_262_000_000), + (862_000_000, 878_000_000), + (878_000_000, 894_000_000), + (894_000_000, 910_000_000), + (910_000_000, 926_000_000), + (942_000_000, 958_000_000), + (958_000_000, 974_000_000), + (974_000_000, 990_000_000), + (990_000_000, 1_006_000_000), + (1_006_000_000, 1_022_000_000), + (1_022_000_000, 1_038_000_000), + (1_038_000_000, 1_054_000_000), + (1_054_000_000, 1_070_000_000), + (1_070_000_000, 1_086_000_000), + (1_086_000_000, 1_102_000_000), + (1_102_000_000, 1_115_000_000), + (1_115_000_000, 1_128_000_000), + (1_128_000_000, 1_141_000_000), + (1_141_000_000, 1_154_000_000), + (1_154_000_000, 1_167_000_000), ] MAINNET_RANGES = [ @@ -196,9 +199,8 @@ def main(runner_no=None, runner_cnt=None, start_version=None, end_version=None): ), "runner_cnt must match the number of runners in the mapping" runner_start = runner_mapping[runner_no][0] runner_end = runner_mapping[runner_no][1] - latest_version = query_backup_latest_version(BACKUP_CONFIG_TEMPLATE_PATH) + warm_cache_and_get_latest_backup_version(BACKUP_CONFIG_TEMPLATE_PATH) if runner_no == runner_cnt - 1: - runner_end = latest_version if runner_end is None: raise Exception("Failed to query latest version from backup") print("runner start %d end %d" % (runner_start, runner_end)) diff --git a/testsuite/verify_core/common.py b/testsuite/verify_core/common.py index a7ae25d42fccc..ed92129d1cd01 100644 --- a/testsuite/verify_core/common.py +++ b/testsuite/verify_core/common.py @@ -39,7 +39,9 @@ def clear_artifacts(): shutil.rmtree(f) -def query_backup_latest_version(backup_config_template_path: str) -> int: +def warm_cache_and_get_latest_backup_version( + backup_config_template_path: str, +) -> int: """query latest version in backup, at the same time, pre-heat metadata cache""" db_backup_result = subprocess.Popen( [ From bd0dd6a823f317090960625dc0af625619e5e929 Mon Sep 17 00:00:00 2001 From: Alexzander Stone Date: Tue, 2 Jul 2024 14:22:50 -0700 Subject: [PATCH 027/469] [aptos-framework] Dispatchable Fungible Asset - Derived Supply addition (#13801) * Dispatchable Fungible Asset - Derived Supply Derived Supply overload added to dispatchable fungible assets. *similar to `derived_balance`* Primary Updates: * Fungible Asset * Updated Errors to include Supply-related features. * `register_dispatch_functions` includes supply override support. * Dispatchable Fungible Asset * `derived_supply` created to retrieve supply (either through function or default fungible asset supply), and re-package into expected Option-wrapped u128 value. * Native function defined for derived supply, with associated spec. Included in pre-existing `make_all` of related `.rs` file. Secondary Updates: * `ten_x_token.move` updated to include tests for `derived_supply`. * `deflation_token.move` and tests updated to include new parameter for `register_dispatch_functions`. * `simple_dispatchable_token.move`, `reentrant_token.move`, `permissioned_token.move`, and `nil_op_token.move` have same additions as above. * * Expected error codes updated in dispatchable tests. * * Seperated derive supply from original dispatchable functions (struct and register/get functions). * New function for registering derive supply dispatch function. * Added inline function for register dispatch function sanity check (gas/execution improvement). * Error codes reverted, and new error code placed as last index + 1. Prevents bytecode issues when upgrading (also, other smart contracts won't be impacted if relying on the error code integer). * Docs updates. --------- Co-authored-by: Alex Stone --- .../doc/dispatchable_fungible_asset.md | 114 +++++++++++ .../aptos-framework/doc/fungible_asset.md | 192 ++++++++++++++++-- .../sources/dispatchable_fungible_asset.move | 34 ++++ .../dispatchable_fungible_asset.spec.move | 4 + .../sources/fungible_asset.move | 97 +++++++-- .../tests/deflation_token_tests.move | 10 +- .../aptos-framework/tests/nil_op_token.move | 2 +- .../tests/permissioned_token.move | 2 +- .../tests/simple_dispatchable_token.move | 2 +- .../aptos-framework/tests/ten_x_token.move | 22 +- .../tests/ten_x_token_tests.move | 8 +- .../natives/dispatchable_fungible_asset.rs | 3 +- 12 files changed, 445 insertions(+), 45 deletions(-) diff --git a/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md b/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md index bb6d62d42743e..b3b76f108128c 100644 --- a/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md +++ b/aptos-move/framework/aptos-framework/doc/dispatchable_fungible_asset.md @@ -23,19 +23,23 @@ See AIP-73 for further discussion - [Resource `TransferRefStore`](#0x1_dispatchable_fungible_asset_TransferRefStore) - [Constants](#@Constants_0) - [Function `register_dispatch_functions`](#0x1_dispatchable_fungible_asset_register_dispatch_functions) +- [Function `register_derive_supply_dispatch_function`](#0x1_dispatchable_fungible_asset_register_derive_supply_dispatch_function) - [Function `withdraw`](#0x1_dispatchable_fungible_asset_withdraw) - [Function `deposit`](#0x1_dispatchable_fungible_asset_deposit) - [Function `transfer`](#0x1_dispatchable_fungible_asset_transfer) - [Function `transfer_assert_minimum_deposit`](#0x1_dispatchable_fungible_asset_transfer_assert_minimum_deposit) - [Function `derived_balance`](#0x1_dispatchable_fungible_asset_derived_balance) +- [Function `derived_supply`](#0x1_dispatchable_fungible_asset_derived_supply) - [Function `borrow_transfer_ref`](#0x1_dispatchable_fungible_asset_borrow_transfer_ref) - [Function `dispatchable_withdraw`](#0x1_dispatchable_fungible_asset_dispatchable_withdraw) - [Function `dispatchable_deposit`](#0x1_dispatchable_fungible_asset_dispatchable_deposit) - [Function `dispatchable_derived_balance`](#0x1_dispatchable_fungible_asset_dispatchable_derived_balance) +- [Function `dispatchable_derived_supply`](#0x1_dispatchable_fungible_asset_dispatchable_derived_supply) - [Specification](#@Specification_1) - [Function `dispatchable_withdraw`](#@Specification_1_dispatchable_withdraw) - [Function `dispatchable_deposit`](#@Specification_1_dispatchable_deposit) - [Function `dispatchable_derived_balance`](#@Specification_1_dispatchable_derived_balance) + - [Function `dispatchable_derived_supply`](#@Specification_1_dispatchable_derived_supply) 

use 0x1::error;
@@ -160,6 +164,36 @@ TransferRefStore doesn't exist on the fungible asset type.
 
 
 
+
+
+
+
+## Function `register_derive_supply_dispatch_function`
+
+
+
+
public fun register_derive_supply_dispatch_function(constructor_ref: &object::ConstructorRef, dispatch_function: option::Option<function_info::FunctionInfo>)
+

+
+
+
+

+Implementation
+
+
+
public fun register_derive_supply_dispatch_function(
+    constructor_ref: &ConstructorRef,
+    dispatch_function: Option<FunctionInfo>
+) {
+    fungible_asset::register_derive_supply_dispatch_function(
+        constructor_ref,
+        dispatch_function
+    );
+}
+

+
+
+
 

 
 
@@ -364,6 +398,45 @@ The semantics of value will be governed by the function specified in DispatchFun
 
 
 
+
+
+
+
+## Function `derived_supply`
+
+Get the derived supply of the fungible asset using the overloaded hook.
+
+The semantics of supply will be governed by the function specified in DeriveSupplyDispatch.
+
+
+
#[view]
+public fun derived_supply<T: key>(metadata: object::Object<T>): option::Option<u128>
+

+
+
+
+

+Implementation
+
+
+
public fun derived_supply<T: key>(metadata: Object<T>): Option<u128> {
+    let func_opt = fungible_asset::derived_supply_dispatch_function(metadata);
+    if (option::is_some(&func_opt)) {
+        assert!(
+            features::dispatchable_fungible_asset_enabled(),
+            error::aborted(ENOT_ACTIVATED)
+        );
+        let func = option::borrow(&func_opt);
+        function_info::load_module_from_function(func);
+        dispatchable_derived_supply(metadata, func)
+    } else {
+        fungible_asset::supply(metadata)
+    }
+}
+

+
+
+
 

 
 
@@ -474,6 +547,31 @@ The semantics of value will be governed by the function specified in DispatchFun
 
 
 
+
+
+
+
+## Function `dispatchable_derived_supply`
+
+
+
+
fun dispatchable_derived_supply<T: key>(store: object::Object<T>, function: &function_info::FunctionInfo): option::Option<u128>
+

+
+
+
+

+Implementation
+
+
+
native fun dispatchable_derived_supply<T: key>(
+    store: Object<T>,
+    function: &FunctionInfo,
+): Option<u128>;
+

+
+
+
 

 
 
@@ -530,6 +628,22 @@ The semantics of value will be governed by the function specified in DispatchFun
 
 
 
+
pragma opaque;
+

+
+
+
+
+
+### Function `dispatchable_derived_supply`
+
+
+
fun dispatchable_derived_supply<T: key>(store: object::Object<T>, function: &function_info::FunctionInfo): option::Option<u128>
+

+
+
+
+
 
pragma opaque;
 

 
diff --git a/aptos-move/framework/aptos-framework/doc/fungible_asset.md b/aptos-move/framework/aptos-framework/doc/fungible_asset.md
index 25a863d8482c9..2c58cdd944dac 100644
--- a/aptos-move/framework/aptos-framework/doc/fungible_asset.md
+++ b/aptos-move/framework/aptos-framework/doc/fungible_asset.md
@@ -13,6 +13,7 @@ metadata object can be any object that equipped with 
+
+## Resource `DeriveSupply`
+
+
+
+
#[resource_group_member(#[group = 0x1::object::ObjectGroup])]
+struct DeriveSupply has key
+

+
+
+
+

+Fields
+
+
+

+

+dispatch_function: option::Option<function_info::FunctionInfo>
+

+

+
+

+

+
+
 

 
 
@@ -892,6 +924,16 @@ Provided derived_balance function type doesn't meet the signature requirement.
 
 
 
+
+
+Provided derived_supply function type doesn't meet the signature requirement.
+
+
+
const EDERIVED_SUPPLY_FUNCTION_SIGNATURE_MISMATCH: u64 = 33;
+

+
+
+
 
 
 Fungible asset and store do not match.
@@ -1403,28 +1445,13 @@ Create a fungible asset store whose transfer rule would be overloaded by the pro
             )
         );
     });
-
-    // Cannot register hook for APT.
-    assert!(
-        object::address_from_constructor_ref(constructor_ref) != @aptos_fungible_asset,
-        error::permission_denied(EAPT_NOT_DISPATCHABLE)
-    );
-    assert!(
-        !object::can_generate_delete_ref(constructor_ref),
-        error::invalid_argument(EOBJECT_IS_DELETABLE)
-    );
+    register_dispatch_function_sanity_check(constructor_ref);
     assert!(
         !exists<DispatchFunctionStore>(
             object::address_from_constructor_ref(constructor_ref)
         ),
         error::already_exists(EALREADY_REGISTERED)
     );
-    assert!(
-        exists<Metadata>(
-            object::address_from_constructor_ref(constructor_ref)
-        ),
-        error::not_found(EFUNGIBLE_METADATA_EXISTENCE),
-    );
 
     let store_obj = &object::generate_signer(constructor_ref);
 
@@ -1442,6 +1469,110 @@ Create a fungible asset store whose transfer rule would be overloaded by the pro
 
 
 
+
+
+
+
+## Function `register_derive_supply_dispatch_function`
+
+Define the derived supply dispatch with the provided function.
+
+
+
public(friend) fun register_derive_supply_dispatch_function(constructor_ref: &object::ConstructorRef, dispatch_function: option::Option<function_info::FunctionInfo>)
+

+
+
+
+

+Implementation
+
+
+
public(friend) fun register_derive_supply_dispatch_function(
+    constructor_ref: &ConstructorRef,
+    dispatch_function: Option<FunctionInfo>
+) {
+    // Verify that caller type matches callee type so wrongly typed function cannot be registered.
+    option::for_each_ref(&dispatch_function, |supply_function| {
+        let function_info = function_info::new_function_info_from_address(
+            @aptos_framework,
+            string::utf8(b"dispatchable_fungible_asset"),
+            string::utf8(b"dispatchable_derived_supply"),
+        );
+        // Verify that caller type matches callee type so wrongly typed function cannot be registered.
+        assert!(
+            function_info::check_dispatch_type_compatibility(
+                &function_info,
+                supply_function
+            ),
+            error::invalid_argument(
+                EDERIVED_SUPPLY_FUNCTION_SIGNATURE_MISMATCH
+            )
+        );
+    });
+    register_dispatch_function_sanity_check(constructor_ref);
+    assert!(
+        !exists<DeriveSupply>(
+            object::address_from_constructor_ref(constructor_ref)
+        ),
+        error::already_exists(EALREADY_REGISTERED)
+    );
+
+
+    let store_obj = &object::generate_signer(constructor_ref);
+
+    // Store the overload function hook.
+    move_to<DeriveSupply>(
+        store_obj,
+        DeriveSupply {
+            dispatch_function
+        }
+    );
+}
+

+
+
+
+

+
+
+
+## Function `register_dispatch_function_sanity_check`
+
+Check the requirements for registering a dispatchable function.
+
+
+
fun register_dispatch_function_sanity_check(constructor_ref: &object::ConstructorRef)
+

+
+
+
+

+Implementation
+
+
+
inline fun register_dispatch_function_sanity_check(
+    constructor_ref: &ConstructorRef,
+)  {
+    // Cannot register hook for APT.
+    assert!(
+        object::address_from_constructor_ref(constructor_ref) != @aptos_fungible_asset,
+        error::permission_denied(EAPT_NOT_DISPATCHABLE)
+    );
+    assert!(
+        !object::can_generate_delete_ref(constructor_ref),
+        error::invalid_argument(EOBJECT_IS_DELETABLE)
+    );
+    assert!(
+        exists<Metadata>(
+            object::address_from_constructor_ref(constructor_ref)
+        ),
+        error::not_found(EFUNGIBLE_METADATA_EXISTENCE),
+    );
+}
+

+
+
+
 

 
 
@@ -2243,6 +2374,35 @@ Return whether a fungible asset type is dispatchable.
 
 
 
+
+
+
+
+## Function `derived_supply_dispatch_function`
+
+
+
+
public(friend) fun derived_supply_dispatch_function<T: key>(metadata: object::Object<T>): option::Option<function_info::FunctionInfo>
+

+
+
+
+

+Implementation
+
+
+
public(friend) fun derived_supply_dispatch_function<T: key>(metadata: Object<T>): Option<FunctionInfo> acquires DeriveSupply {
+    let metadata_addr = object::object_address(&metadata);
+    if (exists<DeriveSupply>(metadata_addr)) {
+        borrow_global<DeriveSupply>(metadata_addr).dispatch_function
+    } else {
+        option::none()
+    }
+}
+

+
+
+
 

 
 
diff --git a/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move b/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move
index aa843a38f7201..5a70aff95d2c1 100644
--- a/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move
+++ b/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.move
@@ -58,6 +58,16 @@ module aptos_framework::dispatchable_fungible_asset {
         );
     }
 
+    public fun register_derive_supply_dispatch_function(
+        constructor_ref: &ConstructorRef,
+        dispatch_function: Option
+    ) {
+        fungible_asset::register_derive_supply_dispatch_function(
+            constructor_ref,
+            dispatch_function
+        );
+    }
+
     /// Withdraw `amount` of the fungible asset from `store` by the owner.
     ///
     /// The semantics of deposit will be governed by the function specified in DispatchFunctionStore.
@@ -162,6 +172,25 @@ module aptos_framework::dispatchable_fungible_asset {
         }
     }
 
+    #[view]
+    /// Get the derived supply of the fungible asset using the overloaded hook.
+    ///
+    /// The semantics of supply will be governed by the function specified in DeriveSupplyDispatch.
+    public fun derived_supply(metadata: Object): Option {
+        let func_opt = fungible_asset::derived_supply_dispatch_function(metadata);
+        if (option::is_some(&func_opt)) {
+            assert!(
+                features::dispatchable_fungible_asset_enabled(),
+                error::aborted(ENOT_ACTIVATED)
+            );
+            let func = option::borrow(&func_opt);
+            function_info::load_module_from_function(func);
+            dispatchable_derived_supply(metadata, func)
+        } else {
+            fungible_asset::supply(metadata)
+        }
+    }
+
     inline fun borrow_transfer_ref(metadata: Object): &TransferRef acquires TransferRefStore {
         let metadata_addr = object::object_address(
             &fungible_asset::store_metadata(metadata)
@@ -191,4 +220,9 @@ module aptos_framework::dispatchable_fungible_asset {
         store: Object,
         function: &FunctionInfo,
     ): u64;
+
+    native fun dispatchable_derived_supply(
+        store: Object,
+        function: &FunctionInfo,
+    ): Option;
 }
diff --git a/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.spec.move b/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.spec.move
index 5932673f6041c..744faced798ac 100644
--- a/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.spec.move
+++ b/aptos-move/framework/aptos-framework/sources/dispatchable_fungible_asset.spec.move
@@ -14,4 +14,8 @@ spec aptos_framework::dispatchable_fungible_asset {
     spec dispatchable_derived_balance{
         pragma opaque;
     }
+
+    spec dispatchable_derived_supply{
+        pragma opaque;
+    }
 }
diff --git a/aptos-move/framework/aptos-framework/sources/fungible_asset.move b/aptos-move/framework/aptos-framework/sources/fungible_asset.move
index 1d86762e01e73..1b12cb49a2c61 100644
--- a/aptos-move/framework/aptos-framework/sources/fungible_asset.move
+++ b/aptos-move/framework/aptos-framework/sources/fungible_asset.move
@@ -85,6 +85,8 @@ module aptos_framework::fungible_asset {
     const EAPT_NOT_DISPATCHABLE: u64 = 31;
     /// Flag for Concurrent Supply not enabled
     const ECONCURRENT_BALANCE_NOT_ENABLED: u64 = 32;
+    /// Provided derived_supply function type doesn't meet the signature requirement.
+    const EDERIVED_SUPPLY_FUNCTION_SIGNATURE_MISMATCH: u64 = 33;
 
     //
     // Constants
@@ -152,6 +154,11 @@ module aptos_framework::fungible_asset {
         derived_balance_function: Option,
     }
 
+    #[resource_group_member(group = aptos_framework::object::ObjectGroup)]
+    struct DeriveSupply has key {
+        dispatch_function: Option
+    }
+
     #[resource_group_member(group = aptos_framework::object::ObjectGroup)]
     /// The store object that holds concurrent fungible asset balance.
     struct ConcurrentFungibleBalance has key {
@@ -348,28 +355,13 @@ module aptos_framework::fungible_asset {
                 )
             );
         });
-
-        // Cannot register hook for APT.
-        assert!(
-            object::address_from_constructor_ref(constructor_ref) != @aptos_fungible_asset,
-            error::permission_denied(EAPT_NOT_DISPATCHABLE)
-        );
-        assert!(
-            !object::can_generate_delete_ref(constructor_ref),
-            error::invalid_argument(EOBJECT_IS_DELETABLE)
-        );
+        register_dispatch_function_sanity_check(constructor_ref);
         assert!(
             !exists(
                 object::address_from_constructor_ref(constructor_ref)
             ),
             error::already_exists(EALREADY_REGISTERED)
         );
-        assert!(
-            exists(
-                object::address_from_constructor_ref(constructor_ref)
-            ),
-            error::not_found(EFUNGIBLE_METADATA_EXISTENCE),
-        );
 
         let store_obj = &object::generate_signer(constructor_ref);
 
@@ -384,6 +376,70 @@ module aptos_framework::fungible_asset {
         );
     }
 
+    /// Define the derived supply dispatch with the provided function.
+    public(friend) fun register_derive_supply_dispatch_function(
+        constructor_ref: &ConstructorRef,
+        dispatch_function: Option
+    ) {
+        // Verify that caller type matches callee type so wrongly typed function cannot be registered.
+        option::for_each_ref(&dispatch_function, |supply_function| {
+            let function_info = function_info::new_function_info_from_address(
+                @aptos_framework,
+                string::utf8(b"dispatchable_fungible_asset"),
+                string::utf8(b"dispatchable_derived_supply"),
+            );
+            // Verify that caller type matches callee type so wrongly typed function cannot be registered.
+            assert!(
+                function_info::check_dispatch_type_compatibility(
+                    &function_info,
+                    supply_function
+                ),
+                error::invalid_argument(
+                    EDERIVED_SUPPLY_FUNCTION_SIGNATURE_MISMATCH
+                )
+            );
+        });
+        register_dispatch_function_sanity_check(constructor_ref);
+        assert!(
+            !exists(
+                object::address_from_constructor_ref(constructor_ref)
+            ),
+            error::already_exists(EALREADY_REGISTERED)
+        );
+
+
+        let store_obj = &object::generate_signer(constructor_ref);
+
+        // Store the overload function hook.
+        move_to(
+            store_obj,
+            DeriveSupply {
+                dispatch_function
+            }
+        );
+    }
+
+    /// Check the requirements for registering a dispatchable function.
+    inline fun register_dispatch_function_sanity_check(
+        constructor_ref: &ConstructorRef,
+    )  {
+        // Cannot register hook for APT.
+        assert!(
+            object::address_from_constructor_ref(constructor_ref) != @aptos_fungible_asset,
+            error::permission_denied(EAPT_NOT_DISPATCHABLE)
+        );
+        assert!(
+            !object::can_generate_delete_ref(constructor_ref),
+            error::invalid_argument(EOBJECT_IS_DELETABLE)
+        );
+        assert!(
+            exists(
+                object::address_from_constructor_ref(constructor_ref)
+            ),
+            error::not_found(EFUNGIBLE_METADATA_EXISTENCE),
+        );
+    }
+
     /// Creates a mint ref that can be used to mint fungible assets from the given fungible object's constructor ref.
     /// This can only be called at object creation time as constructor_ref is only available then.
     public fun generate_mint_ref(constructor_ref: &ConstructorRef): MintRef {
@@ -625,6 +681,15 @@ module aptos_framework::fungible_asset {
         }
     }
 
+    public(friend) fun derived_supply_dispatch_function(metadata: Object): Option acquires DeriveSupply {
+        let metadata_addr = object::object_address(&metadata);
+        if (exists(metadata_addr)) {
+            borrow_global(metadata_addr).dispatch_function
+        } else {
+            option::none()
+        }
+    }
+
     public fun asset_metadata(fa: &FungibleAsset): Object {
         fa.metadata
     }
diff --git a/aptos-move/framework/aptos-framework/tests/deflation_token_tests.move b/aptos-move/framework/aptos-framework/tests/deflation_token_tests.move
index 32f4f153a2cd4..6d7b6d32b3dae 100644
--- a/aptos-move/framework/aptos-framework/tests/deflation_token_tests.move
+++ b/aptos-move/framework/aptos-framework/tests/deflation_token_tests.move
@@ -152,7 +152,7 @@ module 0xcafe::deflation_token_tests {
             &creator_ref,
             option::some(withdraw),
             option::none(),
-            option::none(),
+            option::none()
         );
     }
 
@@ -218,7 +218,7 @@ module 0xcafe::deflation_token_tests {
             &creator_ref,
             option::some(withdraw),
             option::none(),
-            option::some(withdraw),
+            option::some(withdraw)
         );
     }
 
@@ -241,7 +241,7 @@ module 0xcafe::deflation_token_tests {
             &creator_ref,
             option::some(withdraw),
             option::none(),
-            option::none(),
+            option::none()
         );
     }
 
@@ -263,7 +263,7 @@ module 0xcafe::deflation_token_tests {
             &creator_ref,
             option::some(withdraw),
             option::none(),
-            option::none(),
+            option::none()
         );
     }
 
@@ -303,7 +303,7 @@ module 0xcafe::deflation_token_tests {
             &creator_ref,
             option::some(withdraw),
             option::none(),
-            option::none(),
+            option::none()
         );
     }
 
diff --git a/aptos-move/framework/aptos-framework/tests/nil_op_token.move b/aptos-move/framework/aptos-framework/tests/nil_op_token.move
index 6a4a852e19c78..9cdf7b13a6de2 100644
--- a/aptos-move/framework/aptos-framework/tests/nil_op_token.move
+++ b/aptos-move/framework/aptos-framework/tests/nil_op_token.move
@@ -21,7 +21,7 @@ module 0xcafe::nil_op_token {
             constructor_ref,
             option::some(withdraw),
             option::none(),
-            option::none()
+            option::none(),
         );
     }
 
diff --git a/aptos-move/framework/aptos-framework/tests/permissioned_token.move b/aptos-move/framework/aptos-framework/tests/permissioned_token.move
index fe0ea6d3a7dfc..724ee6b628fac 100644
--- a/aptos-move/framework/aptos-framework/tests/permissioned_token.move
+++ b/aptos-move/framework/aptos-framework/tests/permissioned_token.move
@@ -31,7 +31,7 @@ module 0xcafe::permissioned_token {
             constructor_ref,
             option::some(withdraw),
             option::none(),
-            option::none()
+            option::none(),
         );
     }
 
diff --git a/aptos-move/framework/aptos-framework/tests/simple_dispatchable_token.move b/aptos-move/framework/aptos-framework/tests/simple_dispatchable_token.move
index 7c6e05e0fc5ea..3f9057521bcaa 100644
--- a/aptos-move/framework/aptos-framework/tests/simple_dispatchable_token.move
+++ b/aptos-move/framework/aptos-framework/tests/simple_dispatchable_token.move
@@ -28,7 +28,7 @@ module 0xcafe::simple_token {
             constructor_ref,
             option::some(withdraw),
             option::some(deposit),
-            option::none(),
+            option::none()
         );
     }
 
diff --git a/aptos-move/framework/aptos-framework/tests/ten_x_token.move b/aptos-move/framework/aptos-framework/tests/ten_x_token.move
index 7db59d838c9fc..c3841702fe881 100644
--- a/aptos-move/framework/aptos-framework/tests/ten_x_token.move
+++ b/aptos-move/framework/aptos-framework/tests/ten_x_token.move
@@ -6,21 +6,31 @@ module 0xcafe::ten_x_token {
     use aptos_framework::function_info;
 
     use std::option;
+    use std::option::Option;
     use std::signer;
     use std::string;
 
     public fun initialize(account: &signer, constructor_ref: &ConstructorRef) {
         assert!(signer::address_of(account) == @0xcafe, 1);
-        let value = function_info::new_function_info(
+        let balance_value = function_info::new_function_info(
             account,
             string::utf8(b"ten_x_token"),
             string::utf8(b"derived_balance"),
         );
+        let supply_value = function_info::new_function_info(
+            account,
+            string::utf8(b"ten_x_token"),
+            string::utf8(b"derived_supply"),
+        );
         dispatchable_fungible_asset::register_dispatch_functions(
             constructor_ref,
             option::none(),
             option::none(),
-            option::some(value)
+            option::some(balance_value)
+        );
+        dispatchable_fungible_asset::register_derive_supply_dispatch_function(
+            constructor_ref,
+            option::some(supply_value)
         );
     }
 
@@ -28,4 +38,12 @@ module 0xcafe::ten_x_token {
         // Derived value is always 10x!
         fungible_asset::balance(store) * 10
     }
+
+    public fun derived_supply(metadata: Object): Option {
+        // Derived supply is 10x.
+        if(option::is_some(&fungible_asset::supply(metadata))) {
+            return option::some(option::extract(&mut fungible_asset::supply(metadata)) * 10)
+        };
+        option::none()
+    }
 }
diff --git a/aptos-move/framework/aptos-framework/tests/ten_x_token_tests.move b/aptos-move/framework/aptos-framework/tests/ten_x_token_tests.move
index d62278c6c54d0..282783f40c32b 100644
--- a/aptos-move/framework/aptos-framework/tests/ten_x_token_tests.move
+++ b/aptos-move/framework/aptos-framework/tests/ten_x_token_tests.move
@@ -19,13 +19,17 @@ module aptos_framework::ten_x_token_tests {
         ten_x_token::initialize(creator, &creator_ref);
 
         assert!(fungible_asset::supply(metadata) == option::some(0), 1);
+        assert!(dispatchable_fungible_asset::derived_supply(metadata) == option::some(0), 2);
         // Mint
         let fa = fungible_asset::mint(&mint, 100);
-        assert!(fungible_asset::supply(metadata) == option::some(100), 2);
+        assert!(fungible_asset::supply(metadata) == option::some(100), 3);
         // Deposit will cause an re-entrant call into dispatchable_fungible_asset
         dispatchable_fungible_asset::deposit(creator_store, fa);
 
         // The derived value is 10x
-        assert!(dispatchable_fungible_asset::derived_balance(creator_store) == 1000, 5);
+        assert!(dispatchable_fungible_asset::derived_balance(creator_store) == 1000, 4);
+
+        // The derived supply is 10x
+        assert!(dispatchable_fungible_asset::derived_supply(metadata) == option::some(1000), 5);
     }
 }
diff --git a/aptos-move/framework/src/natives/dispatchable_fungible_asset.rs b/aptos-move/framework/src/natives/dispatchable_fungible_asset.rs
index d7d0d76a6784d..9554c5d163377 100644
--- a/aptos-move/framework/src/natives/dispatchable_fungible_asset.rs
+++ b/aptos-move/framework/src/natives/dispatchable_fungible_asset.rs
@@ -11,7 +11,7 @@ use smallvec::SmallVec;
 use std::collections::VecDeque;
 
 /***************************************************************************************************
- * native fun dispatchable_withdraw / dispatchable_deposit / dispatchable_derived_balance
+ * native fun dispatchable_withdraw / dispatchable_deposit / dispatchable_derived_balance / dispatchable_derived_supply
  *
  *   Directs control flow based on the last argument. We use the same native function implementation
  *   for all dispatching native.
@@ -54,6 +54,7 @@ pub fn make_all(
         ("dispatchable_withdraw", native_dispatch as RawSafeNative),
         ("dispatchable_deposit", native_dispatch),
         ("dispatchable_derived_balance", native_dispatch),
+        ("dispatchable_derived_supply", native_dispatch),
     ];
 
     builder.make_named_natives(natives)

From 174bf39995a962d8bca3f6097f0e80634b7fc749 Mon Sep 17 00:00:00 2001
From: Michael Straka 
Date: Tue, 2 Jul 2024 17:44:43 -0500
Subject: [PATCH 028/469] Fix Keyless input_gen.py (#13895)

* calc string bodies in input gen

* add string bodies to input gen

* input gen makes string bodies

* input gen kind of works

* better logs

* edit input gen

* input gen works again

* clean up input gen

* edit

* edit README
---
 keyless/circuit/README.md                     |   2 +-
 .../circuit/templates/helpers/arrays.circom   |   3 -
 .../circuit/templates/helpers/base64.circom   |  22 ++--
 keyless/circuit/tools/input_gen.py            | 116 +++++++++++-------
 4 files changed, 82 insertions(+), 61 deletions(-)

diff --git a/keyless/circuit/README.md b/keyless/circuit/README.md
index a3749bd4fc903..c62c9dcd31c7b 100644
--- a/keyless/circuit/README.md
+++ b/keyless/circuit/README.md
@@ -26,7 +26,7 @@ To generate a sample prover and verifier key pair, run the following commands:
 
 ## Testing
 
-TODO: update `input_gen.py` to match the latest circuit, then provide instructions.
+Unit testing located in `src` may be run using `cargo test`. 
 
 ## Generating a sample proof
 
diff --git a/keyless/circuit/templates/helpers/arrays.circom b/keyless/circuit/templates/helpers/arrays.circom
index dc0996e949621..54fe02a449b12 100644
--- a/keyless/circuit/templates/helpers/arrays.circom
+++ b/keyless/circuit/templates/helpers/arrays.circom
@@ -269,9 +269,6 @@ template ConcatenationCheck(maxFullStringLen, maxLeftStringLen, maxRightStringLe
 template CheckAreASCIIDigits(maxNumDigits) {
     signal input in[maxNumDigits];
     signal input len;
-    for (var i = 0; i < maxNumDigits; i++) {
-        log(in[i]);
-    }
     
     signal selector[maxNumDigits] <== ArraySelector(maxNumDigits)(0, len);
     for (var i = 0; i < maxNumDigits; i++) {
diff --git a/keyless/circuit/templates/helpers/base64.circom b/keyless/circuit/templates/helpers/base64.circom
index 2e63416ac45ae..b6793e8d6f1f1 100644
--- a/keyless/circuit/templates/helpers/base64.circom
+++ b/keyless/circuit/templates/helpers/base64.circom
@@ -62,7 +62,7 @@ template Base64URLLookup() {
     signal sum_underscore <== sum_minus + equal_underscore.out * 63;
 
     out <== sum_underscore;
-    log("sum_underscore (out): ", out);
+    //log("sum_underscore (out): ", out);
 
     // '='
     component equal_eqsign = IsZero();
@@ -73,14 +73,14 @@ template Base64URLLookup() {
     zero_padding.in <== in;
 
 
-    log("zero_padding.out: ", zero_padding.out);
-    log("equal_eqsign.out: ", equal_eqsign.out);
-    log("equal_underscore.out: ", equal_underscore.out);
-    log("equal_minus.out: ", equal_minus.out);
-    log("range_09: ", range_09);
-    log("range_az: ", range_az);
-    log("range_AZ: ", range_AZ);
-    log("< end Base64URLLookup");
+    //log("zero_padding.out: ", zero_padding.out);
+    //log("equal_eqsign.out: ", equal_eqsign.out);
+    //log("equal_underscore.out: ", equal_underscore.out);
+    //log("equal_minus.out: ", equal_minus.out);
+    //log("range_09: ", range_09);
+    //log("range_az: ", range_az);
+    //log("range_AZ: ", range_AZ);
+    //log("< end Base64URLLookup");
 
     signal result <== range_AZ + range_az + range_09 + equal_minus.out + equal_underscore.out + equal_eqsign.out + zero_padding.out;
     1 === result;
@@ -107,8 +107,8 @@ template Base64Decode(N) {
         for (var j = 0; j < 4; j++) {
             bits_in[i\4][j] = Num2Bits(6);
 
-            log(">> calling into Base64URLLookup");
-            log("translate[i\\4][j].in: ", in[i+j]);
+            //log(">> calling into Base64URLLookup");
+            //log("translate[i\\4][j].in: ", in[i+j]);
 
             translate[i\4][j] = Base64URLLookup();
             translate[i\4][j].in <== in[i+j];
diff --git a/keyless/circuit/tools/input_gen.py b/keyless/circuit/tools/input_gen.py
index 1faff3e9fa6ae..df5cbc471b922 100755
--- a/keyless/circuit/tools/input_gen.py
+++ b/keyless/circuit/tools/input_gen.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
-# TODO: update `input_gen.py` to match the latest circuit, then provide instructions.
-# WARNING: This code is guaranteed to work only for the hardcoded JWT present, and is planned to be deprecated soon
+# WARNING: This code is guaranteed to work only for the hardcoded JWT present, and is meant only to be used to provide a quick and easy way to run the full Keyless circuit. Detailed unit tests can be found in `circuit/src`.
 
 import json
 import os
@@ -16,8 +15,20 @@
 from Crypto.Signature.pkcs1_15 import PKCS115_SigScheme
 from Crypto.Hash import SHA256
 import Crypto
-import pprint
+import pprint 
 
+def calc_string_bodies(string):
+    string_bodies = [0] * len(string)
+    string_bodies[1] = int(string[0] == '"')
+
+    for i in range(2,len(string)):
+        if not string_bodies[i-2] and string[i-1] == '"' and string[i-2] != '\\':
+            string_bodies[i] = 1
+        elif string_bodies[i-1] and string[i] == '"' and string[i-1] != '\\':
+            string_bodies[i] = 0
+        else:
+            string_bodies[i] = string_bodies[i-1]
+    return string_bodies
 
 # Returns JSON array of bytes representing the input `string`, 0 padded to `maxLen`
 def pad_string_new(string, maxLen, n="", use_ord=False):
@@ -42,13 +53,16 @@ def pad_string_new(string, maxLen, n="", use_ord=False):
 
 
 # Returns JSON array of bytes representing the input `string`, 0 padded to `maxLen`
-def pad_string(string, maxLen, n="", use_ord=False):
+def pad_string(string, maxLen, n="", use_ord=True):
     string_len = len(string)
     string_to_pad = maxLen - string_len
 
     result = "["
     for c in string:
-        result = result + '"' + str(ord(c)) + '",'
+        if use_ord:
+            result = result + '"' + str(ord(c)) + '",'
+        else:
+            result = result + '"' + str(c) + '",'
 
     for i in range(string_to_pad):
         result = result + '"' + '0' + '",'
@@ -61,7 +75,7 @@ def pad_string(string, maxLen, n="", use_ord=False):
 def format_output(dictionary):
     res = "{"
     for key in dictionary:
-        res = res + key + " : " + dictionary[key] + ","
+        res = res + key + " : " + str(dictionary[key]) + ","
     res = res[:-1] + "}"
     return res
 
@@ -91,39 +105,21 @@ def limbs_to_long(limbs):
 
 
 # iat_value = "1700255944" # Friday, November 17, 2023
-iat_value = "1711552630"
+iat_value = "1719866138"
 
 exp_date_num = 111_111_111_111
 exp_date = str(exp_date_num)  # 12-21-5490
 exp_horizon_num = 999_999_999_999  # ~31,710 years
 exp_horizon = str(exp_horizon_num)
 # nonce_value = "2284473333442251804379681643965308154311773667525398119496797545594705356495"
-nonce_value = "12772123150809496860193457976937182964297283633705872391534946866719681904311"
+nonce_value = "2284473333442251804379681643965308154311773667525398119496797545594705356495"
 public_inputs_hash_value = '"' + str(
-    20184347722831264297183009689956363527052066666845340178129495539169215716642) + '"'
+    990250399590304032496786539443088814495837679250179990478424822100531016130) + '"'
 
 nonce = int(nonce_value)
 
 jwt_max_len = 192 * 8
 
-# Dictionary encoding of the JWT
-# jwt_dict = {
-#    "iss": "https://accounts.google.com",
-#    "azp": "407408718192.apps.googleusercontent.com",
-#    "aud": "407408718192.apps.googleusercontent.com",
-#    "sub": "113990307082899718775",
-#    "hd": "aptoslabs.com",
-#    "email": "michael@aptoslabs.com",
-#    "email_verified": True,
-#    "at_hash": "bxIESuI59IoZb5alCASqBg",
-#    "name": "Michael Straka",
-#    "picture": "https://lh3.googleusercontent.com/a/ACg8ocJvY4kVUBRtLxe1IqKWL5i7tBDJzFp9YuWVXMzwPpbs=s96-c",
-#    "given_name": "Michael",
-#    "family_name": "Straka",
-#    "locale": "en",
-#    "exp":2700259544
-# }
-#
 # jwt_dict = {
 #  "iss": "test.oidc.provider",
 #  "azp": "511276456880-i7i4787c1863damto6899ts989j2e35r.apps.googleusercontent.com",
@@ -141,13 +137,30 @@ def limbs_to_long(limbs):
 #  "exp": 1911556230
 # }
 
-original_b64 = "eyJpc3MiOiJ0ZXN0Lm9pZGMucHJvdmlkZXIiLCJhenAiOiI1MTEyNzY0NTY4ODAtaTdpNDc4N2MxODYzZGFtdG82ODk5dHM5ODlqMmUzNXIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJhdWQiOiI1MTEyNzY0NTY4ODAtaTdpNDc4N2MxODYzZGFtdG82ODk5dHM5ODlqMmUzNXIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJzdWIiOiIxMDI5MDQ2MzAxNzE1OTI1MjA1OTIiLCJlbWFpbCI6Imhlcm8xMjAwMDkxQGdtYWlsLmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJub25jZSI6IjEyNzcyMTIzMTUwODA5NDk2ODYwMTkzNDU3OTc2OTM3MTgyOTY0Mjk3MjgzNjMzNzA1ODcyMzkxNTM0OTQ2ODY2NzE5NjgxOTA0MzExIiwibmJmIjoxNzExNTUyMzMwLCJuYW1lIjoi44Kz44Oz44OJ44Km44OP44Or44KtIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FDZzhvY0lNWmZJa05XR1JCVEQ5MjR4bF9pZWZwTWNjTGd1d2RNSWluTVB6YWo1TDRRPXM5Ni1jIiwiZ2l2ZW5fbmFtZSI6IuODq-OCrSIsImZhbWlseV9uYW1lIjoi44Kz44Oz44OJ44KmIiwiaWF0IjoxNzExNTUyNjMwLCJleHAiOjE5MTE1NTYyMzB9"
+#jwt_dict = {
+#        "iss":"https://accounts.google.com",
+#        "azp":"407408718192.apps.googleusercontent.com",
+#        "aud":"407408718192.apps.googleusercontent.com",
+#        "sub":"113990307082899718775",
+#        "at_hash":"lVeD4xP6Q1ZGrL3gFcCQLQ",
+#        "name":"Michael Straka",
+#        "picture":"https://lh3.googleusercontent.com/a/ACg8ocLVn8F8VnXKNNJhROhTpQuLLjFEdv_uhoe-DUaRTlxKEy9e4w=s96-c",
+#        "given_name":"Michael",
+#        "family_name":"Straka",
+#        "iat":1719866138,
+#        "exp":1719869738,
+#        "nonce":"2284473333442251804379681643965308154311773667525398119496797545594705356495"
+#        }
+
+
+#original_b64 = "eyJpc3MiOiJ0ZXN0Lm9pZGMucHJvdmlkZXIiLCJhenAiOiI1MTEyNzY0NTY4ODAtaTdpNDc4N2MxODYzZGFtdG82ODk5dHM5ODlqMmUzNXIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJhdWQiOiI1MTEyNzY0NTY4ODAtaTdpNDc4N2MxODYzZGFtdG82ODk5dHM5ODlqMmUzNXIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJzdWIiOiIxMDI5MDQ2MzAxNzE1OTI1MjA1OTIiLCJlbWFpbCI6Imhlcm8xMjAwMDkxQGdtYWlsLmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJub25jZSI6IjEyNzcyMTIzMTUwODA5NDk2ODYwMTkzNDU3OTc2OTM3MTgyOTY0Mjk3MjgzNjMzNzA1ODcyMzkxNTM0OTQ2ODY2NzE5NjgxOTA0MzExIiwibmJmIjoxNzExNTUyMzMwLCJuYW1lIjoi44Kz44Oz44OJ44Km44OP44Or44KtIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FDZzhvY0lNWmZJa05XR1JCVEQ5MjR4bF9pZWZwTWNjTGd1d2RNSWluTVB6YWo1TDRRPXM5Ni1jIiwiZ2l2ZW5fbmFtZSI6IuODq-OCrSIsImZhbWlseV9uYW1lIjoi44Kz44Oz44OJ44KmIiwiaWF0IjoxNzExNTUyNjMwLCJleHAiOjE5MTE1NTYyMzB9"
+original_b64 = "eyJpc3MiOiJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20iLCJhenAiOiI0MDc0MDg3MTgxOTIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJhdWQiOiI0MDc0MDg3MTgxOTIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJzdWIiOiIxMTM5OTAzMDcwODI4OTk3MTg3NzUiLCJhdF9oYXNoIjoibFZlRDR4UDZRMVpHckwzZ0ZjQ1FMUSIsIm5hbWUiOiJNaWNoYWVsIFN0cmFrYSIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS9BQ2c4b2NMVm44RjhWblhLTk5KaFJPaFRwUXVMTGpGRWR2X3Vob2UtRFVhUlRseEtFeTllNHc9czk2LWMiLCJnaXZlbl9uYW1lIjoiTWljaGFlbCIsImZhbWlseV9uYW1lIjoiU3RyYWthIiwiaWF0IjoxNzE5ODY2MTM4LCJleHAiOjE3MTk4Njk3MzgsIm5vbmNlIjoiMjI4NDQ3MzMzMzQ0MjI1MTgwNDM3OTY4MTY0Mzk2NTMwODE1NDMxMTc3MzY2NzUyNTM5ODExOTQ5Njc5NzU0NTU5NDcwNTM1NjQ5NSJ9"
 original_str = base64.urlsafe_b64decode(original_b64)
 print("original_str bytes", original_str)
 jwt_dict = json.loads(original_str)
 
-jwt_dict['iat'] = int(iat_value)  # WARNING: the code assumes this is NOT the last field
-jwt_dict['nonce'] = nonce_value  # WARNING: the code assumes this is the last field
+#jwt_dict['iat'] = int(iat_value)  # WARNING: the code assumes this is NOT the last field
+#jwt_dict['nonce'] = nonce_value  # WARNING: the code assumes this is the last field
 
 secret = rsa.generate_private_key(
     backend=crypto_default_backend(),
@@ -174,6 +187,7 @@ def _encode_payload(
 #    crypto_serialization.PublicFormat.OpenSSH
 # )
 jwt_payload_b64 = unsigned_b64_jwt_string[unsigned_b64_jwt_string.rfind(".") + 1:]
+print(jwt_payload_b64)
 jwt_payload = base64.urlsafe_b64decode(jwt_payload_b64)
 jwt_payload = jwt_payload.decode('utf-8')
 print("\njwt_payload bytes  ", jwt_payload, "\n")
@@ -187,8 +201,10 @@ def _encode_payload(
 maxAudKVPairLen = 140
 maxAudNameLen = 40
 maxAudValueLen = 120
-# aud_field_string = "\"aud\":\"407408718192.apps.googleusercontent.com\","
-aud_field_string = "\"aud\":\"511276456880-i7i4787c1863damto6899ts989j2e35r.apps.googleusercontent.com\","
+aud_field_string = "\"aud\":\"407408718192.apps.googleusercontent.com\","
+#aud_field_string = "\"aud\":\"511276456880-i7i4787c1863damto6899ts989j2e35r.apps.googleusercontent.com\","
+aud_string_bodies = calc_string_bodies(aud_field_string)
+aud_string_bodies_value = pad_string(aud_string_bodies, maxAudKVPairLen, use_ord=False)
 aud_field_value = pad_string(aud_field_string, maxAudKVPairLen)
 aud_field_len_value = '"' + str(len(aud_field_string)) + '"'
 aud_index_value = '"' + str(jwt_payload.index("aud") - 1) + '"'  # First '"' character in aud field index in payload
@@ -196,8 +212,8 @@ def _encode_payload(
 aud_colon_index_value = '"' + str(aud_colon_index) + '"'
 aud_value_index_value = '"' + str(aud_colon_index + 2) + '"'  # TODO: This doesn't work if there's whitespace
 aud_name = "aud"
-# aud_value = "407408718192.apps.googleusercontent.com"
-aud_value = "511276456880-i7i4787c1863damto6899ts989j2e35r.apps.googleusercontent.com"
+aud_value = "407408718192.apps.googleusercontent.com"
+#aud_value = "511276456880-i7i4787c1863damto6899ts989j2e35r.apps.googleusercontent.com"
 aud_name_value = pad_string(aud_name, maxAudNameLen)
 aud_value_value = pad_string(aud_value, maxAudValueLen)
 aud_value_len_value = '"' + str(len(aud_value)) + '"'
@@ -230,8 +246,10 @@ def _encode_payload(
 maxUidKVPairLen = 350
 maxUidNameLen = 30
 maxUidValueLen = 330
-# uid_field_string = "\"sub\":\"113990307082899718775\","
-uid_field_string = "\"sub\":\"102904630171592520592\","
+uid_field_string = "\"sub\":\"113990307082899718775\","
+#uid_field_string = "\"sub\":\"102904630171592520592\","
+uid_string_bodies = calc_string_bodies(uid_field_string)
+uid_string_bodies_value = pad_string(uid_string_bodies, maxUidKVPairLen, use_ord=False)
 uid_field_value = pad_string(uid_field_string, maxUidKVPairLen)
 uid_field_len_value = '"' + str(len(uid_field_string)) + '"'
 uid_index_value = '"' + str(jwt_payload.index("sub") - 1) + '"'  # This doesn't work for non-sub user id fields
@@ -241,8 +259,8 @@ def _encode_payload(
 uid_colon_index_value = '"' + str(uid_colon_index) + '"'
 uid_value_index_value = '"' + str(uid_colon_index + 2) + '"'
 uid_name = "sub"
-# uid_value = "113990307082899718775"
-uid_value = "102904630171592520592"
+uid_value = "113990307082899718775"
+#uid_value = "102904630171592520592"
 uid_name_value = pad_string(uid_name, maxUidNameLen)
 uid_value_value = pad_string(uid_value, maxUidValueLen)
 uid_value_len_value = '"' + str(len(uid_value)) + '"'
@@ -261,8 +279,8 @@ def _encode_payload(
 extra_colon_index_value = '"' + str(extra_colon_index) + '"'
 extra_value_index_value = '"' + str(extra_colon_index + 2) + '"'
 extra_name = "family_name"
-# extra_value = "Straka";
-extra_value = "コンドウ"
+extra_value = "Straka";
+#extra_value = "コンドウ"
 extra_name_value = pad_string_new(extra_name, maxEFNameLen)
 extra_value_value = pad_string_new(extra_value, maxEFValueLen)
 extra_value_len_value = '"' + str(len(extra_value)) + '"'
@@ -288,8 +306,10 @@ def _encode_payload(
 maxIssKVPairLen = 140
 maxIssNameLen = 40
 maxIssValueLen = 120
-# iss_field_string = "\"iss\":\"https://accounts.google.com\","
-iss_field_string = "\"iss\":\"test.oidc.provider\","
+iss_field_string = "\"iss\":\"https://accounts.google.com\","
+#iss_field_string = "\"iss\":\"test.oidc.provider\","
+iss_string_bodies = calc_string_bodies(iss_field_string)
+iss_string_bodies_value = pad_string(iss_string_bodies, maxIssKVPairLen, use_ord=False)
 iss_field_value = pad_string(iss_field_string, maxIssKVPairLen)
 iss_field_len_value = '"' + str(len(iss_field_string)) + '"'
 iss_index_value = '"' + str(jwt_payload.index("iss") - 1) + '"'
@@ -298,8 +318,8 @@ def _encode_payload(
 iss_colon_index_value = '"' + str(iss_colon_index) + '"'
 iss_value_index_value = '"' + str(iss_colon_index + 2) + '"'  # TODO: Doesn't work with whitespace
 iss_name = "iss"
-# iss_value = "https://accounts.google.com"
-iss_value = "test.oidc.provider"
+iss_value = "https://accounts.google.com"
+#iss_value = "test.oidc.provider"
 iss_name_value = pad_string(iss_name, maxIssNameLen)
 iss_value_value = pad_string(iss_value, maxIssValueLen)
 iss_value_len_value = '"' + str(len(iss_value)) + '"'
@@ -320,6 +340,8 @@ def _encode_payload(
 maxNonceNameLen = 10
 maxNonceValueLen = 100
 nonce_field_string = "\"nonce\":\"" + nonce_value + "\"}"
+nonce_string_bodies = calc_string_bodies(nonce_field_string)
+nonce_string_bodies_value = pad_string(nonce_string_bodies, maxNonceKVPairLen, use_ord=False)
 nonce_field_value = pad_string(nonce_field_string, maxNonceKVPairLen)
 nonce_field_len_value = '"' + str(len(nonce_field_string)) + '"'
 nonce_index_value = '"' + str(jwt_payload.index("nonce") - 1) + '"'
@@ -463,13 +485,14 @@ def _encode_payload(
     "\"signature\"": sig_value,
     "\"pubkey_modulus\"": mod_value,
     "\"aud_field\"": aud_field_value,
-    "\"aud_field_string_bodies\"": 0,
+    "\"aud_field_string_bodies\"": aud_string_bodies_value,
     "\"aud_field_len\"": aud_field_len_value,
     "\"aud_index\"": aud_index_value,
     "\"aud_value_index\"": aud_value_index_value,
     "\"aud_colon_index\"": aud_colon_index_value,
     "\"aud_name\"": aud_name_value,
     "\"uid_field\"": uid_field_value,
+    "\"uid_field_string_bodies\"": uid_string_bodies_value,
     "\"uid_field_len\"": uid_field_len_value,
     "\"uid_index\"": uid_index_value,
     "\"uid_name_len\"": uid_name_len_value,
@@ -487,6 +510,7 @@ def _encode_payload(
     "\"ev_name\"": ev_name_value,
     "\"ev_value\"": ev_value_value,
     "\"iss_field\"": iss_field_value,
+    "\"iss_field_string_bodies\"": iss_string_bodies_value,
     "\"iss_field_len\"": iss_field_len_value,
     "\"iss_index\"": iss_index_value,
     "\"iss_value_index\"": iss_value_index_value,
@@ -495,6 +519,7 @@ def _encode_payload(
     "\"iss_name\"": iss_name_value,
     "\"iss_value\"": iss_value_value,
     "\"nonce_field\"": nonce_field_value,
+    "\"nonce_field_string_bodies\"": nonce_string_bodies_value,
     "\"nonce_field_len\"": nonce_field_len_value,
     "\"nonce_index\"": nonce_index_value,
     "\"nonce_value_index\"": nonce_value_index_value,
@@ -591,4 +616,3 @@ def _encode_payload(
 
 print("\nPretty-printed JWT payload:")
 jwt_parsed = json.loads(jwt_payload)
-pprint.pprint(jwt_parsed)

From dfdbceb92a2bd0c32eb34dcfe1b9bcba7ccc3d7c Mon Sep 17 00:00:00 2001
From: Greg Nazario 
Date: Fri, 14 Jun 2024 20:26:33 -0700
Subject: [PATCH 029/469] [cli] Add aliases to some move commands

These aliases will make it a little easier for people who are trying
to run commands, and are getting mixed between tools.  They are hidden
from documentation, but make it a little bit easier, and we can remove
them at any time
---
 crates/aptos/src/move_tool/mod.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/crates/aptos/src/move_tool/mod.rs b/crates/aptos/src/move_tool/mod.rs
index d5dee099d6d2e..681b9b6b7d24e 100644
--- a/crates/aptos/src/move_tool/mod.rs
+++ b/crates/aptos/src/move_tool/mod.rs
@@ -85,7 +85,9 @@ pub mod stored_package;
 pub enum MoveTool {
     BuildPublishPayload(BuildPublishPayload),
     Clean(CleanPackage),
+    #[clap(alias = "build")]
     Compile(CompilePackage),
+    #[clap(alias = "build-script")]
     CompileScript(CompileScript),
     #[clap(subcommand)]
     Coverage(coverage::CoveragePackage),
@@ -94,11 +96,13 @@ pub enum MoveTool {
     CreateResourceAccountAndPublishPackage(CreateResourceAccountAndPublishPackage),
     Disassemble(Disassemble),
     Decompile(Decompile),
+    #[clap(alias = "doc")]
     Document(DocumentPackage),
     Download(DownloadPackage),
     Init(InitPackage),
     List(ListPackage),
     Prove(ProvePackage),
+    #[clap(alias = "deploy")]
     Publish(PublishPackage),
     Run(RunFunction),
     RunScript(RunScript),

From 88a6322a1aec74f0119572f1096ec2d63666a7bb Mon Sep 17 00:00:00 2001
From: Greg Nazario 
Date: Fri, 14 Jun 2024 20:30:21 -0700
Subject: [PATCH 030/469] [cli] Add smart contract to short description of move
 tool

---
 crates/aptos/src/move_tool/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crates/aptos/src/move_tool/mod.rs b/crates/aptos/src/move_tool/mod.rs
index 681b9b6b7d24e..6a2a108238718 100644
--- a/crates/aptos/src/move_tool/mod.rs
+++ b/crates/aptos/src/move_tool/mod.rs
@@ -76,7 +76,7 @@ pub mod package_hooks;
 mod show;
 pub mod stored_package;
 
-/// Tool for Move related operations
+/// Tool for Move smart contract related operations
 ///
 /// This tool lets you compile, test, and publish Move code, in addition
 /// to run any other tools that help run, verify, or provide information

From 6024e0d23a8a611fb7fd9cde761541e4a4208fd7 Mon Sep 17 00:00:00 2001
From: Greg Nazario 
Date: Fri, 14 Jun 2024 20:50:24 -0700
Subject: [PATCH 031/469] [cli] Add templates to move init

Move init will put hello_blockchain in place as the example, and
with the extensibility to add any other move examples from the
move examples folder.
---
 crates/aptos/CHANGELOG.md         |  3 ++
 crates/aptos/src/move_tool/mod.rs | 57 +++++++++++++++++++++++++++----
 crates/aptos/src/test/mod.rs      |  1 +
 3 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/crates/aptos/CHANGELOG.md b/crates/aptos/CHANGELOG.md
index 1e1bbd63fab1b..5d3dbd6526674 100644
--- a/crates/aptos/CHANGELOG.md
+++ b/crates/aptos/CHANGELOG.md
@@ -7,6 +7,9 @@ All notable changes to the Aptos CLI will be captured in this file. This project
 - Add network to config file
 - Add explorer links to initialized accounts, and transaction submissions
 
+- Alias some move commands as common misnomers (e.g. build -> compile, deploy -> publish)
+- Add "hello_blockchain" template to move init command
+
 ## [3.4.1] - 2024/05/31
 - Upgraded indexer processors for localnet from ca60e51b53c3be6f9517de7c73d4711e9c1f7236 to 5244b84fa5ed872e5280dc8df032d744d62ad29d. Upgraded Hasura metadata accordingly.
 
diff --git a/crates/aptos/src/move_tool/mod.rs b/crates/aptos/src/move_tool/mod.rs
index 6a2a108238718..bec9ca7f809bc 100644
--- a/crates/aptos/src/move_tool/mod.rs
+++ b/crates/aptos/src/move_tool/mod.rs
@@ -76,6 +76,10 @@ pub mod package_hooks;
 mod show;
 pub mod stored_package;
 
+const HELLO_BLOCKCHAIN_EXAMPLE: &str = include_str!(
+    "../../../../aptos-move/move-examples/hello_blockchain/sources/hello_blockchain.move"
+);
+
 /// Tool for Move smart contract related operations
 ///
 /// This tool lets you compile, test, and publish Move code, in addition
@@ -248,6 +252,11 @@ impl FrameworkPackageArgs {
     }
 }
 
+#[derive(ValueEnum, Clone, Copy, Debug)]
+pub enum Template {
+    HelloBlockchain,
+}
+
 /// Creates a new Move package at the given location
 ///
 /// This will create a directory for a Move package and a corresponding
@@ -276,6 +285,10 @@ pub struct InitPackage {
     )]
     pub(crate) named_addresses: BTreeMap,
 
+    /// Template name for initialization
+    #[clap(long)]
+    pub(crate) template: Option