From 8ba7147b527dd922962e6a2803438e15d383004f Mon Sep 17 00:00:00 2001 From: arvidn Date: Mon, 4 Dec 2023 16:43:42 +0100 Subject: [PATCH] switch rust wrapper for sqlite in chia-tools --- Cargo.lock | 154 ++++++++++----- chia-tools/Cargo.toml | 2 +- chia-tools/src/bin/analyze-chain.rs | 39 ++-- chia-tools/src/bin/test-block-generators.rs | 198 ++++++-------------- chia-tools/src/visit_spends.rs | 36 ++-- 5 files changed, 202 insertions(+), 227 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d80cb777..41d2adcb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.0.2" @@ -26,6 +38,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "anes" version = "0.1.6" @@ -416,7 +434,7 @@ dependencies = [ "clvmr", "hex", "hex-literal", - "sqlite", + "rusqlite", "threadpool", "zstd", ] @@ -467,7 +485,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -477,7 +495,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -550,7 +568,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -565,7 +583,7 @@ version = "0.2.14" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -800,7 +818,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -832,7 +850,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -902,6 +920,18 @@ dependencies = [ "libc", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "ff" version = "0.13.0" @@ -990,7 +1020,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1096,6 +1126,19 @@ name = "hashbrown" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -1288,6 +1331,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.3" @@ -1942,6 +1996,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "rusqlite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" +dependencies = [ + "bitflags 2.3.3", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2081,7 +2149,7 @@ checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2187,36 +2255,6 @@ dependencies = [ "der", ] -[[package]] -name = "sqlite" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ddda64c469a257a3b31298805427784d992c226c94b81003f96e8b122286ad" -dependencies = [ - "libc", - "sqlite3-sys", -] - -[[package]] -name = "sqlite3-src" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfc95a51a1ee38839599371685b9d4a926abb51791f0bc3bf8c3bb7867e6e454" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "sqlite3-sys" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2752c669433e40ebb08fde824146f50d9628aa0b66a3b7fc6be34db82a8063b" -dependencies = [ - "libc", - "sqlite3-src", -] - [[package]] name = "strsim" version = "0.10.0" @@ -2242,9 +2280,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" dependencies = [ "proc-macro2", "quote", @@ -2312,7 +2350,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2540,6 +2578,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -2583,7 +2627,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -2617,7 +2661,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2816,6 +2860,26 @@ dependencies = [ "time", ] +[[package]] +name = "zerocopy" +version = "0.7.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "zeroize" version = "1.6.0" @@ -2833,7 +2897,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] diff --git a/chia-tools/Cargo.toml b/chia-tools/Cargo.toml index 9240cf5f3..ed2cbe6e7 100644 --- a/chia-tools/Cargo.toml +++ b/chia-tools/Cargo.toml @@ -16,7 +16,7 @@ clvm-traits = { path = "../clvm-traits" } chia-wallet = { version = "=0.2.14", path = "../chia-wallet" } clvmr = { version = "=0.3.0", features = ["counters"] } chia = { version = "0.2.14", path = ".." } -sqlite = "=0.31.0" +rusqlite = { version = "=0.30.0", features = ["bundled"] } clap = { version = "=4.3.9", features = ["derive"] } zstd = "=0.12.3" threadpool = "=1.8.1" diff --git a/chia-tools/src/bin/analyze-chain.rs b/chia-tools/src/bin/analyze-chain.rs index d3dcd98a8..7ffdb1d41 100644 --- a/chia-tools/src/bin/analyze-chain.rs +++ b/chia-tools/src/bin/analyze-chain.rs @@ -5,7 +5,7 @@ use chia_traits::Streamable; use std::io::Write; use std::time::SystemTime; -use sqlite::State; +use rusqlite::Connection; use chia::gen::conditions::{parse_spends, MempoolVisitor}; use chia::gen::flags::MEMPOOL_MODE; @@ -41,7 +41,7 @@ struct Args { fn main() { let args = Args::parse(); - let connection = sqlite::open(args.file).expect("failed to open database file"); + let connection = Connection::open(args.file).expect("failed to open database file"); let mut statement = connection .prepare( @@ -51,12 +51,6 @@ fn main() { ORDER BY height", ) .expect("failed to prepare SQL statement enumerating blocks"); - statement - .bind((1, args.start as i64)) - .expect("failed to bind start height to sql query"); - statement - .bind((2, args.end as i64)) - .expect("failed to bind start height to sql query"); let mut block_ref_lookup = connection .prepare("SELECT block FROM full_blocks WHERE height=? and in_main_chain=1") @@ -74,13 +68,12 @@ fn main() { let mut prev_timestamp = 0; - while let Ok(State::Row) = statement.next() { - let height: u32 = statement - .read::(0) - .expect("missing height") - .try_into() - .expect("invalid height in block record"); - let block_buffer = statement.read::, _>(1).expect("invalid block blob"); + let mut rows = statement + .query([args.start, args.end]) + .expect("failed to query blocks"); + while let Ok(Some(row)) = rows.next() { + let height: u32 = row.get::<_, u32>(0).expect("missing height"); + let block_buffer: Vec = row.get(1).expect("invalid block blob"); let start_parse = SystemTime::now(); let block_buffer = @@ -123,18 +116,16 @@ fn main() { // iterate in reverse order since we're building a linked list from // the tail for height in block.transactions_generator_ref_list.iter().rev() { - block_ref_lookup - .reset() - .expect("sqlite reset statement failed"); - block_ref_lookup - .bind((1, *height as i64)) + let mut rows = block_ref_lookup + .query(rusqlite::params![height]) .expect("failed to look up ref-block"); - block_ref_lookup + let row = rows .next() - .expect("failed to fetch block-ref row"); - let ref_block = block_ref_lookup - .read::, _>(0) + .expect("failed to fetch block-ref row") + .expect("get None block-ref row"); + let ref_block = row + .get::<_, Vec>(0) .expect("failed to lookup block reference"); let ref_block = diff --git a/chia-tools/src/bin/test-block-generators.rs b/chia-tools/src/bin/test-block-generators.rs index cf509ba47..b85e79fcb 100644 --- a/chia-tools/src/bin/test-block-generators.rs +++ b/chia-tools/src/bin/test-block-generators.rs @@ -1,13 +1,9 @@ use clap::Parser; -use chia_protocol::FullBlock; -use chia_traits::Streamable; - -use sqlite::State; - use chia::gen::conditions::{EmptyVisitor, NewCoin, Spend, SpendBundleConditions}; use chia::gen::flags::{ALLOW_BACKREFS, MEMPOOL_MODE}; use chia::gen::run_block_generator::{run_block_generator, run_block_generator2}; +use chia_tools::iterate_tx_blocks; use clvmr::allocator::NodePtr; use clvmr::serde::{node_from_bytes, node_to_bytes_backrefs}; use clvmr::Allocator; @@ -122,24 +118,6 @@ fn compare(a: &Allocator, lhs: &SpendBundleConditions, rhs: &SpendBundleConditio fn main() { let args = Args::parse(); - let connection = sqlite::open(args.file).expect("failed to open database file"); - - let mut statement = connection - .prepare( - "SELECT height, block \ - FROM full_blocks \ - WHERE in_main_chain=1 AND height >= ?\ - ORDER BY height", - ) - .expect("failed to prepare SQL statement enumerating blocks"); - statement - .bind((1, args.start_height as i64)) - .expect("failed to bind start-height to SQL statement"); - - let mut block_ref_lookup = connection - .prepare("SELECT block FROM full_blocks WHERE height=? and in_main_chain=1") - .expect("failed to prepare SQL statement looking up ref-blocks"); - if args.validate && !(args.mempool || args.rust_generator || args.test_backrefs) { panic!("it doesn't make sense to validate the output against identical runs. Specify --mempool, --rust-generator or --test-backrefs"); } @@ -162,119 +140,67 @@ fn main() { run_block_generator::<_, EmptyVisitor> }; - while let Ok(State::Row) = statement.next() { - let height: u32 = statement - .read::(0) - .expect("missing height") - .try_into() - .expect("invalid height in block record"); - if let Some(h) = args.max_height { - if height > h { - break; - } - } - - let block_buffer = statement.read::, _>(1).expect("invalid block blob"); - - let block_buffer = - zstd::stream::decode_all(&mut std::io::Cursor::>::new(block_buffer)) - .expect("failed to decompress block"); - let block = - FullBlock::from_bytes_unchecked(&block_buffer).expect("failed to parse FullBlock"); - - let ti = match block.transactions_info { - Some(ti) => ti, - None => { - continue; - } - }; - - let prg = match block.transactions_generator { - Some(prg) => prg, - None => { - continue; - } - }; - - // iterate in reverse order since we're building a linked list from - // the tail - let mut block_refs = Vec::>::new(); - for height in block.transactions_generator_ref_list { - block_ref_lookup - .reset() - .expect("sqlite reset statement failed"); - block_ref_lookup - .bind((1, height as i64)) - .expect("failed to look up ref-block"); - - block_ref_lookup - .next() - .expect("failed to fetch block-ref row"); - let ref_block = block_ref_lookup - .read::, _>(0) - .expect("failed to lookup block reference"); - - let ref_block = - zstd::stream::decode_all(&mut std::io::Cursor::>::new(ref_block)) - .expect("failed to decompress block"); - - let ref_block = - FullBlock::from_bytes_unchecked(&ref_block).expect("failed to parse ref-block"); - let ref_gen = ref_block - .transactions_generator - .expect("block ref has no generator"); - block_refs.push(ref_gen.as_ref().into()); - } - - pool.execute(move || { - let mut a = Allocator::new_limited(500000000, 62500000, 62500000); - - let storage: Vec; - let generator = if args.test_backrefs { - // re-serialize the generator with back-references - let gen = node_from_bytes(&mut a, prg.as_ref()).expect("node_from_bytes"); - storage = node_to_bytes_backrefs(&a, gen).expect("node_to_bytes_backrefs"); - &storage[..] - } else { - prg.as_ref() - }; - - let mut conditions = block_runner(&mut a, generator, &block_refs, ti.cost, flags) - .expect("failed to run block generator"); - - if args.rust_generator || args.test_backrefs { - assert!(conditions.cost <= ti.cost); - assert!(conditions.cost > 0); - - // in order for the comparison below the hold, we need to - // patch up the cost of the rust generator to look like the - // baseline - conditions.cost = ti.cost; - } else { - assert_eq!(conditions.cost, ti.cost); - } - - if args.validate { - let mut baseline = run_block_generator::<_, EmptyVisitor>( - &mut a, - prg.as_ref(), - &block_refs, - ti.cost, - 0, - ) - .expect("failed to run block generator"); - assert_eq!(baseline.cost, ti.cost); - - baseline.spends.sort_by_key(|s| *s.coin_id); - conditions.spends.sort_by_key(|s| *s.coin_id); - - // now ensure the outputs are the same - compare(&a, &baseline, &conditions); - } - }); - - assert_eq!(pool.panic_count(), 0); - } + iterate_tx_blocks( + &args.file, + args.start_height, + args.max_height, + |_height, block, block_refs| { + pool.execute(move || { + let mut a = Allocator::new_limited(500000000, 62500000, 62500000); + + let ti = block.transactions_info.as_ref().expect("transactions_info"); + let prg = block + .transactions_generator + .as_ref() + .expect("transactions_generator"); + + let storage: Vec; + let generator = if args.test_backrefs { + // re-serialize the generator with back-references + let gen = node_from_bytes(&mut a, prg.as_ref()).expect("node_from_bytes"); + storage = node_to_bytes_backrefs(&a, gen).expect("node_to_bytes_backrefs"); + &storage[..] + } else { + prg.as_ref() + }; + + let mut conditions = block_runner(&mut a, generator, &block_refs, ti.cost, flags) + .expect("failed to run block generator"); + + if args.rust_generator || args.test_backrefs { + assert!(conditions.cost <= ti.cost); + assert!(conditions.cost > 0); + + // in order for the comparison below the hold, we need to + // patch up the cost of the rust generator to look like the + // baseline + conditions.cost = ti.cost; + } else { + assert_eq!(conditions.cost, ti.cost); + } + + if args.validate { + let mut baseline = run_block_generator::<_, EmptyVisitor>( + &mut a, + prg.as_ref(), + &block_refs, + ti.cost, + 0, + ) + .expect("failed to run block generator"); + assert_eq!(baseline.cost, ti.cost); + + baseline.spends.sort_by_key(|s| *s.coin_id); + conditions.spends.sort_by_key(|s| *s.coin_id); + + // now ensure the outputs are the same + compare(&a, &baseline, &conditions); + } + }); + + assert_eq!(pool.panic_count(), 0); + }, + ); pool.join(); } diff --git a/chia-tools/src/visit_spends.rs b/chia-tools/src/visit_spends.rs index 9f03071df..036803c0d 100644 --- a/chia-tools/src/visit_spends.rs +++ b/chia-tools/src/visit_spends.rs @@ -10,7 +10,7 @@ use clvmr::reduction::Reduction; use clvmr::run_program::run_program; use clvmr::serde::{node_from_bytes, node_from_bytes_backrefs}; use clvmr::Allocator; -use sqlite::State; +use rusqlite::Connection; pub fn iterate_tx_blocks( db: &str, @@ -18,7 +18,7 @@ pub fn iterate_tx_blocks( max_height: Option, callback: impl Fn(u32, FullBlock, Vec>), ) { - let connection = sqlite::open(db).expect("failed to open database file"); + let connection = Connection::open(db).expect("failed to open database file"); let mut statement = connection .prepare( @@ -28,27 +28,23 @@ pub fn iterate_tx_blocks( ORDER BY height", ) .expect("failed to prepare SQL statement enumerating blocks"); - statement - .bind((1, start_height as i64)) - .expect("failed to bind start-height to SQL statement"); let mut block_ref_lookup = connection .prepare("SELECT block FROM full_blocks WHERE height=? and in_main_chain=1") .expect("failed to prepare SQL statement looking up ref-blocks"); - while let Ok(State::Row) = statement.next() { - let height: u32 = statement - .read::(0) - .expect("missing height") - .try_into() - .expect("invalid height in block record"); + let mut rows = statement + .query([start_height]) + .expect("failed to query blocks"); + while let Ok(Some(row)) = rows.next() { + let height = row.get::<_, u32>(0).expect("missing height"); if let Some(h) = max_height { if height > h { break; } } - let block_buffer = statement.read::, _>(1).expect("invalid block blob"); + let block_buffer: Vec = row.get(1).expect("invalid block blob"); let block_buffer = zstd::stream::decode_all(&mut std::io::Cursor::>::new(block_buffer)) @@ -65,18 +61,16 @@ pub fn iterate_tx_blocks( let mut block_refs = Vec::>::new(); for height in &block.transactions_generator_ref_list { - block_ref_lookup - .reset() - .expect("sqlite reset statement failed"); - block_ref_lookup - .bind((1, *height as i64)) + let mut rows = block_ref_lookup + .query(rusqlite::params![height]) .expect("failed to look up ref-block"); - block_ref_lookup + let row = rows .next() - .expect("failed to fetch block-ref row"); - let ref_block = block_ref_lookup - .read::, _>(0) + .expect("failed to fetch block-ref row") + .expect("get None block-ref row"); + let ref_block = row + .get::<_, Vec>(0) .expect("failed to lookup block reference"); let ref_block =