From 16b3e45ced30edf9e332910ba5fdc9fc48ff398f Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Thu, 24 Oct 2024 13:23:04 -0600 Subject: [PATCH 01/11] feat: programming_books example --- .../examples/programming_books/main.rs | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 crates/proof-of-sql/examples/programming_books/main.rs diff --git a/crates/proof-of-sql/examples/programming_books/main.rs b/crates/proof-of-sql/examples/programming_books/main.rs new file mode 100644 index 000000000..a524fff34 --- /dev/null +++ b/crates/proof-of-sql/examples/programming_books/main.rs @@ -0,0 +1,125 @@ +//! This is a non-interactive example of using Proof of SQL with an extended books dataset. +//! To run this, use `cargo run --example books_extra`. +//! +//! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, +//! you can run `cargo run --example books_extra --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. + +use arrow::datatypes::SchemaRef; +use arrow_csv::{infer_schema_from_files, ReaderBuilder}; +use proof_of_sql::{ + base::database::{ + arrow_schema_utility::get_posql_compatible_schema, OwnedTable, OwnedTableTestAccessor, + TestAccessor, + }, + proof_primitive::dory::{ + DynamicDoryCommitment, DynamicDoryEvaluationProof, ProverSetup, PublicParameters, + VerifierSetup, + }, + sql::{parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::QueryProof}, +}; +use rand::{rngs::StdRng, SeedableRng}; +use std::{fs::File, time::Instant}; + +const DORY_SETUP_MAX_NU: usize = 8; +const DORY_SEED: [u8; 32] = *b"ebab60d58dee4cc69658939b7c2a582d"; + +/// # Panics +/// Will panic if the query does not parse or the proof fails to verify. +fn prove_and_verify_query( + sql: &str, + accessor: &OwnedTableTestAccessor, + prover_setup: &ProverSetup, + verifier_setup: &VerifierSetup, +) { + // Parse the query: + println!("Parsing the query: {sql}..."); + let now = Instant::now(); + let query_plan = QueryExpr::::try_new( + sql.parse().unwrap(), + "books_extra".parse().unwrap(), + accessor, + ) + .unwrap(); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Generate the proof and result: + print!("Generating proof..."); + let now = Instant::now(); + let (proof, provable_result) = QueryProof::::new( + query_plan.proof_expr(), + accessor, + &prover_setup, + ); + println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Verify the result with the proof: + print!("Verifying proof..."); + let now = Instant::now(); + let result = proof + .verify( + query_plan.proof_expr(), + accessor, + &provable_result, + &verifier_setup, + ) + .unwrap(); + let result = apply_postprocessing_steps(result.table, query_plan.postprocessing()); + println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.); + + // Display the result + println!("Query Result:"); + println!("{result:?}"); +} + +fn main() { + let mut rng = StdRng::from_seed(DORY_SEED); + let public_parameters = PublicParameters::rand(DORY_SETUP_MAX_NU, &mut rng); + let prover_setup = ProverSetup::from(&public_parameters); + let verifier_setup = VerifierSetup::from(&public_parameters); + + let filename = "./crates/proof-of-sql/examples/books_extra/books_extra.csv"; + let inferred_schema = + SchemaRef::new(infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap()); + let posql_compatible_schema = get_posql_compatible_schema(&inferred_schema); + + let books_extra_batch = ReaderBuilder::new(posql_compatible_schema) + .with_header(true) + .build(File::open(filename).unwrap()) + .unwrap() + .next() + .unwrap() + .unwrap(); + + // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments. + let mut accessor = + OwnedTableTestAccessor::::new_empty_with_setup(&prover_setup); + accessor.add_table( + "books_extra.books".parse().unwrap(), + OwnedTable::try_from(books_extra_batch).unwrap(), + 0, + ); + + // Query 1: Count the total number of books + prove_and_verify_query( + "SELECT COUNT(*) AS total_books FROM books", + &accessor, + &prover_setup, + &verifier_setup, + ); + + // Query 2: Find books with a rating higher than 4.5 + prove_and_verify_query( + "SELECT title, author FROM books WHERE rating > 4.5", + &accessor, + &prover_setup, + &verifier_setup, + ); + + // Query 3: List all programming books published after 2000 + prove_and_verify_query( + "SELECT title, publication_year FROM books WHERE genre = 'Programming' AND publication_year > 2000", + &accessor, + &prover_setup, + &verifier_setup, + ); +} \ No newline at end of file From e1ae87ecc3847980b48dd949b66b968b75fd8465 Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Thu, 24 Oct 2024 13:24:23 -0600 Subject: [PATCH 02/11] chore: Add workflow for programming books --- .github/workflows/lint-and-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 5e291eda1..9395d5bee 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -122,6 +122,8 @@ jobs: run: cargo run --example dinosaurs - name: Run books example run: cargo run --example books + - name: Run programming books example + run: cargo run --example programmingbooks - name: Run brands example run: cargo run --example brands - name: Run avocado-prices example From b338551a99dd8b12a39d8eec82ae059d93b65970 Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Thu, 24 Oct 2024 13:24:50 -0600 Subject: [PATCH 03/11] chore: Add proper example to cargo file --- crates/proof-of-sql/Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/Cargo.toml b/crates/proof-of-sql/Cargo.toml index 0c783d5eb..e966aa2bd 100644 --- a/crates/proof-of-sql/Cargo.toml +++ b/crates/proof-of-sql/Cargo.toml @@ -111,6 +111,10 @@ required-features = [ "arrow" ] name = "books" required-features = [ "arrow" ] +[[example]] +name = "programming_books" +required-features = ["arrow"] + [[example]] name = "brands" required-features = [ "arrow" ] @@ -148,4 +152,4 @@ required-features = [ "test" ] [[bench]] name = "jaeger_benches" harness = false -required-features = [ "blitzar" ] \ No newline at end of file +required-features = [ "blitzar" ] From dd7232a56c6cf7bd4c7ea575e6d06ed7faf16b2b Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Thu, 24 Oct 2024 13:25:18 -0600 Subject: [PATCH 04/11] chore: Add programming books dataset file --- .../examples/programming_books/programming_books.csv | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 crates/proof-of-sql/examples/programming_books/programming_books.csv diff --git a/crates/proof-of-sql/examples/programming_books/programming_books.csv b/crates/proof-of-sql/examples/programming_books/programming_books.csv new file mode 100644 index 000000000..a520b71c8 --- /dev/null +++ b/crates/proof-of-sql/examples/programming_books/programming_books.csv @@ -0,0 +1,6 @@ +title,author,publication_year,genre,rating +The Pragmatic Programmer,Andrew Hunt,1999,Programming,4.5 +Clean Code,Robert C. Martin,2008,Programming,4.7 +The Clean Coder,Robert C. Martin,2011,Programming,4.6 +Design Patterns,Erich Gamma,1994,Software Engineering,4.8 +Refactoring,Martin Fowler,1999,Programming,4.5 \ No newline at end of file From 5c1000e7d03d1ea641c2273e9beab95e18d9f781 Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Thu, 24 Oct 2024 13:53:04 -0600 Subject: [PATCH 05/11] fix: Fix naming convention --- crates/proof-of-sql/examples/programming_books/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/proof-of-sql/examples/programming_books/main.rs b/crates/proof-of-sql/examples/programming_books/main.rs index a524fff34..a40f557b2 100644 --- a/crates/proof-of-sql/examples/programming_books/main.rs +++ b/crates/proof-of-sql/examples/programming_books/main.rs @@ -1,8 +1,8 @@ //! This is a non-interactive example of using Proof of SQL with an extended books dataset. -//! To run this, use `cargo run --example books_extra`. +//! To run this, use `cargo run --example programming_books`. //! //! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed, -//! you can run `cargo run --example books_extra --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. +//! you can run `cargo run --example programming_books --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation. use arrow::datatypes::SchemaRef; use arrow_csv::{infer_schema_from_files, ReaderBuilder}; @@ -36,7 +36,7 @@ fn prove_and_verify_query( let now = Instant::now(); let query_plan = QueryExpr::::try_new( sql.parse().unwrap(), - "books_extra".parse().unwrap(), + "programming_books".parse().unwrap(), accessor, ) .unwrap(); @@ -77,7 +77,7 @@ fn main() { let prover_setup = ProverSetup::from(&public_parameters); let verifier_setup = VerifierSetup::from(&public_parameters); - let filename = "./crates/proof-of-sql/examples/books_extra/books_extra.csv"; + let filename = "./crates/proof-of-sql/examples/programming_books/programming_books.csv"; let inferred_schema = SchemaRef::new(infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap()); let posql_compatible_schema = get_posql_compatible_schema(&inferred_schema); @@ -94,7 +94,7 @@ fn main() { let mut accessor = OwnedTableTestAccessor::::new_empty_with_setup(&prover_setup); accessor.add_table( - "books_extra.books".parse().unwrap(), + "programming_books.books".parse().unwrap(), OwnedTable::try_from(books_extra_batch).unwrap(), 0, ); From c9835eedae0b6e9952e79476005d3d64afb9787f Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Mon, 28 Oct 2024 11:46:47 -0600 Subject: [PATCH 06/11] chore: Added more programming books to csv file --- .../examples/programming_books/programming_books.csv | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/proof-of-sql/examples/programming_books/programming_books.csv b/crates/proof-of-sql/examples/programming_books/programming_books.csv index a520b71c8..dbad4ba3b 100644 --- a/crates/proof-of-sql/examples/programming_books/programming_books.csv +++ b/crates/proof-of-sql/examples/programming_books/programming_books.csv @@ -3,4 +3,9 @@ The Pragmatic Programmer,Andrew Hunt,1999,Programming,4.5 Clean Code,Robert C. Martin,2008,Programming,4.7 The Clean Coder,Robert C. Martin,2011,Programming,4.6 Design Patterns,Erich Gamma,1994,Software Engineering,4.8 -Refactoring,Martin Fowler,1999,Programming,4.5 \ No newline at end of file +Refactoring,Martin Fowler,1999,Programming,4.5 +Effective Java,Joshua Bloch,2008,Programming,4.7 +Introduction to Algorithms,Thomas H. Cormen,2009,Computer Science,4.8 +Code Complete,Steve McConnell,2004,Programming,4.6 +The Mythical Man-Month,Fred Brooks,1975,Software Engineering,4.3 +Algorithms,Robert Sedgewick,1983,Computer Science,4.5 From 4976dc4e7b48fc2432dea944d175682871e0202d Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Mon, 28 Oct 2024 11:47:58 -0600 Subject: [PATCH 07/11] fix: fixed cargo run command --- .github/workflows/lint-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 9395d5bee..5fee053d9 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -123,7 +123,7 @@ jobs: - name: Run books example run: cargo run --example books - name: Run programming books example - run: cargo run --example programmingbooks + run: cargo run --example programming_books - name: Run brands example run: cargo run --example brands - name: Run avocado-prices example From 2e544c1851677c1e64e29b445550956e4163b1e3 Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Mon, 28 Oct 2024 11:50:39 -0600 Subject: [PATCH 08/11] fix: fix formatting issue with extra line at the end of file --- crates/proof-of-sql/examples/programming_books/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof-of-sql/examples/programming_books/main.rs b/crates/proof-of-sql/examples/programming_books/main.rs index a40f557b2..c22960815 100644 --- a/crates/proof-of-sql/examples/programming_books/main.rs +++ b/crates/proof-of-sql/examples/programming_books/main.rs @@ -122,4 +122,4 @@ fn main() { &prover_setup, &verifier_setup, ); -} \ No newline at end of file +} From ebbdeeb0e5395593b865a1b9be178abef9b8b0fe Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Mon, 28 Oct 2024 11:54:20 -0600 Subject: [PATCH 09/11] chore: add a top 5 authors example --- crates/proof-of-sql/examples/programming_books/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/proof-of-sql/examples/programming_books/main.rs b/crates/proof-of-sql/examples/programming_books/main.rs index c22960815..09af38488 100644 --- a/crates/proof-of-sql/examples/programming_books/main.rs +++ b/crates/proof-of-sql/examples/programming_books/main.rs @@ -122,4 +122,12 @@ fn main() { &prover_setup, &verifier_setup, ); + + // Query 4: Find the top 5 authors with the most books + prove_and_verify_query( + "SELECT author, COUNT(*) AS book_count FROM books GROUP BY author ORDER BY book_count DESC LIMIT 5", + &accessor, + &prover_setup, + &verifier_setup, + ); } From a0e32405d938fa2581afb18be670502ab90fb01a Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Mon, 28 Oct 2024 11:56:33 -0600 Subject: [PATCH 10/11] feat: Add an average rating check for programming books --- crates/proof-of-sql/examples/programming_books/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/proof-of-sql/examples/programming_books/main.rs b/crates/proof-of-sql/examples/programming_books/main.rs index 09af38488..4bbcdbb28 100644 --- a/crates/proof-of-sql/examples/programming_books/main.rs +++ b/crates/proof-of-sql/examples/programming_books/main.rs @@ -130,4 +130,12 @@ fn main() { &prover_setup, &verifier_setup, ); + + // Query 5: Calculate the average rating of books in each genre + prove_and_verify_query( + "SELECT genre, AVG(rating) AS average_rating FROM books GROUP BY genre ORDER BY average_rating DESC", + &accessor, + &prover_setup, + &verifier_setup, + ); } From 4d51ff214f31a0f9d631c2a60ce802867009c8d7 Mon Sep 17 00:00:00 2001 From: Hunter Yarbrough Date: Mon, 28 Oct 2024 12:11:47 -0600 Subject: [PATCH 11/11] chore: Remove average rating query proof --- crates/proof-of-sql/examples/programming_books/main.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/proof-of-sql/examples/programming_books/main.rs b/crates/proof-of-sql/examples/programming_books/main.rs index 4bbcdbb28..09af38488 100644 --- a/crates/proof-of-sql/examples/programming_books/main.rs +++ b/crates/proof-of-sql/examples/programming_books/main.rs @@ -130,12 +130,4 @@ fn main() { &prover_setup, &verifier_setup, ); - - // Query 5: Calculate the average rating of books in each genre - prove_and_verify_query( - "SELECT genre, AVG(rating) AS average_rating FROM books GROUP BY genre ORDER BY average_rating DESC", - &accessor, - &prover_setup, - &verifier_setup, - ); }