Skip to content

Commit

Permalink
Store refactor (#1010)
Browse files Browse the repository at this point in the history 
* WIP raw pointers and store

* const generic store functions

* Finished raw store

* Integrated raw_store into store

* Store simplification

* Renames, comments, tests, simplifications

* Updated some documentation

* Utility macros

* Reverted `Hash3` and `comms` change

* Tag interning optimization

* LEM tag own file and tests

* Better conversion functions
  • Loading branch information
@gabriel-barrett
gabriel-barrett authored Jan 5, 2024
1 parent f1c5064 commit 993e958
Show file tree
Hide file tree
Showing 16 changed files with 844 additions and 596 deletions.
1 change: 1 addition & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ serde = { workspace = true, features = ["derive"] }
serde_bytes = "0.11.12"
serde_json = { workspace = true }
serde_repr = "0.1.14"
strum = { version = "0.25", features = ["derive"] }
tap = "1.0.1"
stable_deref_trait = "1.2.0"
thiserror = { workspace = true }
Expand Down
22 changes: 12 additions & 10 deletions src/cli/repl/meta_cmd.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ use crate::{
lem::{
eval::evaluate_with_env_and_cont,
multiframe::MultiFrame,
pointers::{Ptr, ZPtr},
Tag,
pointers::{Ptr, RawPtr, ZPtr},
store::expect_ptrs,
tag::Tag,
},
package::{Package, SymbolRef},
proof::{
Expand Down Expand Up @@ -262,13 +263,13 @@ where
let (second_io, ..) = repl
.eval_expr(second)
.with_context(|| "evaluating second arg")?;
let Ptr::Atom(Tag::Expr(ExprTag::Num), secret) = first_io[0] else {
let (Tag::Expr(ExprTag::Num), RawPtr::Atom(secret)) = first_io[0].parts() else {
bail!(
"Secret must be a number. Got {}",
first_io[0].fmt_to_string(&repl.store, &repl.state.borrow())
)
};
let secret = *repl.store.expect_f(secret);
let secret = *repl.store.expect_f(*secret);
repl.hide(secret, second_io[0])?;
Ok(())
},
Expand Down Expand Up @@ -572,10 +573,10 @@ where
.get_result()
.expect("evaluation result must have been set");
let (_, comm) = repl.store.car_cdr(result)?;
let Ptr::Atom(Tag::Expr(ExprTag::Comm), hash) = comm else {
let (Tag::Expr(ExprTag::Comm), RawPtr::Atom(hash)) = comm.parts() else {
bail!("Second component of a chain must be a commitment")
};
let hash = *repl.store.expect_f(hash);
let hash = *repl.store.expect_f(*hash);
// retrieve from store to persist
let (secret, fun) = repl
.store
Expand Down Expand Up @@ -778,10 +779,11 @@ where

let (fun, rest) = repl.store.car_cdr(ptcl)?;

let (Ptr::Atom(Tag::Expr(ExprTag::Num), rc_idx), _) = repl.store.car_cdr(&rest)? else {
let (car, _) = repl.store.car_cdr(&rest)?;
let (Tag::Expr(ExprTag::Num), RawPtr::Atom(rc_idx)) = car.parts() else {
bail!("Reduction count must be a Num")
};
let Some(rc) = repl.store.expect_f(rc_idx).to_u64().map(|u| u as usize) else {
let Some(rc) = repl.store.expect_f(*rc_idx).to_u64().map(|u| u as usize) else {
bail!("Invalid value for reduction count")
};
Ok((fun, rc))
Expand Down Expand Up @@ -824,13 +826,13 @@ where
.eval_expr_with_env(apply_call, repl.store.intern_nil())
.with_context(|| "evaluating protocol function call")?;

let Ptr::Tuple2(Tag::Expr(ExprTag::Cons), idx) = &io[0] else {
let (Tag::Expr(ExprTag::Cons), RawPtr::Hash4(idx)) = &io[0].parts() else {
bail!(
"Protocol function must return a pair. Got {}",
io[0].fmt_to_string(&repl.store, &repl.state.borrow())
)
};
let (pre_verify, post_verify) = repl.store.fetch_2_ptrs(*idx).unwrap();
let [pre_verify, post_verify] = &expect_ptrs!(repl.store, 2, *idx);

if pre_verify.is_nil() {
bail!("Pre-verification predicate rejected the input")
Expand Down
6 changes: 3 additions & 3 deletions src/cli/repl/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ use crate::{
eval::{evaluate_simple_with_env, evaluate_with_env},
interpreter::Frame,
multiframe::MultiFrame,
pointers::Ptr,
pointers::{Ptr, RawPtr},
store::Store,
Tag,
tag::Tag,
},
parser,
proof::{
Expand Down Expand Up @@ -457,7 +457,7 @@ where
let (expr_io, ..) = self
.eval_expr(expr)
.with_context(|| "evaluating first arg")?;
let Ptr::Atom(Tag::Expr(ExprTag::Num), hash_idx) = &expr_io[0] else {
let (Tag::Expr(ExprTag::Num), RawPtr::Atom(hash_idx)) = &expr_io[0].parts() else {
bail!("hash must be a number")
};
Ok(self.store.expect_f(*hash_idx))
Expand Down
70 changes: 38 additions & 32 deletions src/cli/zstore.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ use std::collections::{BTreeMap, HashMap, HashSet};
use crate::{
field::{FWrap, LurkField},
lem::{
pointers::{Ptr, ZPtr},
store::Store,
pointers::{Ptr, RawPtr, ZPtr},
store::{expect_ptrs, intern_ptrs_hydrated, Store},
},
};

Expand Down Expand Up @@ -38,17 +38,18 @@ impl<F: LurkField> ZDag<F> {
if let Some(z_ptr) = cache.get(ptr) {
*z_ptr
} else {
let z_ptr = match ptr {
Ptr::Atom(tag, idx) => {
let tag = ptr.tag();
let z_ptr = match ptr.raw() {
RawPtr::Atom(idx) => {
let f = store.expect_f(*idx);
let z_ptr = ZPtr::from_parts(*tag, *f);
self.0.insert(z_ptr, ZPtrType::Atom);
z_ptr
}
Ptr::Tuple2(tag, idx) => {
let (a, b) = store.expect_2_ptrs(*idx);
let a = self.populate_with(a, store, cache);
let b = self.populate_with(b, store, cache);
RawPtr::Hash4(idx) => {
let [a, b] = expect_ptrs!(store, 2, *idx);
let a = self.populate_with(&a, store, cache);
let b = self.populate_with(&b, store, cache);
let z_ptr = ZPtr::from_parts(
*tag,
store.poseidon_cache.hash4(&[
Expand All @@ -61,11 +62,11 @@ impl<F: LurkField> ZDag<F> {
self.0.insert(z_ptr, ZPtrType::Tuple2(a, b));
z_ptr
}
Ptr::Tuple3(tag, idx) => {
let (a, b, c) = store.expect_3_ptrs(*idx);
let a = self.populate_with(a, store, cache);
let b = self.populate_with(b, store, cache);
let c = self.populate_with(c, store, cache);
RawPtr::Hash6(idx) => {
let [a, b, c] = expect_ptrs!(store, 3, *idx);
let a = self.populate_with(&a, store, cache);
let b = self.populate_with(&b, store, cache);
let c = self.populate_with(&c, store, cache);
let z_ptr = ZPtr::from_parts(
*tag,
store.poseidon_cache.hash6(&[
Expand All @@ -80,12 +81,12 @@ impl<F: LurkField> ZDag<F> {
self.0.insert(z_ptr, ZPtrType::Tuple3(a, b, c));
z_ptr
}
Ptr::Tuple4(tag, idx) => {
let (a, b, c, d) = store.expect_4_ptrs(*idx);
let a = self.populate_with(a, store, cache);
let b = self.populate_with(b, store, cache);
let c = self.populate_with(c, store, cache);
let d = self.populate_with(d, store, cache);
RawPtr::Hash8(idx) => {
let [a, b, c, d] = expect_ptrs!(store, 4, *idx);
let a = self.populate_with(&a, store, cache);
let b = self.populate_with(&b, store, cache);
let c = self.populate_with(&c, store, cache);
let d = self.populate_with(&d, store, cache);
let z_ptr = ZPtr::from_parts(
*tag,
store.poseidon_cache.hash8(&[
Expand Down Expand Up @@ -126,26 +127,24 @@ impl<F: LurkField> ZDag<F> {
} else {
let ptr = match self.get_type(z_ptr) {
None => bail!("Couldn't find ZPtr on ZStore"),
Some(ZPtrType::Atom) => {
store.intern_atom_hydrated(*z_ptr.tag(), *z_ptr.value(), *z_ptr)
}
Some(ZPtrType::Atom) => store.intern_atom(*z_ptr.tag(), *z_ptr.value()),
Some(ZPtrType::Tuple2(z1, z2)) => {
let ptr1 = self.populate_store(z1, store, cache)?;
let ptr2 = self.populate_store(z2, store, cache)?;
store.intern_2_ptrs_hydrated(*z_ptr.tag(), ptr1, ptr2, *z_ptr)
intern_ptrs_hydrated!(store, *z_ptr.tag(), *z_ptr, ptr1, ptr2)
}
Some(ZPtrType::Tuple3(z1, z2, z3)) => {
let ptr1 = self.populate_store(z1, store, cache)?;
let ptr2 = self.populate_store(z2, store, cache)?;
let ptr3 = self.populate_store(z3, store, cache)?;
store.intern_3_ptrs_hydrated(*z_ptr.tag(), ptr1, ptr2, ptr3, *z_ptr)
intern_ptrs_hydrated!(store, *z_ptr.tag(), *z_ptr, ptr1, ptr2, ptr3)
}
Some(ZPtrType::Tuple4(z1, z2, z3, z4)) => {
let ptr1 = self.populate_store(z1, store, cache)?;
let ptr2 = self.populate_store(z2, store, cache)?;
let ptr3 = self.populate_store(z3, store, cache)?;
let ptr4 = self.populate_store(z4, store, cache)?;
store.intern_4_ptrs_hydrated(*z_ptr.tag(), ptr1, ptr2, ptr3, ptr4, *z_ptr)
intern_ptrs_hydrated!(store, *z_ptr.tag(), *z_ptr, ptr1, ptr2, ptr3, ptr4)
}
};
cache.insert(*z_ptr, ptr);
Expand Down Expand Up @@ -276,7 +275,11 @@ mod tests {

use crate::{
field::LurkField,
lem::{pointers::Ptr, store::Store, Tag},
lem::{
pointers::Ptr,
store::{intern_ptrs, Store},
tag::Tag,
},
tag::{ContTag, ExprTag, Op1, Op2},
};

Expand All @@ -297,23 +300,26 @@ mod tests {
} else {
match rnd % 4 {
0 => store.intern_atom(tag, Fp::from_u64(rnd)),
1 => store.intern_2_ptrs(
1 => intern_ptrs!(
store,
tag,
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store)
),
2 => store.intern_3_ptrs(
2 => intern_ptrs!(
store,
tag,
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store)
),
3 => store.intern_4_ptrs(
3 => intern_ptrs!(
store,
tag,
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store),
rng_interner(rng, max_depth - 1, store)
),
_ => unreachable!(),
}
Expand Down
25 changes: 14 additions & 11 deletions src/coprocessor/gadgets.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use crate::{
lem::{
circuit::GlobalAllocator,
pointers::{Ptr, ZPtr},
store::Store,
store::{expect_ptrs, Store},
},
tag::{ExprTag, Tag},
};
Expand Down Expand Up @@ -178,7 +178,7 @@ pub(crate) fn deconstruct_tuple2<F: LurkField, CS: ConstraintSystem<F>>(
) -> Result<(AllocatedPtr<F>, AllocatedPtr<F>), SynthesisError> {
let (a, b) = if not_dummy.get_value() == Some(true) {
let idx = get_ptr(tuple, store)?.get_index2().expect("invalid Ptr");
let (a, b) = store.expect_2_ptrs(idx);
let [a, b] = &expect_ptrs!(store, 2, idx);
(store.hash_ptr(a), store.hash_ptr(b))
} else {
(ZPtr::dummy(), ZPtr::dummy())
Expand Down Expand Up @@ -221,7 +221,7 @@ pub(crate) fn deconstruct_tuple3<F: LurkField, CS: ConstraintSystem<F>>(
) -> Result<(AllocatedPtr<F>, AllocatedPtr<F>, AllocatedPtr<F>), SynthesisError> {
let (a, b, c) = if not_dummy.get_value() == Some(true) {
let idx = get_ptr(tuple, store)?.get_index3().expect("invalid Ptr");
let (a, b, c) = store.expect_3_ptrs(idx);
let [a, b, c] = &expect_ptrs!(store, 3, idx);
(store.hash_ptr(a), store.hash_ptr(b), store.hash_ptr(c))
} else {
(ZPtr::dummy(), ZPtr::dummy(), ZPtr::dummy())
Expand Down Expand Up @@ -275,7 +275,7 @@ pub(crate) fn deconstruct_tuple4<F: LurkField, CS: ConstraintSystem<F>>(
> {
let (a, b, c, d) = if not_dummy.get_value() == Some(true) {
let idx = get_ptr(tuple, store)?.get_index4().expect("invalid Ptr");
let (a, b, c, d) = store.expect_4_ptrs(idx);
let [a, b, c, d] = &expect_ptrs!(store, 4, idx);
(
store.hash_ptr(a),
store.hash_ptr(b),
Expand Down Expand Up @@ -538,7 +538,10 @@ mod test {
deconstruct_tuple4,
},
field::LurkField,
lem::{circuit::GlobalAllocator, store::Store},
lem::{
circuit::GlobalAllocator,
store::{intern_ptrs, Store},
},
};

use super::{a_ptr_as_z_ptr, chain_car_cdr, construct_list, deconstruct_tuple2};
Expand All @@ -561,7 +564,7 @@ mod test {
&a_nil,
)
.unwrap();
let nil2_ptr = store.intern_2_ptrs(*nil_tag, nil, nil);
let nil2_ptr = intern_ptrs!(store, *nil_tag, nil, nil);
let z_nil2_ptr = store.hash_ptr(&nil2_ptr);
assert_eq!(a_ptr_as_z_ptr(&nil2), Some(z_nil2_ptr));

Expand All @@ -575,7 +578,7 @@ mod test {
&a_nil,
)
.unwrap();
let nil3_ptr = store.intern_3_ptrs(*nil_tag, nil, nil, nil);
let nil3_ptr = intern_ptrs!(store, *nil_tag, nil, nil, nil);
let z_nil3_ptr = store.hash_ptr(&nil3_ptr);
assert_eq!(a_ptr_as_z_ptr(&nil3), Some(z_nil3_ptr));

Expand All @@ -590,7 +593,7 @@ mod test {
&a_nil,
)
.unwrap();
let nil4_ptr = store.intern_4_ptrs(*nil_tag, nil, nil, nil, nil);
let nil4_ptr = intern_ptrs!(store, *nil_tag, nil, nil, nil, nil);
let z_nil4_ptr = store.hash_ptr(&nil4_ptr);
assert_eq!(a_ptr_as_z_ptr(&nil4), Some(z_nil4_ptr));
}
Expand Down Expand Up @@ -624,7 +627,7 @@ mod test {
let nil_tag = *nil.tag();
let not_dummy = Boolean::Constant(true);

let tuple2 = store.intern_2_ptrs(nil_tag, nil, nil);
let tuple2 = intern_ptrs!(store, nil_tag, nil, nil);
let z_tuple2 = store.hash_ptr(&tuple2);
let a_tuple2 = AllocatedPtr::alloc_infallible(&mut cs.namespace(|| "tuple2"), || z_tuple2);
let (a, b) = deconstruct_tuple2(
Expand All @@ -637,7 +640,7 @@ mod test {
assert_eq!(a_ptr_as_z_ptr(&a), Some(z_nil));
assert_eq!(a_ptr_as_z_ptr(&b), Some(z_nil));

let tuple3 = store.intern_3_ptrs(nil_tag, nil, nil, nil);
let tuple3 = intern_ptrs!(store, nil_tag, nil, nil, nil);
let z_tuple3 = store.hash_ptr(&tuple3);
let a_tuple3 = AllocatedPtr::alloc_infallible(&mut cs.namespace(|| "tuple3"), || z_tuple3);
let (a, b, c) = deconstruct_tuple3(
Expand All @@ -651,7 +654,7 @@ mod test {
assert_eq!(a_ptr_as_z_ptr(&b), Some(z_nil));
assert_eq!(a_ptr_as_z_ptr(&c), Some(z_nil));

let tuple4 = store.intern_4_ptrs(nil_tag, nil, nil, nil, nil);
let tuple4 = intern_ptrs!(store, nil_tag, nil, nil, nil, nil);
let z_tuple4 = store.hash_ptr(&tuple4);
let a_tuple4 = AllocatedPtr::alloc_infallible(&mut cs.namespace(|| "tuple4"), || z_tuple4);
let (a, b, c, d) = deconstruct_tuple4(
Expand Down
6 changes: 3 additions & 3 deletions src/coprocessor/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ pub(crate) mod test {

use super::*;
use crate::circuit::gadgets::constraints::{alloc_equal, mul};
use crate::lem::Tag as LEMTag;
use crate::lem::{pointers::RawPtr, tag::Tag as LEMTag};
use crate::tag::{ExprTag, Tag};
use std::marker::PhantomData;

Expand Down Expand Up @@ -209,11 +209,11 @@ pub(crate) mod test {
}

fn evaluate(&self, s: &Store<F>, args: &[Ptr], env: &Ptr, cont: &Ptr) -> Vec<Ptr> {
let Ptr::Atom(LEMTag::Expr(ExprTag::Num), a) = &args[0] else {
let (LEMTag::Expr(ExprTag::Num), RawPtr::Atom(a)) = args[0].parts() else {
return vec![args[0], *env, s.cont_error()];
};
let a = s.expect_f(*a);
let Ptr::Atom(LEMTag::Expr(ExprTag::Num), b) = &args[1] else {
let (LEMTag::Expr(ExprTag::Num), RawPtr::Atom(b)) = args[1].parts() else {
return vec![args[1], *env, s.cont_error()];
};
let b = s.expect_f(*b);
Expand Down
Loading

1 comment on commit 993e958

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmarks

Table of Contents

Overview

This benchmark report shows the Fibonacci GPU benchmark.
NVIDIA L4
Intel(R) Xeon(R) CPU @ 2.20GHz
125.78 GB RAM
Workflow run: https://github.com/lurk-lab/lurk-rs/actions/runs/7425300731

Benchmark Results

LEM Fibonacci Prove - rc = 100

fib-ref=f1c5064774e419086bc09b8c8148f6fdd03ff0e8 fib-ref=993e958a6a9c0a03ba84914fae7701c873f7bbc0
num-100 2.37 s (✅ 1.00x) 2.37 s (✅ 1.00x faster)
num-200 4.65 s (✅ 1.00x) 4.63 s (✅ 1.00x faster)

LEM Fibonacci Prove - rc = 600

fib-ref=f1c5064774e419086bc09b8c8148f6fdd03ff0e8 fib-ref=993e958a6a9c0a03ba84914fae7701c873f7bbc0
num-100 1.99 s (✅ 1.00x) 1.98 s (✅ 1.00x faster)
num-200 4.49 s (✅ 1.00x) 4.49 s (✅ 1.00x faster)

Made with criterion-table

Please sign in to comment.