diff --git a/crates/proof-of-sql/src/base/database/column.rs b/crates/proof-of-sql/src/base/database/column.rs index 1e633b229..eb478bb09 100644 --- a/crates/proof-of-sql/src/base/database/column.rs +++ b/crates/proof-of-sql/src/base/database/column.rs @@ -365,6 +365,16 @@ impl ColumnType { pub fn bit_size(&self) -> u32 { self.byte_size() as u32 * 8 } + + /// Returns if the column type supports signed values. + pub const fn is_signed(&self) -> bool { + match self { + Self::SmallInt | Self::Int | Self::BigInt | Self::Int128 | Self::TimestampTZ(_, _) => { + true + } + Self::Decimal75(_, _) | Self::Scalar | Self::VarChar | Self::Boolean => false, + } + } } /// Convert ColumnType values to some arrow DataType diff --git a/crates/proof-of-sql/src/proof_primitive/dory/dory_commitment_helper_gpu.rs b/crates/proof-of-sql/src/proof_primitive/dory/dory_commitment_helper_gpu.rs index 82d88fc2e..09d16d830 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/dory_commitment_helper_gpu.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/dory_commitment_helper_gpu.rs @@ -14,33 +14,31 @@ fn compute_dory_commitments_packed_impl( offset: usize, setup: &DoryProverPublicSetup, ) -> Vec { + // Make sure that the committable columns are not empty. if committable_columns.is_empty() { return vec![]; } + // Set the parameters. let num_columns = 1 << setup.sigma(); + let gamma_2 = &setup.prover_setup().Gamma_2; // If the offset is larger than the number of columns, we compute an - // offset for the gamma_2 table to avoid finding sub-commits of zero. + // offset for the gamma_2 table to avoid performing msm on all zeros. let gamma_2_offset = offset / num_columns; let offset = offset % num_columns; - // Get the number of sub-commits for each full commit - let num_sub_commits_per_full_commit = - pack_scalars::sub_commits_per_full_commit(committable_columns, offset, num_columns); - - // Get the bit table and packed scalars for the packed msm + // Pack the scalars and create the bit table. let (bit_table, packed_scalars) = pack_scalars::bit_table_and_scalars_for_packed_msm( committable_columns, offset, num_columns, - num_sub_commits_per_full_commit, ); + // Get sub commits by computing the packed msm. let mut sub_commits_from_blitzar = vec![ElementP2::::default(); bit_table.len()]; - // Compute packed msm if !bit_table.is_empty() { setup.prover_setup().blitzar_packed_msm( &mut sub_commits_from_blitzar, @@ -49,27 +47,51 @@ fn compute_dory_commitments_packed_impl( ); } - // Convert the sub-commits to G1Affine + // Convert the sub-commits to G1Affine. let all_sub_commits: Vec = sub_commits_from_blitzar .par_iter() .map(Into::into) .collect(); - // Modify the signed sub-commits by adding the offset - let modified_sub_commits = pack_scalars::modify_commits( + // Modify the sub-commits to account for signed values that were offset. + let modified_sub_commits_update = pack_scalars::modify_commits( &all_sub_commits, + &bit_table, committable_columns, - num_sub_commits_per_full_commit, + offset, + num_columns, ); - let gamma_2_slice = &setup.prover_setup().Gamma_2.last().unwrap() - [gamma_2_offset..gamma_2_offset + num_sub_commits_per_full_commit]; + // All columns are not guaranteed to have the same number of sub-commits. + // Create a vector that stores the number of sub-commits per full commit. + let num_sub_commits_per_full_commit: Vec = committable_columns + .iter() + .map(|column| pack_scalars::num_sub_commits(column, offset, num_columns)) + .collect(); + + // Compute the cumulative sum of the number of sub-commits per full commit. + // This is used to index into the modified sub-commits in the Doris commitment loop. + let cumulative_sub_commit_sums: Vec = num_sub_commits_per_full_commit + .iter() + .scan(0, |acc, &x| { + let prev = *acc; + *acc += x; + Some(prev) + }) + .collect(); - // Compute the Dory commitments using multi pairing of sub-commits + // Compute the Dory commitments using multi pairing of sub-commits. let span = span!(Level::INFO, "multi_pairing").entered(); - let dc = modified_sub_commits - .par_chunks_exact(num_sub_commits_per_full_commit) - .map(|sub_commits| DoryCommitment(pairings::multi_pairing(sub_commits, gamma_2_slice))) + let dc: Vec = cumulative_sub_commit_sums + .par_iter() + .zip(num_sub_commits_per_full_commit.par_iter()) + .map(|(&idx, &num_sub_commits)| { + let sub_commits = &modified_sub_commits_update[idx..idx + num_sub_commits]; + let gamma_2_slice = + &gamma_2.last().unwrap()[gamma_2_offset..gamma_2_offset + num_sub_commits]; + + DoryCommitment(pairings::multi_pairing(sub_commits, gamma_2_slice)) + }) .collect(); span.exit(); diff --git a/crates/proof-of-sql/src/proof_primitive/dory/pack_scalars.rs b/crates/proof-of-sql/src/proof_primitive/dory/pack_scalars.rs index 5ac572081..4b7689703 100644 --- a/crates/proof-of-sql/src/proof_primitive/dory/pack_scalars.rs +++ b/crates/proof-of-sql/src/proof_primitive/dory/pack_scalars.rs @@ -1,39 +1,51 @@ -use super::{G1Affine, F}; +use super::{G1Affine, G1Projective, F}; use crate::{ base::{commitment::CommittableColumn, database::ColumnType}, proof_primitive::dory::offset_to_bytes::OffsetToBytes, }; -use ark_ec::CurveGroup; use ark_ff::MontFp; use ark_std::ops::Mul; -use rayon::prelude::*; const BYTE_SIZE: usize = 8; +const OFFSET_SIZE: usize = 2; -/// Returns a bit table vector related to each of the committable columns data size. +/// Returns the number of sub commitments needed for +/// each full commitment in the packed_msm function. /// /// # Arguments /// /// * `committable_columns` - A reference to the committable columns. -fn output_bit_table<'a>( - committable_columns: &'a [CommittableColumn], -) -> impl Iterator + 'a { - committable_columns - .iter() - .map(|column| column.column_type().bit_size()) +/// * `offset` - The offset to the data. +/// * `num_matrix_commitment_columns` - The number of generators used for msm. +pub fn num_sub_commits( + column: &CommittableColumn, + offset: usize, + num_matrix_commitment_columns: usize, +) -> usize { + (column.len() + offset + num_matrix_commitment_columns - 1) / num_matrix_commitment_columns } -/// Returns the size of the largest committable column. +/// Returns a bit table vector related to each of the committable columns data size. /// /// # Arguments /// /// * `committable_columns` - A reference to the committable columns. -fn max_committable_column_length(committable_columns: &[CommittableColumn]) -> usize { +/// * `offset` - The offset to the data. +/// * `num_matrix_commitment_columns` - The number of generators used for msm. +fn output_bit_table( + committable_columns: &[CommittableColumn], + offset: usize, + num_matrix_commitment_columns: usize, +) -> Vec { committable_columns .iter() - .map(|column| column.len()) - .max() - .unwrap_or(0) + .flat_map(|column| { + itertools::repeat_n( + column.column_type().bit_size(), + num_sub_commits(column, offset, num_matrix_commitment_columns), + ) + }) + .collect() } /// Returns the minimum value of a column as F. @@ -54,86 +66,81 @@ const fn min_as_f(column_type: ColumnType) -> F { } } -/// Returns a repeated bit table vector that duplicated the -/// bit table for each element by the num_sub_commits_per_full_commit. -/// -/// # Arguments -/// -/// * `bit_table` - A iterable bit table. -/// * `num_sub_commits_per_full_commit` - The number of sub commitments needed for each full commit. -fn repeat_bit_table( - bit_table: impl Iterator, - num_sub_commits_per_full_commit: usize, -) -> Vec { - bit_table - .flat_map(|value| itertools::repeat_n(value, num_sub_commits_per_full_commit)) - .collect() -} - -/// Returns the number of sub commitments needed for -/// each full commitment in the packed_msm function. -/// -/// # Arguments -/// -/// * `committable_columns` - A reference to the committable columns. -/// * `offset` - The offset to the data. -/// * `num_matrix_commitment_columns` - The number of generators used for msm. -pub fn sub_commits_per_full_commit( - committable_columns: &[CommittableColumn], - offset: usize, - num_matrix_commitment_columns: usize, -) -> usize { - // Committable columns may be different sizes, get the max size and add offset. - let max_column_length = max_committable_column_length(committable_columns) + offset; - - // Number of scalar vectors that the size of the generators n. - // Each scalar will be used to call the packed_msm function. - (max_column_length + num_matrix_commitment_columns - 1) / num_matrix_commitment_columns -} - /// Modifies the signed matrix commitment columns by adding the offset to the matrix commitment columns. /// /// # Arguments /// /// * `sub_commits` - A reference to the signed sub-commits. +/// * `bit_table` - A reference to the bit table used by the packed_msm function. /// * `committable_columns` - A reference to the committable columns. -/// * `num_sub_commits_per_full_commit` - The number of sub-commits needed for -/// each full commit for the packed_msm function. -#[tracing::instrument(name = "pack_scalars::modify_commits (gpu)", level = "debug", skip_all)] +/// * `offset` - The offset to the data. +/// * `num_matrix_commitment_columns` - The number of generators used for msm. +#[tracing::instrument(name = "pack_scalars::modify_commits", level = "debug", skip_all)] pub fn modify_commits( sub_commits: &[G1Affine], + bit_table: &[u32], committable_columns: &[CommittableColumn], - num_sub_commits_per_full_commit: usize, + offset: usize, + num_matrix_commitment_columns: usize, ) -> Vec { - let num_full_commits = committable_columns.len(); + // Set parameters + let num_offset_commits = OFFSET_SIZE + committable_columns.len(); + let num_sub_commits_in_full_commit = bit_table.len() - num_offset_commits; assert_eq!( - 2 * num_full_commits * num_sub_commits_per_full_commit, + num_sub_commits_in_full_commit + num_offset_commits, sub_commits.len() ); - // Currently, the packed_scalars doubles the number of sub-commits to deal with - // signed sub-commits. Sub-commit i is offset by the sub-commit at i + num_sub_commits_per_full_commit. - // Spit the sub-commits into signed sub-commits and offset sub-commits. - let num_signed_sub_commits = num_full_commits * num_sub_commits_per_full_commit; - let (signed_sub_commits, offset_sub_commits) = sub_commits.split_at(num_signed_sub_commits); - - // Ensure the packed_scalars were split correctly - assert_eq!(signed_sub_commits.len(), offset_sub_commits.len()); + // Spit the sub-commits and offset commits. + let (signed_sub_commits, offset_sub_commits) = + sub_commits.split_at(num_sub_commits_in_full_commit); + assert_eq!(signed_sub_commits.len(), num_sub_commits_in_full_commit); + assert_eq!(offset_sub_commits.len(), num_offset_commits); + + let mut modifed_commits: Vec = + vec![G1Projective::default(); signed_sub_commits.len()]; + + let mut k = 0; + for (i, column) in committable_columns.iter().enumerate() { + let num_sub_commits = num_sub_commits(column, offset, num_matrix_commitment_columns); + if column.column_type().is_signed() { + let min = min_as_f(column.column_type()); + let offset_sub_commit_first = offset_sub_commits[0].mul(min); + let offset_sub_commit_middle = offset_sub_commits[1].mul(min); + let offset_sub_commit_last = offset_sub_commits[OFFSET_SIZE + i].mul(min); + + match num_sub_commits { + n if n > 2 => { + modifed_commits[k] = signed_sub_commits[k] + offset_sub_commit_first; + k += 1; + (1..num_sub_commits - 1).for_each(|_| { + modifed_commits[k] = signed_sub_commits[k] + offset_sub_commit_middle; + k += 1; + }); + modifed_commits[k] = signed_sub_commits[k] + offset_sub_commit_last; + k += 1; + } + 2 => { + modifed_commits[k] = signed_sub_commits[k] + offset_sub_commit_first; + k += 1; + modifed_commits[k] = signed_sub_commits[k] + offset_sub_commit_last; + k += 1; + } + 1 => { + modifed_commits[k] = signed_sub_commits[k] + offset_sub_commit_last; + k += 1; + } + _ => {} + } + } else { + for _ in 0..num_sub_commits { + modifed_commits[k] = signed_sub_commits[k].into(); + k += 1; + } + } + } - // Add the offset sub-commits multiplied by the min value to the signed sub-commits - signed_sub_commits - .par_iter() - .zip(offset_sub_commits.par_iter()) - .enumerate() - .map(|(index, (first, second))| { - let min = min_as_f( - committable_columns[index / num_sub_commits_per_full_commit].column_type(), - ); - let modified_second = second.mul(min).into_affine(); - *first + modified_second - }) - .map(|point| point.into_affine()) - .collect::>() + modifed_commits.into_iter().map(Into::into).collect() } /// Packs bits of a committable column into the packed scalars array. @@ -159,268 +166,313 @@ fn pack_bit>( ) { let byte_offset = current_bit_table_sum / BYTE_SIZE; column.iter().enumerate().for_each(|(i, value)| { - let row_offset = ((i + offset) % num_columns) * bit_table_sum_in_bytes; - let col_offset = current_byte_size * ((i + offset) / num_columns); - let offset_idx = row_offset + col_offset + byte_offset; + let index = i + offset; + let row_offset = (index % num_columns) * bit_table_sum_in_bytes; + let col_offset = current_byte_size * (index / num_columns); + let offset_index = row_offset + col_offset + byte_offset; - packed_scalars[offset_idx..offset_idx + current_byte_size] + packed_scalars[offset_index..offset_index + current_byte_size] .copy_from_slice(&value.offset_to_bytes()[..]); }); } -/// Packs the offset bits of a committable column into the packed scalars at the end of the array. -/// The offsets are 8-bit values used to handle the signed values. +/// Returns an offset vector to support signed values. /// /// # Arguments /// -/// * `column` - A reference to a signed committable column that needs offsets calculated. -/// * `packed_scalars` - A mutable reference to the array where the packed scalars will be stored. -/// * `current_bit_table_sum` - The current sum of the bit table up to the current column. +/// * `committable_columns` - A reference to the committable columns. /// * `offset` - The offset to the data. -/// * `bit_table_sum_in_bytes` - The full bit table size in bytes. -/// * `num_columns` - The number of columns in a matrix commitment. -fn pack_offset_bit>( - column: &[T], - packed_scalars: &mut [u8], - current_bit_table_sum: usize, +/// * `num_matrix_commitment_columns` - The number of generators used for msm. +/// * `buffer` - Pre-allocated offset column buffer. +fn offset_column( + committable_columns: &[CommittableColumn], offset: usize, - bit_table_sum_in_bytes: usize, - num_columns: usize, + num_matrix_commitment_columns: usize, + buffer: &mut [u8], ) { - let byte_offset = current_bit_table_sum / BYTE_SIZE; - column.iter().enumerate().for_each(|(i, _)| { - let row_offset = ((i + offset) % num_columns) * bit_table_sum_in_bytes; - let col_offset = (i + offset) / num_columns; - let offset_idx = row_offset + col_offset + byte_offset; + assert!( + offset < num_matrix_commitment_columns, + "offset {} must be less than the number of columns {}", + offset, + num_matrix_commitment_columns + ); - packed_scalars[offset_idx] = 1_u8; - }); + assert!( + buffer.len() == (OFFSET_SIZE + committable_columns.len()) * num_matrix_commitment_columns, + "buffer length {} must be equal to the offset size {} plus the number of committable columns {} times the number of columns {}", + buffer.len(), + OFFSET_SIZE, + committable_columns.len(), + num_matrix_commitment_columns + ); + + let has_signed_column = committable_columns + .iter() + .any(|column| column.column_type().is_signed()); + + if has_signed_column { + // Set the offset and the column of all ones in a single loop + (0..num_matrix_commitment_columns).for_each(|i| { + if i >= offset { + buffer[i] = 1_u8; + } + buffer[num_matrix_commitment_columns + i] = 1_u8; + }); + + // Set the remaining columns + for (j, column) in committable_columns.iter().enumerate() { + if column.column_type().is_signed() { + let total_length = offset + column.len(); + + let first_value = if total_length <= num_matrix_commitment_columns { + offset + } else { + 0 + }; + + let last_value = if total_length % num_matrix_commitment_columns == 0 { + num_matrix_commitment_columns + } else { + total_length % num_matrix_commitment_columns + }; + + (first_value..last_value).for_each(|i| { + buffer[((2 + j) * num_matrix_commitment_columns) + i] = 1_u8; + }); + } + } + } } -/// Returns the bit table and packed scalar array to be used in Blitzar's packed_msm function. +/// Packs the offsets into the packed scalar array. /// /// # Arguments /// /// * `committable_columns` - A reference to the committable columns. /// * `offset` - The offset to the data. -/// * `num_columns` - The number of columns in a matrix commitment. -/// * `num_sub_commits_per_full_commit` - The number of sub commits needed for -/// each full commit for the packed_msm function. -/// -/// # Example +/// * `num_matrix_commitment_columns` - The number of generators used for msm. +/// * `bit_table_sub_commits_sum` - The sum of the bit table sub commits. +/// * `bit_table_sum_in_bytes` - The full bit table size in bytes. +/// * `packed_scalars` - A mutable reference to the array where the packed scalars will be stored. +fn pack_offsets( + committable_columns: &[CommittableColumn], + offset: usize, + num_matrix_commitment_columns: usize, + bit_table_sub_commits_sum: usize, + bit_table_sum_in_bytes: usize, + packed_scalars: &mut [u8], +) { + let mut offset_column_buffer = + vec![0_u8; (OFFSET_SIZE + committable_columns.len()) * num_matrix_commitment_columns]; + + offset_column( + committable_columns, + offset, + num_matrix_commitment_columns, + &mut offset_column_buffer, + ); + + pack_bit( + &offset_column_buffer, + packed_scalars, + bit_table_sub_commits_sum, + 0, + 1, + bit_table_sum_in_bytes, + num_matrix_commitment_columns, + ); +} + +/// Creates a cumulative bit table sum used by the bit_table_and_scalars_for_packed_msm inner loop. /// -/// ```ignore -/// let committable_columns = [ -/// CommittableColumn::SmallInt(&[0, 1, 2]), -/// CommittableColumn::SmallInt(&[3, 4, 5, 6, 7]), -/// ]; -/// let offset = 1; -/// let num_columns = 3; +/// # Arguments /// -/// let num_sub_commits_per_full_commit = sub_commits_per_full_commit(&committable_columns, offset, num_columns); +/// * `committable_columns` - A reference to the committable columns. +/// * `offset` - The offset to the data. +/// * `num_matrix_commitment_columns` - The number of generators used for msm. +/// * `bit_table` - The bit table. +fn compute_cumulative_bit_sum_table( + committable_columns: &[CommittableColumn], + offset: usize, + num_matrix_commitment_columns: usize, + bit_table: &[u32], +) -> Vec { + let mut running_sum = 0; + let mut num_sub_commits_completed = 0; + + committable_columns + .iter() + .flat_map(|column| { + let sub_commits = num_sub_commits(column, offset, num_matrix_commitment_columns); + let current_sum = running_sum; + running_sum += bit_table + .iter() + .skip(num_sub_commits_completed) + .take(sub_commits) + .sum::() as usize; + num_sub_commits_completed += sub_commits; + std::iter::once(current_sum) + }) + .collect() +} + +/// Returns the bit table and packed scalar array to be used in Blitzar's packed_msm function. /// -/// let (bit_table, packed_scalars) = bit_table_and_scalars_for_packed_msm( -/// &committable_columns, -/// offset, -/// num_columns, -/// num_sub_commits_per_full_commit, -/// ); +/// # Arguments /// -/// assert_eq!(num_of_commits, 2); -/// assert_eq!(bit_table, [16, 16, 16, 16, 8, 8, 8, 8]); -/// assert_eq!(packed_scalars.len(), 36); // num_columns * bit_table_sum / BYTE_SIZE -/// assert_eq!(packed_scalars, [0, 0, 2, 128, 0, 0, 5, 128, 0, 1, 0, 1, -/// 0, 128, 0, 0, 3, 128, 6, 128, 1, 0, 1, 1, -/// 1, 128, 0, 0, 4, 128, 7, 128, 1, 0, 1, 1]); -/// ``` +/// * `committable_columns` - A reference to the committable columns. +/// * `offset` - The offset to the data. +/// * `num_matrix_commitment_columns` - The number of generators used for msm. #[tracing::instrument( - name = "pack_scalars::bit_table_and_scalars_for_packed_msm (gpu)", + name = "pack_scalars::bit_table_and_scalars_for_packed_msm", level = "debug", skip_all )] pub fn bit_table_and_scalars_for_packed_msm( committable_columns: &[CommittableColumn], offset: usize, - num_columns: usize, - num_sub_commits_per_full_commit: usize, + num_matrix_commitment_columns: usize, ) -> (Vec, Vec) { - // Get a bit table that represented each of the committable columns bit size. - let bit_table_full_commits = output_bit_table(committable_columns); + // Make sure that the committable columns are not empty. + if committable_columns.is_empty() { + return (vec![], vec![]); + } - // Repeat the bit table to account for the appropriate number of sub commitments per full commit. - let mut bit_table = repeat_bit_table(bit_table_full_commits, num_sub_commits_per_full_commit); + // Get the bit table to account for the appropriate number of sub commitments needed in a full commit. + let mut bit_table = + output_bit_table(committable_columns, offset, num_matrix_commitment_columns); let bit_table_sub_commits_sum = bit_table.iter().sum::() as usize; - // Double the bit table to handle handle the BYTE_SIZE offsets. - bit_table.extend(std::iter::repeat(BYTE_SIZE as u32).take(bit_table.len())); - let bit_table_sum_in_bytes = bit_table.iter().sum::() as usize / BYTE_SIZE; + // Add offsets to handle signed values to the bit table. + bit_table + .extend(std::iter::repeat(BYTE_SIZE as u32).take(OFFSET_SIZE + committable_columns.len())); + let bit_table_full_sum_in_bytes = bit_table.iter().sum::() as usize / BYTE_SIZE; + + // Create the packed_scalar array. + let mut packed_scalars = + vec![0_u8; bit_table_full_sum_in_bytes * num_matrix_commitment_columns]; + + // Pack the offsets, used to handle signed values, into the packed_scalars array. + pack_offsets( + committable_columns, + offset, + num_matrix_commitment_columns, + bit_table_sub_commits_sum, + bit_table_full_sum_in_bytes, + &mut packed_scalars, + ); - // Create the packed_scalar vector. - let mut packed_scalars = vec![0_u8; bit_table_sum_in_bytes * num_columns]; + // Pre-compute the cumulative bit sum table to be used in the inner loop. + let cumulative_bit_sum_table = compute_cumulative_bit_sum_table( + committable_columns, + offset, + num_matrix_commitment_columns, + &bit_table, + ); // For each committable column, pack the data into the packed_scalar array. committable_columns .iter() .enumerate() - .for_each(|(i, column)| { - // Get the running sum of the bit table for the signed values. - let current_bit_table_sum = if i > 0 { - bit_table - .iter() - .take(i * num_sub_commits_per_full_commit) - .sum::() as usize - } else { - 0 - }; - - // Get the running sum of the bit table for the offsets. - let bit_table_offset_sum = - bit_table_sub_commits_sum + i * BYTE_SIZE * num_sub_commits_per_full_commit; - - // Get the byte size of the column of data. - let byte_size = committable_columns[i].column_type().byte_size(); - - // Pack the signed bits and offset bits into the packed_scalars array. - match column { - CommittableColumn::SmallInt(column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - pack_offset_bit( - column, - &mut packed_scalars, - bit_table_offset_sum, - offset, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::Int(column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - pack_offset_bit( - column, - &mut packed_scalars, - bit_table_offset_sum, - offset, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::BigInt(column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - pack_offset_bit( - column, - &mut packed_scalars, - bit_table_offset_sum, - offset, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::Int128(column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - pack_offset_bit( - column, - &mut packed_scalars, - bit_table_offset_sum, - offset, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::TimestampTZ(_, _, column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - pack_offset_bit( - column, - &mut packed_scalars, - bit_table_offset_sum, - offset, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::Boolean(column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::Decimal75(_, _, column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::Scalar(column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - } - CommittableColumn::VarChar(column) => { - pack_bit( - column, - &mut packed_scalars, - current_bit_table_sum, - offset, - byte_size, - bit_table_sum_in_bytes, - num_columns, - ); - } + .for_each(|(i, column)| match column { + CommittableColumn::SmallInt(column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::Int(column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::BigInt(column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::Int128(column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::TimestampTZ(_, _, column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::Boolean(column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::Decimal75(_, _, column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::Scalar(column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); + } + CommittableColumn::VarChar(column) => { + pack_bit( + column, + &mut packed_scalars, + cumulative_bit_sum_table[i], + offset, + committable_columns[i].column_type().byte_size(), + bit_table_full_sum_in_bytes, + num_matrix_commitment_columns, + ); } }); @@ -434,115 +486,237 @@ mod tests { use proof_of_sql_parser::posql_time::{PoSQLTimeUnit, PoSQLTimeZone}; #[test] - fn we_can_get_max_committable_column_length_of_the_same_type() { + fn we_can_get_a_bit_table() { let committable_columns = [ - CommittableColumn::Scalar(vec![[1, 2, 3, 4], [5, 6, 7, 8]]), - CommittableColumn::Scalar(vec![[1, 2, 3, 4]]), + CommittableColumn::SmallInt(&[1]), + CommittableColumn::Int(&[1, 2]), + CommittableColumn::BigInt(&[1, 2, 3]), + CommittableColumn::Int128(&[1, 2, 3, 4]), + CommittableColumn::Decimal75( + Precision::new(1).unwrap(), + 0, + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], + ), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1]), ]; - let max_column_length = max_committable_column_length(&committable_columns); - assert_eq!(max_column_length, 2); + let offset = 0; + let num_matrix_commitment_columns = 1; + let bit_table: Vec = + output_bit_table(&committable_columns, offset, num_matrix_commitment_columns); + let expected = [ + 16, 32, 32, 64, 64, 64, 128, 128, 128, 128, 256, 256, 256, 256, 256, 256, 256, 256, + 256, 256, 256, 256, 8, 8, 64, + ]; + assert_eq!(bit_table, expected); } #[test] - fn we_can_get_max_committable_column_length_of_different_types() { + fn we_can_get_a_bit_table_with_offset() { let committable_columns = [ - CommittableColumn::SmallInt(&[1, 2, 3]), - CommittableColumn::Int(&[1, 2, 3]), + CommittableColumn::SmallInt(&[1]), + CommittableColumn::Int(&[1, 2]), CommittableColumn::BigInt(&[1, 2, 3]), - CommittableColumn::Int128(&[1, 2, 3, 4, 5, 6]), + CommittableColumn::Int128(&[1, 2, 3, 4]), CommittableColumn::Decimal75( Precision::new(1).unwrap(), 0, - vec![[1, 0, 0, 0], [2, 0, 0, 0]], + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], ), - CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), - CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), - CommittableColumn::Boolean(&[true, false, true]), - CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1, 2, 3]), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1]), ]; - let max_column_length = max_committable_column_length(&committable_columns); - assert_eq!(max_column_length, 6); + let offset = 1; + let num_matrix_commitment_columns = 1; + let bit_table: Vec = + output_bit_table(&committable_columns, offset, num_matrix_commitment_columns); + let expected = [ + 16, 16, 32, 32, 32, 64, 64, 64, 64, 128, 128, 128, 128, 128, 256, 256, 256, 256, 256, + 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 8, 8, 8, 64, 64, + ]; + assert_eq!(bit_table, expected); } #[test] - fn we_can_get_a_bit_table() { + fn we_can_get_the_num_of_sub_commits_with_more_rows_than_cols_update() { let committable_columns = [ - CommittableColumn::SmallInt(&[1, 2, 3]), - CommittableColumn::Int(&[1, 2, 3]), + CommittableColumn::SmallInt(&[1]), + CommittableColumn::Int(&[1, 2]), CommittableColumn::BigInt(&[1, 2, 3]), - CommittableColumn::Int128(&[1, 2, 3]), + CommittableColumn::Int128(&[1, 2, 3, 4]), CommittableColumn::Decimal75( Precision::new(1).unwrap(), 0, - vec![[1, 0, 0, 0], [2, 0, 0, 0]], + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], ), - CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), - CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), - CommittableColumn::Boolean(&[true, false, true]), - CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1, 2, 3]), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1]), ]; - let bit_table: Vec = output_bit_table(&committable_columns).collect(); - let expected = [16, 32, 64, 128, 64 * 4, 64 * 4, 64 * 4, 8, 64]; - assert_eq!(bit_table, expected); + let offset = 0; + let num_matrix_commitment_columns = 1; + let expected: Vec = vec![1, 2, 3, 4, 5, 4, 3, 2, 1]; + let num_sub_commits: Vec = (0..committable_columns.len()) + .map(|i| { + num_sub_commits( + &committable_columns[i], + offset, + num_matrix_commitment_columns, + ) + }) + .collect(); + + assert_eq!(num_sub_commits, expected); } #[test] - fn we_can_get_a_repeated_bit_table() { + fn we_can_get_the_num_of_sub_commits_offset_with_more_rows_than_cols_update() { let committable_columns = [ - CommittableColumn::SmallInt(&[1, 2, 3]), - CommittableColumn::Int(&[1, 2, 3]), + CommittableColumn::SmallInt(&[1]), + CommittableColumn::Int(&[1, 2]), CommittableColumn::BigInt(&[1, 2, 3]), - CommittableColumn::Int128(&[1, 2, 3]), + CommittableColumn::Int128(&[1, 2, 3, 4]), CommittableColumn::Decimal75( Precision::new(1).unwrap(), 0, - vec![[1, 0, 0, 0], [2, 0, 0, 0]], + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], ), - CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), - CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), - CommittableColumn::Boolean(&[true, false, true]), - CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1, 2, 3]), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1]), ]; - let bit_table = output_bit_table(&committable_columns); - let repeated_bit_table = repeat_bit_table(bit_table, 3); - let expected_bit_table = [ - 16, - 16, - 16, - 32, - 32, - 32, - 64, - 64, - 64, - 128, - 128, - 128, - 64 * 4, - 64 * 4, - 64 * 4, - 64 * 4, - 64 * 4, - 64 * 4, - 64 * 4, - 64 * 4, - 64 * 4, - 8, - 8, - 8, - 64, - 64, - 64, - ]; - assert_eq!(repeated_bit_table, expected_bit_table); + let offset = 2; + let num_matrix_commitment_columns = 1; + let expected: Vec = vec![3, 4, 5, 6, 7, 6, 5, 4, 3]; + let num_sub_commits: Vec = (0..committable_columns.len()) + .map(|i| { + num_sub_commits( + &committable_columns[i], + offset, + num_matrix_commitment_columns, + ) + }) + .collect(); + + assert_eq!(num_sub_commits, expected); } #[test] - fn we_can_get_sub_commits_per_full_commit_with_less_rows_than_columns() { + fn we_can_get_the_num_of_sub_commits_with_less_rows_than_cols_update() { + let committable_columns = [ + CommittableColumn::SmallInt(&[1]), + CommittableColumn::Int(&[1, 2]), + CommittableColumn::BigInt(&[1, 2, 3]), + CommittableColumn::Int128(&[1, 2, 3, 4]), + CommittableColumn::Decimal75( + Precision::new(1).unwrap(), + 0, + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], + ), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1]), + ]; + + let offset = 0; + let num_matrix_commitment_columns = 4; + let expected: Vec = vec![1, 1, 1, 1, 2, 1, 1, 1, 1]; + let num_sub_commits: Vec = (0..committable_columns.len()) + .map(|i| { + num_sub_commits( + &committable_columns[i], + offset, + num_matrix_commitment_columns, + ) + }) + .collect(); + + assert_eq!(num_sub_commits, expected); + } + + #[test] + fn we_can_get_the_num_of_sub_commits_offset_with_less_rows_than_cols_update() { + let committable_columns = [ + CommittableColumn::SmallInt(&[1]), + CommittableColumn::Int(&[1, 2]), + CommittableColumn::BigInt(&[1, 2, 3]), + CommittableColumn::Int128(&[1, 2, 3, 4]), + CommittableColumn::Decimal75( + Precision::new(1).unwrap(), + 0, + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], + ), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false]), + CommittableColumn::TimestampTZ(PoSQLTimeUnit::Second, PoSQLTimeZone::Utc, &[1]), + ]; + + let offset = 1; + let num_matrix_commitment_columns = 4; + let expected: Vec = vec![1, 1, 1, 2, 2, 2, 1, 1, 1]; + let num_sub_commits: Vec = (0..committable_columns.len()) + .map(|i| { + num_sub_commits( + &committable_columns[i], + offset, + num_matrix_commitment_columns, + ) + }) + .collect(); + + assert_eq!(num_sub_commits, expected); + } + + #[test] + fn we_can_get_sub_commits_with_less_rows_than_columns() { let committable_columns = [ CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), CommittableColumn::Scalar(vec![[1, 0, 0, 0]]), @@ -550,27 +724,39 @@ mod tests { let offset = 0; let num_columns = 1 << 2; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 1); + + assert_eq!( + num_sub_commits(&committable_columns[0], offset, num_columns), + 1 + ); + assert_eq!( + num_sub_commits(&committable_columns[1], offset, num_columns), + 1 + ); } #[test] - fn we_can_get_sub_commits_per_full_commit_with_offset_and_less_rows_than_columns() { + fn we_can_get_sub_commits_with_offset_and_less_rows_than_columns() { let committable_columns = [ CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0]]), CommittableColumn::Scalar(vec![[1, 0, 0, 0]]), ]; - let offset = 5; + let offset = 3; let num_columns = 1 << 2; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 2); + + assert_eq!( + num_sub_commits(&committable_columns[0], offset, num_columns), + 2 + ); + assert_eq!( + num_sub_commits(&committable_columns[1], offset, num_columns), + 1 + ); } #[test] - fn we_can_get_sub_commits_per_full_commit_with_more_rows_than_generators() { + fn we_can_get_sub_commits_per_with_more_rows_than_generators() { let committable_columns = [ CommittableColumn::SmallInt(&[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, @@ -581,13 +767,23 @@ mod tests { let offset = 0; let num_columns = 1 << 2; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 5); + + assert_eq!( + num_sub_commits(&committable_columns[0], offset, num_columns), + 5 + ); + assert_eq!( + num_sub_commits(&committable_columns[1], offset, num_columns), + 4 + ); + assert_eq!( + num_sub_commits(&committable_columns[2], offset, num_columns), + 3 + ); } #[test] - fn we_can_get_sub_commits_per_full_commit_with_offset_and_more_rows_than_generators() { + fn we_can_get_sub_commits_with_offset_and_more_rows_than_generators() { let committable_columns = [ CommittableColumn::SmallInt(&[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, @@ -598,9 +794,19 @@ mod tests { let offset = 1; let num_columns = 1 << 2; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 6); + + assert_eq!( + num_sub_commits(&committable_columns[0], offset, num_columns), + 6 + ); + assert_eq!( + num_sub_commits(&committable_columns[1], offset, num_columns), + 4 + ); + assert_eq!( + num_sub_commits(&committable_columns[2], offset, num_columns), + 3 + ); } #[test] @@ -617,36 +823,26 @@ mod tests { ]), ]; - let num_columns = 1 << 2; let offset = 0; + let num_columns = 1 << 2; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 5); - - let (bit_table, packed_scalar) = bit_table_and_scalars_for_packed_msm( - &committable_columns, - offset, - num_columns, - num_sub_commits_per_full_commit, - ); + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); let expected_bit_table = [ - 16, 16, 16, 16, 16, 32, 32, 32, 32, 32, 16, 16, 16, 16, 16, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, + 16, 16, 16, 16, 16, 32, 32, 32, 32, 32, 16, 16, 16, 16, 16, 8, 8, 8, 8, 8, ]; let expected_packed_scalar = [ 0, 128, 4, 128, 8, 128, 12, 128, 16, 128, 19, 0, 0, 128, 23, 0, 0, 128, 27, 0, 0, 128, 31, 0, 0, 128, 35, 0, 0, 128, 38, 128, 42, 128, 46, 128, 50, 128, 54, 128, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 128, 5, 128, 9, 128, 13, 128, 17, 128, 20, 0, 0, - 128, 24, 0, 0, 128, 28, 0, 0, 128, 32, 0, 0, 128, 36, 0, 0, 128, 39, 128, 43, 128, 47, - 128, 51, 128, 55, 128, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 128, 6, 128, 10, - 128, 14, 128, 18, 128, 21, 0, 0, 128, 25, 0, 0, 128, 29, 0, 0, 128, 33, 0, 0, 128, 37, - 0, 0, 128, 40, 128, 44, 128, 48, 128, 52, 128, 56, 128, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 128, 5, 128, 9, 128, 13, 128, 17, 128, 20, 0, 0, 128, 24, 0, 0, 128, 28, 0, 0, + 128, 32, 0, 0, 128, 36, 0, 0, 128, 39, 128, 43, 128, 47, 128, 51, 128, 55, 128, 1, 1, + 1, 1, 1, 2, 128, 6, 128, 10, 128, 14, 128, 18, 128, 21, 0, 0, 128, 25, 0, 0, 128, 29, + 0, 0, 128, 33, 0, 0, 128, 37, 0, 0, 128, 40, 128, 44, 128, 48, 128, 52, 128, 56, 128, 1, 1, 1, 1, 1, 3, 128, 7, 128, 11, 128, 15, 128, 0, 0, 22, 0, 0, 128, 26, 0, 0, 128, 30, 0, 0, 128, 34, 0, 0, 128, 0, 0, 0, 0, 41, 128, 45, 128, 49, 128, 53, 128, 0, 0, 1, - 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, + 1, 0, 0, 0, ]; assert_eq!(bit_table, expected_bit_table); @@ -661,30 +857,19 @@ mod tests { CommittableColumn::SmallInt(&[8, 9, 10, 11]), ]; + let offset = 1; let num_columns = 1 << 2; - let offset = 5; - - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 3); - let (bit_table, packed_scalar) = bit_table_and_scalars_for_packed_msm( - &committable_columns, - offset, - num_columns, - num_sub_commits_per_full_commit, - ); + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); - let expected_bit_table = [ - 16, 16, 16, 32, 32, 32, 16, 16, 16, 8, 8, 8, 8, 8, 8, 8, 8, 8, - ]; + let expected_bit_table = [16, 16, 32, 32, 16, 16, 8, 8, 8, 8, 8]; let expected_packed_scalar = [ - 0, 0, 0, 0, 3, 128, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 128, 0, 0, 0, 0, 11, 128, 0, 0, 1, - 0, 0, 1, 0, 0, 1, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 4, 0, 0, 128, 0, 0, 0, 0, 0, 0, 8, - 128, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 128, 0, 0, 0, 0, 0, 0, 5, 0, 0, 128, 0, - 0, 0, 0, 0, 0, 9, 128, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2, 128, 0, 0, 0, 0, 0, 0, - 6, 0, 0, 128, 0, 0, 0, 0, 0, 0, 10, 128, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 3, 128, 0, 0, 0, 0, 7, 0, 0, 128, 0, 0, 11, 128, 0, 1, 1, 1, 1, 0, 128, 0, 0, 4, + 0, 0, 128, 0, 0, 0, 0, 8, 128, 0, 0, 1, 1, 0, 0, 0, 1, 128, 0, 0, 5, 0, 0, 128, 0, 0, + 0, 0, 9, 128, 0, 0, 1, 1, 0, 0, 0, 2, 128, 0, 0, 6, 0, 0, 128, 0, 0, 0, 0, 10, 128, 0, + 0, 1, 1, 0, 0, 0, ]; assert_eq!(bit_table, expected_bit_table); @@ -696,7 +881,7 @@ mod tests { let committable_columns = []; let (bit_table, packed_scalar) = - bit_table_and_scalars_for_packed_msm(&committable_columns, 0, 1, 0); + bit_table_and_scalars_for_packed_msm(&committable_columns, 0, 1); assert!(bit_table.is_empty()); assert!(packed_scalar.is_empty()); @@ -712,62 +897,46 @@ mod tests { let offset = 0; let num_columns = 1 << 1; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 1); - - let (bit_table, packed_scalar) = bit_table_and_scalars_for_packed_msm( - &committable_columns, - offset, - num_columns, - num_sub_commits_per_full_commit, - ); + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); let expected_packed_scalar = [ - 1, 0, 0, 0, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 0, 128, 1, 1, 2, 0, 0, 0, 0, 0, 0, 128, 4, - 0, 0, 0, 0, 0, 0, 128, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 0, 128, 1, 1, 1, 1, 2, 0, 0, 0, 0, 0, 0, + 128, 4, 0, 0, 0, 0, 0, 0, 128, 1, 1, 1, 1, ]; - let expected_bit_table = [64, 64, 8, 8]; + let expected_bit_table = [64, 64, 8, 8, 8, 8]; assert_eq!(bit_table, expected_bit_table); assert_eq!(packed_scalar, expected_packed_scalar); } #[test] - fn we_can_pack_scalars_with_more_than_one_row() { + fn we_can_pack_scalars_with_one_full_row_update() { let committable_columns = [ CommittableColumn::BigInt(&[1, 2]), CommittableColumn::BigInt(&[3, 4]), ]; let offset = 0; - let num_columns = 1 << 0; - - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 2); + let num_columns = 1 << 1; - let (bit_table, packed_scalar) = bit_table_and_scalars_for_packed_msm( - &committable_columns, - offset, - num_columns, - num_sub_commits_per_full_commit, - ); + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); let expected_packed_scalar = [ - 1, 0, 0, 0, 0, 0, 0, 128, 2, 0, 0, 0, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 0, 128, 4, 0, 0, - 0, 0, 0, 0, 128, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 0, 128, 1, 1, 1, 1, 2, 0, 0, 0, 0, 0, 0, + 128, 4, 0, 0, 0, 0, 0, 0, 128, 1, 1, 1, 1, ]; - let expected_bit_table = [64, 64, 64, 64, 8, 8, 8, 8]; + let expected_bit_table = [64, 64, 8, 8, 8, 8]; assert_eq!(bit_table, expected_bit_table); assert_eq!(packed_scalar, expected_packed_scalar); } #[test] - fn we_can_pack_scalars_with_one_full_row_with_offset() { + fn we_can_pack_scalars_with_offset_one_full_row_update() { let committable_columns = [ CommittableColumn::BigInt(&[1, 2]), CommittableColumn::BigInt(&[3, 4]), @@ -776,21 +945,13 @@ mod tests { let offset = 1; let num_columns = 1 << 1; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 2); - - let (bit_table, packed_scalar) = bit_table_and_scalars_for_packed_msm( - &committable_columns, - offset, - num_columns, - num_sub_commits_per_full_commit, - ); + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); let expected_packed_scalar = [ 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 0, 128, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, + 0, 0, 0, 128, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, ]; let expected_bit_table = [64, 64, 64, 64, 8, 8, 8, 8]; @@ -800,32 +961,182 @@ mod tests { } #[test] - fn we_can_pack_scalars_with_offset_and_more_rows_than_columns() { + fn we_can_get_simple_offset_columns() { let committable_columns = [ CommittableColumn::BigInt(&[1, 2]), CommittableColumn::BigInt(&[3, 4]), ]; + // No offset + let offset = 0; + let num_columns = 1 << 1; + + let expected: Vec = vec![1, 1, 1, 1, 1, 1, 1, 1]; + + let mut buffer = vec![0_u8; (OFFSET_SIZE + committable_columns.len()) * num_columns]; + offset_column(&committable_columns, offset, num_columns, &mut buffer); + + assert_eq!(buffer, expected); + + // Offset let offset = 1; + let num_columns = 1 << 1; + + let expected: Vec = vec![0, 1, 1, 1, 1, 0, 1, 0]; + + let mut buffer = vec![0_u8; (OFFSET_SIZE + committable_columns.len()) * num_columns]; + offset_column(&committable_columns, offset, num_columns, &mut buffer); + + assert_eq!(buffer, expected); + + // One column + let offset = 0; + let num_columns = 1; + + let expected: Vec = vec![1, 1, 1, 1]; + + let mut buffer = vec![0_u8; (OFFSET_SIZE + committable_columns.len()) * num_columns]; + offset_column(&committable_columns, offset, num_columns, &mut buffer); + + assert_eq!(buffer, expected); + } + + #[test] + #[should_panic(expected = "offset 1 must be less than the number of columns 1")] + fn we_can_panic_when_offset_is_not_less_than_num_of_columns_when_getting_offsets() { + let committable_columns = [ + CommittableColumn::BigInt(&[1, 2]), + CommittableColumn::BigInt(&[3, 4]), + ]; + + let offset = 1; + let num_columns = 1; + let mut buffer = vec![0_u8; (OFFSET_SIZE + committable_columns.len()) * num_columns]; + offset_column(&committable_columns, offset, num_columns, &mut buffer); + } + + #[test] + fn we_can_get_complex_offset_columns() { + let committable_columns = [ + CommittableColumn::SmallInt(&[1, 2, 3, 4, 5]), + CommittableColumn::Int(&[1, 2, 3, 4, 5, 6]), + CommittableColumn::BigInt(&[1, 2, 3, 4, 5, 6, 7]), + CommittableColumn::Int128(&[1, 2, 3, 4, 5, 6, 7, 8]), + CommittableColumn::Decimal75( + Precision::new(1).unwrap(), + 0, + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], + ), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false, true, false, true]), + CommittableColumn::TimestampTZ( + PoSQLTimeUnit::Second, + PoSQLTimeZone::Utc, + &[1, 2, 3, 4, 5], + ), + ]; + + let offset = 0; + let num_columns = 1 << 2; + let mut buffer = vec![0_u8; (OFFSET_SIZE + committable_columns.len()) * num_columns]; + offset_column(&committable_columns, offset, num_columns, &mut buffer); + let expected: Vec = vec![ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + ]; + assert_eq!(buffer, expected); + } + + #[test] + fn we_can_get_complex_offset_columns_with_offset() { + let committable_columns = [ + CommittableColumn::SmallInt(&[1, 2, 3, 4, 5]), + CommittableColumn::Int(&[1, 2, 3, 4, 5, 6]), + CommittableColumn::BigInt(&[1, 2, 3, 4, 5, 6, 7]), + CommittableColumn::Int128(&[1, 2, 3, 4, 5, 6, 7, 8]), + CommittableColumn::Decimal75( + Precision::new(1).unwrap(), + 0, + vec![ + [1, 0, 0, 0], + [2, 0, 0, 0], + [3, 0, 0, 0], + [4, 0, 0, 0], + [5, 0, 0, 0], + ], + ), + CommittableColumn::Scalar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0], [4, 0, 0, 0]]), + CommittableColumn::VarChar(vec![[1, 0, 0, 0], [2, 0, 0, 0], [3, 0, 0, 0]]), + CommittableColumn::Boolean(&[true, false, true, false, true]), + CommittableColumn::TimestampTZ( + PoSQLTimeUnit::Second, + PoSQLTimeZone::Utc, + &[1, 2, 3, 4, 5], + ), + ]; + + let offset = 3; + let num_columns = 1 << 2; + let mut buffer = vec![0_u8; (OFFSET_SIZE + committable_columns.len()) * num_columns]; + offset_column(&committable_columns, offset, num_columns, &mut buffer); + let expected: Vec = vec![ + 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + ]; + assert_eq!(buffer, expected); + } + + #[test] + fn we_can_pack_scalars_with_more_than_one_row() { + let committable_columns = [ + CommittableColumn::BigInt(&[1, 2]), + CommittableColumn::BigInt(&[3, 4]), + ]; + + let offset = 0; let num_columns = 1 << 0; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 3); + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); - let (bit_table, packed_scalar) = bit_table_and_scalars_for_packed_msm( - &committable_columns, - offset, - num_columns, - num_sub_commits_per_full_commit, - ); + let expected_packed_scalar = [ + 1, 0, 0, 0, 0, 0, 0, 128, 2, 0, 0, 0, 0, 0, 0, 128, 3, 0, 0, 0, 0, 0, 0, 128, 4, 0, 0, + 0, 0, 0, 0, 128, 1, 1, 1, 1, + ]; + + let expected_bit_table = [64, 64, 64, 64, 8, 8, 8, 8]; + + assert_eq!(bit_table, expected_bit_table); + assert_eq!(packed_scalar, expected_packed_scalar); + } + + #[test] + fn we_can_pack_scalars_with_one_full_row_with_offset() { + let committable_columns = [ + CommittableColumn::BigInt(&[1, 2]), + CommittableColumn::BigInt(&[3, 4]), + ]; + + let offset = 1; + let num_columns = 1 << 1; + + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); let expected_packed_scalar = [ - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 128, 2, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 128, 4, 0, 0, 0, 0, 0, 0, 128, 0, 1, 1, 0, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 0, 128, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, ]; - let expected_bit_table = [64, 64, 64, 64, 64, 64, 8, 8, 8, 8, 8, 8]; + let expected_bit_table = [64, 64, 64, 64, 8, 8, 8, 8]; assert_eq!(bit_table, expected_bit_table); assert_eq!(packed_scalar, expected_packed_scalar); @@ -842,32 +1153,21 @@ mod tests { let offset = 0; let num_columns = 3; - let num_sub_commits_per_full_commit = - sub_commits_per_full_commit(&committable_columns, offset, num_columns); - assert_eq!(num_sub_commits_per_full_commit, 2); - - let (bit_table, packed_scalar) = bit_table_and_scalars_for_packed_msm( - &committable_columns, - offset, - num_columns, - num_sub_commits_per_full_commit, - ); + let (bit_table, packed_scalar) = + bit_table_and_scalars_for_packed_msm(&committable_columns, offset, num_columns); let expected_packed_scalar = [ 0, 128, 3, 128, 6, 0, 0, 128, 9, 0, 0, 128, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, - 128, 4, 128, 7, 0, 0, 128, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 2, 128, 5, - 128, 8, 0, 0, 128, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 128, 4, 128, 7, + 0, 0, 128, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 2, 128, 5, 128, 8, 0, 0, 128, 0, 0, 0, + 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 1, 0, 0, ]; - let expected_bit_table = [16, 16, 32, 32, 64 * 4, 64 * 4, 8, 8, 8, 8, 8, 8]; + let expected_bit_table = [16, 16, 32, 32, 256, 8, 8, 8, 8, 8]; - assert_eq!(bit_table, expected_bit_table); assert_eq!(packed_scalar, expected_packed_scalar); + assert_eq!(bit_table, expected_bit_table); } }