From fcfdfa67ea07f017c5f351836bd383810ad53fcf Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Fri, 6 Dec 2024 14:27:26 -0800 Subject: [PATCH] Hybrid integration (complete) * tmp commit: failing * fix panic from encrypt-hybrid command * progress * distinct filenames for in_the_clear and mpc output * temporary printlns for debugging * make sure to call .wait on command * fix os error with config setup * add query_type lookup * add plumbing for starting a hybrid query * make helpers not silent for debugging * integrate with RoundRobinSubmission, in progress * Per shard submission from report collector * update HistogramValue type to be consistent across helper and report collector * add hybrid_test assert against in the clear results * update comment * Make hybrid run with compact gate * Add required features for Hybrid integration tests * update pre-commit and github check.yaml to use correct flags for hybrid test * no default features for hybrid * check.yaml typo * increase STEP_COUNT_LIMIT * lower step count limit * Update ipa-core/src/bin/report_collector.rs Co-authored-by: Alex Koshelev --------- Co-authored-by: Alex Koshelev Co-authored-by: Alex Koshelev --- .github/workflows/check.yml | 2 +- ipa-core/Cargo.toml | 6 +- ipa-core/src/bin/report_collector.rs | 69 ++++++++++++++++++----- ipa-core/src/cli/crypto/hybrid_encrypt.rs | 8 +-- ipa-core/src/cli/playbook/hybrid.rs | 35 ++++++------ ipa-core/src/cli/playbook/mod.rs | 1 + ipa-core/src/cli/playbook/streaming.rs | 4 +- ipa-core/src/net/http_serde.rs | 4 ++ ipa-core/src/protocol/basics/shard_fin.rs | 6 +- ipa-core/src/protocol/hybrid/mod.rs | 2 +- ipa-core/src/protocol/hybrid/step.rs | 3 + ipa-core/src/protocol/ipa_prf/mod.rs | 2 +- ipa-core/src/query/executor.rs | 19 ++++++- ipa-core/src/query/runner/hybrid.rs | 41 ++++++++++++-- ipa-core/src/query/runner/mod.rs | 2 +- ipa-core/tests/hybrid.rs | 49 ++++++++++------ pre-commit | 2 +- 17 files changed, 185 insertions(+), 70 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e6a61bbd3..99ef7d09f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -182,7 +182,7 @@ jobs: run: cargo test --release --test "helper_networks" --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate" - name: Integration Tests - Hybrid - run: cargo test --release --test "hybrid" --features "cli test-fixture" + run: cargo test --release --test "hybrid" --no-default-features --features "cli compact-gate web-app real-world-infra test-fixture relaxed-dp" - name: Integration Tests - IPA with Relaxed DP run: cargo test --release --test "ipa_with_relaxed_dp" --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate relaxed-dp" diff --git a/ipa-core/Cargo.toml b/ipa-core/Cargo.toml index b70384f37..6c8df1f33 100644 --- a/ipa-core/Cargo.toml +++ b/ipa-core/Cargo.toml @@ -288,6 +288,10 @@ required-features = [ [[test]] name = "hybrid" required-features = [ - "test-fixture", "cli", + "compact-gate", + "web-app", + "real-world-infra", + "test-fixture", + "relaxed-dp", ] diff --git a/ipa-core/src/bin/report_collector.rs b/ipa-core/src/bin/report_collector.rs index aa6da0bb4..9866fb62a 100644 --- a/ipa-core/src/bin/report_collector.rs +++ b/ipa-core/src/bin/report_collector.rs @@ -4,7 +4,7 @@ use std::{ fmt::Debug, fs::{File, OpenOptions}, io, - io::{stdout, Write}, + io::{stdout, BufReader, Write}, ops::Deref, path::{Path, PathBuf}, }; @@ -16,13 +16,20 @@ use ipa_core::{ playbook::{ make_clients, make_sharded_clients, playbook_oprf_ipa, run_hybrid_query_and_validate, run_query_and_validate, validate, validate_dp, HybridQueryResult, InputSource, + RoundRobinSubmission, StreamingSubmission, }, CsvSerializer, IpaQueryResult, Verbosity, }, config::{KeyRegistries, NetworkConfig}, - ff::{boolean_array::BA32, FieldType}, - helpers::query::{ - DpMechanism, HybridQueryParams, IpaQueryConfig, QueryConfig, QuerySize, QueryType, + ff::{ + boolean_array::{BA16, BA32}, + FieldType, + }, + helpers::{ + query::{ + DpMechanism, HybridQueryParams, IpaQueryConfig, QueryConfig, QuerySize, QueryType, + }, + BodyStream, }, net::{Helper, IpaHttpClient}, report::{EncryptedOprfReportStreams, DEFAULT_KEY_ID}, @@ -143,6 +150,10 @@ enum ReportCollectorCommand { #[clap(flatten)] hybrid_query_config: HybridQueryParams, + + /// Number of records to aggregate + #[clap(long, short = 'n')] + count: u32, }, } @@ -255,7 +266,17 @@ async fn main() -> Result<(), Box> { ReportCollectorCommand::MaliciousHybrid { ref encrypted_inputs, hybrid_query_config, - } => hybrid(&args, hybrid_query_config, clients, encrypted_inputs).await?, + count, + } => { + hybrid( + &args, + hybrid_query_config, + clients, + encrypted_inputs, + count.try_into().expect("u32 should fit into usize"), + ) + .await? + } }; Ok(()) @@ -402,20 +423,37 @@ async fn hybrid( hybrid_query_config: HybridQueryParams, helper_clients: Vec<[IpaHttpClient; 3]>, encrypted_inputs: &EncryptedInputs, + count: usize, ) -> Result<(), Box> { let query_type = QueryType::MaliciousHybrid(hybrid_query_config); - let files = [ + let [h1_streams, h2_streams, h3_streams] = [ &encrypted_inputs.enc_input_file1, &encrypted_inputs.enc_input_file2, &encrypted_inputs.enc_input_file3, - ]; - - // despite the name, this is generic enough to work with hybrid - let encrypted_report_streams = EncryptedOprfReportStreams::from(files); + ] + .map(|path| { + let file = File::open(path).unwrap_or_else(|e| panic!("unable to open file {path:?}. {e}")); + RoundRobinSubmission::new(BufReader::new(file)) + }) + .map(|s| s.into_byte_streams(args.shard_count)); + + // create byte streams for each shard + let submissions = h1_streams + .into_iter() + .zip(h2_streams.into_iter()) + .zip(h3_streams.into_iter()) + .map(|((s1, s2), s3)| { + [ + BodyStream::from_bytes_stream(s1), + BodyStream::from_bytes_stream(s2), + BodyStream::from_bytes_stream(s3), + ] + }) + .collect::>(); let query_config = QueryConfig { - size: QuerySize::try_from(encrypted_report_streams.query_size).unwrap(), + size: QuerySize::try_from(count).unwrap(), field_type: FieldType::Fp32BitPrime, query_type, }; @@ -426,12 +464,13 @@ async fn hybrid( .expect("Unable to create query!"); tracing::info!("Starting query for OPRF"); - // the value for histogram values (BA32) must be kept in sync with the server-side + + // the value for histogram values (BA16) must be kept in sync with the server-side // implementation, otherwise a runtime reconstruct error will be generated. // see ipa-core/src/query/executor.rs - let actual = run_hybrid_query_and_validate::( - encrypted_report_streams.streams, - encrypted_report_streams.query_size, + let actual = run_hybrid_query_and_validate::( + submissions, + count, helper_clients, query_id, hybrid_query_config, diff --git a/ipa-core/src/cli/crypto/hybrid_encrypt.rs b/ipa-core/src/cli/crypto/hybrid_encrypt.rs index e0c749faf..7db024681 100644 --- a/ipa-core/src/cli/crypto/hybrid_encrypt.rs +++ b/ipa-core/src/cli/crypto/hybrid_encrypt.rs @@ -58,16 +58,16 @@ impl HybridEncryptArgs { let mut key_registries = KeyRegistries::default(); let network = - NetworkConfig::from_toml_str(&read_to_string(&self.network).unwrap_or_else(|e| { - panic!("Failed to open network file: {:?}. {}", &self.network, e) - })) + NetworkConfig::from_toml_str_sharded(&read_to_string(&self.network).unwrap_or_else( + |e| panic!("Failed to open network file: {:?}. {}", &self.network, e), + )) .unwrap_or_else(|e| { panic!( "Failed to parse network file into toml: {:?}. {}", &self.network, e ) }); - let Some(key_registries) = key_registries.init_from(&network) else { + let Some(key_registries) = key_registries.init_from(&network[0]) else { panic!("could not load network file") }; diff --git a/ipa-core/src/cli/playbook/hybrid.rs b/ipa-core/src/cli/playbook/hybrid.rs index a8c80cffd..c52547f6e 100644 --- a/ipa-core/src/cli/playbook/hybrid.rs +++ b/ipa-core/src/cli/playbook/hybrid.rs @@ -1,6 +1,7 @@ #![cfg(all(feature = "web-app", feature = "cli"))] use std::{ cmp::min, + iter::zip, time::{Duration, Instant}, }; @@ -25,7 +26,7 @@ use crate::{ /// if results are invalid #[allow(clippy::disallowed_methods)] // allow try_join_all pub async fn run_hybrid_query_and_validate( - inputs: [BodyStream; 3], + inputs: Vec<[BodyStream; 3]>, query_size: usize, clients: Vec<[IpaHttpClient; 3]>, query_id: QueryId, @@ -36,28 +37,30 @@ where AdditiveShare: Serializable, { let mpc_time = Instant::now(); - - // for now, submit everything to the leader. TODO: round robin submission - let leader_clients = &clients[0]; - try_join_all( - inputs - .into_iter() - .zip(leader_clients) - .map(|(input_stream, client)| { - client.query_input(QueryInput { - query_id, - input_stream, - }) - }), - ) + assert_eq!(clients.len(), inputs.len()); + // submit inputs to each shard + let _ = try_join_all(zip(clients.iter(), inputs.into_iter()).map( + |(shard_clients, shard_inputs)| { + try_join_all(shard_clients.iter().zip(shard_inputs.into_iter()).map( + |(client, input)| { + client.query_input(QueryInput { + query_id, + input_stream: input, + }) + }, + )) + }, + )) .await .unwrap(); + let leader_clients = &clients[0]; + let mut delay = Duration::from_millis(125); loop { if try_join_all( leader_clients - .iter() + .each_ref() .map(|client| client.query_status(query_id)), ) .await diff --git a/ipa-core/src/cli/playbook/mod.rs b/ipa-core/src/cli/playbook/mod.rs index dc10806e7..fc8b46e9e 100644 --- a/ipa-core/src/cli/playbook/mod.rs +++ b/ipa-core/src/cli/playbook/mod.rs @@ -22,6 +22,7 @@ use tokio::time::sleep; pub use self::{ hybrid::{run_hybrid_query_and_validate, HybridQueryResult}, ipa::{playbook_oprf_ipa, run_query_and_validate}, + streaming::{RoundRobinSubmission, StreamingSubmission}, }; use crate::{ cli::config_parse::HelperNetworkConfigParseExt, diff --git a/ipa-core/src/cli/playbook/streaming.rs b/ipa-core/src/cli/playbook/streaming.rs index 141b25922..f246582ee 100644 --- a/ipa-core/src/cli/playbook/streaming.rs +++ b/ipa-core/src/cli/playbook/streaming.rs @@ -15,7 +15,7 @@ use crate::{ /// Trait for submitting inputs as streams, rather than reading everything /// in memory. Should provide better performance for very large inputs. -trait StreamingSubmission { +pub trait StreamingSubmission { /// Spits itself into `count` instances of [`BytesStream`]. fn into_byte_streams(self, count: usize) -> Vec; } @@ -25,7 +25,7 @@ trait StreamingSubmission { /// and delimited by newlines. The output streams will have /// run-length encoding, meaning that each element will have /// a 2 byte length prefix added to it. -struct RoundRobinSubmission(R); +pub struct RoundRobinSubmission(R); impl RoundRobinSubmission { pub fn new(read_from: R) -> Self { diff --git a/ipa-core/src/net/http_serde.rs b/ipa-core/src/net/http_serde.rs index 2b9dad085..c7c3eb1a4 100644 --- a/ipa-core/src/net/http_serde.rs +++ b/ipa-core/src/net/http_serde.rs @@ -132,6 +132,10 @@ pub mod query { let Query(q) = req.extract().await?; Ok(QueryType::MaliciousOprfIpa(q)) } + QueryType::MALICIOUS_HYBRID_STR => { + let Query(q) = req.extract().await?; + Ok(QueryType::MaliciousHybrid(q)) + } other => Err(Error::bad_query_value("query_type", other)), }?; Ok(QueryConfigQueryParams(QueryConfig { diff --git a/ipa-core/src/protocol/basics/shard_fin.rs b/ipa-core/src/protocol/basics/shard_fin.rs index c5aafd12c..ce817c13a 100644 --- a/ipa-core/src/protocol/basics/shard_fin.rs +++ b/ipa-core/src/protocol/basics/shard_fin.rs @@ -9,7 +9,7 @@ use crate::{ ff::{boolean::Boolean, boolean_array::BooleanArray, Serializable}, helpers::{Message, TotalRecords}, protocol::{ - boolean::step::SixteenBitStep, + boolean::step::ThirtyTwoBitStep, context::{ dzkp_validator::DZKPValidator, DZKPContext, DZKPUpgradedMaliciousContext, DZKPUpgradedSemiHonestContext, MaliciousProtocolSteps, ShardedContext, @@ -288,9 +288,7 @@ where C: 'a, { async move { - // todo: SixteenBit only works for values up to BA16. EightBitStep will panic if we try - // to add larger values - self.values = integer_sat_add::<_, SixteenBitStep, B>( + self.values = integer_sat_add::<_, ThirtyTwoBitStep, B>( ctx, record_id, &self.values, diff --git a/ipa-core/src/protocol/hybrid/mod.rs b/ipa-core/src/protocol/hybrid/mod.rs index e9bf009eb..5217365b6 100644 --- a/ipa-core/src/protocol/hybrid/mod.rs +++ b/ipa-core/src/protocol/hybrid/mod.rs @@ -130,7 +130,7 @@ where let aggregated_reports = aggregate_reports::(ctx.clone(), sharded_reports).await?; let histogram = breakdown_reveal_aggregation::( - ctx.clone(), + ctx.narrow(&Step::Aggregate), aggregated_reports, &dp_padding_params, ) diff --git a/ipa-core/src/protocol/hybrid/step.rs b/ipa-core/src/protocol/hybrid/step.rs index 16c73b7a2..2a98488fc 100644 --- a/ipa-core/src/protocol/hybrid/step.rs +++ b/ipa-core/src/protocol/hybrid/step.rs @@ -19,6 +19,8 @@ pub(crate) enum HybridStep { GroupBySum, #[step(child = crate::protocol::context::step::DzkpValidationProtocolStep)] GroupBySumValidate, + #[step(child = crate::protocol::ipa_prf::aggregation::step::AggregationStep)] + Aggregate, #[step(child = FinalizeSteps)] Finalize, } @@ -33,6 +35,7 @@ pub(crate) enum AggregateReportsStep { #[derive(CompactStep)] pub(crate) enum FinalizeSteps { + #[step(child = crate::protocol::ipa_prf::boolean_ops::step::SaturatedAdditionStep)] Add, #[step(child = crate::protocol::context::step::DzkpValidationProtocolStep)] Validate, diff --git a/ipa-core/src/protocol/ipa_prf/mod.rs b/ipa-core/src/protocol/ipa_prf/mod.rs index 2659d31d4..4ac3abba0 100644 --- a/ipa-core/src/protocol/ipa_prf/mod.rs +++ b/ipa-core/src/protocol/ipa_prf/mod.rs @@ -867,7 +867,7 @@ mod compact_gate_tests { fn step_count_limit() { // This is an arbitrary limit intended to catch changes that unintentionally // blow up the step count. It can be increased, within reason. - const STEP_COUNT_LIMIT: u32 = 24_000; + const STEP_COUNT_LIMIT: u32 = 32_500; assert!( ProtocolStep::STEP_COUNT < STEP_COUNT_LIMIT, "Step count of {actual} exceeds limit of {STEP_COUNT_LIMIT}.", diff --git a/ipa-core/src/query/executor.rs b/ipa-core/src/query/executor.rs index 47a0d3bf5..d626e12bf 100644 --- a/ipa-core/src/query/executor.rs +++ b/ipa-core/src/query/executor.rs @@ -39,7 +39,7 @@ use crate::{ Gate, }, query::{ - runner::{OprfIpaQuery, QueryResult}, + runner::{execute_hybrid_protocol, OprfIpaQuery, QueryResult}, state::RunningQuery, }, sync::Arc, @@ -165,7 +165,22 @@ pub fn execute( ) }, ), - (QueryType::MaliciousHybrid(_), _) => todo!(), + (QueryType::MaliciousHybrid(ipa_config), _) => do_query( + runtime, + config, + gateway, + input, + move |prss, gateway, config, input| { + Box::pin(execute_hybrid_protocol( + prss, + gateway, + input, + ipa_config, + config, + key_registry, + )) + }, + ), } } diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index 199ee385f..9cc5294d8 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -8,31 +8,35 @@ use std::{ use futures::{stream::iter, StreamExt, TryStreamExt}; use generic_array::ArrayLength; +use super::QueryResult; use crate::{ error::{Error, LengthError}, ff::{ boolean::Boolean, - boolean_array::{BooleanArray, BA3, BA8}, + boolean_array::{BooleanArray, BA16, BA3, BA8}, curve_points::RP25519, ec_prime_field::Fp25519, Serializable, U128Conversions, }, helpers::{ - query::{DpMechanism, HybridQueryParams, QuerySize}, - BodyStream, LengthDelimitedStream, + query::{DpMechanism, HybridQueryParams, QueryConfig, QuerySize}, + setup_cross_shard_prss, BodyStream, Gateway, LengthDelimitedStream, }, hpke::PrivateKeyRegistry, protocol::{ basics::{shard_fin::FinalizerContext, BooleanArrayMul, BooleanProtocols, Reveal}, - context::{DZKPUpgraded, MacUpgraded, ShardedContext, UpgradableContext}, + context::{ + DZKPUpgraded, MacUpgraded, ShardedContext, ShardedMaliciousContext, UpgradableContext, + }, hybrid::{ hybrid_protocol, oprf::{CONV_CHUNK, PRF_CHUNK}, step::HybridStep, }, ipa_prf::{oprf_padding::PaddingParameters, prf_eval::PrfSharing, shuffle::Shuffle}, - prss::FromPrss, + prss::{Endpoint, FromPrss}, step::ProtocolStep::Hybrid, + Gate, }, query::runner::reshard_tag::reshard_aad, report::hybrid::{ @@ -42,6 +46,7 @@ use crate::{ replicated::semi_honest::AdditiveShare as Replicated, BitDecomposed, TransposeFrom, Vectorizable, }, + sharding::{ShardConfiguration, Sharded}, }; #[allow(dead_code)] @@ -165,6 +170,32 @@ where } } +pub async fn execute_hybrid_protocol<'a, R: PrivateKeyRegistry>( + prss: &'a Endpoint, + gateway: &'a Gateway, + input: BodyStream, + ipa_config: HybridQueryParams, + config: &QueryConfig, + key_registry: Arc, +) -> QueryResult { + let gate = Gate::default(); + let cross_shard_prss = + setup_cross_shard_prss(gateway, &gate, prss.indexed(&gate), gateway).await?; + let sharded = Sharded { + shard_id: gateway.shard_id(), + shard_count: gateway.shard_count(), + prss: Arc::new(cross_shard_prss), + }; + + let ctx = ShardedMaliciousContext::new_with_gate(prss, gateway, gate, sharded); + + Ok(Box::new( + Query::<_, BA16, R>::new(ipa_config, key_registry) + .execute(ctx, config.size, input) + .await?, + )) +} + #[cfg(all(test, unit_test, feature = "in-memory-infra"))] mod tests { use std::{ diff --git a/ipa-core/src/query/runner/mod.rs b/ipa-core/src/query/runner/mod.rs index 83f033fe4..642e2a846 100644 --- a/ipa-core/src/query/runner/mod.rs +++ b/ipa-core/src/query/runner/mod.rs @@ -15,7 +15,7 @@ pub(super) use sharded_shuffle::execute_sharded_shuffle; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] pub(super) use test_multiply::execute_test_multiply; -pub use self::oprf_ipa::OprfIpaQuery; +pub use self::{hybrid::execute_hybrid_protocol, oprf_ipa::OprfIpaQuery}; use crate::{error::Error, query::ProtocolResult}; pub(super) type QueryResult = Result, Error>; diff --git a/ipa-core/tests/hybrid.rs b/ipa-core/tests/hybrid.rs index 858a25dde..b40f524f0 100644 --- a/ipa-core/tests/hybrid.rs +++ b/ipa-core/tests/hybrid.rs @@ -2,7 +2,10 @@ #[allow(dead_code)] mod common; -use std::process::{Command, Stdio}; +use std::{ + fs::File, + process::{Command, Stdio}, +}; use common::{ spawn_shards, tempdir::TempDir, test_sharded_setup, CommandExt, TerminateOnDropExt, @@ -11,13 +14,13 @@ use common::{ use ipa_core::{cli::playbook::HybridQueryResult, helpers::query::HybridQueryParams}; use rand::thread_rng; use rand_core::RngCore; +use serde_json::from_reader; pub const IN_THE_CLEAR_BIN: &str = env!("CARGO_BIN_EXE_in_the_clear"); // this currently only generates data and runs in the clear // eventaully we'll want to add the MPC as well #[test] -#[ignore] fn test_hybrid() { const INPUT_SIZE: usize = 100; const SHARDS: usize = 5; @@ -34,6 +37,7 @@ fn test_hybrid() { // Gen inputs let input_file = dir.path().join("ipa_inputs.txt"); + let in_the_clear_output_file = dir.path().join("ipa_output_in_the_clear.json"); let output_file = dir.path().join("ipa_output.json"); let mut command = Command::new(TEST_RC_BIN); @@ -51,25 +55,25 @@ fn test_hybrid() { let mut command = Command::new(IN_THE_CLEAR_BIN); command .args(["--input-file".as_ref(), input_file.as_os_str()]) - .args(["--output-file".as_ref(), output_file.as_os_str()]) + .args([ + "--output-file".as_ref(), + in_the_clear_output_file.as_os_str(), + ]) .silent() .stdin(Stdio::piped()); command.status().unwrap_status(); - // set to true to always keep the temp dir after test finishes - let dir = TempDir::new_delete_on_drop(); - let path = dir.path(); - - let sockets = test_sharded_setup::(path); - let _helpers = spawn_shards(path, &sockets, true); + let config_path = dir.path().join("config"); + let sockets = test_sharded_setup::(&config_path); + let _helpers = spawn_shards(&config_path, &sockets, true); // encrypt input let mut command = Command::new(CRYPTO_UTIL_BIN); command .arg("hybrid-encrypt") .args(["--input-file".as_ref(), input_file.as_os_str()]) - .args(["--output-dir".as_ref(), path.as_os_str()]) - .args(["--network".into(), dir.path().join("network.toml")]) + .args(["--output-dir".as_ref(), dir.path().as_os_str()]) + .args(["--network".into(), config_path.join("network.toml")]) .stdin(Stdio::piped()); command.status().unwrap_status(); let enc1 = dir.path().join("helper1.enc"); @@ -79,10 +83,13 @@ fn test_hybrid() { // Run Hybrid let mut command = Command::new(TEST_RC_BIN); command - .args(["--network".into(), dir.path().join("network.toml")]) + .args(["--network".into(), config_path.join("network.toml")]) .args(["--output-file".as_ref(), output_file.as_os_str()]) + .args(["--shard-count", SHARDS.to_string().as_str()]) .args(["--wait", "2"]) - .arg("hybrid") + .arg("malicious-hybrid") + .silent() + .args(["--count", INPUT_SIZE.to_string().as_str()]) .args(["--enc-input-file1".as_ref(), enc1.as_os_str()]) .args(["--enc-input-file2".as_ref(), enc2.as_os_str()]) .args(["--enc-input-file3".as_ref(), enc3.as_os_str()]) @@ -100,11 +107,12 @@ fn test_hybrid() { } command.stdin(Stdio::piped()); - let _test_mpc = command.spawn().unwrap().terminate_on_drop(); + let test_mpc = command.spawn().unwrap().terminate_on_drop(); + test_mpc.wait().unwrap_status(); // basic output checks - output should have the exact size as number of breakdowns let output = serde_json::from_str::( - &std::fs::read_to_string(&output_file).expect("IPA results file exists"), + &std::fs::read_to_string(&output_file).expect("IPA results file should exist"), ) .expect("IPA results file is valid JSON"); @@ -115,5 +123,14 @@ fn test_hybrid() { ); assert_eq!(INPUT_SIZE, usize::from(output.input_size)); - // TODO compare in the clear results with MPC results + let expected_result: Vec = from_reader( + File::open(in_the_clear_output_file) + .expect("file should exist as it's created above in the test"), + ) + .expect("should match hard coded format from in_the_clear"); + assert!(output + .breakdowns + .iter() + .zip(expected_result.iter()) + .all(|(a, b)| a == b)); } diff --git a/pre-commit b/pre-commit index b5319ac91..9a5619875 100755 --- a/pre-commit +++ b/pre-commit @@ -124,7 +124,7 @@ then cargo test --release --test "helper_networks" --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate" check "Slow tests: Hybrid tests" \ - cargo test --release --test "hybrid" --features "cli test-fixture" + cargo test --release --test "hybrid" --no-default-features --features "cli compact-gate web-app real-world-infra test-fixture relaxed-dp" check "Slow tests: IPA with Relaxed DP" \