From da00669cab543ad414d462ab8e1b3f9bde1be19e Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Wed, 18 Sep 2024 11:30:38 -0700 Subject: [PATCH 1/3] Sharded shuffle query type In order for us to test sharded circuits through the HTTP stack, we need a protocol that exercises both shard-to-shard and helper-to-helper path. Shuffle seems like a good fit for that, so this change just plumbs the new query type through the stack --- ipa-core/src/bin/test_mpc.rs | 10 ++++++++++ ipa-core/src/helpers/transport/query/mod.rs | 5 +++++ ipa-core/src/net/http_serde.rs | 1 + ipa-core/src/query/executor.rs | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/ipa-core/src/bin/test_mpc.rs b/ipa-core/src/bin/test_mpc.rs index 74e2e7284..9da4afbb2 100644 --- a/ipa-core/src/bin/test_mpc.rs +++ b/ipa-core/src/bin/test_mpc.rs @@ -85,6 +85,11 @@ enum TestAction { /// All helpers add their shares locally and set the resulting share to be the /// sum. No communication is required to run the circuit. AddInPrimeField, + /// A test protocol for sharded MPCs. The goal here is to use + /// both shard-to-shard and helper-to-helper communication channels. + /// This is exactly what shuffle does and that's why it is picked + /// for this purpose. + ShardedShuffle, } #[tokio::main] @@ -102,6 +107,7 @@ async fn main() -> Result<(), Box> { match args.action { TestAction::Multiply => multiply(&args, &clients).await, TestAction::AddInPrimeField => add(&args, &clients).await, + TestAction::ShardedShuffle => sharded_shuffle(&args, &clients).await, }; Ok(()) @@ -159,3 +165,7 @@ async fn add(args: &Args, helper_clients: &[MpcHelperClient; 3]) { FieldType::Fp32BitPrime => add_in_field::(args, helper_clients).await, }; } + +async fn sharded_shuffle(_args: &Args, _helper_clients: &[MpcHelperClient; 3]) { + unimplemented!() +} diff --git a/ipa-core/src/helpers/transport/query/mod.rs b/ipa-core/src/helpers/transport/query/mod.rs index f509185a5..3cb655173 100644 --- a/ipa-core/src/helpers/transport/query/mod.rs +++ b/ipa-core/src/helpers/transport/query/mod.rs @@ -198,6 +198,8 @@ pub enum QueryType { TestMultiply, #[cfg(any(test, feature = "test-fixture", feature = "cli"))] TestAddInPrimeField, + #[cfg(any(test, feature = "test-fixture", feature = "cli"))] + TestShardedShuffle, SemiHonestOprfIpa(IpaQueryConfig), MaliciousOprfIpa(IpaQueryConfig), } @@ -206,6 +208,7 @@ impl QueryType { /// TODO: strum pub const TEST_MULTIPLY_STR: &'static str = "test-multiply"; pub const TEST_ADD_STR: &'static str = "test-add"; + pub const TEST_SHARDED_SHUFFLE_STR: &'static str = "test-sharded-shuffle"; pub const SEMI_HONEST_OPRF_IPA_STR: &'static str = "semi-honest-oprf-ipa"; pub const MALICIOUS_OPRF_IPA_STR: &'static str = "malicious-oprf-ipa"; } @@ -218,6 +221,8 @@ impl AsRef for QueryType { QueryType::TestMultiply => Self::TEST_MULTIPLY_STR, #[cfg(any(test, feature = "cli", feature = "test-fixture"))] QueryType::TestAddInPrimeField => Self::TEST_ADD_STR, + #[cfg(any(test, feature = "cli", feature = "test-fixture"))] + QueryType::TestShardedShuffle => Self::TEST_SHARDED_SHUFFLE_STR, QueryType::SemiHonestOprfIpa(_) => Self::SEMI_HONEST_OPRF_IPA_STR, QueryType::MaliciousOprfIpa(_) => Self::MALICIOUS_OPRF_IPA_STR, } diff --git a/ipa-core/src/net/http_serde.rs b/ipa-core/src/net/http_serde.rs index 687fd2f19..cef850ae8 100644 --- a/ipa-core/src/net/http_serde.rs +++ b/ipa-core/src/net/http_serde.rs @@ -152,6 +152,7 @@ pub mod query { match self.query_type { #[cfg(any(test, feature = "test-fixture", feature = "cli"))] QueryType::TestMultiply | QueryType::TestAddInPrimeField => Ok(()), + QueryType::TestShardedShuffle => Ok(()), QueryType::SemiHonestOprfIpa(config) | QueryType::MaliciousOprfIpa(config) => { write!( f, diff --git a/ipa-core/src/query/executor.rs b/ipa-core/src/query/executor.rs index c24ab8b5f..a3e7b866d 100644 --- a/ipa-core/src/query/executor.rs +++ b/ipa-core/src/query/executor.rs @@ -94,6 +94,12 @@ pub fn execute( Box::pin(execute_test_multiply::(prss, gateway, input)) }) } + #[cfg(any(test, feature = "cli", feature = "test-fixture"))] + (QueryType::TestShardedShuffle, _) => { + do_query(config, gateway, input, |_prss, _gateway, _config, _input| { + unimplemented!() + }) + } #[cfg(any(test, feature = "weak-field"))] (QueryType::TestAddInPrimeField, FieldType::Fp31) => { do_query(config, gateway, input, |prss, gateway, _config, input| { From a26d58e2788cc5184e7779516e377452b7b82f54 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Wed, 18 Sep 2024 15:07:24 -0700 Subject: [PATCH 2/3] Hybrid query params --- .../src/helpers/transport/query/hybrid.rs | 32 ++++++++++++++++ ipa-core/src/helpers/transport/query/mod.rs | 6 +++ ipa-core/src/net/http_serde.rs | 16 ++++++++ ipa-core/src/query/executor.rs | 26 ++++++++++--- ipa-core/src/query/runner/hybrid.rs | 37 +++++++++++++++++++ ipa-core/src/query/runner/mod.rs | 2 + 6 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 ipa-core/src/helpers/transport/query/hybrid.rs create mode 100644 ipa-core/src/query/runner/hybrid.rs diff --git a/ipa-core/src/helpers/transport/query/hybrid.rs b/ipa-core/src/helpers/transport/query/hybrid.rs new file mode 100644 index 000000000..2b6906d28 --- /dev/null +++ b/ipa-core/src/helpers/transport/query/hybrid.rs @@ -0,0 +1,32 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] +#[cfg_attr(feature = "clap", derive(clap::Args))] +pub struct HybridQueryParams { + #[cfg_attr(feature = "clap", arg(long, default_value = "8"))] + pub per_user_credit_cap: u32, + #[cfg_attr(feature = "clap", arg(long, default_value = "5"))] + pub max_breakdown_key: u32, + #[cfg_attr(feature = "clap", arg(short = 'd', long, default_value = "1"))] + pub with_dp: u32, + #[cfg_attr(feature = "clap", arg(short = 'e', long, default_value = "5.0"))] + pub epsilon: f64, + #[cfg_attr(feature = "clap", arg(long))] + #[serde(default)] + pub plaintext_match_keys: bool, +} + +#[cfg(test)] +impl Eq for HybridQueryParams {} + +impl Default for HybridQueryParams { + fn default() -> Self { + Self { + per_user_credit_cap: 8, + max_breakdown_key: 20, + with_dp: 1, + epsilon: 0.10, + plaintext_match_keys: false, + } + } +} diff --git a/ipa-core/src/helpers/transport/query/mod.rs b/ipa-core/src/helpers/transport/query/mod.rs index 3cb655173..ac70209b3 100644 --- a/ipa-core/src/helpers/transport/query/mod.rs +++ b/ipa-core/src/helpers/transport/query/mod.rs @@ -1,8 +1,11 @@ +mod hybrid; + use std::{ fmt::{Debug, Display, Formatter}, num::NonZeroU32, }; +pub use hybrid::HybridQueryParams; use serde::{Deserialize, Deserializer, Serialize}; use crate::{ @@ -202,6 +205,7 @@ pub enum QueryType { TestShardedShuffle, SemiHonestOprfIpa(IpaQueryConfig), MaliciousOprfIpa(IpaQueryConfig), + SemiHonestHybrid(HybridQueryParams), } impl QueryType { @@ -211,6 +215,7 @@ impl QueryType { pub const TEST_SHARDED_SHUFFLE_STR: &'static str = "test-sharded-shuffle"; pub const SEMI_HONEST_OPRF_IPA_STR: &'static str = "semi-honest-oprf-ipa"; pub const MALICIOUS_OPRF_IPA_STR: &'static str = "malicious-oprf-ipa"; + pub const SEMI_HONEST_HYBRID_STR: &'static str = "semi-honest-hybrid"; } /// TODO: should this `AsRef` impl (used for `Substep`) take into account config of IPA? @@ -225,6 +230,7 @@ impl AsRef for QueryType { QueryType::TestShardedShuffle => Self::TEST_SHARDED_SHUFFLE_STR, QueryType::SemiHonestOprfIpa(_) => Self::SEMI_HONEST_OPRF_IPA_STR, QueryType::MaliciousOprfIpa(_) => Self::MALICIOUS_OPRF_IPA_STR, + QueryType::SemiHonestHybrid(_) => Self::SEMI_HONEST_HYBRID_STR, } } } diff --git a/ipa-core/src/net/http_serde.rs b/ipa-core/src/net/http_serde.rs index cef850ae8..98fae4dc0 100644 --- a/ipa-core/src/net/http_serde.rs +++ b/ipa-core/src/net/http_serde.rs @@ -171,6 +171,22 @@ pub mod query { write!(f, "&attribution_window_seconds={}", window.get())?; } + Ok(()) + } + QueryType::SemiHonestHybrid(config) => { + write!( + f, + "&per_user_credit_cap={}&max_breakdown_key={}&with_dp={}&epsilon={}", + config.per_user_credit_cap, + config.max_breakdown_key, + config.with_dp, + config.epsilon, + )?; + + if config.plaintext_match_keys { + write!(f, "&plaintext_match_keys=true")?; + } + Ok(()) } } diff --git a/ipa-core/src/query/executor.rs b/ipa-core/src/query/executor.rs index a3e7b866d..b3e197e4d 100644 --- a/ipa-core/src/query/executor.rs +++ b/ipa-core/src/query/executor.rs @@ -44,7 +44,7 @@ use crate::{ Gate, }, query::{ - runner::{OprfIpaQuery, QueryResult}, + runner::{HybridQuery, OprfIpaQuery, QueryResult}, state::RunningQuery, }, sync::Arc, @@ -95,11 +95,12 @@ pub fn execute( }) } #[cfg(any(test, feature = "cli", feature = "test-fixture"))] - (QueryType::TestShardedShuffle, _) => { - do_query(config, gateway, input, |_prss, _gateway, _config, _input| { - unimplemented!() - }) - } + (QueryType::TestShardedShuffle, _) => do_query( + config, + gateway, + input, + |_prss, _gateway, _config, _input| unimplemented!(), + ), #[cfg(any(test, feature = "weak-field"))] (QueryType::TestAddInPrimeField, FieldType::Fp31) => { do_query(config, gateway, input, |prss, gateway, _config, input| { @@ -144,6 +145,19 @@ pub fn execute( ) }, ), + (QueryType::SemiHonestHybrid(query_params), _) => do_query( + config, + gateway, + input, + move |prss, gateway, config, input| { + let ctx = SemiHonestContext::new(prss, gateway); + Box::pin( + HybridQuery::<_, BA32, R>::new(query_params, key_registry) + .execute(ctx, config.size, input) + .then(|res| ready(res.map(|out| Box::new(out) as Box))), + ) + }, + ), } } diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs new file mode 100644 index 000000000..73abcaf75 --- /dev/null +++ b/ipa-core/src/query/runner/hybrid.rs @@ -0,0 +1,37 @@ +use std::{marker::PhantomData, sync::Arc}; + +use crate::{ + error::Error, + helpers::{ + query::{HybridQueryParams, QuerySize}, + BodyStream, + }, + hpke::PrivateKeyRegistry, + secret_sharing::{replicated::semi_honest::AdditiveShare as ReplicatedShare, SharedValue}, +}; + +pub struct Query { + _config: HybridQueryParams, + _key_registry: Arc, + phantom_data: PhantomData<(C, HV)>, +} + +impl Query { + pub fn new(query_params: HybridQueryParams, key_registry: Arc) -> Self { + Self { + _config: query_params, + _key_registry: key_registry, + phantom_data: PhantomData, + } + } + + #[tracing::instrument("hybrid_query", skip_all, fields(sz=%query_size))] + pub async fn execute( + self, + _ctx: C, + query_size: QuerySize, + _input_stream: BodyStream, + ) -> Result>, Error> { + unimplemented!() + } +} diff --git a/ipa-core/src/query/runner/mod.rs b/ipa-core/src/query/runner/mod.rs index 4c7240cbb..9e5935c20 100644 --- a/ipa-core/src/query/runner/mod.rs +++ b/ipa-core/src/query/runner/mod.rs @@ -1,11 +1,13 @@ #[cfg(any(test, feature = "cli", feature = "test-fixture"))] mod add_in_prime_field; +mod hybrid; mod oprf_ipa; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] mod test_multiply; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] pub(super) use add_in_prime_field::execute as test_add_in_prime_field; +pub use hybrid::Query as HybridQuery; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] pub(super) use test_multiply::execute_test_multiply; From 0aa1e5032795ac9bac5b5028fe79f94e45672530 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Wed, 18 Sep 2024 15:35:15 -0700 Subject: [PATCH 3/3] Fix compile errors in release --- ipa-core/src/net/http_serde.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/ipa-core/src/net/http_serde.rs b/ipa-core/src/net/http_serde.rs index 98fae4dc0..927ae4a4d 100644 --- a/ipa-core/src/net/http_serde.rs +++ b/ipa-core/src/net/http_serde.rs @@ -152,6 +152,7 @@ pub mod query { match self.query_type { #[cfg(any(test, feature = "test-fixture", feature = "cli"))] QueryType::TestMultiply | QueryType::TestAddInPrimeField => Ok(()), + #[cfg(any(test, feature = "test-fixture", feature = "cli"))] QueryType::TestShardedShuffle => Ok(()), QueryType::SemiHonestOprfIpa(config) | QueryType::MaliciousOprfIpa(config) => { write!(