Hybrid event gen (#1291)

* fix compilation bugs

* add integration test for hybrid in the clear

* add another cfg flag

* remove test mod

* add test to Cargo.toml

* Update ipa-core/Cargo.toml

ipa-core/Cargo.toml

@@ -197,6 +197,11 @@ name = "crypto_util"
 required-features = ["cli", "test-fixture", "web-app"]
 bench = false
 
+[[bin]]
+name = "in_the_clear"
+required-features = ["cli", "test-fixture", "web-app"]
+bench = false
+
 [[bench]]
 name = "criterion_arithmetic"
 path = "benches/ct/arithmetic_circuit.rs"
@@ -250,3 +255,10 @@ required-features = [
  "real-world-infra",
  "test-fixture",
 ]
+
+[[test]]
+name = "hybrid"
+required-features = [
+ "test-fixture",
+ "cli",
+]

ipa-core/src/bin/in_the_clear.rs

@@ -0,0 +1,72 @@
+use std::{error::Error, fs::File, io::Write, num::NonZeroU32, path::PathBuf};
+
+use clap::Parser;
+use ipa_core::{
+ cli::{playbook::InputSource, Verbosity},
+ test_fixture::hybrid::{hybrid_in_the_clear, TestHybridRecord},
+};
+
+#[derive(Debug, Parser)]
+pub struct CommandInput {
+ #[arg(
+ long,
+ help = "Read the input from the provided file, instead of standard input"
+ )]
+ input_file: Option<PathBuf>,
+}
+
+impl From<&CommandInput> for InputSource {
+ fn from(source: &CommandInput) -> Self {
+ if let Some(ref file_name) = source.input_file {
+ InputSource::from_file(file_name)
+ } else {
+ InputSource::from_stdin()
+ }
+ }
+}
+
+#[derive(Debug, Parser)]
+#[clap(name = "in_the_clear", about = "In the Clear CLI")]
+#[command(about)]
+struct Args {
+ #[clap(flatten)]
+ logging: Verbosity,
+
+ #[clap(flatten)]
+ input: CommandInput,
+
+ /// The destination file for output.
+ #[arg(long, value_name = "OUTPUT_FILE")]
+ output_file: PathBuf,
+
+ #[arg(long, default_value = "20")]
+ max_breakdown_key: NonZeroU32,
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+ let args = Args::parse();
+ let _handle = args.logging.setup_logging();
+
+ let input = InputSource::from(&args.input);
+
+ let input_rows = input.iter::<TestHybridRecord>().collect::<Vec<_>>();
+ let expected = hybrid_in_the_clear(
+ &input_rows,
+ usize::try_from(args.max_breakdown_key.get()).unwrap(),
+ );
+
+ let mut file = File::options()
+ .write(true)
+ .create_new(true)
+ .open(&args.output_file)
+ .map_err(|e| {
+ format!(
+ "Failed to create output file {}: {e}",
+ &args.output_file.display()
+ )
+ })?;
+
+ write!(file, "{}", serde_json::to_string_pretty(&expected)?)?;
+
+ Ok(())
+}

ipa-core/src/cli/playbook/input.rs

@@ -7,8 +7,9 @@ use std::{
 };
 
 use crate::{
- cli::playbook::generator::U128Generator, ff::U128Conversions,
- test_fixture::ipa::TestRawDataRecord,
+ cli::playbook::generator::U128Generator,
+ ff::U128Conversions,
+ test_fixture::{hybrid::TestHybridRecord, ipa::TestRawDataRecord},
 };
 
 pub trait InputItem {
@@ -56,6 +57,40 @@ impl InputItem for TestRawDataRecord {
  }
 }
 
+impl InputItem for TestHybridRecord {
+ fn from_str(s: &str) -> Self {
+ if let [event_type, match_key, number] = s.splitn(3, ',').collect::<Vec<_>>()[..] {
+ let match_key: u64 = match_key
+ .parse()
+ .unwrap_or_else(|e| panic!("Expected an u64, got {match_key}: {e}"));
+
+ let number: u32 = number
+ .parse()
+ .unwrap_or_else(|e| panic!("Expected an u32, got {number}: {e}"));
+
+ match event_type {
+ "i" => TestHybridRecord::TestImpression {
+ match_key,
+ breakdown_key: number,
+ },
+
+ "c" => TestHybridRecord::TestConversion {
+ match_key,
+ value: number,
+ },
+ _ => panic!(
+ "{}",
+ format!(
+ "Invalid input. Rows should start with 'i' or 'c'. Did not expect {event_type}"
+ )
+ ),
+ }
+ } else {
+ panic!("{s} is not a valid {}", type_name::<Self>())
+ }
+ }
+}
+
 pub struct InputSource {
  inner: Box<dyn BufRead>,
  sz: Option<u64>,

ipa-core/tests/hybrid.rs

@@ -0,0 +1,48 @@
+// some pub functions in `common` to be compiled, and rust complains about dead code.
+#[allow(dead_code)]
+mod common;
+
+use std::process::{Command, Stdio};
+
+use common::{tempdir::TempDir, CommandExt, UnwrapStatusExt, TEST_RC_BIN};
+use rand::thread_rng;
+use rand_core::RngCore;
+
+pub const IN_THE_CLEAR_BIN: &str = env!("CARGO_BIN_EXE_in_the_clear");
+
+// this currently only generates data and runs in the clear
+// eventaully we'll want to add the MPC as well
+#[test]
+fn test_hybrid() {
+ const INPUT_SIZE: usize = 100;
+ const MAX_CONVERSION_VALUE: usize = 5;
+ const MAX_BREAKDOWN_KEY: usize = 20;
+ const MAX_CONVS_PER_IMP: usize = 10;
+
+ let dir = TempDir::new_delete_on_drop();
+
+ // Gen inputs
+ let input_file = dir.path().join("ipa_inputs.txt");
+ let output_file = dir.path().join("ipa_output.json");
+
+ let mut command = Command::new(TEST_RC_BIN);
+ command
+ .args(["--output-file".as_ref(), input_file.as_os_str()])
+ .arg("gen-hybrid-inputs")
+ .args(["--count", &INPUT_SIZE.to_string()])
+ .args(["--max-conversion-value", &MAX_CONVERSION_VALUE.to_string()])
+ .args(["--max-breakdown-key", &MAX_BREAKDOWN_KEY.to_string()])
+ .args(["--max-convs-per-imp", &MAX_CONVS_PER_IMP.to_string()])
+ .args(["--seed", &thread_rng().next_u64().to_string()])
+ .silent()
+ .stdin(Stdio::piped());
+ command.status().unwrap_status();
+
+ let mut command = Command::new(IN_THE_CLEAR_BIN);
+ command
+ .args(["--input-file".as_ref(), input_file.as_os_str()])
+ .args(["--output-file".as_ref(), output_file.as_os_str()])
+ .silent()
+ .stdin(Stdio::piped());
+ command.status().unwrap_status();
+}