Tool to convert Wycheproof test vectors to .blb files (#280)

Cargo.toml

@@ -18,5 +18,6 @@ members = [
     "pkcs5",
     "pkcs8",
     "spki",
+    "wycheproof2blb",
     "x509"
 ]

blobby/examples/convert.rs

@@ -1,86 +1,19 @@
 //! Convert utility
-use blobby::BlobIterator;
-use std::collections::HashMap;
+use blobby::{encode_blobs, BlobIterator};
 use std::io::{self, BufRead, BufReader, BufWriter, Write};
 use std::{env, error::Error, fs::File};
 
-const NEXT_MASK: u8 = 0b1000_0000;
-const VAL_MASK: u8 = 0b0111_1111;
-
-fn encode_vlq(mut val: usize, buf: &mut [u8; 4]) -> &[u8] {
-    macro_rules! step {
-        ($n:expr) => {
-            buf[$n] = if $n == 3 {
-                (val & (VAL_MASK as usize)) as u8
-            } else {
-                val -= 1;
-                NEXT_MASK | (val & (VAL_MASK as usize)) as u8
-            };
-            val >>= 7;
-            if val == 0 {
-                return &buf[$n..];
-            }
-        };
-    }
-
-    step!(3);
-    step!(2);
-    step!(1);
-    step!(0);
-    panic!("integer is too big")
-}
-
 fn encode(reader: impl BufRead, mut writer: impl Write) -> io::Result<usize> {
     let mut blobs = Vec::new();
     for line in reader.lines() {
         let blob = hex::decode(line?.as_str())
             .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
         blobs.push(blob);
     }
-
-    let mut idx_map = HashMap::new();
-    for blob in blobs.iter().filter(|b| b.len() != 0) {
-        let v = idx_map.entry(blob.as_slice()).or_insert(0);
-        *v += 1;
-    }
-
-    let mut idx: Vec<&[u8]> = idx_map
-        .iter()
-        .filter(|(_, &v)| v > 1)
-        .map(|(&k, _)| k)
-        .collect();
-    idx.sort_by_key(|e| {
-        let k = match e {
-            &[0] => 2,
-            &[1] => 1,
-            _ => 0,
-        };
-        (k, idx_map.get(e).unwrap())
-    });
-    idx.reverse();
-
-    let rev_idx: HashMap<&[u8], usize> = idx.iter().enumerate().map(|(i, &e)| (e, i)).collect();
-
-    println!("Index len: {:?}", idx.len());
-    let mut buf = [0u8; 4];
-    writer.write_all(encode_vlq(idx.len(), &mut buf))?;
-    for e in idx {
-        writer.write_all(encode_vlq(e.len(), &mut buf))?;
-        writer.write_all(e)?;
-    }
-
-    for blob in blobs.iter() {
-        if let Some(dup_pos) = rev_idx.get(blob.as_slice()) {
-            let n = (dup_pos << 1) + 1usize;
-            writer.write_all(encode_vlq(n, &mut buf))?;
-        } else {
-            let n = blob.len() << 1;
-            writer.write_all(encode_vlq(n, &mut buf))?;
-            writer.write_all(blob)?;
-        }
-    }
-
-    Ok(blobs.len())
+    let (data, idx_len) = encode_blobs(&blobs);
+    let data_len = data.len();
+    println!("Index len: {:?}", idx_len);
+    writer.write_all(&data).map(|_| data_len)
 }
 
 fn decode<R: BufRead, W: Write>(mut reader: R, mut writer: W) -> io::Result<usize> {

blobby/src/lib.rs

@@ -51,7 +51,7 @@
 )]
 extern crate alloc;
 
-use alloc::{boxed::Box, vec, vec::Vec};
+use alloc::{boxed::Box, collections::BTreeMap, vec, vec::Vec};
 use core::iter::Iterator;
 
 /// Iterator over binary blobs
@@ -114,6 +114,100 @@ fn read_vlq(data: &[u8], pos: &mut usize) -> Result<usize, Error> {
     Ok(val)
 }
 
+/// Write a git-flavoured VLQ value into `buf`.
+///
+/// Returns the slice within `buf` that holds the value.
+fn encode_vlq(mut val: usize, buf: &mut [u8; 4]) -> &[u8] {
+    macro_rules! step {
+        ($n:expr) => {
+            buf[$n] = if $n == 3 {
+                (val & (VAL_MASK as usize)) as u8
+            } else {
+                val -= 1;
+                NEXT_MASK | (val & (VAL_MASK as usize)) as u8
+            };
+            val >>= 7;
+            if val == 0 {
+                return &buf[$n..];
+            }
+        };
+    }
+
+    step!(3);
+    step!(2);
+    step!(1);
+    step!(0);
+    panic!("integer is too big")
+}
+
+/// Encode the given collection of binary blobs in .blb format into `writer`.
+/// Returns the encoded data together with a count of the number of blobs included in the index.
+///
+/// The encoded file format is:
+///  - count of index entries=N
+///  - N x index entries, each encoded as:
+///      - size L of index entry (VLQ)
+///      - index blob contents (L bytes)
+///  - repeating encoded blobs, each encoded as:
+///      - VLQ value that is either:
+///         - (J << 1) & 0x01: indicates this blob is index entry J
+///         - (L << 1) & 0x00: indicates an explicit blob of len L
+///      - (in the latter case) explicit blob contents (L bytes)
+pub fn encode_blobs<'a, I, T: 'a>(blobs: &'a I) -> (Vec<u8>, usize)
+where
+    &'a I: IntoIterator<Item = &'a T>,
+    T: AsRef<[u8]>,
+{
+    let mut idx_map = BTreeMap::new();
+    blobs
+        .into_iter()
+        .map(|v| v.as_ref())
+        .filter(|blob| !blob.is_empty())
+        .for_each(|blob| {
+            let v = idx_map.entry(blob.as_ref()).or_insert(0);
+            *v += 1;
+        });
+
+    let mut idx: Vec<&[u8]> = idx_map
+        .iter()
+        .filter(|(_, &v)| v > 1)
+        .map(|(&k, _)| k)
+        .collect();
+    idx.sort_by_key(|e| {
+        let k = match e {
+            [0] => 2,
+            [1] => 1,
+            _ => 0,
+        };
+        (k, idx_map.get(e).unwrap())
+    });
+    idx.reverse();
+    let idx_len = idx.len();
+
+    let rev_idx: BTreeMap<&[u8], usize> = idx.iter().enumerate().map(|(i, &e)| (e, i)).collect();
+
+    let mut out_buf = Vec::new();
+    let mut buf = [0u8; 4];
+    out_buf.extend_from_slice(encode_vlq(idx.len(), &mut buf));
+    for e in idx {
+        out_buf.extend_from_slice(encode_vlq(e.len(), &mut buf));
+        out_buf.extend_from_slice(e);
+    }
+
+    for blob in blobs.into_iter().map(|v| v.as_ref()) {
+        if let Some(dup_pos) = rev_idx.get(blob) {
+            let n = (dup_pos << 1) + 1usize;
+            out_buf.extend_from_slice(encode_vlq(n, &mut buf));
+        } else {
+            let n = blob.len() << 1;
+            out_buf.extend_from_slice(encode_vlq(n, &mut buf));
+            out_buf.extend_from_slice(blob);
+        }
+    }
+
+    (out_buf, idx_len)
+}
+
 impl<'a> BlobIterator<'a> {
     /// Create new `BlobIterator` for given `data`.
     pub fn new(data: &'a [u8]) -> Result<Self, Error> {

wycheproof2blb/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "wycheproof2blb"
+version = "0.1.0"
+authors = ["RustCrypto Developers"]
+license = "MIT OR Apache-2.0"
+description = "A utility for converting Wycheproof test vectors to the blobby format"
+repository = "https://github.com/RustCrypto/utils"
+keywords = ["test", "Wycheproof"]
+categories = ["cryptography", "no-std"]
+edition = "2018"
+
+[dependencies]
+blobby = { version = "*", path = "../blobby" }
+hex = "*"
+serde = { version = "*", features = ["derive"] }
+serde_json = "*"

wycheproof2blb/src/aead.rs

@@ -0,0 +1,89 @@
+use crate::wycheproof;
+use crate::wycheproof::{case_result, description, hex_string};
+use crate::TestInfo;
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+pub struct TestSuite {
+    #[serde(flatten)]
+    pub suite: wycheproof::Suite,
+    #[serde(rename = "testGroups")]
+    pub test_groups: Vec<TestGroup>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TestGroup {
+    #[serde(flatten)]
+    pub group: wycheproof::Group,
+    #[serde(rename = "ivSize")]
+    pub iv_size: u32,
+    #[serde(rename = "keySize")]
+    pub key_size: u32,
+    #[serde(rename = "tagSize")]
+    pub tag_size: u32,
+    pub tests: Vec<TestCase>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TestCase {
+    #[serde(flatten)]
+    pub case: wycheproof::Case,
+    #[serde(with = "hex_string")]
+    pub aad: Vec<u8>,
+    #[serde(with = "hex_string")]
+    pub ct: Vec<u8>,
+    #[serde(with = "hex_string")]
+    pub iv: Vec<u8>,
+    #[serde(with = "hex_string")]
+    pub key: Vec<u8>,
+    #[serde(with = "hex_string")]
+    pub msg: Vec<u8>,
+    #[serde(with = "hex_string")]
+    pub tag: Vec<u8>,
+}
+
+pub fn aes_gcm_generator(data: &[u8], algorithm: &str, key_size: u32) -> Vec<TestInfo> {
+    generator(data, algorithm, key_size, 12 * 8)
+}
+
+pub fn chacha20_poly1305(data: &[u8], algorithm: &str, _key_size: u32) -> Vec<TestInfo> {
+    generator(data, algorithm, 256, 12 * 8)
+}
+
+pub fn xchacha20_poly1305(data: &[u8], algorithm: &str, _key_size: u32) -> Vec<TestInfo> {
+    generator(data, algorithm, 256, 24 * 8)
+}
+
+fn generator(data: &[u8], algorithm: &str, key_size: u32, iv_size: u32) -> Vec<TestInfo> {
+    let suite: TestSuite = serde_json::from_slice(data).unwrap();
+    assert_eq!(algorithm, suite.suite.algorithm);
+
+    let mut infos = vec![];
+    for g in &suite.test_groups {
+        for tc in &g.tests {
+            if key_size != 0 && g.key_size != key_size {
+                continue;
+            }
+            if g.iv_size != iv_size {
+                println!(" skipping tests for iv_size={}", g.iv_size);
+                continue;
+            }
+            let mut combined_ct = Vec::new();
+            combined_ct.extend_from_slice(&tc.ct);
+            combined_ct.extend_from_slice(&tc.tag);
+
+            infos.push(TestInfo {
+                data: vec![
+                    tc.key.clone(),
+                    tc.iv.clone(),
+                    tc.aad.clone(),
+                    tc.msg.clone(),
+                    combined_ct,
+                    vec![case_result(&tc.case)],
+                ],
+                desc: description(&suite.suite, &tc.case),
+            });
+        }
+    }
+    infos
+}