Implement unrolled CurlP (#745)

* implement `Sponge` for `UnrolledCurlP81`

* use default when possible

* add tests and correct errors

* add benchmarks

* split modules in different files

* do some optimizations

* address clippy lints

* Fix typo

* Reorg curl modules

* Nits

* More nits

* Clippy fix

Co-authored-by: Thibault Martinez <thibault@iota.org>

bee-crypto/Cargo.toml

@@ -22,5 +22,5 @@ tiny-keccak = { version = "2.0", features = [ "keccak" ] }
 criterion = "0.3"
 
 [[bench]]
-name = "batched_hash"
+name = "raw_speed"
 harness = false

bee-crypto/benches/raw_speed.rs

@@ -0,0 +1,74 @@
+// Copyright 2021 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+use bee_crypto::ternary::sponge::{BatchHasher, CurlP81, CurlPRounds, Sponge, UnrolledCurlP81, BATCH_SIZE};
+use bee_ternary::{T1B1Buf, T5B1Buf, TritBuf, TryteBuf};
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+
+fn batched_hasher_t5b1(input: &TritBuf<T5B1Buf>) {
+    let mut hasher = BatchHasher::new(input.len(), CurlPRounds::Rounds81);
+
+    for _ in 0..BATCH_SIZE {
+        hasher.add(input.clone());
+    }
+
+    for _ in hasher.hash_batched() {}
+}
+
+fn regular_hasher_t5b1(input: &TritBuf<T5B1Buf>) {
+    let mut hasher = CurlP81::new();
+
+    for _ in 0..BATCH_SIZE {
+        hasher.digest(&input.encode::<T1B1Buf>()).unwrap();
+    }
+}
+
+fn unrolled_hasher_t5b1(input: &TritBuf<T5B1Buf>) {
+    let mut hasher = UnrolledCurlP81::new();
+
+    for _ in 0..BATCH_SIZE {
+        hasher.digest(&input.encode::<T1B1Buf>()).unwrap();
+    }
+}
+
+fn bench_hasher(c: &mut Criterion) {
+    let input_243 = "HHPELNTNJIOKLYDUW9NDULWPHCWFRPTDIUWLYUHQWWJVPAKKGKOAZFJPQJBLNDPALCVXGJLRBFSHATF9C";
+    let input_8019 = "AQUIC9VCKYJWSGMGHMPGRMHLLCKBOCMLOKMLXAMWSCBEDOWQS9HJBYWNABSUHARMZHQFOMKNBUEKB9YWC9UWQVWIDRFPNUF9QGFORSXLLK9MBVVHSRQMOWEJIKGNMKTKKZLXXKFMSVZDUMMWGYEAWPGXRHJZOYYVOYUFQDELKPTFTFXFGN9KUCLXPSVVX9PXCKEGWBWMMYWVWBXUTAAZZALALFVCJWFP9HKIVGVZSBZSESSIEGTPPLNJZIIXJRRYUREWVOGOCGWMSJUISJHIRNTNCUV9CHRLVFBMCQSVB9DLMPYSJIOBWVJIXEVQORQJHSQLUPNJFGSXAOULRMIIXGVNSPFGFZZTCVHRTXKPHFLAXKKSDXDUTVSWXFIGSYMMSHWWAKTJSCYZHXEBLMMJX9XJUOXGBCZ9RTDPHMRGAHHQNNNK9SNTEFNAMOBPYGIN9OZLJANAWFVUIXZJAJMHJQP9UVANIGSXTFGSANSTXRILDWQOHTUQRXKAPFUA9PDNAEZHTMZIMGNGHESYDXSODNBYVCETFVVBCF9FEPHUSGK9YSFYMDRLBDLLZXQKY9ZXRWVBXGICVNNNCRJLRAWTJYEBJNPSX9ZZLZPF9AISXIOSVVYQVICDLMVKOFUHAKWUKGMPDYJNZCHSNXSFVQPJXWCXN9JBEAGNMWRVTGNOWKADPJ9AJBTVYYVXTFNHTKAYHUGMEBOHYXIKPUYCKNHUAOQUVGWXAEIIT9YWJMCCGIPAQYNUZWWWWRFTF9KRDDDOEFOOJWMRZXZHTPUBH9IKXKGXLUIBFAFLXMIYDXJSAFFGAURXLSDUNZYJYEZLMEGUMQFVKZWBTVRLBDS9RSEQPUSMXLHUBMTYVFMM999VIRVELDUOCDYZRXVQCYB9AJLJYDBIIKCRSQSRQLI9JORWAOFUNEUGEUBNQFMASBWTUTPHJTGFYGHO9PDQDJXZSOC9RRZGRREQY9IYURJGTZRYVCLBGBUVCANFIQWDARHJHUPXJIHZX9GTNOWGJBPVZRYYNWRIHSBPCED9NDWEYUZNHPXVQNNFIUAGABXESBENVWDALPSCLQTXANWTXIGORBGR9XHNDITOLWFZVFZXJCIZGYGDORYIZYALLQETQQMSWEDPYX9EDZKJKDVBNNAK9BHNIVIFMPBN9FJFVFCK9XSMGFDSOXNDELLVYNVVTDDBTYFBMPMMYYDI9LCFFXMHXJQSUFUFASBLIBHWSCFHZWFZOBJOROVMWJ9FSIFJAEVDJZSXEPSLJMNVXYWWIIUJUPYIPVWOBAUSOBCIUAECILFJVNCCPQOZZSIHIWZMXLOXFZUVSXYRWPKCXDTYTHSFHXNDEDC9BREQBOIPECTGMQFEYIJTVXMBFLLJWZJMKMJDOZ9ECKWDOPWPYGV9PQIBTXOWPCTHPVGPVQUBBPSC9NVJLEZOITPJE9ZZNG9KOCMEZEHF9JZZSMPDRRAD9CSV9UVFXTZTTDOCWDYRIWMHAOYUO9SHRKW9MACEO9LPGZBSAWYQVKWMZDUNEE9ONRTDWTZEZVQEZXWHSPFKGAHBIKYBWGYCLZRZBDSVJIUXERPSF9UFQOZSLMMTGS9UCQASYAECTDZFWRBOAWMXOMQFQODFJQJGVOXHNJDLPWSGDDUPBXAOLFFUUYUAOVDMMTIEFTJWMFE9OPANAKKWIMXRHLNHMRZH9TCEONTVCSLADVJYZXWYHDMVBXFOCYFTOXHVZVCELZMUIJZHRALVHZ9NSVIK9VMBGRXX9GOUSGHSERBFIZGC9X9HVWV9VVGEWCPVQI9CFRGAYVPSVULWQNKTCJUZBYYVPBNHSTIMXVZOCUVRYJIRCT9LFANYATSAPPDFORPAYWXWFG9CECXRKTV9PMQSZMOVVYYKX9JBAVRYSCMXWFM9QVS9QUUPKNJSUSOMYCUIOOAAD9NHZZKOYMMZNSQYSDYBBYMRTEWQUYUHFLVHUUFWIXQRAWSPZPKETGKJSFGMAKFMVSTQEDLTUJYQONNBWVJDHTLVIGOSKKPQPDSHYUAAFAOVMOAXOMRWOBTAOIAGVBRTFELKIFZNFSADZYSGHBWLTOEJCUDFVRPPLWDMHSOJBWEBRTCRIEEMDSEKKXPRZGGIJYHOWYQKSJH9MFCHYKBGXUWYXQGJRXFLCDHVV9TKDITLCAIRZTJPACOPINMUC9RLFTYVALBRVA9OBMOYMFHQWC9TAISJODUMBIWC9RSPSVZGAJOZXTULEOPTKCGYYOOKDEOSKLDEGDXRQHIEMGZUXMOW999999999999999999999999999GD9999999999999999999999999JAYNICD99999999999B99999999YGOQEYWBEGZJGLJSPPFFVWA9RIDDZRFEGDZRZGATZTKSETYBXQHKQZGYLJVOCIQEDVCWAWBK9TUPK9DIWCEWQXNQCKQQEAAOFSEUGCGPMXCIBPCR9ASZEDDPRGWBLOVDSKMDCAUBYFLDTRJRUCAPCWSSSVILNZ9999UQJFWFRNIWMVPSICVZSWBW9JMEVVDHQRJWOPVHRBDMNEGRCVNYRLKPHZLQMLVYYBFVUXAIXYRRLW99999999999999999999999999999999USHXKUSRF999999999K99999999RC999999999PTA9999999999999";
+
+    let input_243 = TryteBuf::try_from_str(input_243)
+        .unwrap()
+        .as_trits()
+        .encode::<T5B1Buf>();
+    let input_8019 = TryteBuf::try_from_str(input_8019)
+        .unwrap()
+        .as_trits()
+        .encode::<T5B1Buf>();
+
+    let mut group = c.benchmark_group("CurlP");
+    group.throughput(Throughput::Elements(BATCH_SIZE as u64));
+    for input in [input_243, input_8019].iter() {
+        let length = input.len();
+
+        // Using T5B1 directly.
+        group.bench_with_input(
+            BenchmarkId::new("Batched", format!("{} T5B1", length)),
+            input,
+            |b, i| b.iter(|| batched_hasher_t5b1(i)),
+        );
+        group.bench_with_input(
+            BenchmarkId::new("Regular", format!("{} T5B1", length)),
+            input,
+            |b, i| b.iter(|| regular_hasher_t5b1(i)),
+        );
+        group.bench_with_input(
+            BenchmarkId::new("Unrolled", format!("{} T5B1", length)),
+            input,
+            |b, i| b.iter(|| unrolled_hasher_t5b1(i)),
+        );
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_hasher);
+criterion_main!(benches);

bee-crypto/src/ternary/sponge/batched_curlp/bct_curlp.rs → bee-crypto/src/ternary/sponge/curlp/batched/bct_curlp.rs

@@ -3,7 +3,7 @@
 
 use crate::ternary::{
     sponge::{
-        batched_curlp::{
+        curlp::batched::{
             bct::{BcTrit, BcTritArr, BcTrits},
             HIGH_BITS,
         },

bee-crypto/src/ternary/sponge/curlp.rs → bee-crypto/src/ternary/sponge/curlp/mod.rs

@@ -1,6 +1,12 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+mod batched;
+mod unrolled;
+
+pub use batched::{BatchHasher, BATCH_SIZE};
+pub use unrolled::UnrolledCurlP81;
+
 use crate::ternary::{sponge::Sponge, HASH_LENGTH};
 
 use bee_ternary::{Btrit, TritBuf, Trits};

bee-crypto/src/ternary/sponge/curlp/unrolled/mod.rs

@@ -0,0 +1,108 @@
+// Copyright 2021 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+mod transform;
+mod u256;
+
+use u256::U256;
+
+use super::{Sponge, HASH_LENGTH};
+
+use bee_ternary::{Btrit, Trits};
+
+use std::convert::Infallible;
+
+enum SpongeDirection {
+    Absorb,
+    Squeeze,
+}
+
+/// Unrolled [`CurlP`] with a fixed number of 81 rounds.
+pub struct UnrolledCurlP81 {
+    p: [U256; 3],
+    n: [U256; 3],
+    direction: SpongeDirection,
+}
+
+impl UnrolledCurlP81 {
+    /// Creates a new [`UnrolledCurlP81`].
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    fn squeeze_aux(&mut self, hash: &mut Trits) {
+        if let SpongeDirection::Squeeze = self.direction {
+            self.transform();
+        }
+
+        self.direction = SpongeDirection::Squeeze;
+
+        for i in 0..HASH_LENGTH {
+            // SAFETY: `U256::bit` returns an `i8` between `0` and `1`.
+            // Substracting two bits will produce an `i8` between `-1` and `1` and matches the `repr` of `Btrit`.
+            let trit = unsafe { std::mem::transmute::<i8, Btrit>(self.p[0].bit(i) - self.n[0].bit(i)) };
+            hash.set(i, trit);
+        }
+    }
+
+    fn transform(&mut self) {
+        transform::transform(&mut self.p, &mut self.n)
+    }
+}
+
+impl Default for UnrolledCurlP81 {
+    fn default() -> Self {
+        Self {
+            p: Default::default(),
+            n: Default::default(),
+            direction: SpongeDirection::Absorb,
+        }
+    }
+}
+
+impl Sponge for UnrolledCurlP81 {
+    type Error = Infallible;
+
+    fn reset(&mut self) {
+        *self = Self::new();
+    }
+
+    fn absorb(&mut self, input: &Trits) -> Result<(), Self::Error> {
+        if input.is_empty() || input.len() % HASH_LENGTH != 0 {
+            panic!("trits slice length must be multiple of {}", HASH_LENGTH);
+        }
+
+        if let SpongeDirection::Squeeze = self.direction {
+            panic!("absorb after squeeze");
+        }
+
+        for chunk in input.chunks(HASH_LENGTH) {
+            let mut p = U256::default();
+            let mut n = U256::default();
+
+            for (i, trit) in chunk.iter().enumerate() {
+                match trit {
+                    Btrit::PlusOne => p.set_bit(i),
+                    Btrit::Zero => (),
+                    Btrit::NegOne => n.set_bit(i),
+                }
+            }
+
+            self.p[0] = p;
+            self.n[0] = n;
+            self.transform();
+        }
+
+        Ok(())
+    }
+
+    fn squeeze_into(&mut self, buf: &mut Trits) -> Result<(), Self::Error> {
+        assert_eq!(buf.len() % HASH_LENGTH, 0, "Invalid squeeze length");
+
+        for chunk in buf.chunks_mut(HASH_LENGTH) {
+            self.squeeze_aux(chunk);
+        }
+
+        Ok(())
+    }
+}

bee-crypto/src/ternary/sponge/curlp/unrolled/transform.rs

@@ -0,0 +1,126 @@
+// Copyright 2021 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{u256::U256, HASH_LENGTH};
+
+use lazy_static::lazy_static;
+
+const NUM_ROUNDS: usize = 81;
+const ROTATION_OFFSET: usize = 364;
+const STATE_SIZE: usize = HASH_LENGTH * 3;
+
+#[derive(Clone, Copy)]
+struct StateRotation {
+    offset: usize,
+    shift: usize,
+}
+
+lazy_static! {
+    static ref STATE_ROTATIONS: [StateRotation; NUM_ROUNDS] = {
+        let mut rotation = ROTATION_OFFSET;
+
+        let mut state_rotations = [StateRotation { offset: 0, shift: 0 }; NUM_ROUNDS];
+
+        for state_rotation in &mut state_rotations {
+            state_rotation.offset = rotation / HASH_LENGTH;
+            state_rotation.shift = rotation % HASH_LENGTH;
+            rotation = (rotation * ROTATION_OFFSET) % STATE_SIZE;
+        }
+
+        state_rotations
+    };
+}
+
+pub(super) fn transform(p: &mut [U256; 3], n: &mut [U256; 3]) {
+    for state_rotation in STATE_ROTATIONS.iter() {
+        let (p2, n2) = rotate_state(p, n, state_rotation.offset, state_rotation.shift);
+
+        macro_rules! compute {
+            ($i: expr, $j: expr) => {
+                let tmp = batch_box(p[$i][$j], n[$i][$j], p2[$i][$j], n2[$i][$j]);
+                p[$i][$j] = tmp.0;
+                n[$i][$j] = tmp.1;
+            };
+        }
+
+        compute!(0, 0);
+        compute!(0, 1);
+        compute!(0, 2);
+        compute!(0, 3);
+        compute!(1, 0);
+        compute!(1, 1);
+        compute!(1, 2);
+        compute!(1, 3);
+        compute!(2, 0);
+        compute!(2, 1);
+        compute!(2, 2);
+        compute!(2, 3);
+
+        p[0].norm243();
+        p[1].norm243();
+        p[2].norm243();
+        n[0].norm243();
+        n[1].norm243();
+        n[2].norm243();
+    }
+
+    reorder(p, n);
+}
+
+fn rotate_state(p: &[U256; 3], n: &[U256; 3], offset: usize, shift: usize) -> ([U256; 3], [U256; 3]) {
+    let mut p2 = <[U256; 3]>::default();
+    let mut n2 = <[U256; 3]>::default();
+
+    macro_rules! rotate {
+        ($p:expr, $p2:expr, $i:expr) => {
+            $p2[$i]
+                .shr_into(&$p[($i + offset) % 3], shift)
+                .shl_into(&$p[(($i + 1) + offset) % 3], 243 - shift);
+        };
+    }
+
+    rotate!(p, p2, 0);
+    rotate!(p, p2, 1);
+    rotate!(p, p2, 2);
+
+    rotate!(n, n2, 0);
+    rotate!(n, n2, 1);
+    rotate!(n, n2, 2);
+
+    (p2, n2)
+}
+
+fn batch_box(x_p: u64, x_n: u64, y_p: u64, y_n: u64) -> (u64, u64) {
+    let tmp = x_n ^ y_p;
+    (tmp & !x_p, !tmp & !(x_p ^ y_n))
+}
+
+fn reorder(p: &mut [U256; 3], n: &mut [U256; 3]) {
+    const M0: u64 = 0x9249249249249249;
+    const M1: u64 = M0 << 1;
+    const M2: u64 = M0 << 2;
+
+    let mut p2 = <[U256; 3]>::default();
+    let mut n2 = <[U256; 3]>::default();
+
+    for i in 0..3 {
+        macro_rules! compute {
+            ($p:expr, $p2:expr, $j:expr, $m0:expr, $m1:expr, $m2:expr) => {
+                $p2[i][$j] = ($p[i][$j] & $m0) | ($p[(1 + i) % 3][$j] & $m1) | ($p[(2 + i) % 3][$j] & $m2);
+            };
+        }
+
+        compute!(p, p2, 0, M0, M1, M2);
+        compute!(p, p2, 1, M2, M0, M1);
+        compute!(p, p2, 2, M1, M2, M0);
+        compute!(p, p2, 3, M0, M1, M2);
+
+        compute!(n, n2, 0, M0, M1, M2);
+        compute!(n, n2, 1, M2, M0, M1);
+        compute!(n, n2, 2, M1, M2, M0);
+        compute!(n, n2, 3, M0, M1, M2);
+    }
+
+    *p = p2;
+    *n = n2;
+}