Refactor scoring PoWs and deprecate old hash functions (#826)

* Store hash functions for POW internally * Formatting and usage in `miner` * Clarify intent * Start deprecating Bee's `CurlP` wrapper * Get `iota-crypto` directly from git repo * Patch `bee-ternary` to local version * Switch `CurlP` in benchmarks * Deprecation notices * Formatting * More Formatting * Include benchmark to compare legacy/new * Provide `PoWScorer::default` * Reorganize imports * Formatting * Add changelog entries * Allow deprecation * Update changelog * Small change to changelog * Bump `iota-crypto` version * Deprecated `Kerl` * Toying around with counting allocations * Add script the count number of allocations * Switch `UnrolledCurlP81` for performance reasons * Formatting * Deprecate all the things * Draft releases for `bee-crypto` and `bee-pow` * Deprecate `Sponge` and `SpongeKind` * Formatting * Update bee-pow/Cargo.toml Co-authored-by: Thibault Martinez <thibault.martinez.30@gmail.com> * Put allocation example into `bee-pow` * Revert "Put allocation example into `bee-pow`" This reverts commit b7e9036. * Update changelogs * Get rid of conflict * Use correct version of `iota-crypto` * Bump `iota-crypto` version * Deprecate more things * Get rid of warnings * Make PR nicer to review 😄 * More cleanup * More clean up * More cleanup * Reduce commit size * Final fixes * Bump version of `iota-crypto` * Review comments * Date * Thanks @Alex6323 Co-authored-by: Thibault Martinez <thibault.martinez.30@gmail.com>
iotaledger · Nov 19, 2021 · 9ce096c · 9ce096c
1 parent 5e3c86e
commit 9ce096c
Show file tree

Hide file tree

Showing 38 changed files with 251 additions and 63 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -27,3 +27,7 @@ panic = "abort"
 
 [profile.release]
 panic = "abort"
+
+# `iota-crypto` uses bee-ternary. We patch it to use our local version to avoid importing it twice.
+[patch.crates-io]
+bee-ternary = { path = "bee-ternary" }
diff --git a/bee-crypto/CHANGELOG.md b/bee-crypto/CHANGELOG.md
@@ -19,6 +19,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security -->
 
+## 0.3.0 - 2021-11-19
+
+### Deprecated
+
+The whole `bee-signing` crate is deprecated and will not be supported in the future.
+
 ## 0.2.1-alpha - 2021-04-07
 
 ### Changed

diff --git a/bee-crypto/Cargo.toml b/bee-crypto/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "bee-crypto"
-version = "0.2.1-alpha"
+version = "0.3.0"
 authors = [ "IOTA Stiftung" ]
 edition = "2021"
 description = "Cryptographic primitives of the IOTA protocol"
@@ -14,6 +14,7 @@ homepage = "https://www.iota.org"
 bee-ternary = { version = "0.5.2", path = "../bee-ternary", default-features = false }
 
 byteorder = {version = "1.4.3", default-features = false }
+iota-crypto = { version = "0.9.1", default-features = false, features = [ "curl-p" ] }
 lazy_static = {version = "1.4.0", default-features = false }
 thiserror = {version = "1.0.30", default-features = false }
 tiny-keccak = { version = "2.0.2", default-features = false, features = [ "keccak" ] }

diff --git a/bee-crypto/benches/batched_hash.rs b/bee-crypto/benches/batched_hash.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::sponge::{BatchHasher, CurlPRounds, BATCH_SIZE};
 use bee_ternary::{T1B1Buf, T5B1Buf, TritBuf, TryteBuf};
 

diff --git a/bee-crypto/benches/raw_speed.rs b/bee-crypto/benches/raw_speed.rs
@@ -1,10 +1,13 @@
 // Copyright 2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::sponge::{BatchHasher, CurlP81, CurlPRounds, Sponge, UnrolledCurlP81, BATCH_SIZE};
 use bee_ternary::{T1B1Buf, T5B1Buf, TritBuf, TryteBuf};
 
 use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use crypto::hashes::ternary::curl_p::CurlP;
 
 fn batched_hasher_t5b1(input: &TritBuf<T5B1Buf>) {
     let mut hasher = BatchHasher::new(input.len(), CurlPRounds::Rounds81);
@@ -17,6 +20,14 @@ fn batched_hasher_t5b1(input: &TritBuf<T5B1Buf>) {
 }
 
 fn regular_hasher_t5b1(input: &TritBuf<T5B1Buf>) {
+    let mut hasher = CurlP::new();
+
+    for _ in 0..BATCH_SIZE {
+        hasher.digest(&input.encode::<T1B1Buf>());
+    }
+}
+
+fn regular_hasher_t5b1_legacy(input: &TritBuf<T5B1Buf>) {
     let mut hasher = CurlP81::new();
 
     for _ in 0..BATCH_SIZE {
@@ -61,6 +72,11 @@ fn bench_hasher(c: &mut Criterion) {
             input,
             |b, i| b.iter(|| regular_hasher_t5b1(i)),
         );
+        group.bench_with_input(
+            BenchmarkId::new("Regular (legacy)", format!("{} T5B1", length)),
+            input,
+            |b, i| b.iter(|| regular_hasher_t5b1_legacy(i)),
+        );
         group.bench_with_input(
             BenchmarkId::new("Unrolled", format!("{} T5B1", length)),
             input,

diff --git a/bee-crypto/src/lib.rs b/bee-crypto/src/lib.rs
@@ -3,11 +3,15 @@
 
 //! Cryptographic primitives of the IOTA protocol.
 
+#![allow(deprecated)] // This is ok, because we are going to deprecate everything here anyways.
 #![deny(clippy::cast_lossless, clippy::checked_conversions)]
 #![warn(
     missing_docs,
     clippy::cast_possible_truncation,
     clippy::cast_sign_loss,
     clippy::cast_possible_wrap
 )]
+#![deprecated(
+    note = "`bee-crypto` will not be supported in future versions. You can use functions from `iota-crypto` instead."
+)]
 pub mod ternary;
diff --git a/bee-crypto/src/ternary/sponge/curlp/unrolled/mod.rs b/bee-crypto/src/ternary/sponge/curlp/unrolled/mod.rs
@@ -6,7 +6,7 @@ mod u256;
 
 use u256::U256;
 
-use super::{Sponge, HASH_LENGTH};
+use crate::ternary::sponge::{Sponge, HASH_LENGTH};
 
 use bee_ternary::{Btrit, Trits};
 

diff --git a/bee-crypto/tests/batched_curlp.rs b/bee-crypto/tests/batched_curlp.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::{
     sponge::{BatchHasher, CurlPRounds, BATCH_SIZE},
     HASH_LENGTH,

diff --git a/bee-crypto/tests/binary_to_binary.rs b/bee-crypto/tests/binary_to_binary.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::bigint::{binary_representation::U32Repr, endianness::LittleEndian, I384, U384};
 
 #[test]

diff --git a/bee-crypto/tests/binary_to_ternary.rs b/bee-crypto/tests/binary_to_ternary.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::bigint::{binary_representation::U8Repr, endianness::BigEndian, I384, T242, T243};
 use bee_ternary::{Trits, T1B1};
 

diff --git a/bee-crypto/tests/binary_to_ternary_roundtrip.rs b/bee-crypto/tests/binary_to_ternary_roundtrip.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::bigint::{binary_representation::U32Repr, endianness::LittleEndian, I384, T243, U384};
 use bee_ternary::{Btrit, Utrit};
 

diff --git a/bee-crypto/tests/i384.rs b/bee-crypto/tests/i384.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 #[macro_use]
 mod test_macros;
 

diff --git a/bee-crypto/tests/sponge.rs b/bee-crypto/tests/sponge.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::sponge::{CurlP27, CurlP81, Kerl, Sponge, SpongeKind, UnrolledCurlP81};
 use bee_ternary::{T1B1Buf, T3B1Buf, TritBuf, TryteBuf};
 

diff --git a/bee-crypto/tests/ternary_hash.rs b/bee-crypto/tests/ternary_hash.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::{Hash, HASH_LENGTH};
 use bee_ternary::Btrit;
 

diff --git a/bee-crypto/tests/ternary_to_binary_roundtrip.rs b/bee-crypto/tests/ternary_to_binary_roundtrip.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::bigint::{binary_representation::U32Repr, endianness::LittleEndian, I384, T242, T243, U384};
 use bee_ternary::{Btrit, Utrit};
 

diff --git a/bee-crypto/tests/trytes_to_bigint.rs b/bee-crypto/tests/trytes_to_bigint.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 use bee_crypto::ternary::bigint::{binary_representation::U8Repr, endianness::BigEndian, I384, T243};
 use bee_ternary::{T1B1Buf, TritBuf, TryteBuf};
 

diff --git a/bee-crypto/tests/u384.rs b/bee-crypto/tests/u384.rs
@@ -1,6 +1,8 @@
 // Copyright 2020-2021 IOTA Stiftung
 // SPDX-License-Identifier: Apache-2.0
 
+#![allow(deprecated)]
+
 #[macro_use]
 mod test_macros;
 

diff --git a/bee-ledger/Cargo.toml b/bee-ledger/Cargo.toml
@@ -22,7 +22,7 @@ digest = { version = "0.9.0", default-features = false, optional = true }
 futures = { version = "0.3.17", default-features = false, optional = true }
 hashbrown = { version = "0.11.2", default-features = false, optional = true }
 hex = { version = "0.4.3", default-features = false, optional = true }
-iota-crypto = { version = "0.7.0", default-features = false, features = [ "blake2b" ], optional = true }
+iota-crypto = { version = "0.9.1", default-features = false, features = [ "blake2b" ], optional = true }
 log = { version = "0.4.14", default-features = false, optional = true }
 ref-cast = { version = "1.0.6", default-features = false, optional = true }
 reqwest = { version = "0.11.5", default-features = false, features = [ "default-tls", "stream" ], optional = true }

diff --git a/bee-message/CHANGELOG.md b/bee-message/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Example that counts the number of allocations while scoring PoWs
+
 ### Changed
 
 ### Deprecated

diff --git a/bee-message/Cargo.toml b/bee-message/Cargo.toml
@@ -12,14 +12,14 @@ homepage = "https://www.iota.org"
 
 [dependencies]
 bee-common = { version = "0.5.0", path = "../bee-common/bee-common", default-features = false }
-bee-pow = { version = "0.1.0", path = "../bee-pow", default-features = false }
+bee-pow = { version = "0.2.0", path = "../bee-pow", default-features = false }
 bee-ternary = { version = "0.5.2", path = "../bee-ternary", default-features = false, features = [ "serde1" ] }
 
 bech32 = { version = "0.8.1", default-features = false }
 bytemuck = { version = "1.7.2", default-features = false }
 digest = { version = "0.9.0", default-features = false }
 hex = { version = "0.4.3", default-features = false }
-iota-crypto = { version = "0.7.0", default-features = false, features = [ "ed25519", "blake2b" ] }
+iota-crypto = { version = "0.9.1", default-features = false, features = [ "ed25519", "blake2b" ] }
 serde = { version = "1.0.130", default-features = false, optional = true }
 thiserror = { version = "1.0.30", default-features = false }
 

diff --git a/bee-message/examples/allocations_while_scoring.rs b/bee-message/examples/allocations_while_scoring.rs
@@ -0,0 +1,56 @@
+// Copyright 2020-2021 IOTA Stiftung
+// SPDX-License-Identifier: Apache-2.0
+
+//! This script aims at counting the number of allocations that are performed when we score the Proof of Work of a
+//! message by wrapping `GlobalAlloc`. Ideally, this method should not allocate at all, which would lead to a better
+//! performance.
+//!
+//! The code was adapted from: https://kanejaku.org/posts/2021/01/2021-01-27/ (CC-BY 4.0)
+
+use bee_common::packable::Packable;
+use bee_message::prelude::*;
+use bee_pow::{
+    providers::{miner::MinerBuilder, NonceProviderBuilder},
+    score::PoWScorer,
+};
+use bee_test::rand::parents::rand_parents;
+
+use std::{
+    alloc::{GlobalAlloc, Layout, System},
+    sync::atomic::{AtomicUsize, Ordering::SeqCst},
+};
+
+struct CheckAlloc;
+
+static ALLOCATED: AtomicUsize = AtomicUsize::new(0);
+
+unsafe impl GlobalAlloc for CheckAlloc {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        ALLOCATED.fetch_add(1, SeqCst);
+        System.alloc(layout)
+    }
+
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        System.dealloc(ptr, layout);
+    }
+}
+
+#[global_allocator]
+static A: CheckAlloc = CheckAlloc;
+
+fn main() {
+    let message = MessageBuilder::new()
+        .with_network_id(0)
+        .with_parents(rand_parents())
+        .with_nonce_provider(MinerBuilder::new().with_num_workers(num_cpus::get()).finish(), 10000f64)
+        .finish()
+        .unwrap();
+
+    let message_bytes = message.pack_new();
+
+    let before_count = ALLOCATED.load(SeqCst);
+    let _score = PoWScorer::new().score(&message_bytes);
+    let after_count = ALLOCATED.load(SeqCst);
+
+    println!("Number of allocations: {}", after_count - before_count);
+}
diff --git a/bee-message/tests/message.rs b/bee-message/tests/message.rs
@@ -8,7 +8,7 @@ use bee_pow::{
         miner::{Miner, MinerBuilder},
         NonceProviderBuilder,
     },
-    score::compute_pow_score,
+    score::PoWScorer,
 };
 use bee_test::rand::{
     message::rand_message_ids,
@@ -26,7 +26,7 @@ fn pow_default_provider() {
         .unwrap();
 
     let message_bytes = message.pack_new();
-    let score = compute_pow_score(&message_bytes);
+    let score = PoWScorer::new().score(&message_bytes);
 
     assert!(score >= 4000f64);
 }
@@ -41,7 +41,7 @@ fn pow_provider() {
         .unwrap();
 
     let message_bytes = message.pack_new();
-    let score = compute_pow_score(&message_bytes);
+    let score = PoWScorer::new().score(&message_bytes);
 
     assert!(score >= 10000f64);
 }

diff --git a/bee-pow/CHANGELOG.md b/bee-pow/CHANGELOG.md
@@ -19,6 +19,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security -->
 
+## 0.2.0 - 2021-11-19
+
+### Changed
+
+- Scoring of Proof of Work can now reuse hash functions;
+
 ## 0.1.0 - 2021-04-13
 
 ### Added