chore!: Replace computing hashes in circuits wasm, with computing the…

…m in ts via bb.js pedersen call. (#3114)

* This was originally going to use our TS pedersen lib to do all
pedersen computations, but wasm is 10x faster, so we still want to use
wasm.
* Introduce `elliptic` version of pedersen as it can be much faster than
`@noble/curves` (WARNING: Jest totally messes with benchmarks and can't
be trusted for timings with this stuff). Leaving noble version in for
now in case want to revisit, but not exported via index. The `elliptic`
version is also just there as reference (maybe useful in future).
* The pedersen api required being passed a wasm, and that wasm was
obtained via async call. This creates a tonne of async function overhead
and is generally a bit of a faff.
* This adds `foundation/crypto/pedersen/pedersen.wasm.ts` (and defaults
to using it via index.ts) that mimics our pure TS pedersen api, but uses
the bb.js wasm.
* Adds a hand rolled synchronous pedersen bind caller to bb.js (I should
probably bring back making bindgen produce the sync api, but also
figured might adopt Adams msgpack approach so not worth effort).
* `foundation/crypto/pedersen` leverages top-level await to load the
wasm as a global, so we can remove a tonne of async await calls.
* We cleanup some of the c_binds in barretenberg.
* We still use circuits.wasm for a couple of more complex calls ive not
tackled yet, e.g. `hashVK`.
* We now have dependency on bb.js, pulled in via `portal` yarn thingy.
*What does this mean for releases?* (edit: This would probably have
broken canary. This PR disabled canary.
#3244)
* Pulls the bbmalloc WASM_EXPORTS into own header so it doesn't mess
with the bindgen util.
* Broke some circular import dependencies as I was hitting weird errors.
* All pedersen calls now made on `foundation/crypto/pedersen`, got rid
of `circuits/barretenberg/crypto/pedersen` wrapper.
* Enable colorizing debug output in e2e tests via `DEBUG_COLOR=1`.
* Change some `test()` to `it()` (was faffing with other test runner
when profiling pedersen).
* Timer uses `performance.now` rather than `Date.getTime()`
* Introduces `madge` as a tool to detect circular deps.

TODO in subsequent PR's.
* Move all the hashing functions in `abis.ts` to exist as hash functions
on the various types.
* Implement hash functions for e.g. vkHash and a couple others that were
to complicated for doing right now.
* Move the types out of circuits.js into types project.
* Ultimately remove need for circuits.js / circuits.wasm?

.circleci/config.yml

@@ -1177,6 +1177,7 @@ workflows:
           requires:
             - circuits-wasm-linux-clang
             - l1-contracts
+            - bb-js
           <<: *defaults
       - yarn-project:
           requires:

barretenberg/cpp/src/barretenberg/barretenberg.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 // External Barretenberg C++ API
+#include "common/bbmalloc.hpp"
 #include "common/container.hpp"
 #include "common/map.hpp"
 #include "common/mem.hpp"

barretenberg/cpp/src/barretenberg/common/mem.cpp → barretenberg/cpp/src/barretenberg/common/bbmalloc.cpp

@@ -1,6 +1,5 @@
-#include "./mem.hpp"
+#include "./bbmalloc.hpp"
 #include "./slab_allocator.hpp"
-#include "./wasm_export.hpp"
 
 WASM_EXPORT void* bbmalloc(size_t size)
 {

barretenberg/cpp/src/barretenberg/common/bbmalloc.hpp

@@ -0,0 +1,7 @@
+#pragma once
+#include "./wasm_export.hpp"
+#include <cstddef>
+
+WASM_EXPORT void* bbmalloc(size_t size);
+
+WASM_EXPORT void bbfree(void* ptr);

barretenberg/cpp/src/barretenberg/common/mem.hpp

@@ -75,7 +75,4 @@ inline void aligned_free(void* mem)
 //     info("Total allocated space (uordblks): ", minfo.uordblks);
 //     info("Total free space (fordblks): ", minfo.fordblks);
 //     info("Top-most, releasable space (keepcost): ", minfo.keepcost);
-// }
-
-WASM_EXPORT void* bbmalloc(size_t size);
-WASM_EXPORT void bbfree(void* ptr);
+// }

barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp

@@ -3,11 +3,16 @@
 #include "barretenberg/common/serialize.hpp"
 #include "pedersen.hpp"
 
-WASM_EXPORT void pedersen__commit(uint8_t const* inputs_buffer, uint8_t* output)
+extern "C" {
+
+using namespace barretenberg;
+
+WASM_EXPORT void pedersen_commit(fr::vec_in_buf inputs_buffer, affine_element::out_buf output)
 {
     std::vector<grumpkin::fq> to_commit;
     read(inputs_buffer, to_commit);
     grumpkin::g1::affine_element pedersen_commitment = crypto::pedersen_commitment::commit_native(to_commit);
 
     serialize::write(output, pedersen_commitment);
+}
 }

barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp

@@ -1,7 +1,12 @@
 #pragma once
-#include "barretenberg/common/mem.hpp"
-#include "barretenberg/common/serialize.hpp"
-#include "barretenberg/common/streams.hpp"
-#include "barretenberg/common/timer.hpp"
+#include "barretenberg/common/wasm_export.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 
-WASM_EXPORT void pedersen__commit(uint8_t const* inputs_buffer, uint8_t* output);
+extern "C" {
+
+using namespace barretenberg;
+using affine_element = grumpkin::g1::affine_element;
+
+WASM_EXPORT void pedersen_commit(fr::vec_in_buf inputs_buffer, affine_element::out_buf output);
+}

barretenberg/cpp/src/barretenberg/crypto/pedersen_commitment/pedersen.test.cpp

@@ -1,4 +1,5 @@
 #include "pedersen.hpp"
+#include "barretenberg/common/timer.hpp"
 #include "barretenberg/crypto/generators/generator_data.hpp"
 #include <gtest/gtest.h>
 
@@ -16,4 +17,38 @@ TEST(Pedersen, Commitment)
     EXPECT_EQ(r, expected);
 }
 
-} // namespace crypto
+TEST(Pedersen, CommitmentWithZero)
+{
+    auto x = pedersen_commitment::Fq::zero();
+    auto y = pedersen_commitment::Fq::one();
+    auto r = pedersen_commitment::commit_native({ x, y });
+    auto expected =
+        grumpkin::g1::affine_element(fr(uint256_t("054aa86a73cb8a34525e5bbed6e43ba1198e860f5f3950268f71df4591bde402")),
+                                     fr(uint256_t("209dcfbf2cfb57f9f6046f44d71ac6faf87254afc7407c04eb621a6287cac126")));
+    EXPECT_EQ(r, expected);
+}
+
+TEST(Pedersen, CommitmentProf)
+{
+    GTEST_SKIP() << "Skipping mini profiler.";
+    auto x = fr::random_element();
+    auto y = fr::random_element();
+    Timer t;
+    for (int i = 0; i < 10000; ++i) {
+        pedersen_commitment::commit_native({ x, y });
+    }
+    info(t.nanoseconds() / 1000 / 10000);
+}
+
+// Useful for pasting into ts version of pedersen.
+TEST(Pedersen, GeneratorPrinter)
+{
+    GTEST_SKIP() << "Skipping generator-for-ts printer.";
+    pedersen_commitment::GeneratorContext ctx;
+    auto generators = ctx.generators->get_default_generators()->get(128);
+    for (auto g : generators) {
+        info("[", g.x, "n, ", g.y, "n],");
+    }
+}
+
+}; // namespace crypto

barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp

@@ -5,12 +5,12 @@
 
 extern "C" {
 
-WASM_EXPORT void pedersen__hash_with_hash_index(uint8_t const* inputs_buffer, uint32_t hash_index, uint8_t* output)
+WASM_EXPORT void pedersen_hash(uint8_t const* inputs_buffer, uint32_t const* hash_index, uint8_t* output)
 {
     std::vector<grumpkin::fq> to_hash;
     read(inputs_buffer, to_hash);
     crypto::GeneratorContext<curve::Grumpkin> ctx;
-    ctx.offset = static_cast<size_t>(hash_index);
+    ctx.offset = static_cast<size_t>(ntohl(*hash_index));
     auto r = crypto::pedersen_hash::hash(to_hash, ctx);
     barretenberg::fr::serialize_to_buffer(r, output);
 }

barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp

@@ -7,7 +7,5 @@ extern "C" {
 
 using namespace barretenberg;
 
-WASM_EXPORT void pedersen_hash_with_hash_index(fr::vec_in_buf inputs_buffer,
-                                               uint32_t const* hash_index,
-                                               fr::out_buf output);
+WASM_EXPORT void pedersen_hash(fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, fr::out_buf output);
 }

barretenberg/cpp/src/barretenberg/crypto/schnorr/c_bind.cpp

@@ -1,48 +1,58 @@
+#include "c_bind.hpp"
 #include "multisig.hpp"
 #include "schnorr.hpp"
 
-#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
+extern "C" {
 
-WASM_EXPORT void compute_public_key(uint8_t const* private_key, uint8_t* public_key_buf)
+using namespace barretenberg;
+using affine_element = grumpkin::g1::affine_element;
+using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
+using multisig_public_key = typename multisig::MultiSigPublicKey;
+
+WASM_EXPORT void schnorr_compute_public_key(uint8_t const* private_key, uint8_t* public_key_buf)
 {
     auto priv_key = from_buffer<grumpkin::fr>(private_key);
     grumpkin::g1::affine_element pub_key = grumpkin::g1::one * priv_key;
     serialize::write(public_key_buf, pub_key);
 }
 
-WASM_EXPORT void negate_public_key(uint8_t const* public_key_buffer, uint8_t* output)
+WASM_EXPORT void schnorr_negate_public_key(uint8_t const* public_key_buffer, uint8_t* output)
 {
     // Negate the public key (effectively negating the y-coordinate of the public key) and return the resulting public
     // key.
     auto account_public_key = from_buffer<grumpkin::g1::affine_element>(public_key_buffer);
     serialize::write(output, -account_public_key);
 }
 
-WASM_EXPORT void construct_signature(
-    uint8_t const* message, size_t msg_len, uint8_t const* private_key, uint8_t* s, uint8_t* e)
+WASM_EXPORT void schnorr_construct_signature(uint8_t const* message_buf,
+                                             uint8_t const* private_key,
+                                             uint8_t* s,
+                                             uint8_t* e)
 {
+    auto message = from_buffer<std::string>(message_buf);
     auto priv_key = from_buffer<grumpkin::fr>(private_key);
     grumpkin::g1::affine_element pub_key = grumpkin::g1::one * priv_key;
     crypto::schnorr::key_pair<grumpkin::fr, grumpkin::g1> key_pair = { priv_key, pub_key };
-    auto sig = crypto::schnorr::construct_signature<Blake2sHasher, grumpkin::fq>(std::string((char*)message, msg_len),
-                                                                                 key_pair);
+    auto sig = crypto::schnorr::construct_signature<Blake2sHasher, grumpkin::fq>(message, key_pair);
     write(s, sig.s);
     write(e, sig.e);
 }
 
-WASM_EXPORT bool verify_signature(
-    uint8_t const* message, size_t msg_len, uint8_t const* pub_key, uint8_t const* sig_s, uint8_t const* sig_e)
+WASM_EXPORT void schnorr_verify_signature(
+    uint8_t const* message_buf, uint8_t const* pub_key, uint8_t const* sig_s, uint8_t const* sig_e, bool* result)
 {
     auto pubk = from_buffer<grumpkin::g1::affine_element>(pub_key);
-    std::array<uint8_t, 32> s, e;
+    auto message = from_buffer<std::string>(message_buf);
+    std::array<uint8_t, 32> s;
+    std::array<uint8_t, 32> e;
     std::copy(sig_s, sig_s + 32, s.begin());
     std::copy(sig_e, sig_e + 32, e.begin());
     crypto::schnorr::signature sig = { s, e };
-    return crypto::schnorr::verify_signature<Blake2sHasher, grumpkin::fq, grumpkin::fr, grumpkin::g1>(
-        std::string((char*)message, msg_len), pubk, sig);
+    *result =
+        crypto::schnorr::verify_signature<Blake2sHasher, grumpkin::fq, grumpkin::fr, grumpkin::g1>(message, pubk, sig);
 }
 
-WASM_EXPORT void multisig_create_multisig_public_key(uint8_t const* private_key, uint8_t* multisig_pubkey_buf)
+WASM_EXPORT void schnorr_multisig_create_multisig_public_key(uint8_t const* private_key, uint8_t* multisig_pubkey_buf)
 {
     using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
     using multisig_public_key = typename multisig::MultiSigPublicKey;
@@ -55,23 +65,26 @@ WASM_EXPORT void multisig_create_multisig_public_key(uint8_t const* private_key,
     serialize::write(multisig_pubkey_buf, agg_pubkey);
 }
 
-WASM_EXPORT bool multisig_validate_and_combine_signer_pubkeys(uint8_t const* signer_pubkey_buf,
-                                                              uint8_t* combined_key_buf)
+WASM_EXPORT void schnorr_multisig_validate_and_combine_signer_pubkeys(uint8_t const* signer_pubkey_buf,
+                                                                      affine_element::out_buf combined_key_buf,
+                                                                      bool* success)
 {
     using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
-    std::vector<multisig::MultiSigPublicKey> pubkeys =
-        from_buffer<std::vector<multisig::MultiSigPublicKey>>(signer_pubkey_buf);
+    auto pubkeys = from_buffer<std::vector<multisig::MultiSigPublicKey>>(signer_pubkey_buf);
+
+    auto combined_key = multisig::validate_and_combine_signer_pubkeys(pubkeys);
 
-    if (auto combined_key = multisig::validate_and_combine_signer_pubkeys(pubkeys); combined_key) {
+    if (combined_key) {
         serialize::write(combined_key_buf, *combined_key);
-        return true;
+        *success = true;
     } else {
-        return false;
+        serialize::write(combined_key_buf, affine_element::one());
+        *success = false;
     }
 }
 
-WASM_EXPORT void multisig_construct_signature_round_1(uint8_t* round_one_public_output_buf,
-                                                      uint8_t* round_one_private_output_buf)
+WASM_EXPORT void schnorr_multisig_construct_signature_round_1(uint8_t* round_one_public_output_buf,
+                                                              uint8_t* round_one_private_output_buf)
 {
     using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
 
@@ -80,15 +93,16 @@ WASM_EXPORT void multisig_construct_signature_round_1(uint8_t* round_one_public_
     serialize::write(round_one_private_output_buf, private_output);
 }
 
-WASM_EXPORT bool multisig_construct_signature_round_2(uint8_t const* message,
-                                                      size_t msg_len,
-                                                      uint8_t* const private_key,
-                                                      uint8_t* const signer_round_one_private_buf,
-                                                      uint8_t* const signer_pubkeys_buf,
-                                                      uint8_t* const round_one_public_buf,
-                                                      uint8_t* round_two_buf)
+WASM_EXPORT void schnorr_multisig_construct_signature_round_2(uint8_t const* message_buf,
+                                                              uint8_t const* private_key,
+                                                              uint8_t const* signer_round_one_private_buf,
+                                                              uint8_t const* signer_pubkeys_buf,
+                                                              uint8_t const* round_one_public_buf,
+                                                              uint8_t* round_two_buf,
+                                                              bool* success)
 {
     using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
+    auto message = from_buffer<std::string>(message_buf);
     auto priv_key = from_buffer<grumpkin::fr>(private_key);
     grumpkin::g1::affine_element pub_key = grumpkin::g1::one * priv_key;
     crypto::schnorr::key_pair<grumpkin::fr, grumpkin::g1> key_pair = { priv_key, pub_key };
@@ -97,39 +111,40 @@ WASM_EXPORT bool multisig_construct_signature_round_2(uint8_t const* message,
     auto round_one_outputs = from_buffer<std::vector<multisig::RoundOnePublicOutput>>(round_one_public_buf);
 
     auto round_one_private = from_buffer<multisig::RoundOnePrivateOutput>(signer_round_one_private_buf);
-    auto round_two_output = multisig::construct_signature_round_2(
-        std::string((char*)message, msg_len), key_pair, round_one_private, signer_pubkeys, round_one_outputs);
+    auto round_two_output =
+        multisig::construct_signature_round_2(message, key_pair, round_one_private, signer_pubkeys, round_one_outputs);
 
     if (round_two_output.has_value()) {
         write(round_two_buf, *round_two_output);
-        return true;
+        *success = true;
     } else {
-        return false;
+        *success = false;
     }
 }
 
-WASM_EXPORT bool multisig_combine_signatures(uint8_t const* message,
-                                             size_t msg_len,
-                                             uint8_t* const signer_pubkeys_buf,
-                                             uint8_t* const round_one_buf,
-                                             uint8_t* const round_two_buf,
-                                             uint8_t* s,
-                                             uint8_t* e)
+WASM_EXPORT void schnorr_multisig_combine_signatures(uint8_t const* message_buf,
+                                                     uint8_t const* signer_pubkeys_buf,
+                                                     uint8_t const* round_one_buf,
+                                                     uint8_t const* round_two_buf,
+                                                     uint8_t* s,
+                                                     uint8_t* e,
+                                                     bool* success)
 {
     using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
 
+    auto message = from_buffer<std::string>(message_buf);
     auto signer_pubkeys = from_buffer<std::vector<multisig::MultiSigPublicKey>>(signer_pubkeys_buf);
     auto round_one_outputs = from_buffer<std::vector<multisig::RoundOnePublicOutput>>(round_one_buf);
     auto round_two_outputs = from_buffer<std::vector<multisig::RoundTwoPublicOutput>>(round_two_buf);
 
-    auto sig = multisig::combine_signatures(
-        std::string((char*)message, msg_len), signer_pubkeys, round_one_outputs, round_two_outputs);
+    auto sig = multisig::combine_signatures(message, signer_pubkeys, round_one_outputs, round_two_outputs);
 
     if (sig.has_value()) {
         write(s, (*sig).s);
         write(e, (*sig).e);
-        return true;
+        *success = true;
     } else {
-        return false;
+        *success = false;
     }
 }
+}