Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Poseidon2 stdlib impl #3551

Merged
merged 43 commits into from
Jan 11, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
aa536b5
initial files and impl
lucasxia01 Dec 4, 2023
89f4bc4
getting it to compile
lucasxia01 Dec 5, 2023
e5298b4
sponge stdlib
lucasxia01 Dec 5, 2023
e80739d
added unconstrained permutation
lucasxia01 Dec 5, 2023
751d74b
added poseidon2 end gate
lucasxia01 Dec 5, 2023
4b3ca57
using the gates in poseidon2 permutation (with bugs)
lucasxia01 Dec 5, 2023
4b68594
compile bug with templating fixed (thanks adam)
lucasxia01 Dec 5, 2023
aaec935
added initial external mul gates
lucasxia01 Dec 6, 2023
45dc5d4
renaming field_t<Builder> to FF
lucasxia01 Dec 6, 2023
d933eef
fixed compile problems
lucasxia01 Dec 6, 2023
171b0e8
naming: poseidon2_hash->poseidon2
lucasxia01 Dec 6, 2023
dc5f3f9
hash_buffers function to native poseidon2
lucasxia01 Dec 6, 2023
26b1930
new stdlib test file copied from pedersen
lucasxia01 Dec 6, 2023
0e312e9
passing builder into everything
lucasxia01 Dec 7, 2023
4126d04
turned values into witnesses
lucasxia01 Dec 8, 2023
41aa7c1
Merge branch 'master' into lx/poseidon2-stdlib
lucasxia01 Dec 8, 2023
8f8b799
busread fix
lucasxia01 Dec 8, 2023
7266f35
other witness creations
lucasxia01 Dec 8, 2023
f766e2b
fixing stdlib poseidon2 hash_buffers
lucasxia01 Dec 8, 2023
db908d7
compile fix
lucasxia01 Dec 8, 2023
9dc8392
added comments
lucasxia01 Dec 8, 2023
cc9edd9
Merge branch 'master' into lx/poseidon2-stdlib
lucasxia01 Dec 11, 2023
6ddfcae
trying to split into source file (wip)
lucasxia01 Jan 5, 2024
e15be66
fixed linker error
lucasxia01 Jan 5, 2024
222e532
updated comments, removed debug printing
lucasxia01 Jan 5, 2024
eedcefb
updated type names, comments
lucasxia01 Jan 5, 2024
c24436f
circleci gcc fix
lucasxia01 Jan 5, 2024
13807f1
circleci fix
lucasxia01 Jan 5, 2024
7dbb45c
adding comments, refactoring to source file
lucasxia01 Jan 5, 2024
8b25996
stupid ci fix
lucasxia01 Jan 5, 2024
bf75776
Merge branch 'master' into lx/poseidon2-stdlib
lucasxia01 Jan 8, 2024
6f04eb0
compile fix
lucasxia01 Jan 8, 2024
ff22a5f
minor update, comments
lucasxia01 Jan 8, 2024
2419db5
small comment update
lucasxia01 Jan 8, 2024
aeaf1a1
revamped poseidon2 stdlib tests
lucasxia01 Jan 9, 2024
f134a94
Merge branch 'master' into lx/poseidon2-stdlib
lucasxia01 Jan 9, 2024
6b67f11
fixed hash consistency test
lucasxia01 Jan 11, 2024
e9fd7da
updated tests to include hash_buffer
lucasxia01 Jan 11, 2024
2c66d1f
added poseidon2 tests to bb-tests
lucasxia01 Jan 11, 2024
4b7d151
added poseidon2 to stdlib tests
lucasxia01 Jan 11, 2024
8bd4df0
Merge branch 'master' into lx/poseidon2-stdlib
lucasxia01 Jan 11, 2024
af5e017
added missing stdlib pedersen hash tests
lucasxia01 Jan 11, 2024
8aa065f
undo weird merge? update to circuit/ files
lucasxia01 Jan 11, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,21 @@ template <typename FF> struct poseidon2_external_gate_ {
uint32_t b;
uint32_t c;
uint32_t d;
uint32_t round_idx;
size_t round_idx;
};

template <typename FF> struct poseidon2_internal_gate_ {
uint32_t a;
uint32_t b;
uint32_t c;
uint32_t d;
uint32_t round_idx;
size_t round_idx;
};

template <typename FF> struct poseidon2_end_gate_ {
uint32_t a;
uint32_t b;
uint32_t c;
uint32_t d;
};
} // namespace proof_system
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,29 @@ void GoblinUltraCircuitBuilder_<FF>::create_poseidon2_internal_gate(const poseid
++this->num_gates;
}

template <typename FF> void GoblinUltraCircuitBuilder_<FF>::create_poseidon2_end_gate(const poseidon2_end_gate_<FF>& in)
{
this->w_l.emplace_back(in.a);
this->w_r.emplace_back(in.b);
this->w_o.emplace_back(in.c);
this->w_4.emplace_back(in.d);
this->q_m.emplace_back(0);
this->q_1.emplace_back(0);
this->q_2.emplace_back(0);
this->q_3.emplace_back(0);
this->q_c.emplace_back(0);
this->q_arith.emplace_back(0);
this->q_4.emplace_back(0);
this->q_sort.emplace_back(0);
this->q_lookup_type.emplace_back(0);
this->q_elliptic.emplace_back(0);
this->q_aux.emplace_back(0);
this->q_busread.emplace_back(0);
this->q_poseidon2_external.emplace_back(0);
this->q_poseidon2_internal.emplace_back(0);
++this->num_gates;
}

template <typename FF>
inline FF GoblinUltraCircuitBuilder_<FF>::compute_poseidon2_external_identity(FF q_poseidon2_external_value,
FF q_1_value,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ template <typename FF> class GoblinUltraCircuitBuilder_ : public UltraCircuitBui
}
void create_poseidon2_external_gate(const poseidon2_external_gate_<FF>& in);
void create_poseidon2_internal_gate(const poseidon2_internal_gate_<FF>& in);
void create_poseidon2_end_gate(const poseidon2_end_gate_<FF>& in);

FF compute_poseidon2_external_identity(FF q_poseidon2_external_value,
FF q_1_value,
Expand Down
3 changes: 2 additions & 1 deletion barretenberg/cpp/src/barretenberg/stdlib/hash/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ add_subdirectory(blake3s)
add_subdirectory(pedersen)
add_subdirectory(sha256)
add_subdirectory(keccak)
add_subdirectory(benchmarks)
add_subdirectory(benchmarks)
add_subdirectory(poseidon2)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
barretenberg_module(stdlib_poseidon2_hash stdlib_primitives)
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#include "barretenberg/stdlib/hash/poseidon2/poseidon2.hpp"
#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
namespace proof_system::plonk::stdlib {

using namespace barretenberg;
using namespace proof_system;

template <typename C> field_t<C> poseidon2_hash<C>::hash(const std::vector<field_t>& inputs)
{

/* Run the sponge by absorbing all the input and squeezing one output.
* This should just call the sponge variable length hash function
*
*/
auto input{ inputs };
return Sponge::hash_fixed_length(input);
}

/**
* Hash a byte_array.
*
*/
template <typename C> field_t<C> poseidon2_hash<C>::hash_buffer(const stdlib::byte_array<C>& input)
{
const size_t num_bytes = input.size();
const size_t bytes_per_element = 31;
size_t num_elements = static_cast<size_t>(num_bytes % bytes_per_element != 0) + (num_bytes / bytes_per_element);

std::vector<field_t> elements;
for (size_t i = 0; i < num_elements; ++i) {
size_t bytes_to_slice = 0;
if (i == num_elements - 1) {
bytes_to_slice = num_bytes - (i * bytes_per_element);
} else {
bytes_to_slice = bytes_per_element;
}
auto element = static_cast<field_t>(input.slice(i * bytes_per_element, bytes_to_slice));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you not use input.slice in the native implementation as well to keep the two as close as possible?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

input.slice is a function that is specific to byte_array, so we have to define another slice one for the native hash_buffer

elements.emplace_back(element);
}
for (auto& x : elements) {
std::cout << x << std::endl;
}
field_t hashed;
if (elements.size() < 2) {
hashed = hash(elements);
} else {
hashed = hash({ elements[0], elements[1] });
for (size_t i = 2; i < elements.size(); ++i) {
hashed = hash({ hashed, elements[i] });
}
}
return hashed;
}
template class poseidon2_hash<proof_system::GoblinUltraCircuitBuilder>;

} // namespace proof_system::plonk::stdlib
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#pragma once
#include "barretenberg/crypto/poseidon2/poseidon2_params.hpp"
#include "barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp"
#include "barretenberg/stdlib/primitives/byte_array/byte_array.hpp"
#include "barretenberg/stdlib/primitives/field/field.hpp"

#include "../../primitives/circuit_builders/circuit_builders.hpp"

namespace proof_system::plonk::stdlib {

using namespace barretenberg;
/**
* @brief stdlib class that evaluates in-circuit poseidon2 hashes, consistent with behavior in
* crypto::poseidon2_hash
*
* @tparam Builder
*/
template <typename Builder> class poseidon2_hash {

private:
using field_t = stdlib::field_t<Builder>;
using bool_t = stdlib::bool_t<Builder>;
using Params = crypto::Poseidon2Bn254ScalarFieldParams;
using Permutation = Poseidon2Permutation<Params, Builder>;
using Sponge = FieldSponge<Params::t - 1, 1, Params::t, Permutation, Builder>;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you explain where the -1 comes from?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we choose the rate to be t-1 (so 3 field elements) and our capacity to be 1. Rate and capacity should always add to t.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah ok, can you add a comment above please?


public:
static field_t hash(const std::vector<field_t>& in);
static field_t hash_buffer(const stdlib::byte_array<Builder>& input);
};

extern template class poseidon2_hash<proof_system::GoblinUltraCircuitBuilder>;

} // namespace proof_system::plonk::stdlib
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
#pragma once
#include <array>
#include <cstddef>
#include <cstdint>

#include "barretenberg/crypto/poseidon2/poseidon2_permutation.hpp"
#include "barretenberg/proof_system/arithmetization/gate_data.hpp"
#include "barretenberg/stdlib/primitives/field/field.hpp"

namespace proof_system::plonk::stdlib {

using namespace proof_system;
template <typename Params, typename Builder> class Poseidon2Permutation {
public:
using NativePermutation = crypto::Poseidon2Permutation<Params>;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

discussed with Kesha, this is fine

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh ok

// t = sponge permutation size (in field elements)
// t = rate + capacity
// capacity = 1 field element (256 bits)
// rate = number of field elements that can be compressed per permutation
static constexpr size_t t = Params::t;
// d = degree of s-box polynomials. For a given field, `d` is the smallest element of `p` such that gdc(d, p - 1) =
// 1 (excluding 1) For bn254/grumpkin, d = 5
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've seen this is not done in native implementation either but we should cite where we got this 5 from

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

doesn't it say that d=5 is the smallest element such that gcd(d, p-1) = 1

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh my bad was looking somewhere else in the native implementation

static constexpr size_t d = Params::d;
// sbox size = number of bits in p
static constexpr size_t sbox_size = Params::sbox_size;
// number of full sbox rounds
static constexpr size_t rounds_f = Params::rounds_f;
// number of partial sbox rounds
static constexpr size_t rounds_p = Params::rounds_p;
static constexpr size_t NUM_ROUNDS = Params::rounds_f + Params::rounds_p;

using FF = typename Params::FF;
using State = std::array<field_t<Builder>, t>;
using NativeState = std::array<FF, t>;

using RoundConstants = std::array<FF, t>;
using RoundConstantsContainer = std::array<RoundConstants, NUM_ROUNDS>;
static constexpr RoundConstantsContainer round_constants = Params::round_constants;
static State permutation(const State& input)
{
Builder* ctx = input[0].get_context();
assert(ctx != nullptr);
// deep copy
State current_state(input);
NativeState current_native_state;
for (size_t i = 0; i < t; ++i) {
current_native_state[i] = current_state[i].get_value();
}

// Apply 1st linear layer
NativePermutation::matrix_multiplication_external(current_native_state);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should you not create external/internal gates for the first linear layer as well?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh, if i understand correctly, you create them at the first iteration of the foor loop

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The first linear layer is just an external matrix mul, and the gates are created in initial_external_matrix_multiplication, not in the for loop.


constexpr size_t rounds_f_beginning = rounds_f / 2;
for (size_t i = 0; i < rounds_f_beginning; ++i) {
poseidon2_external_gate_<FF> in{ current_state[0].witness_index,
current_state[1].witness_index,
current_state[2].witness_index,
current_state[3].witness_index,
i };
ctx->create_poseidon2_external_gate(in);
// calculate the new witnesses
NativePermutation::add_round_constants(current_native_state, round_constants[i]);
NativePermutation::apply_sbox(current_native_state);
NativePermutation::matrix_multiplication_external(current_native_state);
for (size_t j = 0; j < t; ++j) {
current_state[j] = field_t<Builder>(ctx, current_native_state[j]);
}
}

const size_t p_end = rounds_f_beginning + rounds_p;
for (size_t i = rounds_f_beginning; i < p_end; ++i) {
poseidon2_internal_gate_<FF> in{ current_state[0].witness_index,
current_state[1].witness_index,
current_state[2].witness_index,
current_state[3].witness_index,
i };
ctx->create_poseidon2_internal_gate(in);
current_native_state[0] += round_constants[i][0];
NativePermutation::apply_single_sbox(current_native_state[0]);
NativePermutation::matrix_multiplication_internal(current_native_state);
for (size_t j = 0; j < t; ++j) {
current_state[j] = field_t<Builder>(ctx, current_native_state[j]);
}
}

for (size_t i = p_end; i < NUM_ROUNDS; ++i) {
poseidon2_external_gate_<FF> in{ current_state[0].witness_index,
current_state[1].witness_index,
current_state[2].witness_index,
current_state[3].witness_index,
i };
ctx->create_poseidon2_external_gate(in);
// calculate the new witnesses
NativePermutation::add_round_constants(current_native_state, round_constants[i]);
NativePermutation::apply_sbox(current_native_state);
NativePermutation::matrix_multiplication_external(current_native_state);
for (size_t j = 0; j < t; ++j) {
current_state[j] = field_t<Builder>(ctx, current_native_state[j]);
}
}
// need to add an extra row here to ensure that things check out
poseidon2_end_gate_<FF> in{
current_state[0].witness_index,
current_state[1].witness_index,
current_state[2].witness_index,
current_state[3].witness_index,
};
ctx->create_poseidon2_end_gate(in);
return current_state;
}
};
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could just create normal add gate here instead

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this got moved to a different line for some reason, but it was referring to the rounds which only use 3 of the wires and only do an add (gates 4 and 6)

} // namespace proof_system::plonk::stdlib
Loading