Use Zerokit RLN module in nwaku #1030
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -789,9 +789,9 @@ procSuite "WakuNode": | |
| await node1.publish(rlnRelayPubSubTopic, message) | ||
| await sleepAsync(2000.millis) | ||
|
|
||
|
There was a problem hiding this comment. I think this line is removed unintentionally. There was a problem hiding this comment. The previous extra line doesn't seem to be consistent with the remaining tests. |
||
| ## Current circom takes more time to load the circuit.. | ||
| check: | ||
| (await completionFut.withTimeout(200.seconds)) == true | ||
|
There was a problem hiding this comment. Just double checking that There was a problem hiding this comment. Not necessary anymore with zerokit rln perfomance fixes. |
||
|
|
||
| await node1.stop() | ||
| await node2.stop() | ||
|
|
@@ -892,9 +892,10 @@ procSuite "WakuNode": | |
| await node1.publish(rlnRelayPubSubTopic, message) | ||
| await sleepAsync(2000.millis) | ||
|
|
||
| ## Current circom takes more time to load the circuit... | ||
| check: | ||
| # the relayHandler of node3 never gets called | ||
| (await completionFut.withTimeout(200.seconds)) == false | ||
|
|
||
| await node1.stop() | ||
| await node2.stop() | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,7 +6,7 @@ import | |
| os, | ||
| waku_rln_relay_types | ||
|
|
||
| const libPath = "vendor/zerokit/target/debug/" | ||
| when defined(Windows): | ||
| const libName* = libPath / "rln.dll" | ||
| elif defined(Linux): | ||
|
|
@@ -17,72 +17,100 @@ elif defined(MacOsX): | |
| # all the following procedures are Nim wrappers for the functions defined in libName | ||
| {.push dynlib: libName, raises: [Defect].} | ||
|
|
||
| ## Buffer struct is taken from | ||
| # https://github.com/celo-org/celo-threshold-bls-rs/blob/master/crates/threshold-bls-ffi/src/ffi.rs | ||
| type Buffer* = object | ||
| `ptr`*: ptr uint8 | ||
| len*: uint | ||
|
|
||
| #------------------------------ Merkle Tree operations ----------------------------------------- | ||
| proc update_next_member*(ctx: ptr RLN, input_buffer: ptr Buffer): bool {.importc: "set_next_leaf".} | ||
| ## adds an element in the merkle tree to the next available position | ||
| ## input_buffer points to the id commitment byte seq | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc delete_member*(ctx: ptr RLN, index: uint): bool {.importc: "delete_leaf".} | ||
| ## index is the position of the id commitment key to be deleted from the tree | ||
| ## the deleted id commitment key is replaced with a zero leaf | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc get_root*(ctx: ptr RLN, output_buffer: ptr Buffer): bool {.importc: "get_root".} | ||
|
There was a problem hiding this comment. How come this is difference? There was a problem hiding this comment. Current RLN object instantiation is not parametrized by the curve equation (it's hardcoded to Bn254), but this will likely change in the future. Passing the pointer rather than the object is much more efficient and makes more sense IMO since its instantiation and processing should be managed by the RLN library and the object should not be copied around by nim implementation (also the pointer is what is returned by the new_circuit call). |
||
| ## get_root populates the passed pointer output_buffer with the current tree root | ||
| ## the output_buffer holds the Merkle tree root of size 32 bytes | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc get_merkle_proof*(ctx: ptr RLN, index: uint, output_buffer: ptr Buffer): bool {.importc: "get_proof".} | ||
|
There was a problem hiding this comment. I'd probably keep this as qualified merkle proof, as "get proof" could be a ZK proof There was a problem hiding this comment. So is fine like this, or you mean changing the zerokit RLN api name to get_merkle_proof too? There was a problem hiding this comment. This nim wrapper is already named |
||
| ## populates the passed pointer output_buffer with the merkle proof for the leaf at position index in the tree stored by ctx | ||
| ## the output_buffer holds a serialized Merkle proof (vector of 32 bytes nodes) | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc set_leaf*(ctx: ptr RLN, index: uint, input_buffer: ptr Buffer): bool {.importc: "set_leaf".} | ||
| ## sets the leaf at position index in the tree stored by ctx to the value passed by input_buffer | ||
| ## the input_buffer holds a serialized leaf of 32 bytes | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc set_leaves*(ctx: ptr RLN, input_buffer: ptr Buffer): bool {.importc: "set_leaves".} | ||
| ## sets multiple leaves in the tree stored by ctx to the value passed by input_buffer | ||
| ## the input_buffer holds a serialized vector of leaves (32 bytes each) | ||
| ## leaves are set one after each other starting from index 0 | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc reset_tree*(ctx: ptr RLN, tree_height: uint): bool {.importc: "set_tree".} | ||
| ## resets the tree stored by ctx to the the empty tree (all leaves set to 0) of height tree_height | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| #---------------------------------------------------------------------------------------------- | ||
|
|
||
| #-------------------------------- zkSNARKs operations ----------------------------------------- | ||
| proc key_gen*(ctx: ptr RLN, output_buffer: ptr Buffer): bool {.importc: "key_gen".} | ||
| ## generates id key and id commitment key serialized inside keypair_buffer as | id_key <32 bytes>| id_commitment_key <32 bytes> | | ||
| ## id commitment is the poseidon hash of the id key | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc generate_proof*(ctx: ptr RLN, | ||
| input_buffer: ptr Buffer, | ||
| output_buffer: ptr Buffer): bool {.importc: "generate_rln_proof".} | ||
| ## input_buffer has to be serialized as [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal<var> ] | ||
| ## output_buffer holds the proof data and should be parsed as [ proof<128> | share_y<32> | nullifier<32> | root<32> | epoch<32> | share_x<32> | rln_identifier<32> ] | ||
|
There was a problem hiding this comment. How come proof size is different?I find the order of share_/share_y/nullifier a bit odd, any particular reason for changing this order? There was a problem hiding this comment. Proof is 128 bytes since it's compressed: 254 bits for each X coordinate of the 3 proof points + 1 bit for the sign of the y coordinate = 32 bytes per coordinate x 4 coordinates (2x1 over G1, 1x2 over G2) = 128 bytes. Compressed proofs are the default returned by ark-circom create_random_proof_with_reduction and verify_proof, so to make the API the same I should have gone through a decompression and re-compression adding useless overhead. There is probably a way to directly process proofs in uncompressed form (to avoid square roots operation to recover the right y coordinate), but here we discussed how we can benchmark if this makes actually sense. The comment is wrong with respect to the implementation: the current version is The reason for such order of variables was in order to follow the order of the circuit's variables, cf. https://github.com/privacy-scaling-explorations/rln/blob/616ee9b0b085bdf14e7f39df884496b8e77ddc2f/circuits/rln.circom#L5 and https://github.com/privacy-scaling-explorations/rln/blob/616ee9b0b085bdf14e7f39df884496b8e77ddc2f/circuits/rln-base.circom#L60-L62. For serialization/deserialization purposes, this order makes more sense to me since this is the order parameters should be passed to the verify_proof function. However, ultimately, I didn't want to change too much nwaku code, so I ultimately decided the keep the previously implemented order and add rln_identifier at the end. Comment updated in 80dddff |
||
| ## integers wrapped in <> indicate value sizes in bytes | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc verify*(ctx: ptr RLN, | ||
| proof_buffer: ptr Buffer, | ||
| proof_is_valid_ptr: ptr bool): bool {.importc: "verify_rln_proof".} | ||
| ## proof_buffer has to be serialized as [ proof<128> | share_y<32> | nullifier<32> | root<32> | epoch<32> | share_x<32> | rln_identifier<32> | signal_len<8> | signal<var> ] | ||
| ## the return bool value indicates the success or failure of the call to the verify function | ||
| ## the verification of the zk proof is available in result_ptr, where 0 indicates success and 1 is failure | ||
|
|
||
| proc zk_prove*(ctx: ptr RLN, | ||
| input_buffer: ptr Buffer, | ||
| output_buffer: ptr Buffer): bool {.importc: "prove".} | ||
| ## input_buffer serialized as input_data is [ id_key<32> | id_index<8> | epoch<32> | signal_len<8> | signal<var> ] | ||
| ## output_buffer holds the proof data and should be parsed as [ proof<128> | share_y<32> | nullifier<32> | root<32> | epoch<32> | share_x<32> | rln_identifier<32> ] | ||
| ## integers wrapped in <> indicate value sizes in bytes | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
|
|
||
|
|
||
| proc zk_verify*(ctx: ptr RLN, | ||
| proof_buffer: ptr Buffer, | ||
| proof_is_valid_ptr: ptr bool): bool {.importc: "verify".} | ||
|
|
||
| #---------------------------------------------------------------------------------------------- | ||
|
|
||
| #-------------------------------- Common procedures ------------------------------------------- | ||
| proc new_circuit*(tree_height: uint, input_buffer: ptr Buffer, ctx: ptr (ptr RLN)): bool {.importc: "new".} | ||
| ## creates an instance of rln object as defined by the zerokit RLN lib | ||
| ## merkle_depth represent the depth of the Merkle tree | ||
| ## input_buffer contains a serialization of the path where the circuit resources can be found (.r1cs, .wasm, .zkey and optionally the verification_key.json) | ||
| ## ctx holds the final created rln object | ||
| ## the return bool value indicates the success or failure of the operation | ||
|
|
||
| proc hash*(ctx: ptr RLN, | ||
| input_buffer: ptr Buffer, | ||
| output_buffer: ptr Buffer): bool {.importc: "hash".} | ||
| ## it hashes (sha256) the plain text supplied in inputs_buffer and then maps it to a field element | ||
| ## this proc is used to map arbitrary signals to field element for the sake of proof generation | ||
| ## inputs_buffer holds the hash input as a byte seq | ||
| ## the hash output is generated and populated inside output_buffer | ||
| ## the output_buffer contains 32 bytes hash output | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,23 +9,20 @@ import | |
| stew/arrayops, | ||
| ../../utils/protobuf | ||
|
|
||
|
There was a problem hiding this comment. How come we don't have Bn256 anymore? There was a problem hiding this comment. In zerokit RLN I used as default Bn254, the same curve used by circom for generating zkSNARK proofs. Hopefully in the next future, RLN object implementation and API can be extended so that can be instantiatied with any curve from ark-ec. |
||
| ## RLN is a Nim wrapper for the data types used in zerokit RLN | ||
| type RLN* {.incompleteStruct.} = object | ||
|
|
||
| type | ||
| # identity key as defined in https://hackmd.io/tMTLMYmTR5eynw2lwK9n1w?view#Membership | ||
| IDKey* = array[32, byte] | ||
| # hash of identity key as defined ed in https://hackmd.io/tMTLMYmTR5eynw2lwK9n1w?view#Membership | ||
| IDCommitment* = array[32, byte] | ||
|
|
||
| type | ||
| MerkleNode* = array[32, byte] # Each node of the Merkle tee is a Poseidon hash which is a 32 byte value | ||
| RlnIdentifier* = array[32, byte] | ||
| Nullifier* = array[32, byte] | ||
| ZKSNARK* = array[128, byte] | ||
| Epoch* = array[32, byte] | ||
|
|
||
| # Custom data types defined for waku rln relay ------------------------- | ||
|
|
@@ -55,6 +52,8 @@ type RateLimitProof* = object | |
| ## nullifier enables linking two messages published during the same epoch | ||
| ## see details in https://hackmd.io/tMTLMYmTR5eynw2lwK9n1w?view#Nullifiers | ||
| nullifier*: Nullifier | ||
| ## Application specific RLN Identifier | ||
| rlnIdentifier*: RlnIdentifier | ||
|
There was a problem hiding this comment. Good call, we should probably add this to RLN relay spec @staheri14 |
||
|
|
||
| type MembershipIndex* = uint | ||
|
|
||
|
|
@@ -76,7 +75,7 @@ type WakuRLNRelay* = ref object | |
| # TODO may need to erase this ethAccountPrivateKey when is not used | ||
| # TODO may need to make ethAccountPrivateKey mandatory | ||
| ethAccountPrivateKey*: PrivateKey | ||
| rlnInstance*: ptr RLN | ||
| pubsubTopic*: string # the pubsub topic for which rln relay is mounted | ||
| # contentTopic should be of type waku_message.ContentTopic, however, due to recursive module dependency, the underlying type of ContentTopic is used instead | ||
| # TODO a long-term solution is to place types with recursive dependency inside one file | ||
|
|
@@ -98,6 +97,9 @@ const | |
| ETH_CLIENT* = "ws://localhost:8540/" | ||
|
|
||
| const | ||
| # The relative folder where the circuit, proving and verification key for RLN can be found | ||
| # Note that resources has to be compiled with respect to the above MERKLE_TREE_DEPTH | ||
| RLN_RESOURCE_FOLDER* = "vendor/zerokit/rln/resources/tree_height_" & $MERKLE_TREE_DEPTH & "/" | ||
|
There was a problem hiding this comment. In order to not have duplicates of the circuit in nwaku, I thought that it is better to point to zerokit resource folder directly. Since nwaku is calling zerokit API from a different path than the latter, the resource folders should be passed to correctly locate the circuit (i.e., this string cannot be hardcoded in zerokit) There was a problem hiding this comment. Hm, I don't know how exactly but I suspect this might fail when we do distribution. In general vendoring and static resources should be kept separate. Perhaps OK for initial integration where we compile stuff ourselves. cc @jm-clius |
||
| # the size of poseidon hash output in bits | ||
| HASH_BIT_SIZE* = 256 | ||
| # the size of poseidon hash output as the number hex digits | ||
|
|
@@ -314,7 +316,7 @@ const | |
| # STATIC_GROUP_MERKLE_ROOT is the root of the Merkle tree constructed from the STATIC_GROUP_KEYS above | ||
| # only identity commitments are used for the Merkle tree construction | ||
| # the root is created locally, using createMembershipList proc from waku_rln_relay_utils module, and the result is hardcoded in here | ||
| STATIC_GROUP_MERKLE_ROOT* = "2af1e8ca39f5b1c3b097d37c2281da47b5a9b5cdde0525fe51dd929c27cb5400" | ||
|
|
||
| const EPOCH_UNIT_SECONDS* = float64(10) # the rln-relay epoch length in seconds | ||
| const MAX_CLOCK_GAP_SECONDS* = 20.0 # the maximum clock difference between peers in seconds | ||
|
|
@@ -351,6 +353,10 @@ proc init*(T: type RateLimitProof, buffer: seq[byte]): ProtoResult[T] = | |
| discard ? pb.getField(6, nullifier) | ||
| discard nsp.nullifier.copyFrom(nullifier) | ||
|
|
||
| var rlnIdentifier: seq[byte] | ||
| discard ? pb.getField(7, rlnIdentifier) | ||
| discard nsp.rlnIdentifier.copyFrom(rlnIdentifier) | ||
|
|
||
| return ok(nsp) | ||
|
|
||
| proc encode*(nsp: RateLimitProof): ProtoBuffer = | ||
|
|
@@ -362,6 +368,7 @@ proc encode*(nsp: RateLimitProof): ProtoBuffer = | |
| output.write3(4, nsp.shareX) | ||
| output.write3(5, nsp.shareY) | ||
| output.write3(6, nsp.nullifier) | ||
| output.write3(7, nsp.rlnIdentifier) | ||
|
There was a problem hiding this comment. Requires spec change I believe There was a problem hiding this comment. Tracking issue: vacp2p/rfc#523 |
||
|
|
||
| output.finish3() | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How come this isn't needed anymore?