Rln-relay integration into chat2 (#835)

* adds ProofMetadata

* adds EPOCH_INTERVAL

* adds messageLog field

* adds updateLog, toEpoch, fromEpoch, getEpoch, compareTo

* adds unit test for toEpoch and fromEpoch

* adds unit test for Epoch comparison

* adds result codes for updateLog

* adds unit test for update log

* renames epoch related consts

* modifies updateLog with new return type and new logic of spam detection

* adds unit text for the modified updateLog

* changes max epoch gap type size

* splits updateLog into two procs isSpam and updateLog

* updates unittests

* fixes a bug, returns false when the message is not spam

* renames messageLog to nullifierLog

* renames isSpam to hasDuplicate

* updates the rln validator, adds comments

* adds appendRLNProof proc plus some code beatification

* unit test for validate message

* adds unhappy test to validateMessage unit test

* renames EPOCH_UNIT_SECONDS

* renames MAX_CLOCK_GAP_SECONDS

* WIP: integration test

* fixes compile errors

* sets a real epoch value

* updates on old unittests

* adds comments to the rln relay tests

* adds more comments

* makes rln import conditional

* adds todos

* adds more todos

* adds rln-relay mount process into chat2

* further todos

* logs contentTopic

* introduces rln relay configs

* changes default pubsub topic

* adds contentTopic config

* imports rln relay dependencies

* consolidates imports

* removes module identifier from ContentTopic

* adds contentTopic field

* adds contentTopic argument to mountRlnRelay calls

* appends rln proof to chat2 messages

* changes the default chat2 contentTopic

* adds missing content topic fields

* fixes a bug

* adds a new logic about empty content topics

* appends proof only when rln flag is active

* removes unnecessary todos

* fixes an indentation issue

* adds log messages

* verifies the proof against the concatenation of msg payload and content topic

* a bug fix

* removes duplicate epoch time calculation

* updates log level to trace

* updates default rln-relay content topic

* adds support for empty content topics

* updates changelog

* changelog updates

* removes a commented code block

* updates addRLNRelayValidator string doc

CHANGELOG.md

@@ -1,3 +1,22 @@
+##  Next Release
+Release highlights:
+
+- 
+
+The full list of changes is below.
+
+### Features
+
+- The `waku-rln-relay` now supports spam-protection for a specific combination of `pubsubTopic` and `contentTopic` (available under the `rln` compiler flag).  
+- The `waku-rln-relay` protocol in integrated into `chat2` (available under the`rln` compiler flag)
+
+### Changes
+
+-  ...
+### Fixes
+
+- ...
+
 ##  2021-01-19 v0.7
 
 Release highlights:

examples/v2/chat2.nim

@@ -27,6 +27,13 @@ import   ../../waku/v2/node/[wakunode2, waku_payload],
          ../../waku/common/utils/nat,
          ./config_chat2
 
+when defined(rln):
+  import
+    libp2p/protocols/pubsub/rpc/messages,
+    libp2p/protocols/pubsub/pubsub,
+    web3,
+    ../../waku/v2/protocol/waku_rln_relay/[rln, waku_rln_relay_utils]
+
 const Help = """
   Commands: /[?|help|connect|nick|exit]
   help: Prints this help
@@ -191,7 +198,8 @@ proc readNick(transp: StreamTransport): Future[string] {.async.} =
 
 proc publish(c: Chat, line: string) =
   # First create a Chat2Message protobuf with this line of text
-  let chat2pb = Chat2Message(timestamp: getTime().toUnix(),
+  let time = getTime().toUnix()
+  let chat2pb = Chat2Message(timestamp: time,
                              nick: c.nick,
                              payload: line.toBytes()).encode()
 
@@ -206,8 +214,17 @@ proc publish(c: Chat, line: string) =
       version = 1'u32
       encodedPayload = payload.encode(version, c.node.rng[])
     if encodedPayload.isOk():
-      let message = WakuMessage(payload: encodedPayload.get(),
+      var message = WakuMessage(payload: encodedPayload.get(),
         contentTopic: c.contentTopic, version: version)
+      when defined(rln):
+        if  not isNil(c.node.wakuRlnRelay):
+          # for future version when we support more than one rln protected content topic, 
+          # we should check the message content topic as well
+          let success = c.node.wakuRlnRelay.appendRLNProof(message, float64(time))
+          if not success:
+            debug "could not append rate limit proof to the message", success=success
+          else:
+            debug "rate limit proof is appended to the message", success=success
       if not c.node.wakuLightPush.isNil():
         # Attempt lightpush
         asyncSpawn c.node.lightpush(DefaultTopic, message, handler)
@@ -217,8 +234,18 @@ proc publish(c: Chat, line: string) =
       warn "Payload encoding failed", error = encodedPayload.error
   else:
     # No payload encoding/encryption from Waku
-    let message = WakuMessage(payload: chat2pb.buffer,
+    var message = WakuMessage(payload: chat2pb.buffer,
       contentTopic: c.contentTopic, version: 0)
+    when defined(rln):
+      if  not isNil(c.node.wakuRlnRelay):
+        # for future version when we support more than one rln protected content topic, 
+        # we should check the message content topic as well
+        let success = c.node.wakuRlnRelay.appendRLNProof(message, float64(time))
+        if not success:
+          debug "could not append rate limit proof to the message", success=success
+        else:
+          debug "rate limit proof is appended to the message", success=success
+
     if not c.node.wakuLightPush.isNil():
       # Attempt lightpush
       asyncSpawn c.node.lightpush(DefaultTopic, message, handler)
@@ -326,7 +353,6 @@ proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
       wsBindPort = Port(uint16(conf.websocketPort) + conf.portsShift),
       wsEnabled = conf.websocketSupport,
       wssEnabled = conf.websocketSecureSupport)
-
   await node.start()
 
   node.mountRelay(conf.topics.split(" "),
@@ -465,6 +491,33 @@ proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
     let topic = cast[Topic](DefaultTopic)
     node.subscribe(topic, handler)
 
+    when defined(rln): 
+      if conf.rlnRelay:
+        info "WakuRLNRelay is enabled"
+
+        # set up rln relay inputs
+        let (groupOpt, memKeyPairOpt, memIndexOpt) = rlnRelaySetUp(conf.rlnRelayMemIndex)
+        if memIndexOpt.isNone:
+          error "failed to mount WakuRLNRelay"
+        else:
+          # mount rlnrelay in offline mode (for now)
+          waitFor node.mountRlnRelay(groupOpt = groupOpt, memKeyPairOpt = memKeyPairOpt, memIndexOpt= memIndexOpt, onchainMode = false, pubsubTopic = conf.rlnRelayPubsubTopic, contentTopic = conf.rlnRelayContentTopic)
+
+          trace "membership id key", idkey=memKeyPairOpt.get().idKey.toHex
+          trace "membership id commitment key", idCommitmentkey=memKeyPairOpt.get().idCommitment.toHex
+
+          # check the correct construction of the tree by comparing the calculated root against the expected root
+          # no error should happen as it is already captured in the unit tests
+          # TODO have added this check to account for unseen corner cases, will remove it later 
+          let 
+            root = node.wakuRlnRelay.rlnInstance.getMerkleRoot.value.toHex() 
+            expectedRoot = STATIC_GROUP_MERKLE_ROOT
+          if root != expectedRoot:
+            error "root mismatch: something went wrong not in Merkle tree construction"
+          trace "the calculated root", root
+          trace "WakuRLNRelay is mounted successfully", pubsubtopic=conf.rlnRelayPubsubTopic, contentTopic=conf.rlnRelayContentTopic
+
+
   await chat.readWriteLoop()
 
   if conf.keepAlive:

examples/v2/config_chat2.nim

@@ -5,8 +5,9 @@ import
   libp2p/crypto/crypto,
   libp2p/crypto/secp,
   nimcrypto/utils,
-  eth/keys
-
+  eth/keys,
+  ../../waku/v2/protocol/waku_rln_relay/waku_rln_relay_types,
+  ../../waku/v2/protocol/waku_message
 type
   Fleet* =  enum
     none
@@ -75,11 +76,6 @@ type
       defaultValue: true
       name: "relay" }: bool
 
-    rlnRelay* {.
-      desc: "Enable spam protection through rln-relay: true|false",
-      defaultValue: false
-      name: "rln-relay" }: bool
-
     staticnodes* {.
       desc: "Peer multiaddr to directly connect with. Argument may be repeated."
       name: "staticnode" }: seq[string]
@@ -231,6 +227,28 @@ type
       defaultValue: false
       name: "websocket-secure-support" }: bool
 
+    ## rln-relay configuration
+
+    rlnRelay* {.
+      desc: "Enable spam protection through rln-relay: true|false",
+      defaultValue: false
+      name: "rln-relay" }: bool
+
+    rlnRelayMemIndex* {.
+      desc: "(experimental) the index of node in the rln-relay group: a value between 0-99 inclusive",
+      defaultValue: MembershipIndex(0)
+      name: "rln-relay-membership-index" }: MembershipIndex
+
+    rlnRelayContentTopic* {.
+      desc: "the pubsub topic for which rln-relay gets enabled",
+      defaultValue: "waku/2/rln-relay/proto"
+      name: "rln-relay-content-topic" }: ContentTopic
+
+    rlnRelayPubsubTopic* {.
+      desc: "the pubsub topic for which rln-relay gets enabled",
+      defaultValue: "/waku/2/default-waku/proto"
+      name: "rln-relay-pubsub-topic" }: string
+
 # NOTE: Keys are different in nim-libp2p
 proc parseCmdArg*(T: type crypto.PrivateKey, p: TaintedString): T =
   try:

tests/v2/test_waku_rln_relay.nim

@@ -12,6 +12,7 @@ import
   ./test_utils
 
 const RLNRELAY_PUBSUB_TOPIC = "waku/2/rlnrelay/proto"
+const RLNRELAY_CONTENT_TOPIC = "waku/2/rlnrelay/proto"
 
 # POSEIDON_HASHER_CODE holds the bytecode of Poseidon hasher solidity smart contract: 
 # https://github.com/kilic/rlnapp/blob/master/packages/contracts/contracts/crypto/PoseidonHasher.sol 
@@ -250,7 +251,14 @@ procSuite "Waku rln relay":
 
       # start rln-relay
       node.mountRelay(@[RLNRELAY_PUBSUB_TOPIC])
-      await node.mountRlnRelay(ethClientAddrOpt = some(ETH_CLIENT), ethAccAddrOpt =  some(ethAccountAddress), memContractAddOpt =  some(membershipContractAddress), groupOpt = some(group), memKeyPairOpt = some(keypair.get()),  memIndexOpt = some(index), pubsubTopic = RLNRELAY_PUBSUB_TOPIC)
+      await node.mountRlnRelay(ethClientAddrOpt = some(EthClient), 
+                              ethAccAddrOpt =  some(ethAccountAddress), 
+                              memContractAddOpt =  some(membershipContractAddress), 
+                              groupOpt = some(group), 
+                              memKeyPairOpt = some(keypair.get()),  
+                              memIndexOpt = some(index), 
+                              pubsubTopic = RLNRELAY_PUBSUB_TOPIC,
+                              contentTopic = RLNRELAY_CONTENT_TOPIC)
       let calculatedRoot = node.wakuRlnRelay.rlnInstance.getMerkleRoot().value().toHex
       debug "calculated root ", calculatedRoot
 
@@ -286,7 +294,12 @@ procSuite "Waku rln relay":
 
     # -------- mount rln-relay in the off-chain mode
     node.mountRelay(@[RLNRELAY_PUBSUB_TOPIC])
-    await node.mountRlnRelay(groupOpt = some(groupIDCommitments), memKeyPairOpt = some(groupKeyPairs[index]),  memIndexOpt = some(index), onchainMode = false, pubsubTopic = RLNRELAY_PUBSUB_TOPIC)
+    await node.mountRlnRelay(groupOpt = some(groupIDCommitments),
+                            memKeyPairOpt = some(groupKeyPairs[index]),
+                            memIndexOpt = some(index), 
+                            onchainMode = false, 
+                            pubsubTopic = RLNRELAY_PUBSUB_TOPIC,
+                            contentTopic = RLNRELAY_CONTENT_TOPIC)
 
     # get the root of Merkle tree which is constructed inside the mountRlnRelay proc
     let calculatedRoot = node.wakuRlnRelay.rlnInstance.getMerkleRoot().value().toHex

tests/v2/test_wakunode.nim

@@ -2,6 +2,7 @@
 
 import
   testutils/unittests,
+  std/sequtils,
   chronicles, chronos, stew/shims/net as stewNet, stew/byteutils, std/os,
   libp2p/crypto/crypto,
   libp2p/crypto/secp,
@@ -617,7 +618,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt1, 
                                   memIndexOpt= memIndexOpt1, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node1.start() 
 
       # node 2
@@ -628,7 +630,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt2, 
                                   memIndexOpt= memIndexOpt2, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node2.start()
 
       # node 3
@@ -639,7 +642,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt3, 
                                   memIndexOpt= memIndexOpt3, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node3.start()
 
       # connect them together
@@ -663,19 +667,11 @@ procSuite "WakuNode":
       let payload = "Hello".toBytes()
 
       # prepare the epoch
-      let epoch = getCurrentEpoch()
+      let epoch = getCurrentEpoch()    
 
-      # prepare the proof
-      let rateLimitProofRes = node1.wakuRlnRelay.rlnInstance.proofGen(data = payload, 
-                                                                memKeys = node1.wakuRlnRelay.membershipKeyPair, 
-                                                                memIndex = node1.wakuRlnRelay.membershipIndex, 
-                                                                epoch = epoch)
-      doAssert(rateLimitProofRes.isOk())
-      let rateLimitProof = rateLimitProofRes.value     
-
-      let message = WakuMessage(payload: @payload, 
-                                contentTopic: contentTopic,  
-                                proof: rateLimitProof)
+      var message = WakuMessage(payload: @payload, 
+                                contentTopic: contentTopic)
+      doAssert(node1.wakuRlnRelay.appendRLNProof(message, epochTime()))
 
 
       ## node1 publishes a message with a rate limit proof, the message is then relayed to node2 which in turn 
@@ -715,7 +711,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt1, 
                                   memIndexOpt= memIndexOpt1, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node1.start() 
 
       # node 2
@@ -726,7 +723,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt2, 
                                   memIndexOpt= memIndexOpt2, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node2.start()
 
       # node 3
@@ -737,7 +735,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt3, 
                                   memIndexOpt= memIndexOpt3, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node3.start()
 
       # connect them together
@@ -765,7 +764,10 @@ procSuite "WakuNode":
       let epoch = getCurrentEpoch()
 
       # prepare the proof
-      let rateLimitProofRes = node1.wakuRlnRelay.rlnInstance.proofGen(data = payload, 
+      let 
+        contentTopicBytes = contentTopic.toBytes
+        input = concat(payload, contentTopicBytes)
+        rateLimitProofRes = node1.wakuRlnRelay.rlnInstance.proofGen(data = input, 
                                                                 memKeys = node1.wakuRlnRelay.membershipKeyPair, 
                                                                 memIndex = MembershipIndex(4), 
                                                                 epoch = epoch)
@@ -817,7 +819,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt1, 
                                   memIndexOpt= memIndexOpt1, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node1.start() 
 
       # node 2
@@ -828,7 +831,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt2, 
                                   memIndexOpt= memIndexOpt2, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node2.start()
 
       # node 3
@@ -839,7 +843,8 @@ procSuite "WakuNode":
                                   memKeyPairOpt = memKeyPairOpt3, 
                                   memIndexOpt= memIndexOpt3, 
                                   onchainMode = false, 
-                                  pubsubTopic = rlnRelayPubSubTopic)
+                                  pubsubTopic = rlnRelayPubSubTopic,
+                                  contentTopic = contentTopic)
       await node3.start()
 
       # connect the nodes together node1 <-> node2 <-> node3
@@ -850,15 +855,15 @@ procSuite "WakuNode":
       let time = epochTime()
       #  create some messages with rate limit proofs
       var 
-        wm1 = WakuMessage(payload: "message 1".toBytes())
+        wm1 = WakuMessage(payload: "message 1".toBytes(), contentTopic: contentTopic)
         proofAdded1 = node3.wakuRlnRelay.appendRLNProof(wm1, time)
         # another message in the same epoch as wm1, it will break the messaging rate limit
-        wm2 = WakuMessage(payload: "message2".toBytes())
+        wm2 = WakuMessage(payload: "message2".toBytes(), contentTopic: contentTopic)
         proofAdded2 = node3.wakuRlnRelay.appendRLNProof(wm2, time)
         #  wm3 points to the next epoch 
-        wm3 = WakuMessage(payload: "message 3".toBytes())
+        wm3 = WakuMessage(payload: "message 3".toBytes(), contentTopic: contentTopic)
         proofAdded3 = node3.wakuRlnRelay.appendRLNProof(wm3, time+EPOCH_UNIT_SECONDS)
-        wm4 = WakuMessage(payload: "message4".toBytes())
+        wm4 = WakuMessage(payload: "message4".toBytes(), contentTopic: contentTopic)
 
       #  check proofs are added correctly
       check:

waku/v2/node/config.nim

@@ -6,7 +6,8 @@ import
   libp2p/crypto/secp,
   nimcrypto/utils,
   eth/keys,
-  ../protocol/waku_rln_relay/[waku_rln_relay_types]
+  ../protocol/waku_rln_relay/waku_rln_relay_types,
+  ../protocol/waku_message
 
 type
   WakuNodeConf* = object
@@ -83,8 +84,13 @@ type
 
     rlnRelayPubsubTopic* {.
       desc: "the pubsub topic for which rln-relay gets enabled",
-      defaultValue: "waku/2/rlnrelay/proto"
+      defaultValue: "/waku/2/default-waku/proto"
       name: "rln-relay-pubsub-topic" }: string
+
+    rlnRelayContentTopic* {.
+      desc: "the pubsub topic for which rln-relay gets enabled",
+      defaultValue: "/toy-chat/2/huilong/proto"
+      name: "rln-relay-content-topic" }: ContentTopic
 
     staticnodes* {.
       desc: "Peer multiaddr to directly connect with. Argument may be repeated."