feat: autosharding core algorithm (#1854)

- basic rendezvous hashing
- content topic parsing
- sharding config
- tests

apps/wakubridge/message_compat.nim

@@ -4,6 +4,7 @@ else:
   {.push raises: [].}
 
 import
+  std/options,
   stew/[byteutils, results],
   libp2p/crypto/crypto
 import
@@ -35,6 +36,8 @@ proc toV2ContentTopic*(v1Topic: waku_protocol.Topic): ContentTopic =
   ## <v1-topic-bytes-as-hex> should be prefixed with `0x`
   var namespacedTopic = NsContentTopic()
 
+  namespacedTopic.generation = none(int)
+  namespacedTopic.bias = Unbiased
   namespacedTopic.application = ContentTopicApplication
   namespacedTopic.version = ContentTopicAppVersion
   namespacedTopic.name = v1Topic.to0xHex()

tests/v2/test_utils_compat.nim

@@ -3,7 +3,8 @@
 import
   testutils/unittests
 import
-  ../../waku/v2/waku_core,
+  ../../waku/v2/waku_core/message,
+  ../../waku/v2/waku_core/time,
   ../../waku/v2/utils/compat,
   ./testlib/common
 

tests/v2/waku_core/test_namespaced_topics.nim

@@ -1,6 +1,7 @@
 {.used.}
 
 import
+  std/options,
   stew/results,
   testutils/unittests
 import
@@ -11,6 +12,8 @@ suite "Waku Message - Content topics namespacing":
   test "Stringify namespaced content topic":
     ## Given
     var ns = NsContentTopic()
+    ns.generation = none(int)
+    ns.bias = Unbiased
     ns.application = "toychat"
     ns.version = "2"
     ns.name = "huilong"
@@ -31,10 +34,31 @@ suite "Waku Message - Content topics namespacing":
     let nsRes = NsContentTopic.parse(topic)
 
     ## Then
-    check nsRes.isOk()
+    assert nsRes.isOk(), $nsRes.error
 
     let ns = nsRes.get()
     check:
+      ns.generation == none(int)
+      ns.bias == Unbiased
+      ns.application == "toychat"
+      ns.version == "2"
+      ns.name == "huilong"
+      ns.encoding == "proto"
+
+  test "Parse content topic string - Valid string with sharding":
+    ## Given
+    let topic = "/0/lower20/toychat/2/huilong/proto"
+
+    ## When
+    let nsRes = NsContentTopic.parse(topic)
+
+    ## Then
+    assert nsRes.isOk(), $nsRes.error
+
+    let ns = nsRes.get()
+    check:
+      ns.generation == some(0)
+      ns.bias == Lower20
       ns.application == "toychat"
       ns.version == "2"
       ns.name == "huilong"
@@ -48,7 +72,8 @@ suite "Waku Message - Content topics namespacing":
     let ns = NsContentTopic.parse(topic)
 
     ## Then
-    check ns.isErr()
+    assert ns.isErr(), $ns.get()
+
     let err = ns.tryError()
     check:
       err.kind == ParsingErrorKind.InvalidFormat
@@ -62,13 +87,13 @@ suite "Waku Message - Content topics namespacing":
     let ns = NsContentTopic.parse(topic)
 
     ## Then
-    check ns.isErr()
+    assert ns.isErr(), $ns.get()
+
     let err = ns.tryError()
     check:
       err.kind == ParsingErrorKind.InvalidFormat
       err.cause == "invalid topic structure"
 
-
   test "Parse content topic string - Invalid string: missing encoding part":
     ## Given
     let topic = "/toychat/2/huilong"
@@ -77,26 +102,57 @@ suite "Waku Message - Content topics namespacing":
     let ns = NsContentTopic.parse(topic)
 
     ## Then
-    check ns.isErr()
+    assert ns.isErr(), $ns.get()
+
     let err = ns.tryError()
     check:
       err.kind == ParsingErrorKind.InvalidFormat
       err.cause == "invalid topic structure"
 
-  test "Parse content topic string - Invalid string: too many parts":
+  test "Parse content topic string - Invalid string: wrong extra parts":
     ## Given
     let topic = "/toychat/2/huilong/proto/33"
 
     ## When
     let ns = NsContentTopic.parse(topic)
 
     ## Then
-    check ns.isErr()
+    assert ns.isErr(), $ns.get()
+
     let err = ns.tryError()
     check:
       err.kind == ParsingErrorKind.InvalidFormat
       err.cause == "invalid topic structure"
 
+  test "Parse content topic string - Invalid string: non numeric generation":
+    ## Given
+    let topic = "/first/unbiased/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic)
+
+    ## Then
+    assert ns.isErr(), $ns.get()
+
+    let err = ns.tryError()
+    check:
+      err.kind == ParsingErrorKind.InvalidFormat
+      err.cause == "generation should be a numeric value"
+
+  test "Parse content topic string - Invalid string: invalid bias":
+    ## Given
+    let topic = "/0/no/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic)
+
+    ## Then
+    assert ns.isErr(), $ns.get()
+
+    let err = ns.tryError()
+    check:
+      err.kind == ParsingErrorKind.InvalidFormat
+      err.cause == "bias should be one of; unbiased, lower20 or higher80"
 
 suite "Waku Message - Pub-sub topics namespacing":
 
@@ -178,7 +234,6 @@ suite "Waku Message - Pub-sub topics namespacing":
       err.kind == ParsingErrorKind.MissingPart
       err.part == "shard_cluster_index"
 
-
   test "Parse static sharding pub-sub topic string - Invalid string: cluster value":
     ## Given
     let topic = "/waku/2/rs/xx/77"

tests/v2/waku_core/test_sharding.nim

@@ -0,0 +1,178 @@
+{.used.}
+
+import
+  std/options,
+  std/strutils,
+  std/sugar,
+  std/algorithm,
+  std/random,
+  stew/results,
+  testutils/unittests
+import
+  ../../../waku/v2/waku_core/topics
+
+suite "Waku Sharding":
+
+  randomize()
+
+  const WordLength = 5
+
+  proc randomContentTopic(): NsContentTopic =
+    var app = ""
+
+    for n in 0..<WordLength:
+      let letter = sample(Letters)
+      app.add(letter)
+
+    let version = "1"
+
+    var name = ""
+
+    for n in 0..<WordLength:
+      let letter = sample(Letters)
+      name.add(letter)
+
+    let enc = "cbor"
+
+    NsContentTopic.init(none(int), Unbiased, app, version, name, enc)
+
+  test "Implicit content topic generation":
+    ## Given
+    let topic = "/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardCount(ns)
+
+    ## Then
+    assert paramRes.isOk(), paramRes.error
+
+    let count = paramRes.get()
+    check:
+      count == GenerationZeroShardsCount
+      ns.bias == Unbiased
+
+  test "Valid content topic":
+    ## Given
+    let topic = "/0/lower20/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardCount(ns)
+
+    ## Then
+    assert paramRes.isOk(), paramRes.error
+
+    let count = paramRes.get()
+    check:
+      count == GenerationZeroShardsCount
+      ns.bias == Lower20
+
+  test "Invalid content topic generation":
+    ## Given
+    let topic = "/1/unbiased/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardCount(ns)
+
+    ## Then
+    assert paramRes.isErr(), $paramRes.get()
+
+    let err = paramRes.error
+    check:
+      err == "Generation > 0 are not supported yet"
+
+  test "Weigths bias":
+    ## Given
+    let count = 5
+
+    ## When
+    let anonWeigths = biasedWeights(count, ShardingBias.Lower20)
+    let speedWeigths = biasedWeights(count, ShardingBias.Higher80)
+
+    ## Then
+    check:
+      anonWeigths[0] == 2.0
+      anonWeigths[1] == 1.0
+      anonWeigths[2] == 1.0
+      anonWeigths[3] == 1.0
+      anonWeigths[4] == 1.0
+
+      speedWeigths[0] == 1.0
+      speedWeigths[1] == 2.0
+      speedWeigths[2] == 2.0
+      speedWeigths[3] == 2.0
+      speedWeigths[4] == 2.0
+
+  test "Sorted shard list":
+    ## Given
+    let topic = "/0/unbiased/toychat/2/huilong/proto"
+
+    ## When
+    let contentTopic = NsContentTopic.parse(topic).expect("Parsing")
+    let count = shardCount(contentTopic).expect("Valid parameters")
+    let weights = biasedWeights(count, contentTopic.bias)
+
+    let shardsRes = weightedShardList(contentTopic, count, weights)
+
+    ## Then
+    assert shardsRes.isOk(), shardsRes.error
+
+    let shards = shardsRes.get()
+    check:
+      shards.len == count
+      isSorted(shards, hashOrder)
+
+  test "Shard Choice Reproducibility":
+    ## Given
+    let topic = "/toychat/2/huilong/proto"
+
+    ## When
+    let contentTopic = NsContentTopic.parse(topic).expect("Parsing")
+
+    let res = singleHighestWeigthShard(contentTopic)
+
+    ## Then
+    assert res.isOk(), res.error
+
+    let pubsubTopic = res.get()
+
+    check:
+      pubsubTopic == NsPubsubTopic.staticSharding(ClusterIndex, 3)
+
+  test "Shard Choice Simulation":
+    ## Given
+    let topics = collect:
+      for i in 0..<100000:
+        randomContentTopic()
+
+    var counts = newSeq[0](GenerationZeroShardsCount)
+
+    ## When
+    for topic in topics:
+      let pubsub = singleHighestWeigthShard(topic).expect("Valid Topic")
+      counts[pubsub.shard] += 1
+
+    ## Then
+    for i in 1..<GenerationZeroShardsCount:
+      check:
+        float64(counts[i - 1]) <= (float64(counts[i]) * 1.05)
+        float64(counts[i]) <= (float64(counts[i - 1]) * 1.05)
+        float64(counts[i - 1]) >= (float64(counts[i]) * 0.95)
+        float64(counts[i]) >= (float64(counts[i - 1]) * 0.95)
+
+    #echo counts
+
+
+
+
+
+
+
+
+
+

waku/v2/waku_core/topics.nim

@@ -1,7 +1,9 @@
 import
   ./topics/content_topic,
-  ./topics/pubsub_topic
+  ./topics/pubsub_topic,
+  ./topics/sharding
 
 export
   content_topic,
-  pubsub_topic
+  pubsub_topic,
+  sharding