feat: autosharding core algorithm

apps/wakubridge/message_compat.nim

@@ -35,6 +35,8 @@ proc toV2ContentTopic*(v1Topic: waku_protocol.Topic): ContentTopic =
   ## <v1-topic-bytes-as-hex> should be prefixed with `0x`
   var namespacedTopic = NsContentTopic()
 
+  namespacedTopic.generation = "0"
+  namespacedTopic.bias = "none"
   namespacedTopic.application = ContentTopicApplication
   namespacedTopic.version = ContentTopicAppVersion
   namespacedTopic.name = v1Topic.to0xHex()

tests/v2/test_utils_compat.nim

@@ -3,7 +3,8 @@
 import
   testutils/unittests
 import
-  ../../waku/v2/waku_core,
+  ../../waku/v2/waku_core/message,
+  ../../waku/v2/waku_core/time,
   ../../waku/v2/utils/compat,
   ./testlib/common
 

tests/v2/waku_core/test_namespaced_topics.nim

@@ -11,6 +11,8 @@ suite "Waku Message - Content topics namespacing":
   test "Stringify namespaced content topic":
     ## Given
     var ns = NsContentTopic()
+    ns.generation = "0"
+    ns.bias = "none"
     ns.application = "toychat"
     ns.version = "2"
     ns.name = "huilong"
@@ -21,11 +23,11 @@ suite "Waku Message - Content topics namespacing":
 
     ## Then
     check:
-      topic == "/toychat/2/huilong/proto"
+      topic == "/0/none/toychat/2/huilong/proto"
 
   test "Parse content topic string - Valid string":
     ## Given
-    let topic = "/toychat/2/huilong/proto"
+    let topic = "/0/none/toychat/2/huilong/proto"
 
     ## When
     let nsRes = NsContentTopic.parse(topic)
@@ -35,6 +37,8 @@ suite "Waku Message - Content topics namespacing":
 
     let ns = nsRes.get()
     check:
+      ns.generation == "0"
+      ns.bias == "none"
       ns.application == "toychat"
       ns.version == "2"
       ns.name == "huilong"
@@ -68,7 +72,6 @@ suite "Waku Message - Content topics namespacing":
       err.kind == ParsingErrorKind.InvalidFormat
       err.cause == "invalid topic structure"
 
-
   test "Parse content topic string - Invalid string: missing encoding part":
     ## Given
     let topic = "/toychat/2/huilong"
@@ -97,6 +100,20 @@ suite "Waku Message - Content topics namespacing":
       err.kind == ParsingErrorKind.InvalidFormat
       err.cause == "invalid topic structure"
 
+  test "Parse content topic string - Invalid string: missing sharding data":
+    ## Given
+    let topic = "/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic)
+
+    ## Then
+    check ns.isErr()
+    let err = ns.tryError()
+    check:
+      err.kind == ParsingErrorKind.InvalidFormat
+      err.cause == "invalid topic structure"
+
 
 suite "Waku Message - Pub-sub topics namespacing":
 
@@ -178,7 +195,6 @@ suite "Waku Message - Pub-sub topics namespacing":
       err.kind == ParsingErrorKind.MissingPart
       err.part == "shard_cluster_index"
 
-
   test "Parse static sharding pub-sub topic string - Invalid string: cluster value":
     ## Given
     let topic = "/waku/2/rs/xx/77"

tests/v2/waku_core/test_sharding.nim

@@ -0,0 +1,180 @@
+{.used.}
+
+import
+  std/strutils,
+  std/sequtils,
+  std/sugar,
+  std/algorithm,
+  std/random,
+  stew/results,
+  testutils/unittests
+import
+  ../../../waku/v2/waku_core/topics
+
+suite "Waku Sharding":
+
+  randomize()
+
+  const WordLength = 5
+
+  proc randomContentTopic(): NsContentTopic =
+    let gen = "0"
+    let bias = "none"
+
+    var app = ""
+
+    for n in 0..<WordLength:
+      let letter = sample(Letters)
+      app.add(letter)
+
+    let version = "1"
+
+    var name = ""
+
+    for n in 0..<WordLength:
+      let letter = sample(Letters)
+      name.add(letter)
+
+    let enc = "cbor"
+
+    NsContentTopic.init(gen, bias, app, version, name, enc)
+
+
+  test "Valid generation & sharding bias":
+    ## Given
+    let topic = "/0/none/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardingParam(ns)
+
+    ## Then
+    check paramRes.isOk()
+
+    let (count, bias) = paramRes.get()
+    check:
+      count == GenerationZeroShardsCount
+      bias == ShardingBias.None
+
+  test "Invalid generation":
+    ## Given
+    let topic = "/1/none/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardingParam(ns)
+
+    ## Then
+    check paramRes.isErr()
+    let err = paramRes.tryError()
+    check:
+      err == "Generation > 0 are not supported yet"
+
+  test "Invalid bias":
+    ## Given
+    let topic = "/0/kanonymity/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardingParam(ns)
+
+    ## Then
+    check paramRes.isErr()
+    let err = paramRes.tryError()
+    check:
+      err.startsWith("Cannot parse sharding bias: ")
+
+  test "Weigths bias":
+    ## Given
+    let count = 5
+
+    ## When
+    let anonWeigths = biasedWeights(count, ShardingBias.Kanonymity)
+    let speedWeigths = biasedWeights(count, ShardingBias.Throughput)
+
+    ## Then
+    check:
+      anonWeigths[0] == 2.0
+      anonWeigths[1] == 1.0
+      anonWeigths[2] == 1.0
+      anonWeigths[3] == 1.0
+      anonWeigths[4] == 1.0
+
+      speedWeigths[0] == 1.0
+      speedWeigths[1] == 2.0
+      speedWeigths[2] == 2.0
+      speedWeigths[3] == 2.0
+      speedWeigths[4] == 2.0
+
+  test "Sorted shard list":
+    ## Given
+    let topic = "/0/none/toychat/2/huilong/proto"
+
+    ## When
+    let contentTopic = NsContentTopic.parse(topic).expect("Parsing")
+    let (count, bias) = shardingParam(contentTopic).expect("Valid parameters")
+
+    let weigths = biasedWeights(count, bias)
+
+    let shardsRes = weightedShardList(contentTopic, count, weigths)
+
+    ## Then
+    check shardsRes.isOk()
+
+    let shards = shardsRes.get()
+    check:
+      shards.len == count
+      isSorted(shards, hashOrder)
+
+  test "Shard Choice Reproducibility":
+    ## Given
+    let topic = "/0/none/toychat/2/huilong/proto"
+
+    ## When
+    let contentTopic = NsContentTopic.parse(topic).expect("Parsing")
+
+    let res = singleHighestWeigthShard(contentTopic)
+
+    ## Then
+    check res.isOk()
+
+    let pubsubTopic = res.get()
+
+    check:
+      pubsubTopic == NsPubsubTopic.staticSharding(ClusterIndex, 2)
+
+  test "Shard Choice Simulation":
+    ## Given
+    let topics = collect:
+      for i in 0..<100000:
+        randomContentTopic()
+
+    var counts = newSeq[0](GenerationZeroShardsCount)
+
+    ## When
+    for topic in topics:
+      let pubsub = singleHighestWeigthShard(topic).expect("Valid Topic")
+      counts[pubsub.shard] += 1
+
+    ## Then
+    for i in 1..<GenerationZeroShardsCount:
+      check:
+        float64(counts[i - 1]) <= (float64(counts[i]) * 1.05)
+        float64(counts[i]) <= (float64(counts[i - 1]) * 1.05)
+        float64(counts[i - 1]) >= (float64(counts[i]) * 0.95)
+        float64(counts[i]) >= (float64(counts[i - 1]) * 0.95)
+
+    #echo counts
+
+
+
+
+
+
+
+
+
+

tests/wakubridge/test_message_compat.nim

@@ -19,14 +19,14 @@ suite "WakuBridge - Message compat":
     # Expected cases
 
     check:
-      toV1Topic(ContentTopic("/waku/1/0x00000000/rfc26")) == [byte 0x00, byte 0x00, byte 0x00, byte 0x00]
-      toV2ContentTopic([byte 0x00, byte 0x00, byte 0x00, byte 0x00]) == ContentTopic("/waku/1/0x00000000/rfc26")
-      toV1Topic(ContentTopic("/waku/1/0xffffffff/rfc26")) == [byte 0xff, byte 0xff, byte 0xff, byte 0xff]
-      toV2ContentTopic([byte 0xff, byte 0xff, byte 0xff, byte 0xff]) == ContentTopic("/waku/1/0xffffffff/rfc26")
-      toV1Topic(ContentTopic("/waku/1/0x1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
-      toV2ContentTopic([byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]) == ContentTopic("/waku/1/0x1a2b3c4d/rfc26")
+      toV1Topic(ContentTopic("/0/none/waku/1/0x00000000/rfc26")) == [byte 0x00, byte 0x00, byte 0x00, byte 0x00]
+      toV2ContentTopic([byte 0x00, byte 0x00, byte 0x00, byte 0x00]) == ContentTopic("/0/none/waku/1/0x00000000/rfc26")
+      toV1Topic(ContentTopic("/0/none/waku/1/0xffffffff/rfc26")) == [byte 0xff, byte 0xff, byte 0xff, byte 0xff]
+      toV2ContentTopic([byte 0xff, byte 0xff, byte 0xff, byte 0xff]) == ContentTopic("/0/none/waku/1/0xffffffff/rfc26")
+      toV1Topic(ContentTopic("/0/none/waku/1/0x1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
+      toV2ContentTopic([byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]) == ContentTopic("/0/none/waku/1/0x1a2b3c4d/rfc26")
       # Topic conversion should still work where '0x' prefix is omitted from <v1 topic byte array>
-      toV1Topic(ContentTopic("/waku/1/1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
+      toV1Topic(ContentTopic("/0/none/waku/1/1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
 
     # Invalid cases
 
@@ -37,18 +37,18 @@ suite "WakuBridge - Message compat":
 
     expect ValueError:
       # Content topic name too short
-      discard toV1Topic(ContentTopic("/waku/1/0x112233/rfc26"))
+      discard toV1Topic(ContentTopic("/0/none/waku/1/0x112233/rfc26"))
 
     expect ValueError:
       # Content topic name not hex
-      discard toV1Topic(ContentTopic("/waku/1/my-content/rfc26"))
+      discard toV1Topic(ContentTopic("/0/none/waku/1/my-content/rfc26"))
 
   test "Verify that WakuMessages are on bridgeable content topics":
     let
-      validCT = ContentTopic("/waku/1/my-content/rfc26")
+      validCT = ContentTopic("/0/none/waku/1/my-content/rfc26")
       unnamespacedCT = ContentTopic("just_a_bunch_of_words")
-      invalidAppCT = ContentTopic("/facebook/1/my-content/rfc26")
-      invalidVersionCT = ContentTopic("/waku/2/my-content/rfc26")
+      invalidAppCT = ContentTopic("/0/none/facebook/1/my-content/rfc26")
+      invalidVersionCT = ContentTopic("/0/none/waku/2/my-content/rfc26")
 
     check:
       WakuMessage(contentTopic: validCT).isBridgeable() == true

tests/wakubridge/test_wakubridge.nim

@@ -68,7 +68,7 @@ procSuite "WakuBridge":
       builder.withNetworkConfigurationDetails(ValidIpAddress.init("0.0.0.0"), Port(62203)).tryGet()
       builder.build().tryGet()
 
-    contentTopic = ContentTopic("/waku/1/0x1a2b3c4d/rfc26")
+    contentTopic = ContentTopic("/0/none/waku/1/0x1a2b3c4d/rfc26")
     topic = [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
     payloadV1 = "hello from V1".toBytes()
     payloadV2 = "hello from V2".toBytes()

waku/v2/waku_core/topics.nim

@@ -1,7 +1,9 @@
 import
   ./topics/content_topic,
-  ./topics/pubsub_topic
+  ./topics/pubsub_topic,
+  ./topics/sharding
 
 export
   content_topic,
-  pubsub_topic
+  pubsub_topic,
+  sharding