feat: autosharding core algorithm

apps/wakubridge/message_compat.nim

@@ -35,6 +35,8 @@ proc toV2ContentTopic*(v1Topic: waku_protocol.Topic): ContentTopic =
   ## <v1-topic-bytes-as-hex> should be prefixed with `0x`
   var namespacedTopic = NsContentTopic()
 
+  namespacedTopic.generation = "0"
+  namespacedTopic.bias = "none"
   namespacedTopic.application = ContentTopicApplication
   namespacedTopic.version = ContentTopicAppVersion
   namespacedTopic.name = v1Topic.to0xHex()

tests/v2/test_utils_compat.nim

@@ -3,7 +3,8 @@
 import
   testutils/unittests
 import
-  ../../waku/v2/waku_core,
+  ../../waku/v2/waku_core/message,
+  ../../waku/v2/waku_core/time,
   ../../waku/v2/utils/compat,
   ./testlib/common
 

tests/v2/waku_core/test_namespaced_topics.nim

@@ -11,6 +11,8 @@ suite "Waku Message - Content topics namespacing":
   test "Stringify namespaced content topic":
     ## Given
     var ns = NsContentTopic()
+    ns.generation = "0"
+    ns.bias = "none"
     ns.application = "toychat"
     ns.version = "2"
     ns.name = "huilong"
@@ -21,11 +23,11 @@ suite "Waku Message - Content topics namespacing":
 
     ## Then
     check:
-      topic == "/toychat/2/huilong/proto"
+      topic == "/0/none/toychat/2/huilong/proto"
 
   test "Parse content topic string - Valid string":
     ## Given
-    let topic = "/toychat/2/huilong/proto"
+    let topic = "/0/none/toychat/2/huilong/proto"
 
     ## When
     let nsRes = NsContentTopic.parse(topic)
@@ -35,6 +37,8 @@ suite "Waku Message - Content topics namespacing":
 
     let ns = nsRes.get()
     check:
+      ns.generation == "0"
+      ns.bias == "none"
       ns.application == "toychat"
       ns.version == "2"
       ns.name == "huilong"
@@ -68,7 +72,6 @@ suite "Waku Message - Content topics namespacing":
       err.kind == ParsingErrorKind.InvalidFormat
       err.cause == "invalid topic structure"
 
-
   test "Parse content topic string - Invalid string: missing encoding part":
     ## Given
     let topic = "/toychat/2/huilong"
@@ -97,6 +100,20 @@ suite "Waku Message - Content topics namespacing":
       err.kind == ParsingErrorKind.InvalidFormat
       err.cause == "invalid topic structure"
 
+  test "Parse content topic string - Invalid string: missing sharding data":
+    ## Given
+    let topic = "/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic)
+
+    ## Then
+    check ns.isErr()
+    let err = ns.tryError()
+    check:
+      err.kind == ParsingErrorKind.InvalidFormat
+      err.cause == "invalid topic structure"
+
 
 suite "Waku Message - Pub-sub topics namespacing":
 
@@ -178,7 +195,6 @@ suite "Waku Message - Pub-sub topics namespacing":
       err.kind == ParsingErrorKind.MissingPart
       err.part == "shard_cluster_index"
 
-
   test "Parse static sharding pub-sub topic string - Invalid string: cluster value":
     ## Given
     let topic = "/waku/2/rs/xx/77"

tests/v2/waku_core/test_sharding.nim

@@ -0,0 +1,111 @@
+{.used.}
+
+import
+  std/strutils,
+  std/algorithm,
+  stew/results,
+  testutils/unittests
+import
+  ../../../waku/v2/waku_core/topics
+
+suite "Waku Sharding":
+
+  test "Valid generation & sharding bias":
+    ## Given
+    let topic = "/0/none/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardingParam(ns)
+
+    ## Then
+    check paramRes.isOk()
+
+    let (count, bias) = paramRes.get()
+    check:
+      count == GenerationZeroShardsCount
+      bias == ShardingBias.None
+
+  test "Invalid generation":
+    ## Given
+    let topic = "/1/none/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardingParam(ns)
+
+    ## Then
+    check paramRes.isErr()
+    let err = paramRes.tryError()
+    check:
+      err == "Generation > 0 are not supported yet"
+
+  test "Invalid bias":
+    ## Given
+    let topic = "/0/kanonymity/toychat/2/huilong/proto"
+
+    ## When
+    let ns = NsContentTopic.parse(topic).expect("Parsing")
+
+    let paramRes = shardingParam(ns)
+
+    ## Then
+    check paramRes.isErr()
+    let err = paramRes.tryError()
+    check:
+      err.startsWith("Cannot parse sharding bias: ")
+
+  test "Weigths bias":
+    ## Given
+    let count = 5
+
+    ## When
+    let anonWeigths = biasedWeights(count, ShardingBias.Kanonymity)
+    let speedWeigths = biasedWeights(count, ShardingBias.Throughput)
+
+    ## Then
+    check:
+      anonWeigths[0] == 2.0
+      anonWeigths[1] == 1.0
+      anonWeigths[2] == 1.0
+      anonWeigths[3] == 1.0
+      anonWeigths[4] == 1.0
+
+      speedWeigths[0] == 1.0
+      speedWeigths[1] == 2.0
+      speedWeigths[2] == 2.0
+      speedWeigths[3] == 2.0
+      speedWeigths[4] == 2.0
+
+  test "Sorted shard list":
+    ## Given
+    let topic = "/0/none/toychat/2/huilong/proto"
+
+    ## When
+    let contentTopic = NsContentTopic.parse(topic).expect("Parsing")
+    let (count, bias) = shardingParam(contentTopic).expect("Valid parameters")
+
+    let weigths = biasedWeights(count, bias)
+
+    let shardsRes = weightedShardList(contentTopic, count, weigths)
+
+    ## Then
+    check shardsRes.isOk()
+
+    let shards = shardsRes.get()
+    check:
+      shards.len == count
+      isSorted(shards, hashOrder)
+
+
+
+
+
+
+
+
+
+
+

tests/wakubridge/test_message_compat.nim

@@ -19,14 +19,14 @@ suite "WakuBridge - Message compat":
     # Expected cases
 
     check:
-      toV1Topic(ContentTopic("/waku/1/0x00000000/rfc26")) == [byte 0x00, byte 0x00, byte 0x00, byte 0x00]
-      toV2ContentTopic([byte 0x00, byte 0x00, byte 0x00, byte 0x00]) == ContentTopic("/waku/1/0x00000000/rfc26")
-      toV1Topic(ContentTopic("/waku/1/0xffffffff/rfc26")) == [byte 0xff, byte 0xff, byte 0xff, byte 0xff]
-      toV2ContentTopic([byte 0xff, byte 0xff, byte 0xff, byte 0xff]) == ContentTopic("/waku/1/0xffffffff/rfc26")
-      toV1Topic(ContentTopic("/waku/1/0x1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
-      toV2ContentTopic([byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]) == ContentTopic("/waku/1/0x1a2b3c4d/rfc26")
+      toV1Topic(ContentTopic("/0/none/waku/1/0x00000000/rfc26")) == [byte 0x00, byte 0x00, byte 0x00, byte 0x00]
+      toV2ContentTopic([byte 0x00, byte 0x00, byte 0x00, byte 0x00]) == ContentTopic("/0/none/waku/1/0x00000000/rfc26")
+      toV1Topic(ContentTopic("/0/none/waku/1/0xffffffff/rfc26")) == [byte 0xff, byte 0xff, byte 0xff, byte 0xff]
+      toV2ContentTopic([byte 0xff, byte 0xff, byte 0xff, byte 0xff]) == ContentTopic("/0/none/waku/1/0xffffffff/rfc26")
+      toV1Topic(ContentTopic("/0/none/waku/1/0x1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
+      toV2ContentTopic([byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]) == ContentTopic("/0/none/waku/1/0x1a2b3c4d/rfc26")
       # Topic conversion should still work where '0x' prefix is omitted from <v1 topic byte array>
-      toV1Topic(ContentTopic("/waku/1/1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
+      toV1Topic(ContentTopic("/0/none/waku/1/1a2b3c4d/rfc26")) == [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
 
     # Invalid cases
 
@@ -37,18 +37,18 @@ suite "WakuBridge - Message compat":
 
     expect ValueError:
       # Content topic name too short
-      discard toV1Topic(ContentTopic("/waku/1/0x112233/rfc26"))
+      discard toV1Topic(ContentTopic("/0/none/waku/1/0x112233/rfc26"))
 
     expect ValueError:
       # Content topic name not hex
-      discard toV1Topic(ContentTopic("/waku/1/my-content/rfc26"))
+      discard toV1Topic(ContentTopic("/0/none/waku/1/my-content/rfc26"))
 
   test "Verify that WakuMessages are on bridgeable content topics":
     let
-      validCT = ContentTopic("/waku/1/my-content/rfc26")
+      validCT = ContentTopic("/0/none/waku/1/my-content/rfc26")
       unnamespacedCT = ContentTopic("just_a_bunch_of_words")
-      invalidAppCT = ContentTopic("/facebook/1/my-content/rfc26")
-      invalidVersionCT = ContentTopic("/waku/2/my-content/rfc26")
+      invalidAppCT = ContentTopic("/0/none/facebook/1/my-content/rfc26")
+      invalidVersionCT = ContentTopic("/0/none/waku/2/my-content/rfc26")
 
     check:
       WakuMessage(contentTopic: validCT).isBridgeable() == true

tests/wakubridge/test_wakubridge.nim

@@ -68,7 +68,7 @@ procSuite "WakuBridge":
       builder.withNetworkConfigurationDetails(ValidIpAddress.init("0.0.0.0"), Port(62203)).tryGet()
       builder.build().tryGet()
 
-    contentTopic = ContentTopic("/waku/1/0x1a2b3c4d/rfc26")
+    contentTopic = ContentTopic("/0/none/waku/1/0x1a2b3c4d/rfc26")
     topic = [byte 0x1a, byte 0x2b, byte 0x3c, byte 0x4d]
     payloadV1 = "hello from V1".toBytes()
     payloadV2 = "hello from V2".toBytes()

waku/v2/waku_core/topics.nim

@@ -1,7 +1,9 @@
 import
   ./topics/content_topic,
-  ./topics/pubsub_topic
+  ./topics/pubsub_topic,
+  ./topics/sharding
 
 export
   content_topic,
-  pubsub_topic
+  pubsub_topic,
+  sharding

waku/v2/waku_core/topics/content_topic.nim

@@ -27,13 +27,17 @@ const DefaultContentTopic* = ContentTopic("/waku/2/default-content/proto")
 
 type
   NsContentTopic* = object
+    generation*: string
+    bias*: string
     application*: string
     version*: string
     name*: string
     encoding*: string
 
-proc init*(T: type NsContentTopic, application, version, name, encoding: string): T =
+proc init*(T: type NsContentTopic, generation, bias, application, version, name, encoding: string): T =
   NsContentTopic(
+    generation: generation,
+    bias: bias,
     application: application,
     version: version,
     name: name,
@@ -46,41 +50,50 @@ proc init*(T: type NsContentTopic, application, version, name, encoding: string)
 proc `$`*(topic: NsContentTopic): string =
   ## Returns a string representation of a namespaced topic
   ## in the format `/<application>/<version>/<topic-name>/<encoding>`
-  "/" & topic.application & "/" & topic.version & "/" & topic.name & "/" & topic.encoding
+  ## Autosharding adds 2 prefix `/<gen#>/<bias-name>
+  "/" & topic.generation & "/" & topic.bias & "/" & topic.application & "/" & topic.version & "/" & topic.name & "/" & topic.encoding
 
 
 # Deserialization
 
 proc parse*(T: type NsContentTopic, topic: ContentTopic|string): ParsingResult[NsContentTopic] =
   ## Splits a namespaced topic string into its constituent parts.
   ## The topic string has to be in the format `/<application>/<version>/<topic-name>/<encoding>`
+  ## Autosharding adds 2 prefix `/<gen#>/<bias-name>
 
   if not topic.startsWith("/"):
     return err(ParsingError.invalidFormat("topic must start with slash"))
 
   let parts = topic[1..<topic.len].split("/")
-  if parts.len != 4:
+  if parts.len != 6:
     return err(ParsingError.invalidFormat("invalid topic structure"))
 
+  let gen = parts[0]
+  if gen.len == 0:
+    return err(ParsingError.missingPart("generation"))
 
-  let app = parts[0]
+  let bias = parts[1]
+  if bias.len == 0:
+    return err(ParsingError.missingPart("sharding-bias"))
+
+  let app = parts[2]
   if app.len == 0:
     return err(ParsingError.missingPart("appplication"))
 
-  let ver = parts[1]
+  let ver = parts[3]
   if ver.len == 0:
     return err(ParsingError.missingPart("version"))
 
-  let name = parts[2]
+  let name = parts[4]
   if name.len == 0:
     return err(ParsingError.missingPart("topic-name"))
 
-  let enc = parts[3]
+  let enc = parts[5]
   if enc.len == 0:
     return err(ParsingError.missingPart("encoding"))
 
 
-  ok(NsContentTopic.init(app, ver, name, enc))
+  ok(NsContentTopic.init(gen, bias, app, ver, name, enc))
 
 
 # Content topic compatibility