Feat/rework zk proving

.gitmodules

@@ -231,3 +231,16 @@
 	url = https://github.com/vacp2p/nim-ngtcp2.git
 	ignore = untracked
 	branch = master
+[submodule "vendor/nim-groth16"]
+	path = vendor/nim-groth16
+	url = https://github.com/codex-storage/nim-groth16.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-goldilocks-hash"]
+	path = vendor/nim-goldilocks-hash
+	url = https://github.com/codex-storage/nim-goldilocks-hash.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/circom-witnessgen"]
+	path = vendor/circom-witnessgen
+	url = https://github.com/codex-storage/circom-witnessgen.git

codex.nim

@@ -10,7 +10,6 @@
 import pkg/chronos
 import pkg/questionable
 import pkg/confutils
-import pkg/confutils/defs
 import pkg/confutils/std/net
 import pkg/confutils/toml/defs as confTomlDefs
 import pkg/confutils/toml/std/net as confTomlNet

codex/codex.nim

@@ -214,22 +214,19 @@ proc new*(
     .withTcpTransport({ServerFlags.ReuseAddr})
     .build()
 
-  var
-    cache: CacheStore = nil
-    taskpool: Taskpool
-
-  try:
-    if config.numThreads == ThreadCount(0):
-      taskpool = Taskpool.new(numThreads = min(countProcessors(), 16))
+  let numThreads =
+    if int(config.numThreads) == 0:
+      countProcessors()
     else:
-      taskpool = Taskpool.new(numThreads = int(config.numThreads))
-    info "Threadpool started", numThreads = taskpool.numThreads
-  except CatchableError as exc:
-    raiseAssert("Failure in taskpool initialization:" & exc.msg)
+      int(config.numThreads)
+
+  var tp =
+    try:
+      Taskpool.new(numThreads)
+    except CatchableError as exc:
+      raiseAssert("Failure in tp initialization:" & exc.msg)
 
-  if config.cacheSize > 0'nb:
-    cache = CacheStore.new(cacheSize = config.cacheSize)
-    ## Is unused?
+  info "Threadpool started", numThreads = tp.numThreads
 
   let discoveryDir = config.dataDir / CodexDhtNamespace
 
@@ -305,9 +302,8 @@ proc new*(
     store = NetworkStore.new(engine, repoStore)
     prover =
       if config.prover:
-        let backend =
-          config.initializeBackend().expect("Unable to create prover backend.")
-        some Prover.new(store, backend, config.numProofSamples)
+        let prover = config.initializeProver(tp).expect("Unable to create prover.")
+        some prover
       else:
         none Prover
 
@@ -317,7 +313,7 @@ proc new*(
       engine = engine,
       discovery = discovery,
       prover = prover,
-      taskPool = taskpool,
+      taskPool = tp,
     )
 
     restServer = RestServerRef
@@ -337,5 +333,5 @@ proc new*(
     restServer: restServer,
     repoStore: repoStore,
     maintenance: maintenance,
-    taskpool: taskpool,
+    taskpool: tp,
   )

codex/conf.nim

@@ -18,6 +18,7 @@ import std/terminal # Is not used in tests
 import std/options
 import std/strutils
 import std/typetraits
+import std/cpuinfo
 
 import pkg/chronos
 import pkg/chronicles/helpers
@@ -54,9 +55,7 @@ export
   DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval,
   DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas
 
-type ThreadCount* = distinct Natural
-
-proc `==`*(a, b: ThreadCount): bool {.borrow.}
+type ThreadCount* = range[0 .. 256]
 
 proc defaultDataDir*(): string =
   let dataDir =
@@ -76,7 +75,6 @@ const
 
   DefaultDataDir* = defaultDataDir()
   DefaultCircuitDir* = defaultDataDir() / "circuits"
-  DefaultThreadCount* = ThreadCount(0)
 
 type
   StartUpCmd* {.pure.} = enum
@@ -87,6 +85,13 @@ type
     noCmd
     prover
 
+  ProverBackendCmd* {.pure.} = enum
+    nimgroth16
+    circomcompat
+
+  Curves* {.pure.} = enum
+    bn128 = "bn128"
+
   LogKind* {.pure.} = enum
     Auto = "auto"
     Colors = "colors"
@@ -193,7 +198,8 @@ type
     numThreads* {.
       desc:
         "Number of worker threads (\"0\" = use as many threads as there are CPU cores available)",
-      defaultValue: DefaultThreadCount,
+      defaultValueDesc: "0",
+      defaultValue: ThreadCount(0),
       name: "num-threads"
     .}: ThreadCount
 
@@ -389,15 +395,40 @@ type
           name: "circuit-dir"
         .}: OutDir
 
+        proverBackend* {.
+          desc:
+            "The backend to use for the prover. " &
+            "Must be one of: nimgroth16, circomcompat",
+          defaultValue: ProverBackendCmd.nimgroth16,
+          defaultValueDesc: "nimgroth16",
+          name: "prover-backend"
+        .}: ProverBackendCmd
+
+        curve* {.
+          desc: "The curve to use for the storage circuit",
+          defaultValue: Curves.bn128,
+          defaultValueDesc: $Curves.bn128,
+          name: "curve"
+        .}: Curves
+
         circomR1cs* {.
           desc: "The r1cs file for the storage circuit",
           defaultValue: $DefaultCircuitDir / "proof_main.r1cs",
           defaultValueDesc: $DefaultCircuitDir & "/proof_main.r1cs",
           name: "circom-r1cs"
         .}: InputFile
 
+        circomGraph* {.
+          desc:
+            "The graph file for the storage circuit (only used with nimgroth16 backend)",
+          defaultValue: $DefaultCircuitDir / "proof_main.bin",
+          defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.bin",
+          name: "circom-graph"
+        .}: InputFile
+
         circomWasm* {.
-          desc: "The wasm file for the storage circuit",
+          desc:
+            "The wasm file for the storage circuit (only used with circomcompat backend)",
           defaultValue: $DefaultCircuitDir / "proof_main.wasm",
           defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm",
           name: "circom-wasm"
@@ -410,11 +441,11 @@ type
           name: "circom-zkey"
         .}: InputFile
 
-        # TODO: should probably be hidden and behind a feature flag
         circomNoZkey* {.
           desc: "Ignore the zkey file - use only for testing!",
           defaultValue: false,
-          name: "circom-no-zkey"
+          name: "circom-no-zkey",
+          hidden
         .}: bool
 
         numProofSamples* {.
@@ -505,7 +536,7 @@ const
 
 proc parseCmdArg*(
     T: typedesc[MultiAddress], input: string
-): MultiAddress {.upraises: [ValueError].} =
+): MultiAddress {.raises: [ValueError].} =
   var ma: MultiAddress
   try:
     let res = MultiAddress.init(input)
@@ -519,12 +550,8 @@ proc parseCmdArg*(
     quit QuitFailure
   ma
 
-proc parseCmdArg*(T: type ThreadCount, input: string): T {.upraises: [ValueError].} =
-  let count = parseInt(input)
-  if count != 0 and count < 2:
-    warn "Invalid number of threads", input = input
-    quit QuitFailure
-  ThreadCount(count)
+proc parseCmdArg*(T: type ThreadCount, val: string): T {.raises: [ValueError].} =
+  ThreadCount(val.parseUInt())
 
 proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
   var res: SignedPeerRecord
@@ -586,7 +613,7 @@ proc parseCmdArg*(T: type Duration, val: string): T =
 
 proc readValue*(
     r: var TomlReader, val: var EthAddress
-) {.upraises: [SerializationError, IOError].} =
+) {.raises: [SerializationError, IOError].} =
   val = EthAddress.init(r.readValue(string)).get()
 
 proc readValue*(r: var TomlReader, val: var SignedPeerRecord) =
@@ -614,7 +641,7 @@ proc readValue*(r: var TomlReader, val: var MultiAddress) =
 
 proc readValue*(
     r: var TomlReader, val: var NBytes
-) {.upraises: [SerializationError, IOError].} =
+) {.raises: [SerializationError, IOError].} =
   var value = 0'i64
   var str = r.readValue(string)
   let count = parseSize(str, value, alwaysBin = true)
@@ -625,7 +652,7 @@ proc readValue*(
 
 proc readValue*(
     r: var TomlReader, val: var ThreadCount
-) {.upraises: [SerializationError, IOError].} =
+) {.raises: [SerializationError, IOError].} =
   var str = r.readValue(string)
   try:
     val = parseCmdArg(ThreadCount, str)
@@ -634,7 +661,7 @@ proc readValue*(
 
 proc readValue*(
     r: var TomlReader, val: var Duration
-) {.upraises: [SerializationError, IOError].} =
+) {.raises: [SerializationError, IOError].} =
   var str = r.readValue(string)
   var dur: Duration
   let count = parseDuration(str, dur)
@@ -701,7 +728,7 @@ proc stripAnsi*(v: string): string =
 
   res
 
-proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} =
+proc updateLogLevel*(logLevel: string) {.raises: [ValueError].} =
   # Updates log levels (without clearing old ones)
   let directives = logLevel.split(";")
   try:

codex/erasure/erasure.nim

@@ -362,7 +362,7 @@ proc asyncEncode*(
 
 proc encodeData(
     self: Erasure, manifest: Manifest, params: EncodingParams
-): Future[?!Manifest] {.async.} =
+): Future[?!Manifest] {.async: (raises: [CancelledError]).} =
   ## Encode blocks pointed to by the protected manifest
   ##
   ## `manifest` - the manifest to encode
@@ -461,7 +461,7 @@ proc encode*(
     blocks: Natural,
     parity: Natural,
     strategy = SteppedStrategy,
-): Future[?!Manifest] {.async.} =
+): Future[?!Manifest] {.async: (raises: [CancelledError]).} =
   ## Encode a manifest into one that is erasure protected.
   ##
   ## `manifest`   - the original manifest to be encoded
@@ -554,7 +554,7 @@ proc asyncDecode*(
 
 proc decodeInternal(
     self: Erasure, encoded: Manifest
-): Future[?!(ref seq[Cid], seq[Natural])] {.async.} =
+): Future[?!(ref seq[Cid], seq[Natural])] {.async: (raises: [CancelledError]).} =
   logScope:
     steps = encoded.steps
     rounded_blocks = encoded.rounded
@@ -638,7 +638,9 @@ proc decodeInternal(
 
   return (cids, recoveredIndices).success
 
-proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} =
+proc decode*(
+    self: Erasure, encoded: Manifest
+): Future[?!Manifest] {.async: (raises: [CancelledError]).} =
   ## Decode a protected manifest into it's original
   ## manifest
   ##

codex/nat.nim

@@ -423,10 +423,12 @@
           it.remapAddr(ip = newIP, port = tcp)
         else:
           # NAT mapping failed - use original address
-          echo "Failed to get external IP, using original address", it
+          # TODO: `trace` breaks in the mapIt template
+          # trace "Failed to get external IP, using original address", it
           discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(ipPart.get, udpPort))
           it
       else:
         # Invalid multiaddress format - return as is
         it
+
   (newAddrs, discoveryAddrs)