Feat/rework zk proving eric

.github/actions/nimbus-build-system/action.yml

@@ -29,6 +29,7 @@ runs:
       shell: ${{ inputs.shell }} {0}
       run: |
         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --default-toolchain=${{ inputs.rust_version }} -y
+        echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
 
     - name: APT (Linux amd64/arm64)
       if: inputs.os == 'linux' && (inputs.cpu == 'amd64' || inputs.cpu == 'arm64')
@@ -89,7 +90,7 @@ runs:
 
     - name: Install gcc 14 on Linux
       # We don't want to install gcc 14 for coverage (Ubuntu 20.04)
-      if : ${{ inputs.os == 'linux' && inputs.coverage != 'true' }}
+      if: ${{ inputs.os == 'linux' && inputs.coverage != 'true' }}
       shell: ${{ inputs.shell }} {0}
       run: |
         # Skip for older Ubuntu versions
@@ -107,10 +108,17 @@ runs:
       if: inputs.os == 'linux' || inputs.os == 'macos'
       uses: hendrikmuhs/ccache-action@v1.2
       with:
-        create-symlink: true
+        create-symlink: false
         key: ${{ inputs.os }}-${{ inputs.builder }}-${{ inputs.cpu }}-${{ inputs.tests }}-${{ inputs.nim_version }}
         evict-old-files: 7d
 
+    - name: Add ccache to path on Linux/Mac
+      if: inputs.os == 'linux' || inputs.os == 'macos'
+      shell: ${{ inputs.shell }} {0}
+      run: |
+        echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> "$GITHUB_PATH"
+        echo "/usr/local/opt/ccache/libexec" >> "$GITHUB_PATH"
+
     - name: Install ccache on Windows
       if: inputs.os == 'windows'
       uses: hendrikmuhs/ccache-action@v1.2
@@ -123,11 +131,11 @@ runs:
       shell: ${{ inputs.shell }} {0}
       run: |
         CCACHE_DIR=$(dirname $(which ccache))/ccached
-        mkdir ${CCACHE_DIR}
-        ln -s $(which ccache) ${CCACHE_DIR}/gcc.exe
-        ln -s $(which ccache) ${CCACHE_DIR}/g++.exe
-        ln -s $(which ccache) ${CCACHE_DIR}/cc.exe
-        ln -s $(which ccache) ${CCACHE_DIR}/c++.exe
+        mkdir -p ${CCACHE_DIR}
+        ln -sf $(which ccache) ${CCACHE_DIR}/gcc.exe
+        ln -sf $(which ccache) ${CCACHE_DIR}/g++.exe
+        ln -sf $(which ccache) ${CCACHE_DIR}/cc.exe
+        ln -sf $(which ccache) ${CCACHE_DIR}/c++.exe
         echo "export PATH=${CCACHE_DIR}:\$PATH" >> $HOME/.bash_profile # prefix path in MSYS2
 
     - name: Derive environment variables
@@ -208,7 +216,7 @@ runs:
     - name: Restore Nim toolchain binaries from cache
       id: nim-cache
       uses: actions/cache@v4
-      if : ${{ inputs.coverage != 'true'  }}
+      if: ${{ inputs.coverage != 'true' }}
       with:
         path: NimBinaries
         key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_version }}-cache-${{ env.cache_nonce }}-${{ github.run_id }}
@@ -231,4 +239,4 @@ runs:
         gcc --version
         make -j${ncpu} CI_CACHE=NimBinaries ${ARCH_OVERRIDE} QUICK_AND_DIRTY_COMPILER=1 update
         echo
-        ./env.sh nim --version
+        ./env.sh nim --version

.github/workflows/ci-reusable.yml

@@ -43,6 +43,46 @@ jobs:
           nim_version: ${{ matrix.nim_version }}
           coverage: false
 
+      - name: Check runner resources for parallel integration tests
+        run: |
+          echo "Determining runner"
+          case "${{ matrix.os }}" in
+            linux)    CPU=$(nproc --all)
+                      RAM=$(awk '/MemTotal/ {print int($2 / 1024 / 1024 + 0.5)}' /proc/meminfo)
+                      ;;
+            macos)    CPU=$(sysctl -n hw.ncpu)
+                      RAM=$(sysctl -n hw.memsize | awk '{print $0/1073741824}')
+                      sysctl -n hw.ncpu
+                      ;;
+            windows)  CPU=$NUMBER_OF_PROCESSORS
+                      RAM=$(systeminfo | awk '/Total Physical Memory:/ { gsub(/,/,"."); print int($4 + 0.5) }')
+                      ;;
+            *)        CPU=2
+                      RAM=8
+                      echo "Unknown runner"
+                      ;;
+          esac
+          echo "CPU=${CPU}" >> $GITHUB_ENV
+          echo "RAM=${RAM}" >> $GITHUB_ENV
+          echo "TYPE=${RUNNER_ENVIRONMENT}" >> $GITHUB_ENV
+
+          # Set PARALLEL=1 if the runner has enough resources
+          if [[ ("${{ matrix.os }}" == "linux" || "${{ matrix.os }}" == "windows") && "${CPU}" -ge 16 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          elif [[ "${{ matrix.os }}" == "macos" && "${CPU}" -ge 6 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          else
+            echo "PARALLEL=0" >> $GITHUB_ENV
+          fi
+
+      - name: Show runner information
+        run: |
+          echo "OS: ${{ matrix.os }}"
+          echo "CPU: ${{ env.CPU }}"
+          echo "RAM: ${{ env.RAM }} GB"
+          echo "TYPE: ${{ env.TYPE }}"
+          echo "PARALLEL: ${{ env.PARALLEL }}"
+
       ## Part 1 Tests ##
       - name: Unit tests
         if: matrix.tests == 'unittest' || matrix.tests == 'all'
@@ -53,13 +93,20 @@ jobs:
         with:
           node-version: 20
 
-      - name: Start Ethereum node with Codex contracts
+      - name: Install Ethereum node dependencies
         if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all'
         working-directory: vendor/codex-contracts-eth
         env:
           MSYS2_PATH_TYPE: inherit
         run: |
           npm install
+
+      - name: Run Ethereum node with Codex contracts
+        if: matrix.tests == 'contract' || (matrix.tests == 'integration' && env.PARALLEL != 1) || matrix.tests == 'tools' || matrix.tests == 'all'
+        working-directory: vendor/codex-contracts-eth
+        env:
+          MSYS2_PATH_TYPE: inherit
+        run: |
           npm start &
 
       ## Part 2 Tests ##
@@ -70,13 +117,13 @@ jobs:
       ## Part 3 Tests ##
       - name: Integration tests
         if: matrix.tests == 'integration' || matrix.tests == 'all'
-        run: make -j${ncpu} testIntegration
+        run: make -j${ncpu} PARALLEL=${{ env.PARALLEL }} DEBUG=${{ runner.debug }} testIntegration
 
       - name: Upload integration tests log files
         uses: actions/upload-artifact@v4
         if: (matrix.tests == 'integration' || matrix.tests == 'all') && always()
         with:
-          name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-integration-tests-logs
+          name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}-tests-logs
           path: tests/integration/logs/
           retention-days: 1
 

.github/workflows/ci.yml

@@ -10,34 +10,46 @@ on:
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
   nim_version: v2.2.4
+  builder_integration_from_vars: true
+  builder_integration_linux: runner-node-01-linux-03-eu-hel1
+  builder_integration_macos: macos-14-xlarge
+  builder_integration_windows: windows-latest-amd64-32vcpu
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   matrix:
+    name: Compute matrix
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
       cache_nonce: ${{ env.cache_nonce }}
     steps:
+      - name: Compute builders
+        if: ${{ env.builder_integration_from_vars == 'true' }}
+        run: |
+          if [[ -n "${{ vars.builder_integration_linux }}" ]]; then echo "builder_integration_linux=${{ vars.builder_integration_linux }}"; fi >> $GITHUB_ENV
+          if [[ -n "${{ vars.builder_integration_macos }}" ]]; then echo "builder_integration_macos=${{ vars.builder_integration_macos }}"; fi >> $GITHUB_ENV
+          if [[ -n "${{ vars.builder_integration_windows }}" ]]; then echo "builder_integration_windows=${{ vars.builder_integration_windows }}"; fi >> $GITHUB_ENV
+
       - name: Compute matrix
         id: matrix
         uses: fabiocaccamo/create-matrix-action@v5
         with:
           matrix: |
-            os {linux},   cpu {amd64}, builder {ubuntu-latest},  tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {linux},   cpu {amd64}, builder {ubuntu-latest},  tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {linux},   cpu {amd64}, builder {ubuntu-latest},  tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {linux},   cpu {amd64}, builder {ubuntu-latest},  tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {${{ env.builder_integration_linux }}},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {arm64}, builder {macos-14},       tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {arm64}, builder {macos-14},       tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {macos},   cpu {arm64}, builder {macos-14},       tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {arm64}, builder {${{ env.builder_integration_macos }}},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {arm64}, builder {macos-14},       tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest},    nim_version {${{ env.nim_version }}}, shell {msys2}
             os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract},    nim_version {${{ env.nim_version }}}, shell {msys2}
-            os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
+            os {windows}, cpu {amd64}, builder {${{ env.builder_integration_windows }}}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
             os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools},       nim_version {${{ env.nim_version }}}, shell {msys2}
 
   build:

.gitmodules

@@ -231,3 +231,16 @@
 	url = https://github.com/vacp2p/nim-ngtcp2.git
 	ignore = untracked
 	branch = master
+[submodule "vendor/nim-groth16"]
+	path = vendor/nim-groth16
+	url = https://github.com/codex-storage/nim-groth16.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-goldilocks-hash"]
+	path = vendor/nim-goldilocks-hash
+	url = https://github.com/codex-storage/nim-goldilocks-hash.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/circom-witnessgen"]
+	path = vendor/circom-witnessgen
+	url = https://github.com/codex-storage/circom-witnessgen.git

Makefile

@@ -140,10 +140,24 @@ testContracts: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
 
+TEST_PARAMS :=
+ifdef DEBUG
+	TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:NoCodexLogFilters=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG)
+  TEST_PARAMS := $(TEST_PARAMS) -d:DebugHardhat=$(DEBUG)
+endif
+ifdef TEST_TIMEOUT
+  TEST_PARAMS := $(TEST_PARAMS) -d:TestTimeout=$(TEST_TIMEOUT)
+endif
+ifdef PARALLEL
+  TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(PARALLEL)
+endif
+
 # Builds and runs the integration tests
 testIntegration: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
+		$(ENV_SCRIPT) nim testIntegration $(TEST_PARAMS) $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
 
 # Builds and runs all tests (except for Taiko L2 tests)
 testAll: | build deps

build.nims

@@ -1,8 +1,13 @@
 mode = ScriptMode.Verbose
 
 import std/os except commandLineParams
+import std/strutils
 
 ### Helper functions
+proc truthy(val: string): bool =
+  const truthySwitches = @["yes", "1", "on", "true"]
+  return val in truthySwitches
+
 proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
   if not dirExists "build":
     mkDir "build"
@@ -45,11 +50,18 @@ task testContracts, "Build & run Codex Contract tests":
 task testIntegration, "Run integration tests":
   buildBinary "codex",
     params =
-      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
-  test "testIntegration"
+      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_disabled_topics=JSONRPC-HTTP-CLIENT,websock,libp2p,discv5 -d:codex_enable_proof_failures=true"
+  var sinks = @["textlines[nocolors,file]"]
+  for i in 2 ..< paramCount():
+    if "DebugTestHarness" in paramStr(i) and truthy paramStr(i).split('=')[1]:
+      sinks.add "textlines[stdout]"
+      break
+  var testParams =
+    "-d:chronicles_log_level=TRACE -d:chronicles_sinks=\"" & sinks.join(",") & "\""
+  test "testIntegration", params = testParams
   # use params to enable logging from the integration test executable
   # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " &
-  #   "-d:chronicles_enabled_topics:integration:TRACE"  
+  #   "-d:chronicles_enabled_topics:integration:TRACE"
 
 task build, "build codex binary":
   codexTask()

codex.nim

@@ -10,7 +10,6 @@
 import pkg/chronos
 import pkg/questionable
 import pkg/confutils
-import pkg/confutils/defs
 import pkg/confutils/std/net
 import pkg/confutils/toml/defs as confTomlDefs
 import pkg/confutils/toml/std/net as confTomlNet

codex/blockexchange/engine/engine.nim

@@ -68,6 +68,12 @@ const
   DefaultMaxPeersPerRequest* = 10
   DefaultTaskQueueSize = 100
   DefaultConcurrentTasks = 10
+    # DefaultMaxRetries = 3
+    # DefaultConcurrentDiscRequests = 10
+    # DefaultConcurrentAdvertRequests = 10
+    # DefaultDiscoveryTimeout = 1.minutes
+    # DefaultMaxQueriedBlocksCache = 1000
+    # DefaultMinPeersPerBlock = 3
 
 type
   TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.}

codex/codex.nim

@@ -208,22 +208,19 @@ proc new*(
     .withTcpTransport({ServerFlags.ReuseAddr})
     .build()
 
-  var
-    cache: CacheStore = nil
-    taskpool: Taskpool
-
-  try:
-    if config.numThreads == ThreadCount(0):
-      taskpool = Taskpool.new(numThreads = min(countProcessors(), 16))
+  let numThreads =
+    if int(config.numThreads) == 0:
+      countProcessors()
     else:
-      taskpool = Taskpool.new(numThreads = int(config.numThreads))
-    info "Threadpool started", numThreads = taskpool.numThreads
-  except CatchableError as exc:
-    raiseAssert("Failure in taskpool initialization:" & exc.msg)
+      int(config.numThreads)
+
+  var tp =
+    try:
+      Taskpool.new(numThreads)
+    except CatchableError as exc:
+      raiseAssert("Failure in tp initialization:" & exc.msg)
 
-  if config.cacheSize > 0'nb:
-    cache = CacheStore.new(cacheSize = config.cacheSize)
-    ## Is unused?
+  info "Threadpool started", numThreads = tp.numThreads
 
   let discoveryDir = config.dataDir / CodexDhtNamespace
 
@@ -299,9 +296,8 @@ proc new*(
     store = NetworkStore.new(engine, repoStore)
     prover =
       if config.prover:
-        let backend =
-          config.initializeBackend().expect("Unable to create prover backend.")
-        some Prover.new(store, backend, config.numProofSamples)
+        let prover = config.initializeProver(tp).expect("Unable to create prover.")
+        some prover
       else:
         none Prover
 
@@ -311,7 +307,7 @@ proc new*(
       engine = engine,
       discovery = discovery,
       prover = prover,
-      taskPool = taskpool,
+      taskPool = tp,
     )
 
     restServer = RestServerRef