WASI-20. Low memory prover (512k possible). Native pthread support (1…

…5s wasm time). (#434)

* Remove foundation submodule, add build-system
* Initial bindgen work.
* GPT4 created rust bindgen.
* Debug build preset wasm-dbg for stack traces etc.
* Async call support. Refactor binds. Common is module. WASM stubs unused wasi.
* Move some c_bind to module root and make use new abi.
* Initial working pthreads.
* Polynomial cache
* Custom poly allocator. Prealloc.
* Cleanup. Several parallel_for solutions. Might be worth keeping them around until threading work stabilises.
* Improve mem usage by slab allocating large parts of pippenger runtime state, and allowing slab reuse by removing from pk.
* Don't need binaryen.
* Bump alpine to get clang 16
* Backwards compatible cbinds.
* Move slab allocator to common. Init slab allocator just before creating pk.
* Dont need bleeding edge fork of libc anymore
* bb.js is standalone prover that handles noir constraint/witness output. (#471)
* format msgpack serialization and excldue msgpack-c from clang-format (#467)
* Fix horrific nasty memory bug.

---------

Co-authored-by: Suyash Bagad <suyashnbagad1997@gmail.com>
Co-authored-by: Maxim Vezenov <mvezenov@gmail.com>
Co-authored-by: Maddiaa <47148561+Maddiaa0@users.noreply.github.com>
Co-authored-by: ledwards2225 <98505400+ledwards2225@users.noreply.github.com>
Co-authored-by: Innokentii Sennovskii <isennovskiy@gmail.com>
Co-authored-by: kevaundray <kevtheappdev@gmail.com>
Co-authored-by: zac-williamson <blorktronics@gmail.com>
Co-authored-by: codygunton <codygunton@gmail.com>

.circleci/config.yml

@@ -11,8 +11,6 @@
 #
 # The docker executor uses a custom image build in `build_image`. It's specifically streamlined for fast download
 # with just enough tools to execute the build system, and launch EC2 instances etc.
-#
-# There are some `join` steps that are just noops. They are just used to produce cleaner graph rendering in CCI.
 
 version: 2.1
 
@@ -37,11 +35,6 @@ checkout: &checkout
       chmod 0700 .ssh
       ssh-keyscan -t rsa github.com >> .ssh/known_hosts
 
-      # A read only key for cloning the repository.
-      echo $GIT_CHECKOUT_KEY | base64 -d > .ssh/id_rsa
-
-      chmod 0600 .ssh/id_rsa
-
       # IF YOU'RE CHANGING THIS, YOU ALSO WANT TO CHANGE: build-system/remote_build/remote_build
       # Shallow checkout this commit.
       mkdir -p project
@@ -246,6 +239,28 @@ jobs:
           name: "Test"
           command: store_test_benchmark_logs barretenberg-x86_64-linux-clang-assert
 
+  bb-js:
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: large
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build"
+          command: build bb.js
+
+  bb-js-tests:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Test"
+          command: cond_spot_run_tests bb.js
+
   ###################################
   # Aztec integration tests
   circuits-wasm-linux-clang-builder-runner:
@@ -348,8 +363,16 @@ workflows:
               only:
                 - master
           <<: *defaults
+      - bb-js:
+          requires:
+            - wasm-linux-clang
+          <<: *defaults
+      - bb-js-tests:
+          requires:
+            - bb-js
+          <<: *defaults
       - circuits-wasm-linux-clang-builder-runner: *notmaster
-      - circuits-x86_64-linux-clang-builder-runner:  *notmaster
+      - circuits-x86_64-linux-clang-builder-runner: *notmaster
       - circuits-wasm-tests:
           requires:
             - circuits-wasm-linux-clang-builder-runner

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "foundation"]
-	path = foundation
-	url = git@github.com:AztecProtocol/foundation.git
 [submodule "sol/lib/forge-std"]
 	path = sol/lib/forge-std
 	url = https://github.com/foundry-rs/forge-std

.vscode/c_cpp_properties.json

@@ -4,7 +4,7 @@
       "name": "Linux",
       "includePath": [
         "${workspaceFolder}/**",
-        "${workspaceFolder}/cpp/src/aztec"
+        "${workspaceFolder}/cpp/src/barretenberg"
       ],
       "defines": [],
       "compilerPath": "/usr/bin/clang",

.vscode/settings.json

@@ -104,18 +104,21 @@
     "charconv": "cpp",
     "span": "cpp",
     "*.inc": "cpp",
-    "*.macros": "cpp"
+    "*.macros": "cpp",
+    "cuchar": "cpp"
   },
   "solidity.compileUsingRemoteVersion": "v0.6.10+commit.00c0fcaf",
   "search.exclude": {
     "**/.yarn": true,
-    "**/.pnp.*": true
+    "**/.pnp.*": true,
+    "**/msgpack-c/**": true
   },
   "typescript.tsdk": "ts/.yarn/sdks/typescript/lib",
   "typescript.enablePromptUseWorkspaceTsdk": true,
   "eslint.nodePath": "ts/.yarn/sdks",
   "prettier.prettierPath": "ts/.yarn/sdks/prettier/index.js",
-  "[cpp]": { // doesn't conflict with barratenberg.code-workspace settings.
+  "[cpp]": {
+    // doesn't conflict with barratenberg.code-workspace settings.
     "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd"
   },
   "[terraform]": {
@@ -128,4 +131,4 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "cmake.sourceDirectory": "${workspaceFolder}/cpp"
-}
+}

README.md

@@ -114,7 +114,8 @@ CMake can be passed various build options on its command line:
 - `-DCMAKE_BUILD_TYPE=Debug | Release | RelWithAssert`: Build types.
 - `-DDISABLE_ASM=ON | OFF`: Enable/disable x86 assembly.
 - `-DDISABLE_ADX=ON | OFF`: Enable/disable ADX assembly instructions (for older cpu support).
-- `-DMULTITHREADING=ON | OFF`: Enable/disable multithreading using OpenMP.
+- `-DMULTITHREADING=ON | OFF`: Enable/disable multithreading.
+- `-DOMP_MULTITHREADING=ON | OFF`: Enable/disable multithreading that uses OpenMP.
 - `-DTESTING=ON | OFF`: Enable/disable building of tests.
 - `-DBENCHMARK=ON | OFF`: Enable/disable building of benchmarks.
 - `-DFUZZING=ON | OFF`: Enable building various fuzzers.

build_manifest.json

@@ -34,5 +34,10 @@
     "dockerfile": "dockerfiles/Dockerfile.circuits-wasm-linux-clang-builder-runner",
     "rebuildPatterns": ["^cpp/"],
     "dependencies": []
+  },
+  "bb.js": {
+    "buildDir": "ts",
+    "rebuildPatterns": ["^ts/"],
+    "dependencies": ["barretenberg-wasm-linux-clang"]
   }
 }

cpp/.aztec-packages-commit

@@ -1 +1 @@
-84754bbb47e32c722a9c13de17c328ae15e58f01
+1dbca53b4cf6c83d1a25be66bb7b7f4a9d5e484b

cpp/.dockerignore

@@ -1,12 +1,8 @@
 build*
 docker*
-scripts
 .*
 src/wasi-sdk*
 src/barretenberg/rollup/proofs/root_*/fixtures/account
 src/barretenberg/rollup/proofs/root_*/fixtures/join_split
 src/barretenberg/rollup/proofs/root_*/fixtures/**/proving_key
 srs_db/ignition/transcript*
-srs_db/lagrange
-srs_db/coset_lagrange
-srs_db/modified_lagrange

cpp/CMakeLists.txt

@@ -13,6 +13,7 @@ project(
 option(DISABLE_ASM "Disable custom assembly" OFF)
 option(DISABLE_ADX "Disable ADX assembly variant" OFF)
 option(MULTITHREADING "Enable multi-threading" ON)
+option(OMP_MULTITHREADING "Enable OMP multi-threading" ON)
 option(TESTING "Build tests" ON)
 option(BENCHMARKS "Build benchmarks" ON)
 option(FUZZING "Build fuzzing harnesses" OFF)
@@ -23,8 +24,6 @@ option(ENABLE_ASAN "Address sanitizer for debugging tricky memory corruption" OF
 option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF)
 option(INSTALL_BARRETENBERG "Enable installation of barretenberg. (Projects embedding barretenberg may want to turn this OFF.)" ON)
 
-message(STATUS "Building barretenberg for UltraPlonk Composer.")
-
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
     message(STATUS "Compiling for ARM.")
     set(ARM ON)
@@ -74,7 +73,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "wasm32")
     message(STATUS "Compiling for WebAssembly.")
     set(WASM ON)
     set(DISABLE_ASM ON)
-    set(MULTITHREADING OFF)
+    set(OMP_MULTITHREADING OFF)
     set(BENCHMARKS OFF)
     set(DISABLE_TBB 1)
     add_compile_definitions(_WASI_EMULATED_PROCESS_CLOCKS=1)
@@ -99,7 +98,6 @@ else()
 endif()
 
 if(COVERAGE)
-
     # We've only set up LLVM coverage
     if(NOT(CMAKE_CXX_COMPILER_ID MATCHES "Clang"))
         message(FATAL_ERROR "Creating coverage is only available for clang")
@@ -141,4 +139,4 @@ include(cmake/threading.cmake)
 include(cmake/gtest.cmake)
 include(cmake/benchmark.cmake)
 include(cmake/module.cmake)
-add_subdirectory(src)
+add_subdirectory(src)

cpp/CMakePresets.json

@@ -47,6 +47,10 @@
       "inherits": "clang15",
       "environment": {
         "CMAKE_BUILD_TYPE": "Debug"
+      },
+      "cacheVariables": {
+        "ENABLE_ASAN": "OFF",
+        "DISABLE_ASM": "ON"
       }
     },
     {
@@ -100,12 +104,12 @@
     {
       "name": "wasm",
       "displayName": "Build for WASM",
-      "description": "Build with a specific wasm-sdk to create wasm",
+      "description": "Build with wasi-sdk to create wasm",
       "binaryDir": "build-wasm",
       "generator": "Ninja",
       "toolchainFile": "cmake/toolchains/wasm32-wasi.cmake",
       "environment": {
-        "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk-12.0",
+        "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk-20.0",
         "CC": "$env{WASI_SDK_PREFIX}/bin/clang",
         "CXX": "$env{WASI_SDK_PREFIX}/bin/clang++",
         "AR": "$env{WASI_SDK_PREFIX}/bin/llvm-ar",
@@ -118,7 +122,31 @@
         "CMAKE_FIND_ROOT_PATH_MODE_INCLUDE": "ONLY",
         "CMAKE_FIND_ROOT_PATH_MODE_PACKAGE": "ONLY",
         "CMAKE_C_COMPILER_WORKS": "ON",
-        "CMAKE_CXX_COMPILER_WORKS": "ON"
+        "CMAKE_CXX_COMPILER_WORKS": "ON",
+        "MULTITHREADING": "OFF"
+      }
+    },
+    {
+      "name": "wasm-dbg",
+      "displayName": "Build for debug WASM",
+      "description": "Build with wasi-sdk to create debug wasm",
+      "inherits": "wasm",
+      "environment": {
+        "CMAKE_BUILD_TYPE": "Debug"
+      }
+    },
+    {
+      "name": "wasm-threads",
+      "displayName": "Build for pthread enabled WASM",
+      "description": "Build for pthread enabled WASM",
+      "inherits": "wasm",
+      "binaryDir": "build-wasm-threads",
+      "environment": {
+        "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk-20.0+threads",
+        "CMAKE_BUILD_TYPE": "Release"
+      },
+      "cacheVariables": {
+        "MULTITHREADING": "ON"
       }
     }
   ],
@@ -174,9 +202,21 @@
       "configurePreset": "wasm",
       "inheritConfigureEnvironment": true,
       "jobs": 0,
-      "targets": [
-        "barretenberg.wasm"
-      ]
+      "targets": ["barretenberg.wasm"]
+    },
+    {
+      "name": "wasm-dbg",
+      "configurePreset": "wasm-dbg",
+      "inheritConfigureEnvironment": true,
+      "jobs": 0,
+      "targets": ["barretenberg.wasm"]
+    },
+    {
+      "name": "wasm-threads",
+      "configurePreset": "wasm-threads",
+      "inheritConfigureEnvironment": true,
+      "jobs": 0,
+      "targets": ["barretenberg.wasm"]
     }
   ],
   "testPresets": [
@@ -231,4 +271,4 @@
       "inheritConfigureEnvironment": true
     }
   ]
-}
+}

cpp/bootstrap.sh

@@ -4,7 +4,6 @@ set -eu
 # Clean.
 rm -rf ./build
 rm -rf ./build-wasm
-rm -rf ./src/wasi-sdk-*
 
 # Install formatting git hook.
 HOOKS_DIR=$(git rev-parse --git-path hooks)
@@ -63,12 +62,13 @@ echo "#################################"
 cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
 cmake --build --preset $PRESET ${@/#/--target }
 
-# Install the webassembly toolchain.
-WASI_VERSION=12
-cd ./src
-curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_VERSION/wasi-sdk-$WASI_VERSION.0-$OS.tar.gz | tar zxfv -
-cd ..
+# Install wasi-sdk.
+./scripts/install-wasi-sdk.sh
 
 # Build WASM.
 cmake --preset wasm
 cmake --build --preset wasm
+
+# Build WASM with new threading.
+cmake --preset wasm-threads
+cmake --build --preset wasm-threads