feat(bb): integrate tracy memory/cpu profiler

.github/workflows/ci.yml

@@ -279,7 +279,7 @@ jobs:
   # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: ${{ needs.changes.outputs.barretenberg-cpp == 'true' }}
     steps:

barretenberg/cpp/CMakeLists.txt

@@ -37,6 +37,7 @@ option(ENABLE_ASAN "Address sanitizer for debugging tricky memory corruption" OF
 option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF)
 # Note: Must do 'sudo apt-get install libdw-dev' or equivalent
 option(CHECK_CIRCUIT_STACKTRACES "Enable (slow) stack traces for check circuit" OFF)
+option(ENABLE_TRACY "Enable low-medium overhead profiling for memory and performance with tracy" OFF)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
     message(STATUS "Compiling for ARM.")
@@ -52,6 +53,10 @@ if(CHECK_CIRCUIT_STACKTRACES)
     add_compile_options(-DCHECK_CIRCUIT_STACKTRACES)
 endif()
 
+if(ENABLE_TRACY)
+    add_compile_options(-DTRACY_ENABLE)
+endif()
+
 if(ENABLE_ASAN)
     add_compile_options(-fsanitize=address)
     add_link_options(-fsanitize=address)
@@ -139,6 +144,7 @@ include(cmake/build.cmake)
 include(GNUInstallDirs)
 include(cmake/arch.cmake)
 include(cmake/threading.cmake)
+include(cmake/tracy.cmake)
 include(cmake/gtest.cmake)
 include(cmake/benchmark.cmake)
 include(cmake/module.cmake)

barretenberg/cpp/CMakePresets.json

@@ -83,6 +83,26 @@
         "DISABLE_ASM": "ON"
       }
     },
+    {
+      "name": "tracy",
+      "displayName": "Release build with tracy",
+      "description": "Release build with tracy",
+      "inherits": "clang16",
+      "binaryDir": "build-tracy",
+      "cacheVariables": {
+        "ENABLE_TRACY": "ON"
+      }
+    },
+    {
+      "name": "wasm-tracy",
+      "displayName": "WASM build with tracy",
+      "description": "WASM build with tracy",
+      "inherits": "clang16",
+      "binaryDir": "build-wasm-tracy",
+      "cacheVariables": {
+        "ENABLE_TRACY": "ON"
+      }
+    },
     {
       "name": "clang16-dbg-fast",
       "displayName": "Optimized debug build with Clang-16",
@@ -329,7 +349,8 @@
         "CMAKE_BUILD_TYPE": "Release"
       },
       "cacheVariables": {
-        "MULTITHREADING": "ON"
+        "MULTITHREADING": "ON",
+        "Threads_FOUND": "ON"
       }
     },
     {
@@ -410,6 +431,11 @@
       "inherits": "default",
       "configurePreset": "clang16-dbg"
     },
+    {
+      "name": "tracy",
+      "inherits": "default",
+      "configurePreset": "tracy"
+    },
     {
       "name": "clang16-dbg-fast",
       "inherits": "default",

barretenberg/cpp/cmake/module.cmake

@@ -53,6 +53,7 @@ function(barretenberg_module MODULE_NAME)
             ${MODULE_NAME}
             PUBLIC
             ${ARGN}
+            Tracy::TracyClient
             ${TBB_IMPORTED_TARGETS}
         )
 
@@ -89,6 +90,7 @@ function(barretenberg_module MODULE_NAME)
         target_link_libraries(
             ${MODULE_NAME}_test_objects
             PRIVATE
+            Tracy::TracyClient
             GTest::gtest
             GTest::gtest_main
             GTest::gmock_main
@@ -147,6 +149,7 @@ function(barretenberg_module MODULE_NAME)
             GTest::gtest
             GTest::gtest_main
             GTest::gmock_main
+            Tracy::TracyClient
             ${TBB_IMPORTED_TARGETS}
         )
 
@@ -238,6 +241,7 @@ function(barretenberg_module MODULE_NAME)
                 ${BENCHMARK_NAME}_bench_objects
                 PRIVATE
                 benchmark::benchmark
+                Tracy::TracyClient
                 ${TBB_IMPORTED_TARGETS}
             )
 
@@ -253,6 +257,7 @@ function(barretenberg_module MODULE_NAME)
                 ${MODULE_LINK_NAME}
                 ${ARGN}
                 benchmark::benchmark
+                Tracy::TracyClient
                 ${TBB_IMPORTED_TARGETS}
             )
             if(CHECK_CIRCUIT_STACKTRACES)

barretenberg/cpp/cmake/msgpack.cmake

@@ -1,6 +1,6 @@
 include(ExternalProject)
 
-# External project: Download msgpack-c from GitHu
+# External project: Download msgpack-c from GitHub
 set(MSGPACK_PREFIX "${CMAKE_BINARY_DIR}/_deps/msgpack-c")
 set(MSGPACK_INCLUDE "${MSGPACK_PREFIX}/src/msgpack-c/include")
 

barretenberg/cpp/cmake/tracy.cmake

@@ -0,0 +1,10 @@
+include(FetchContent)
+
+set(TRACY_INCLUDE "${CMAKE_BINARY_DIR}/_deps/tracy-src/public")
+
+FetchContent_Declare(tracy
+    GIT_REPOSITORY https://github.com/wolfpld/tracy
+    GIT_TAG ffb98a972401c246b2348fb5341252e2ba855d00
+    SYSTEM          # optional, the tracy include directory will be treated as system directory
+)
+FetchContent_MakeAvailable(tracy)

barretenberg/cpp/scripts/benchmark_tracy.sh

@@ -0,0 +1,32 @@
+
+# NOTE: intended to be ran from one's external computer, connecting to Aztec mainframe
+# the benchmark runs with headless capture and then we copy the trace file and run tracy profiler
+# This is thus only really useful internally at Aztec, sorry external folks. It can be easily tweaked
+# however for any SSH setup, especially an ubuntu one, and of course if you are just tracing on the
+# same machine you can use the normal interactive tracy workflow.
+set -eux
+USER=$1
+BOX=$USER-box
+BENCHMARK=${2:-ultra_plonk_bench}
+COMMAND=${3:-./bin/$BENCHMARK}
+
+ssh $BOX "
+	set -eux ;
+	! [ -d ~/tracy ] && git clone https://github.com/wolfpld/tracy ~/tracy ;
+	cd ~/tracy/capture ;
+	sudo apt-get install libdbus-1-dev libdbus-glib-1-dev ;
+	mkdir -p build && cd build && cmake .. && make -j ;
+	./tracy-capture -a 127.0.0.1 -f -o trace-$BENCHMARK & ;
+	sleep 0.1 ;
+	cd ~/aztec-packages/barretenberg/cpp/ ;
+	cmake --preset tracy && cmake --build --preset tracy --parallel $BENCHMARK ;
+	cd build-tracy ;
+	ninja $BENCHMARK ;
+	$COMMAND ;
+"
+! [ -d ~/tracy ] && git clone https://github.com/wolfpld/tracy ~/tracy
+cd ~/tracy
+cmake -B profiler/build -S profiler -DCMAKE_BUILD_TYPE=Release
+cmake --build profiler/build --parallel
+scp $BOX:/mnt/user-data/$USER/tracy/capture/build/trace-$BENCHMARK .
+~/tracy/profiler/build/tracy-profiler trace-$BENCHMARK

barretenberg/cpp/src/CMakeLists.txt

@@ -41,7 +41,7 @@ if(WASM)
     add_link_options(-Wl,--export-memory,--import-memory,--stack-first,-z,stack-size=1048576,--max-memory=4294967296)
 endif()
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE})
 
 # I feel this should be limited to ecc, however it's currently used in headers that go across libraries,
 # and there currently isn't an easy way to inherit the DDISABLE_ASM parameter.

barretenberg/cpp/src/barretenberg/bb/CMakeLists.txt

@@ -12,6 +12,7 @@ if (NOT(FUZZING) AND NOT(WASM))
         barretenberg
         env
         circuit_checker
+        Tracy::TracyClient
         libdeflate::libdeflate_static
     )
     if(CHECK_CIRCUIT_STACKTRACES)

barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp

@@ -7,6 +7,7 @@
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
 #include "barretenberg/ultra_honk/ultra_verifier.hpp"
+#include "tracy/Tracy.hpp"
 
 using namespace benchmark;
 using namespace bb;
@@ -30,6 +31,8 @@ class ClientIVCBench : public benchmark::Fixture {
         bb::srs::init_grumpkin_crs_factory("../srs_db/grumpkin");
     }
 
+    void TearDown([[maybe_unused]] const ::benchmark::State& state) override {}
+
     /**
      * @brief Compute verification key for each circuit in the IVC based on the number of desired function circuits
      * @details Assumes the following circuit ordering: one initial function circuit followed by pairs of {function,

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -56,6 +56,7 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verific
  */
 ClientIVC::Proof ClientIVC::prove()
 {
+    ZoneScoped();
     max_block_size_tracker.print(); // print minimum structured sizes for each block
     return { fold_output.proof, decider_prove(), goblin.prove() };
 };
@@ -85,7 +86,7 @@ bool ClientIVC::verify(const Proof& proof,
  * @param proof
  * @return bool
  */
-bool ClientIVC::verify(Proof& proof, const std::vector<std::shared_ptr<VerifierInstance>>& verifier_instances)
+bool ClientIVC::verify(Proof& proof, const std::vector<std::shared_ptr<VerifierInstance>>& verifier_instances) const
 {
     auto eccvm_vk = std::make_shared<ECCVMVerificationKey>(goblin.get_eccvm_proving_key());
     auto translator_vk = std::make_shared<TranslatorVerificationKey>(goblin.get_translator_proving_key());

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp

@@ -87,7 +87,7 @@ class ClientIVC {
                        const std::shared_ptr<ClientIVC::ECCVMVerificationKey>& eccvm_vk,
                        const std::shared_ptr<ClientIVC::TranslatorVerificationKey>& translator_vk);
 
-    bool verify(Proof& proof, const std::vector<std::shared_ptr<VerifierInstance>>& verifier_instances);
+    bool verify(Proof& proof, const std::vector<std::shared_ptr<VerifierInstance>>& verifier_instances) const;
 
     bool prove_and_verify();
 

barretenberg/cpp/src/barretenberg/common/mem.cpp

@@ -0,0 +1,18 @@
+#include "barretenberg/common/mem.hpp"
+
+void* operator new(std::size_t count)
+{
+    // NOLINTBEGIN(cppcoreguidelines-no-malloc)
+    void* ptr = malloc(count);
+    // NOLINTEND(cppcoreguidelines-no-malloc)
+    TRACY_ALLOC(ptr, count);
+    return ptr;
+}
+
+void operator delete(void* ptr) noexcept
+{
+    TRACY_FREE(ptr);
+    // NOLINTBEGIN(cppcoreguidelines-no-malloc)
+    free(ptr);
+    // NOLINTEND(cppcoreguidelines-no-malloc)
+}

barretenberg/cpp/src/barretenberg/common/mem.hpp

@@ -1,10 +1,17 @@
 #pragma once
 #include "log.hpp"
 #include "memory.h"
+#include "tracy/Tracy.hpp"
 #include "wasm_export.hpp"
 #include <cstdlib>
 #include <memory>
-// #include <malloc.h>
+
+// This can be altered to capture stack traces, though more expensive
+// This is the only reason we wrap TracyAlloc or TracyAllocS
+#define TRACY_ALLOC(t, size) TracyAllocS(t, size, /*stack depth*/ 10)
+#define TRACY_FREE(t) TracyFreeS(t, /*stack depth*/ 10)
+// #define TRACY_ALLOC(t, size) TracyAlloc(t, size)
+// #define TRACY_FREE(t) TracyFree(t)
 
 #define pad(size, alignment) (size - (size % alignment) + ((size % alignment) == 0 ? 0 : alignment))
 
@@ -17,11 +24,13 @@ inline void* aligned_alloc(size_t alignment, size_t size)
         info("bad alloc of size: ", size);
         std::abort();
     }
+    TRACY_ALLOC(t, size);
     return t;
 }
 
 inline void aligned_free(void* mem)
 {
+    TRACY_FREE(mem);
     free(mem);
 }
 #endif
@@ -41,13 +50,15 @@ inline void* protected_aligned_alloc(size_t alignment, size_t size)
         info("bad alloc of size: ", size);
         std::abort();
     }
+    TRACY_ALLOC(t, size);
     return t;
 }
 
 #define aligned_alloc protected_aligned_alloc
 
 inline void aligned_free(void* mem)
 {
+    TRACY_FREE(mem);
     // NOLINTNEXTLINE(cppcoreguidelines-owning-memory, cppcoreguidelines-no-malloc)
     free(mem);
 }
@@ -56,11 +67,14 @@ inline void aligned_free(void* mem)
 #ifdef _WIN32
 inline void* aligned_alloc(size_t alignment, size_t size)
 {
-    return _aligned_malloc(size, alignment);
+    void* t = _aligned_malloc(size, alignment);
+    TRACY_ALLOC(t, size);
+    return t;
 }
 
 inline void aligned_free(void* mem)
 {
+    TRACY_FREE(mem);
     _aligned_free(mem);
 }
 #endif
@@ -79,4 +93,19 @@ inline void aligned_free(void* mem)
 //     info("Total allocated space (uordblks): ", minfo.uordblks);
 //     info("Total free space (fordblks): ", minfo.fordblks);
 //     info("Top-most, releasable space (keepcost): ", minfo.keepcost);
-// }
+// }
+
+inline void* tracy_malloc(size_t size)
+{
+    // NOLINTNEXTLINE(cppcoreguidelines-owning-memory, cppcoreguidelines-no-malloc)
+    void* t = malloc(size);
+    TRACY_ALLOC(t, size);
+    return t;
+}
+
+inline void tracy_free(void* mem)
+{
+    TRACY_FREE(mem);
+    // NOLINTNEXTLINE(cppcoreguidelines-owning-memory, cppcoreguidelines-no-malloc)
+    free(mem);
+}

barretenberg/cpp/src/barretenberg/common/slab_allocator.cpp

@@ -181,7 +181,7 @@ std::shared_ptr<void> SlabAllocator::get(size_t req_size)
         return { aligned_alloc(32, req_size), aligned_free };
     }
     // NOLINTNEXTLINE(cppcoreguidelines-no-malloc)
-    return { malloc(req_size), free };
+    return { tracy_malloc(req_size), tracy_free };
 }
 
 size_t SlabAllocator::get_total_size()
@@ -209,11 +209,6 @@ void init_slab_allocator(size_t circuit_subgroup_size)
     allocator.init(circuit_subgroup_size);
 }
 
-// auto init = ([]() {
-//     init_slab_allocator(524288);
-//     return 0;
-// })();
-
 std::shared_ptr<void> get_mem_slab(size_t size)
 {
     return allocator.get(size);