Proposed 2.4.0-b1

.github/workflows/instrumentation.yml

@@ -0,0 +1,103 @@
+name: instrumentation
+on:
+  pull_request:
+  push:
+    # If the branches list is ever changed, be sure to change it on all
+    # build/test jobs (nix, macos, windows, instrumentation)
+    branches:
+      # Always build the package branches
+      - develop
+      - release
+      - master
+      # Branches that opt-in to running
+      - 'ci/**'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  # NOTE we are not using dependencies built inside nix because nix is lagging
+  # with compiler versions. Instrumentation requires clang version 16 or later
+
+  instrumentation-build:
+    env:
+      CLANG_RELEASE: 16
+    strategy:
+      fail-fast: false
+    runs-on: [self-hosted, heavy]
+    container: debian:bookworm
+    steps:
+        - name: install prerequisites
+          env:
+            DEBIAN_FRONTEND: noninteractive
+          run: |
+            apt-get update
+            apt-get install --yes --no-install-recommends \
+              clang-${CLANG_RELEASE} clang++-${CLANG_RELEASE} \
+              python3-pip python-is-python3 make cmake git wget
+            apt-get clean
+            update-alternatives --install \
+              /usr/bin/clang clang /usr/bin/clang-${CLANG_RELEASE} 100 \
+              --slave /usr/bin/clang++ clang++ /usr/bin/clang++-${CLANG_RELEASE}
+            update-alternatives --auto clang
+            pip install --no-cache --break-system-packages "conan<2"
+
+        - name: checkout
+          uses: actions/checkout@v4
+
+        - name: prepare environment
+          run: |
+            mkdir ${GITHUB_WORKSPACE}/.build
+            echo "SOURCE_DIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV
+            echo "BUILD_DIR=$GITHUB_WORKSPACE/.build" >> $GITHUB_ENV
+            echo "CC=/usr/bin/clang" >> $GITHUB_ENV
+            echo "CXX=/usr/bin/clang++" >> $GITHUB_ENV
+
+        - name: configure Conan
+          run: |
+            conan profile new --detect default
+            conan profile update settings.compiler=clang default
+            conan profile update settings.compiler.version=${CLANG_RELEASE} default
+            conan profile update settings.compiler.libcxx=libstdc++11 default
+            conan profile update settings.compiler.cppstd=20 default
+            conan profile update options.rocksdb=False default
+            conan profile update \
+              'conf.tools.build:compiler_executables={"c": "/usr/bin/clang", "cpp": "/usr/bin/clang++"}' default
+            conan profile update 'env.CXXFLAGS="-DBOOST_ASIO_DISABLE_CONCEPTS"' default
+            conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]' default
+            conan export external/snappy snappy/1.1.10@
+            conan export external/soci soci/4.0.3@
+
+        - name: build dependencies
+          run: |
+            cd ${BUILD_DIR}
+            conan install ${SOURCE_DIR} \
+              --output-folder ${BUILD_DIR} \
+              --install-folder ${BUILD_DIR} \
+              --build missing \
+              --settings build_type=Debug
+
+        - name: build with instrumentation
+          run: |
+            cd ${BUILD_DIR}
+            cmake -S ${SOURCE_DIR} -B ${BUILD_DIR} \
+              -Dvoidstar=ON \
+              -Dtests=ON \
+              -Dxrpld=ON \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DSECP256K1_BUILD_BENCHMARK=OFF \
+              -DSECP256K1_BUILD_TESTS=OFF \
+              -DSECP256K1_BUILD_EXHAUSTIVE_TESTS=OFF \
+              -DCMAKE_TOOLCHAIN_FILE=${BUILD_DIR}/build/generators/conan_toolchain.cmake
+            cmake --build .  --parallel $(nproc)
+
+        - name: verify instrumentation enabled
+          run: |
+            cd ${BUILD_DIR}
+            ./rippled --version | grep libvoidstar
+
+        - name: run unit tests
+          run: |
+            cd ${BUILD_DIR}
+            ./rippled -u --unittest-jobs $(( $(nproc)/4 ))

.github/workflows/macos.yml

@@ -3,7 +3,7 @@ on:
   pull_request:
   push:
     # If the branches list is ever changed, be sure to change it on all
-    # build/test jobs (nix, macos, windows)
+    # build/test jobs (nix, macos, windows, instrumentation)
     branches:
       # Always build the package branches
       - develop

.github/workflows/nix.yml

@@ -3,7 +3,7 @@ on:
   pull_request:
   push:
     # If the branches list is ever changed, be sure to change it on all
-    # build/test jobs (nix, macos, windows)
+    # build/test jobs (nix, macos, windows, instrumentation)
     branches:
       # Always build the package branches
       - develop

.github/workflows/windows.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
   push:
     # If the branches list is ever changed, be sure to change it on all
-    # build/test jobs (nix, macos, windows)
+    # build/test jobs (nix, macos, windows, instrumentation)
     branches:
       # Always build the package branches
       - develop

API-CHANGELOG.md

@@ -83,11 +83,13 @@ The [commandline](https://xrpl.org/docs/references/http-websocket-apis/api-conve
 
 The `network_id` field was added in the `server_info` response in version 1.5.0 (2019), but it is not returned in [reporting mode](https://xrpl.org/rippled-server-modes.html#reporting-mode). However, use of reporting mode is now discouraged, in favor of using [Clio](https://github.com/XRPLF/clio) instead.
 
-## XRP Ledger server version 2.2.0
+## XRP Ledger server version 2.4.0
 
-The following is a non-breaking addition to the API.
+### Addition in 2.4
 
-- The `feature` method now has a non-admin mode for users. (It was previously only available to admin connections.) The method returns an updated list of amendments, including their names and other information. ([#4781](https://github.com/XRPLF/rippled/pull/4781))
+- `ledger_entry`: `state` is added an alias for `ripple_state`.
+
+## XRP Ledger server version 2.3.0
 
 ### Breaking change in 2.3
 
@@ -105,6 +107,12 @@ The following additions are non-breaking (because they are purely additive).
 - In `Payment` transactions, `DeliverMax` has been added. This is a replacement for the `Amount` field, which should not be used. Typically, the `delivered_amount` (in transaction metadata) should be used. To ease the transition, `DeliverMax` is present regardless of API version, since adding a field is non-breaking.
 - API version 2 has been moved from beta to supported, meaning that it is generally available (regardless of the `beta_rpc_api` setting).
 
+## XRP Ledger server version 2.2.0
+
+The following is a non-breaking addition to the API.
+
+- The `feature` method now has a non-admin mode for users. (It was previously only available to admin connections.) The method returns an updated list of amendments, including their names and other information. ([#4781](https://github.com/XRPLF/rippled/pull/4781))
+
 ## XRP Ledger server version 1.12.0
 
 [Version 1.12.0](https://github.com/XRPLF/rippled/releases/tag/1.12.0) was released on Sep 6, 2023. The following additions are non-breaking (because they are purely additive).

Builds/levelization/results/loops.txt

@@ -4,9 +4,6 @@ Loop: test.jtx test.toplevel
 Loop: test.jtx test.unit_test
   test.unit_test == test.jtx
 
-Loop: xrpl.basics xrpl.json
-  xrpl.json == xrpl.basics
-
 Loop: xrpld.app xrpld.core
   xrpld.app > xrpld.core
 

Builds/levelization/results/ordering.txt

@@ -1,5 +1,4 @@
 libxrpl.basics > xrpl.basics
-libxrpl.basics > xrpl.protocol
 libxrpl.crypto > xrpl.basics
 libxrpl.json > xrpl.basics
 libxrpl.json > xrpl.json
@@ -130,6 +129,7 @@ test.shamap > xrpl.protocol
 test.toplevel > test.csf
 test.toplevel > xrpl.json
 test.unit_test > xrpl.basics
+xrpl.json > xrpl.basics
 xrpl.protocol > xrpl.basics
 xrpl.protocol > xrpl.json
 xrpl.resource > xrpl.basics

CMakeLists.txt

@@ -73,6 +73,7 @@ set(SECP256K1_INSTALL TRUE)
 add_subdirectory(external/secp256k1)
 add_library(secp256k1::secp256k1 ALIAS secp256k1)
 add_subdirectory(external/ed25519-donna)
+add_subdirectory(external/antithesis-sdk)
 find_package(gRPC REQUIRED)
 find_package(lz4 REQUIRED)
 # Target names with :: are not allowed in a generator expression.

CONTRIBUTING.md

@@ -343,6 +343,68 @@ pip3 install pre-commit
 pre-commit install
 ```
 
+## Contracts and instrumentation
+
+We are using [Antithesis](https://antithesis.com/) for continuous fuzzing,
+and keep a copy of [Antithesis C++ SDK](https://github.com/antithesishq/antithesis-sdk-cpp/)
+in `external/antithesis-sdk`. One of the aims of fuzzing is to identify bugs
+by finding external conditions which cause contracts violations inside `rippled`.
+The contracts are expressed as `XRPL_ASSERT` or `UNREACHABLE` (defined in
+`include/xrpl/beast/utility/instrumentation.h`), which are effectively (outside
+of Antithesis) wrappers for `assert(...)` with added name. The purpose of name
+is to provide contracts with stable identity which does not rely on line numbers.
+
+When `rippled` is built with the Antithesis instrumentation enabled
+(using `voidstar` CMake option) and ran on the Antithesis platform, the
+contracts become
+[test properties](https://antithesis.com/docs/using_antithesis/properties.html);
+otherwise they are just like a regular `assert`.
+To learn more about Antithesis, see
+[How Antithesis Works](https://antithesis.com/docs/introduction/how_antithesis_works.html)
+and [C++ SDK](https://antithesis.com/docs/using_antithesis/sdk/cpp/overview.html#)
+
+We continue to use the old style `assert` or `assert(false)` in certain
+locations, where the reporting of contract violations on the Antithesis
+platform is either not possible or not useful.
+
+For this reason:
+* The locations where `assert` or `assert(false)` contracts should continue to be used:
+  * `constexpr` functions
+  * unit tests i.e. files under `src/test`
+  * unit tests-related modules (files under `beast/test` and `beast/unit_test`)
+* Outside of the listed locations, do not use `assert`; use `XRPL_ASSERT` instead,
+  giving it unique name, with the short description of the contract.
+* Outside of the listed locations, do not use `assert(false)`; use
+  `UNREACHABLE` instead, giving it unique name, with the description of the
+  condition being violated
+* The contract name should start with a full name (including scope) of the
+  function, optionally a named lambda, followed by a colon ` : ` and a brief
+  (typically at most five words) description. `UNREACHABLE` contracts
+  can use slightly longer descriptions. If there are multiple overloads of the
+  function, use common sense to balance both brevity and unambiguity of the
+  function name. NOTE: the purpose of name is to provide stable means of
+  unique identification of every contract; for this reason try to avoid elements
+  which can change in some obvious refactors or when reinforcing the condition.
+* Contract description typically (except for `UNREACHABLE`) should describe the
+  _expected_ condition, as in "I assert that _expected_ is true".
+* Contract description for `UNREACHABLE` should describe the _unexpected_
+  situation which caused the line to have been reached.
+* Example good name for an
+  `UNREACHABLE` macro `"Json::operator==(Value, Value) : invalid type"`; example
+  good name for an `XRPL_ASSERT` macro `"Json::Value::asCString : valid type"`.
+* Example **bad** name
+  `"RFC1751::insert(char* s, int x, int start, int length) : length is greater than or equal zero"`
+  (missing namespace, unnecessary full function signature, description too verbose).
+  Good name: `"ripple::RFC1751::insert : minimum length"`.
+* In **few** well-justified cases a non-standard name can be used, in which case a
+  comment should be placed to explain the rationale (example in `contract.cpp`)
+* Do **not** rename a contract without a good reason (e.g. the name no longer
+  reflects the location or the condition being checked)
+* Do not use `std::unreachable`
+* Do not put contracts where they can be violated by an external condition
+  (e.g. timing, data payload before mandatory validation etc.) as this creates
+  bogus bug reports (and causes crashes of Debug builds)
+
 ## Unit Tests
 To execute all unit tests:
 

cmake/RippledCompiler.cmake

@@ -120,7 +120,7 @@ else ()
   target_link_libraries (common
     INTERFACE
       -rdynamic
-      $<$<BOOL:${is_linux}>:-Wl,-z,relro,-z,now>
+      $<$<BOOL:${is_linux}>:-Wl,-z,relro,-z,now,--build-id>
       # link to static libc/c++ iff:
       #   * static option set and
       #   * NOT APPLE (AppleClang does not support static libc/c++) and
@@ -131,6 +131,17 @@ else ()
       >)
 endif ()
 
+# Antithesis instrumentation will only be built and deployed using machines running Linux.
+if (voidstar)
+  if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message(FATAL_ERROR "Antithesis instrumentation requires Debug build type, aborting...")
+  elseif (NOT is_linux)
+    message(FATAL_ERROR "Antithesis instrumentation requires Linux, aborting...")
+  elseif (NOT (is_clang AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 16.0))
+    message(FATAL_ERROR "Antithesis instrumentation requires Clang version 16 or later, aborting...")
+  endif ()
+endif ()
+
 if (use_mold)
   # use mold linker if available
   execute_process (