realm · nicola-cab · Feb 2, 2023 · Oct 12, 2022 · Oct 12, 2022 · Oct 13, 2022
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,6 +36,7 @@
 * Remove `File::is_removed` ([#6222](https://github.com/realm/realm-core/pull/6222))
 * Client reset recovery froze Realms for the callbacks in an invalid way. It is unclear if this resulted in any actual problems.
 * Fix default enabled debug output during realm-sync-tests ([#6233](https://github.com/realm/realm-core/issues/6233))
+* Create the fuzzer framework project in order to run fuzz testing on evergreen ([PR #5940](https://github.com/realm/realm-core/pull/5940))
 * Update ClientImpl::Connection and DefaultWebSocketImpl to use the new WebSocketObserver callbacks ([PR #6219](https://github.com/realm/realm-core/pull/6219))
 * Switched client reset tests to using private `force_client_reset` server API ([PR #6216](https://github.com/realm/realm-core/pull/6216))
 

diff --git a/evergreen/config.yml b/evergreen/config.yml
@@ -32,7 +32,7 @@ functions:
  script: |-
  set -o errexit
  git submodule update --init --recursive
-
+ 
  "compile":
  - command: shell.exec
  params:
@@ -81,6 +81,10 @@ functions:
  set_cmake_var realm_vars REALM_ENABLE_ALLOC_SET_ZERO BOOL On
  fi
 
+ if [ -n "${enable_fuzzer|}" ]; then
+ set_cmake_var realm_vars REALM_LIBFUZZER BOOL On
+ fi
+
  if [ -z "${disable_sync|}" ]; then
  set_cmake_var realm_vars REALM_ENABLE_SYNC BOOL On
  fi
@@ -224,6 +228,35 @@ functions:
  content_type: text/text
  display_name: install baas output
  optional: true
+
+ "upload fuzzer results":
+ - command: shell.exec
+ params:
+ working_dir: realm-core/build/test/realm-fuzzer
+ script: |-
+ if ls crash-*> /dev/null 2>&1; then
+ echo "Found crash file"
+ mv crash-* realm-fuzzer-crash.txt
+ fi
+
+ - command: s3.put
+ params:
+ working_dir: realm-core/build/test/realm-fuzzer
+ aws_key: '${artifacts_aws_access_key}'
+ aws_secret: '${artifacts_aws_secret_key}'
+ local_file: 'realm-core/build/test/realm-fuzzer/realm-fuzzer-crash.txt'
+ remote_file: '${project}/${branch_name}/${task_id}/${execution}/realm-fuzzer-crash.txt'
+ bucket: mciuploads
+ permissions: public-read
+ content_type: text/text
+ display_name: Fuzzer crash report
+ optional: true
+
+ - command: shell.exec
+ params:
+ working_dir: realm-core/build/test/realm-fuzzer
+ script: |-
+ rm realm-fuzzer-crash.txt
 
  "run hang analyzer":
  - command: shell.exec
@@ -659,6 +692,17 @@ tasks:
  echo $out
  exit 1
 
+- name: fuzzer
+ tags: [ "test_suite"]
+ commands:
+ - command: shell.exec
+ timeout_secs: 3600
+ params:
+ working_dir: realm-core/build/test/realm-fuzzer
+ shell: /bin/bash
+ script: |-
+ ${cmake_build_type|Debug}/realm-libfuzz
+
 task_groups:
 - name: compile_test_and_package
  max_hosts: 1
@@ -730,6 +774,19 @@ task_groups:
  tasks:
  - long-running-core-tests
 
+- name: fuzzer-tests
+ setup_group_can_fail_task: true
+ setup_group:
+ - func: "fetch source"
+ - func: "fetch binaries"
+ - func: "compile"
+ vars:
+ target_to_build: realm-libfuzz
+ teardown_task:
+ - func: "upload fuzzer results"
+ tasks:
+ - fuzzer
+
 buildvariants:
 - name: ubuntu2004
  display_name: "Ubuntu 20.04 x86_64 (Clang 11)"
@@ -862,6 +919,27 @@ buildvariants:
  distros:
  - ubuntu2004-large
 
+- name: ubuntu2004-fuzzer
+ display_name: "Ubuntu 20.04 x86_64 (Clang 11 Fuzzer)"
+ run_on: ubuntu2004-small
+ expansions:
+ clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz"
+ cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-linux-x86_64.tar.gz"
+ cmake_bindir: "./cmake_binaries/bin"
+ fetch_missing_dependencies: On
+ run_tests_against_baas: On
+ enable_ubsan: On
+ c_compiler: "./clang_binaries/bin/clang"
+ cxx_compiler: "./clang_binaries/bin/clang++"
+ cmake_build_type: RelWithDebInfo
+ run_with_encryption: On
+ enable_fuzzer: On
+ tasks:
+ - name: fuzzer-tests
+ cron: "@daily"
+ patchable: false
+
+
 - name: rhel70
  display_name: "RHEL 7 x86_64"
  run_on: rhel70-small

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -2,9 +2,10 @@ add_subdirectory(util)
 add_custom_target(benchmarks)
 add_subdirectory(object-store)
 
-# AFL not yet supported by Windows
+# AFL and LIBFUZZER not yet supported by Windows
 if(NOT CMAKE_SYSTEM_NAME MATCHES "^Windows")
  add_subdirectory(fuzzy)
+ add_subdirectory(realm-fuzzer)
 endif()
 
 add_subdirectory(benchmark-common-tasks)

diff --git a/test/fuzzy/libfuzzer_entry.cpp b/test/fuzzy/libfuzzer_entry.cpp
@@ -1,11 +1,5 @@
 #include <realm/db.hpp>
 #include <realm/history.hpp>
-
-#include <ctime>
-#include <cstdio>
-#include <fstream>
-#include <iostream>
-
 #include "../fuzz_group.hpp"
 #include "../util/test_path.hpp"
 

diff --git a/test/object-store/CMakeLists.txt b/test/object-store/CMakeLists.txt
@@ -126,36 +126,23 @@ target_include_directories(ObjectStoreTests PRIVATE
 # on Apple platforms we use the built-in CFRunLoop
 # everywhere else it's libuv, except UWP where it doesn't build
 if(NOT APPLE AND NOT WINDOWS_STORE)
- option(REALM_FETCH_MISSING_DEPENDENCIES "Download missing dependencies with CMake's FetchContent where possible" ON)
- if(REALM_FETCH_MISSING_DEPENDENCIES)
- find_package(LibUV)
- else()
- find_package(LibUV REQUIRED)
+ include(FetchContent)
+ set(libUV_Git_TAG "v1.35.0")
+ if(MSVC)
+ set(liUV_Git_TAG "v1.43.0")
  endif()
- if(LibUV_FOUND)
- set(libuv_target LibUV::LibUV)
- elseif(REALM_FETCH_MISSING_DEPENDENCIES)
- message(STATUS "LibUV not found, building from source with FetchContent")
- include(FetchContent)
- set(libUV_Git_TAG "v1.35.0")
- if(MSVC)
- set(liUV_Git_TAG "v1.43.0")
- endif()
- FetchContent_Declare(
- libuv
- GIT_REPOSITORY https://github.com/libuv/libuv.git
- GIT_TAG ${libUV_Git_TAG}
- )
- # Don't use FetchContent_MakeAvailable since it wants to build libuv.so as well
- FetchContent_Populate(libuv)
- add_subdirectory(${libuv_SOURCE_DIR} ${libuv_BINARY_DIR} EXCLUDE_FROM_ALL)
- set(libuv_target uv_a)
- endif()
- target_link_libraries(ObjectStoreTests ${libuv_target})
+ FetchContent_Declare(
+ libuv
+ GIT_REPOSITORY https://github.com/libuv/libuv.git
+ GIT_TAG ${libUV_Git_TAG}
+ )
+ # Don't use FetchContent_MakeAvailable since it wants to build libuv.so as well
+ FetchContent_Populate(libuv)
+ add_subdirectory(${libuv_SOURCE_DIR} ${libuv_BINARY_DIR} EXCLUDE_FROM_ALL)
+ target_link_libraries(ObjectStoreTests uv_a)
  # FIXME: ObjectStore itself shouldn't care about this, but we need to refactor scheduler.cpp to make it happen
  target_compile_definitions(ObjectStore PUBLIC REALM_HAVE_UV=1)
- get_property(libuv_include_dir TARGET ${libuv_target} PROPERTY INCLUDE_DIRECTORIES)
- target_include_directories(ObjectStore PRIVATE ${libuv_include_dir})
+ target_include_directories(ObjectStore PRIVATE $<TARGET_PROPERTY:uv_a,INCLUDE_DIRECTORIES>)
 endif()
 
 add_subdirectory(notifications-fuzzer)

diff --git a/test/realm-fuzzer/.gitignore b/test/realm-fuzzer/.gitignore
@@ -0,0 +1,5 @@
+findings/*
+fuzzy-tests*
+generate-fuzzy-input*
+*.d
+*.o
diff --git a/test/realm-fuzzer/CMakeLists.txt b/test/realm-fuzzer/CMakeLists.txt
@@ -0,0 +1,45 @@
+set(TEST_AFL_SOURCES
+ afl_runner.cpp
+ fuzz_engine.cpp
+ fuzz_object.cpp
+ fuzz_configurator.cpp
+) # TEST_AFL_SOURCES_OBJECT_STORE
+
+set(TEST_LIBFUZZER_SOURCES
+ libfuzzer_runner.cpp
+ fuzz_engine.cpp
+ fuzz_object.cpp
+ fuzz_configurator.cpp
+) # TEST_LIBFUZZER_SOURCES_OBJECT_STORE
+
+file(GLOB FUZZER_RUN_SCRIPTS
+ "scripts/start_fuzz_afl.sh"
+ "scripts/start_lib_fuzzer.sh")
+
+file(COPY ${FUZZER_RUN_SCRIPTS}
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(GLOB AFL_SEEDS "testcases/*")
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testcases)
+file(COPY ${AFL_SEEDS}
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/testcases)
+
+add_executable(realm-afl++ ${TEST_AFL_SOURCES})
+target_link_libraries(realm-afl++ TestUtil ObjectStore)
+
+if(REALM_LIBFUZZER)
+ if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ add_executable(realm-libfuzz ${TEST_LIBFUZZER_SOURCES})
+ target_link_libraries(realm-libfuzz TestUtil ObjectStore)
+ endif()
+endif()
+
+# on Apple platforms we use the built-in CFRunLoop
+# everywhere else it's libuv, except UWP where it doesn't build
+if(NOT APPLE AND NOT WINDOWS_STORE)
+ target_link_libraries(realm-afl++ uv_a)
+ if(REALM_LIBFUZZER)
+ if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ target_link_libraries(realm-libfuzz uv_a)
+ endif()
+ endif()
+endif()
diff --git a/test/realm-fuzzer/README.md b/test/realm-fuzzer/README.md
@@ -0,0 +1,72 @@
+# The Fuzz Framework project
+
+This project is an attempt to put together all the small fuzzers we have already scattered around the code.
+There are two goals:
+ 1. To be able to run all the fuzzers, collect crashes reports and fix possible bugs that the fuzzer might find.
+ 2. To be able to replace libfuzzer with google fuzz test (https://github.com/google/fuzztest) at some point.
+
+AFL++ support is not dropped yet, but since we want to integrate things inside evergreen and follow the same approach we implement for address/thread sanitazer we prefer to use libfuzzer and clang.
+## Prerequisites
+
+In case you want to use AFL++, then you should install the latest version of the American Fuzzy Lop ++ (AFL++).
+Please use this quick guide: https://aflplus.plus/building/ it requires llvm >= 9.0.
+
+For using libfuzzer, the only pre-requisite is having a recent version of clang.
+## Running
+If you don't want to build manually, you can skip this section and jump to the `Scripts` section. \
+Run the fuzzer via AFL++:
+
+```
+cd <realm-core-src> 
+mkdir build 
+cd build
+cmake -D CMAKE_BUILD_TYPE=${build_mode} 
+ -D CMAKE_C_COMPILER=afl-cc 
+ -D CMAKE_CXX_COMPILER=afl-c++ 
+ -D REALM_ENABLE_ENCRYPTION=ON 
+ -G Ninja 
+ ..
+cmake --build . --target realm-afl++
+afl-fuzz -t "$time_out" 
+ -m "$memory" 
+ -i "${ROOT_DIR}/test/fuzzy_object_store/testcases" 
+ -o "${FINDINGS_DIR}" 
+ realm-afl++ @@
+```
+
+Run the fuzzer via libFuzzer (only with Clang)
+```
+cd <realm-core-src>
+mkdir build
+cd build
+cmake -D REALM_LIBFUZZER=ON 
+ -D CMAKE_BUILD_TYPE=${build_mode} 
+ -D CMAKE_C_COMPILER=clang 
+ -D CMAKE_CXX_COMPILER=clang++ 
+ -D REALM_ENABLE_ENCRYPTION=ON 
+ -G Ninja 
+ ..
+cmake --build . --target realm-libfuzz
+./realm_libfuzz <corpus>
+```
+
+## Scripts
+
+`sh start_fuzz_afl.sh`
+Builds `realm-core` and `object-store` in `Debug` mode using the afl++ compiler `afl-cc` and starts 1 instance of `afl-fuzz`.
+It expects `AFLPlusPlus` to be installed in your system and in general added to your `PATH`. 
+Optionally, the following arguments can be passed to the script:
+1) `<num_fuzzers>` the number of fuzzers to launch (by default 1).
+2) `<build_mode>` either `Release` or `Debug`.
+
+`sh start_lib_fuzzer.sh`
+Builds `realm-core` and `object-store` in `Debug` mode using the clang compiler and starts `realm-libfuzz`.
+Optionally, the following arguments can be passed to the script:
+1) `<build_mode>` either `Release` or `Debug`. 
+2) `<corpus>` essentially initial set of inputs for improving fuzzer efficiency.
+
+## See Also
+
+[AFL++ github](https://github.com/AFLplusplus/AFLplusplus) \
+[LibFuzzer](https://github.com/google/fuzzing/blob/master/tutorial/libFuzzerTutorial.md) \
+[Google Fuzz Test](https://github.com/google/fuzztest)
diff --git a/test/realm-fuzzer/afl_runner.cpp b/test/realm-fuzzer/afl_runner.cpp
@@ -0,0 +1,36 @@
+/*************************************************************************
+ *
+ * Copyright 2022 Realm Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ **************************************************************************/
+
+#include "fuzz_engine.hpp"
+
+int main(int argc, const char* argv[])
+{
+ FuzzEngine fuzz_engine;
+ bool enable_logging = false;
+ std::string path = "realm-afl.txt";
+ size_t input_index = 0;
+ for (size_t i = 0; i < (size_t)argc; ++i) {
+ if (strcmp(argv[i], "--log") == 0) {
+ enable_logging = true;
+ }
+ else {
+ input_index = i;
+ }
+ }
+ return fuzz_engine.run_fuzzer(argv[input_index], "realm_afl", enable_logging, path);
+}