Fuzzer Framework

CHANGELOG.md

@@ -21,6 +21,7 @@
 ### Internals
 * Updates for upcoming Platform Networking feature, including new SyncSocketProvider class. ([PR #6096](https://github.com/realm/realm-core/pull/6096))
 * Updated namespaces for files moved to realm/sync/network ([PR #6109](https://github.com/realm/realm-core/pull/6109))
+* Create the fuzzer framework project in order to run fuzz testing on evergreen.
 
 ----------------------------------------------
 

test/CMakeLists.txt

@@ -5,6 +5,7 @@ add_subdirectory(object-store)
 # AFL not yet supported by Windows
 if(NOT CMAKE_SYSTEM_NAME MATCHES "^Windows")
  add_subdirectory(fuzzy)
+ add_subdirectory(fuzzy_object_store)
 endif()
 
 add_subdirectory(benchmark-common-tasks)

test/fuzzy_object_store/.gitignore

@@ -0,0 +1,5 @@
+findings/*
+fuzzy-tests*
+generate-fuzzy-input*
+*.d
+*.o

test/fuzzy_object_store/CMakeLists.txt

@@ -0,0 +1,81 @@
+set(TEST_AFL_SOURCES
+ afl_runner.cpp
+ fuzz_engine.cpp
+ fuzz_object.cpp
+ fuzz_configurator.cpp
+) # TEST_AFL_SOURCES_OBJECT_STORE
+
+set(TEST_LIBFUZZER_SOURCES
+ libfuzzer_runner.cpp
+ fuzz_engine.cpp
+ fuzz_object.cpp
+ fuzz_configurator.cpp
+) # TEST_LIBFUZZER_SOURCES_OBJECT_STORE
+
+file(GLOB FUZZER_RUN_SCRIPTS
+ "scripts/start_fuzz_afl.sh"
+ "scripts/start_lib_fuzzer.sh")
+
+file(COPY ${FUZZER_RUN_SCRIPTS}
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(GLOB AFL_SEEDS "testcases/*")
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testcases)
+file(COPY ${AFL_SEEDS}
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/testcases)
+
+add_executable(realm-afl++ ${TEST_AFL_SOURCES})
+target_link_libraries(realm-afl++ TestUtil ObjectStore)
+
+if(REALM_LIBFUZZER)
+ if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ add_executable(realm-libfuzz ${TEST_LIBFUZZER_SOURCES})
+ target_link_libraries(realm-libfuzz TestUtil ObjectStore)
+ endif()
+endif()
+
+# on Apple platforms we use the built-in CFRunLoop
+# everywhere else it's libuv, except UWP where it doesn't build
+if(NOT APPLE AND NOT WINDOWS_STORE)
+ if(REALM_FETCH_MISSING_DEPENDENCIES)
+ find_package(LibUV)
+ else()
+ find_package(LibUV REQUIRED)
+ endif()
+
+ if(LibUV_FOUND)
+ set(libuv_target LibUV::LibUV)
+ elseif(REALM_FETCH_MISSING_DEPENDENCIES)
+ message(STATUS "LibUV not found, building from source with FetchContent")
+ include(FetchContent)
+ set(libUV_Git_TAG "v1.35.0")
+
+ if(MSVC)
+ set(liUV_Git_TAG "v1.43.0")
+ endif()
+
+ FetchContent_Declare(
+ libuv
+ GIT_REPOSITORY https://github.com/libuv/libuv.git
+ GIT_TAG ${libUV_Git_TAG}
+ )
+
+ # Don't use FetchContent_MakeAvailable since it wants to build libuv.so as well
+ FetchContent_Populate(libuv)
+ add_subdirectory(${libuv_SOURCE_DIR} ${libuv_BINARY_DIR} EXCLUDE_FROM_ALL)
+ set(libuv_target uv_a)
+ endif()
+
+ target_link_libraries(realm-afl++ ${libuv_target})
+ target_compile_definitions(realm-afl++ PUBLIC REALM_HAVE_UV=1)
+ get_property(libuv_include_dir TARGET ${libuv_target} PROPERTY INCLUDE_DIRECTORIES)
+ target_include_directories(realm-afl++ PRIVATE ${libuv_include_dir})
+
+ if(REALM_LIBFUZZER)
+ if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ target_link_libraries(realm-libfuzz ${libuv_target})
+ target_compile_definitions(realm-libfuzz PUBLIC REALM_HAVE_UV=1)
+ get_property(libuv_include_dir TARGET ${libuv_target} PROPERTY INCLUDE_DIRECTORIES)
+ target_include_directories(realm-libfuzz PRIVATE ${libuv_include_dir})
+ endif()
+ endif()
+endif()

test/fuzzy_object_store/README.md

@@ -0,0 +1,77 @@
+# The Fuzz Framework project
+
+This project is an attempt to put together all the small fuzzers we have already scattered around the code.
+There are two goals:
+ 1. To be able to run all the fuzzers, collect crashes reports and fix possible bugs that the fuzzer might find.
+ 2. To be able to replace libfuzzer with google fuzz test (https://github.com/google/fuzztest) at some point.
+
+AFL++ support is not dropped yet, but since we want to integrate things inside evergreen and follow the same approach we implement for address/thread sanitazer we prefer to use libfuzzer and clang.
+## Prerequisites
+
+In case you want to use AFL++, then you should install the latest version of the American Fuzzy Lop ++ (AFL++).
+Please use this quick guide: https://aflplus.plus/building/ it requires llvm >= 9.0.
+
+For using libfuzzer, the only pre-requisite is having a recent version of clang.
+## Running
+Note REALM_MAX_BPNODE_SIZE is the max number of nodes contained in the b+tree. It determines the depth of the tree and its fanout. \
+This number should be random generated.
+
+If you don't want to build manually, you can skip this section and jump to the `Scripts` section. \
+Run the fuzzer via AFL++:
+
+```
+cd <realm-core-src> 
+mkdir build 
+cd build
+cmake -D CMAKE_BUILD_TYPE=${build_mode} 
+ -D CMAKE_C_COMPILER=afl-cc 
+ -D CMAKE_CXX_COMPILER=afl-c++ 
+ -D REALM_MAX_BPNODE_SIZE="${REALM_MAX_BPNODE_SIZE}" 
+ -D REALM_ENABLE_ENCRYPTION=ON 
+ -G Ninja 
+ ..
+cmake --build . --target realm-afl++
+afl-fuzz -t "$time_out" 
+ -m "$memory" 
+ -i "${ROOT_DIR}/test/fuzzy_object_store/testcases" 
+ -o "${FINDINGS_DIR}" 
+ realm-afl++ @@
+```
+
+Run the fuzzer via libFuzzer (only with Clang)
+```
+cd <realm-core-src>
+mkdir build
+cd build
+cmake -D REALM_LIBFUZZER=ON 
+ -D CMAKE_BUILD_TYPE=${build_mode} 
+ -D CMAKE_C_COMPILER=clang 
+ -D CMAKE_CXX_COMPILER=clang++ 
+ -D REALM_MAX_BPNODE_SIZE="${REALM_MAX_BPNODE_SIZE}" 
+ -D REALM_ENABLE_ENCRYPTION=ON 
+ -G Ninja 
+ ..
+cmake --build . --target realm-libfuzz
+./realm_libfuzz <corpus>
+```
+
+## Scripts
+
+`sh start_fuzz_afl.sh`
+Builds `realm-core` and `object-store` in `Debug` mode using the afl++ compiler `afl-cc` and starts 1 instance of `afl-fuzz`.
+It expects `AFLPlusPlus` to be installed in your system and in general added to your `PATH`. 
+Optionally, the following arguments can be passed to the script:
+1) `<num_fuzzers>` the number of fuzzers to launch (by default 1).
+2) `<build_mode>` either `Release` or `Debug`.
+
+`sh start_lib_fuzzer.sh`
+Builds `realm-core` and `object-store` in `Debug` mode using the clang compiler and starts `realm-libfuzz`.
+Optionally, the following arguments can be passed to the script:
+1) `<build_mode>` either `Release` or `Debug`. 
+2) `<corpus>` essentially initial set of inputs for improving fuzzer efficiency.
+
+## See Also
+
+[AFL++ github](https://github.com/AFLplusplus/AFLplusplus) \
+[LibFuzzer](https://github.com/google/fuzzing/blob/master/tutorial/libFuzzerTutorial.md) \
+[Google Fuzz Test](https://github.com/google/fuzztest)

test/fuzzy_object_store/afl_runner.cpp

@@ -0,0 +1,43 @@
+/*************************************************************************
+ *
+ * Copyright 2022 Realm Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ **************************************************************************/
+
+#include "fuzz_engine.hpp"
+#include <vector>
+#include <string>
+#include <iostream>
+
+#if REALM_USE_UV
+#include <uv.h>
+#endif
+
+int main(int argc, const char* argv[])
+{
+ FuzzEngine fuzz_engine;
+ bool enable_logging = false;
+ std::string path = "real-afl.txt";
+ size_t input_index = 0;
+ for (size_t i = 0; i < (size_t)argc; ++i) {
+ if (strcmp(argv[i], "--log") == 0) {
+ enable_logging = true;
+ }
+ else {
+ input_index = i;
+ }
+ }
+ return fuzz_engine.run_fuzzer(argv[input_index], "realm_afl", enable_logging, path);
+}

test/fuzzy_object_store/fuzz_configurator.cpp

@@ -0,0 +1,109 @@
+/*************************************************************************
+ *
+ * Copyright 2022 Realm Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ **************************************************************************/
+#include "fuzz_configurator.hpp"
+#include "fuzz_object.hpp"
+#include "../util/test_path.hpp"
+#include <iostream>
+
+FuzzConfigurator::FuzzConfigurator(FuzzObject& fuzzer, const std::string& input, bool use_input_file,
+ const std::string& name)
+ : m_used_input_file(use_input_file)
+ , m_fuzzer(fuzzer)
+ , m_fuzz_name(name)
+{
+ realm::disable_sync_to_disk();
+ init(input);
+ setup_realm_config();
+}
+
+void FuzzConfigurator::setup_realm_config()
+{
+ m_config.path = m_path;
+ m_config.schema_version = 0;
+ if (m_use_encryption) {
+ const char* key = m_fuzzer.get_encryption_key();
+ const char* i = key;
+ while (*i != '\0') {
+ m_config.encryption_key.push_back(*i);
+ i++;
+ }
+ }
+}
+
+const realm::Realm::Config& FuzzConfigurator::get_config() const
+{
+ return m_config;
+}
+
+FuzzObject& FuzzConfigurator::get_fuzzer()
+{
+ return m_fuzzer;
+}
+
+const std::string& FuzzConfigurator::get_realm_path() const
+{
+ return m_path;
+}
+
+FuzzLog& FuzzConfigurator::get_logger()
+{
+ return m_log;
+}
+
+State& FuzzConfigurator::get_state()
+{
+ return m_state;
+}
+
+void FuzzConfigurator::init(const std::string& input)
+{
+ std::string db_name = "fuzz-test";
+ realm::test_util::RealmPathInfo test_context{db_name};
+ SHARED_GROUP_TEST_PATH(path);
+ m_path = path.c_str();
+ if (m_used_input_file) {
+ std::ifstream in(input, std::ios::in | std::ios::binary);
+ if (!in.is_open()) {
+ std::cerr << "Could not open file for reading: " << input << "\n";
+ throw;
+ }
+ std::string contents((std::istreambuf_iterator<char>(in)), (std::istreambuf_iterator<char>()));
+ set_state(contents);
+ }
+ else {
+ set_state(input);
+ }
+}
+
+void FuzzConfigurator::set_state(const std::string& input)
+{
+ m_state = State{input, 0};
+ m_use_encryption = m_fuzzer.get_next_token(m_state) % 2 == 0;
+}
+
+void FuzzConfigurator::print_cnf()
+{
+ m_log << "// Fuzzer: " << m_fuzz_name << "\n";
+ m_log << "// Test case generated in " REALM_VER_CHUNK " on " << m_fuzzer.get_current_time_stamp() << ".\n";
+ m_log << "// REALM_MAX_BPNODE_SIZE is " << REALM_MAX_BPNODE_SIZE << "\n";
+ m_log << "// ----------------------------------------------------------------------\n";
+ const auto& printable_key =
+ !m_use_encryption ? "nullptr" : std::string("\"") + m_config.encryption_key.data() + "\"";
+ m_log << "// const char* key = " << printable_key << ";\n";
+ m_log << "\n";
+}

test/fuzzy_object_store/fuzz_configurator.hpp

@@ -0,0 +1,52 @@
+/*************************************************************************
+ *
+ * Copyright 2022 Realm Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ **************************************************************************/
+#ifndef FUZZ_CONFIG_HPP
+#define FUZZ_CONFIG_HPP
+
+#include "util.hpp"
+#include "fuzz_logger.hpp"
+#include <realm/object-store/shared_realm.hpp>
+#include <string>
+#include <vector>
+
+class FuzzObject;
+class FuzzConfigurator {
+public:
+ FuzzConfigurator(FuzzObject& fuzzer, const std::string& input, bool use_input_file, const std::string& name);
+ const realm::Realm::Config& get_config() const;
+ FuzzObject& get_fuzzer();
+ const std::string& get_realm_path() const;
+ FuzzLog& get_logger();
+ State& get_state();
+ void set_state(const std::string& input);
+ void print_cnf();
+
+private:
+ void init(const std::string&);
+ void setup_realm_config();
+
+ realm::Realm::Config m_config;
+ std::string m_path;
+ FuzzLog m_log;
+ bool m_use_encryption{false};
+ bool m_used_input_file{false};
+ FuzzObject& m_fuzzer;
+ State m_state;
+ std::string m_fuzz_name;
+};
+#endif