Improve configurations for sanitized builds

CHANGELOG.md

@@ -6,7 +6,7 @@
 
 ### Fixed
 * <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)
-* None.
+* Potential stack-use-after-scope issue on changesets integration with msvc-2019 and mpack code ([PR #6911](https://github.com/realm/realm-core/pull/6911))
 
 ### Breaking changes
 * None.
@@ -17,7 +17,7 @@
 -----------
 
 ### Internals
-* None.
+* REALM_[ATMU]SAN cmake flags no longer override compilation options and can be combined with Debug|RelWithDebInfo|etc. build types. Rel[ATMU]SAN build type shortcuts are now all slightly optimized debug-based builds with sanitizers. REALM_ASAN now works with msvc (2019/2022) builds. ([PR #6911](https://github.com/realm/realm-core/pull/6911))
 
 ----------------------------------------------
 

Jenkinsfile

@@ -196,8 +196,6 @@ jobWrapper {
  buildAndroidArm64Debug : doAndroidBuildInDocker('arm64-v8a', 'Debug'),
  buildAndroidTestsArmeabi: doAndroidBuildInDocker('armeabi-v7a', 'Debug', TestAction.Build),
  buildEmscripten : doBuildEmscripten('Debug'),
- threadSanitizer : doCheckSanity(buildOptions + [enableSync: true, sanitizeMode: 'thread']),
- addressSanitizer : doCheckSanity(buildOptions + [enableSync: true, sanitizeMode: 'address']),
  ]
  if (releaseTesting) {
  extendedChecks = [
@@ -312,63 +310,6 @@ def doCheckInDocker(Map options = [:]) {
  }
 }
 
-def doCheckSanity(Map options = [:]) {
- def privileged = '';
-
- def cmakeOptions = [
- CMAKE_BUILD_TYPE: options.buildType,
- REALM_MAX_BPNODE_SIZE: options.maxBpNodeSize,
- REALM_ENABLE_SYNC: options.enableSync,
- ]
-
- if (options.sanitizeMode.contains('thread')) {
- cmakeOptions << [
- REALM_TSAN: "ON",
- ]
- }
- else if (options.sanitizeMode.contains('address')) {
- privileged = '--privileged'
- cmakeOptions << [
- REALM_ASAN: "ON",
- ]
- }
-
- def cmakeDefinitions = cmakeOptions.collect { k,v -> "-D$k=$v" }.join(' ')
-
- return {
- rlmNode('docker') {
- getArchive()
-
- def environment = environment() + [
- 'CC=clang',
- 'CXX=clang++',
- 'UNITTEST_XML=unit-test-report.xml',
- "UNITTEST_SUITE_NAME=Linux-${options.buildType}",
- "TSAN_OPTIONS=\"suppressions=${WORKSPACE}/test/tsan.suppress\""
- ]
- buildDockerEnv('linux.Dockerfile').inside(privileged) {
- withEnv(environment) {
- try {
- dir('build-dir') {
- sh "cmake ${cmakeDefinitions} -G Ninja .."
- runAndCollectWarnings(
- script: 'ninja',
- parser: "clang",
- name: "linux-clang-${options.buildType}-${options.sanitizeMode}",
- filters: warningFilters,
- )
- sh "${ctest_cmd}"
- }
-
- } finally {
- junit testResults: 'build-dir/test/unit-test-report.xml'
- }
- }
- }
- }
- }
-}
-
 def doBuildLinux(String buildType) {
  return {
  rlmNode('docker') {

bindgen/src/realm_helpers.h

@@ -221,8 +221,12 @@ struct Helpers {
  static void simulate_sync_error(SyncSession& session, const int& code, const std::string& message,
  const std::string& type, bool is_fatal)
  {
- sync::SessionErrorInfo error(Status{type == "realm::sync::ProtocolError" ? ErrorCodes::SyncClientResetRequired : ErrorCodes::UnknownError, message}, sync::IsFatal(is_fatal));
- error.server_requests_action = code == 211 ? sync::ProtocolErrorInfo::Action::ClientReset : sync::ProtocolErrorInfo::Action::Warning;
+ sync::SessionErrorInfo error(Status{type == "realm::sync::ProtocolError" ? ErrorCodes::SyncClientResetRequired
+ : ErrorCodes::UnknownError,
+ message},
+ sync::IsFatal(is_fatal));
+ error.server_requests_action =
+ code == 211 ? sync::ProtocolErrorInfo::Action::ClientReset : sync::ProtocolErrorInfo::Action::Warning;
  SyncSession::OnlyForTesting::handle_error(session, std::move(error));
  }
 

evergreen/config.yml

@@ -1214,6 +1214,7 @@ buildvariants:
  cxx_compiler: "./clang_binaries/bin/clang++"
  run_with_encryption: On
  enable_tsan: On
+ cmake_build_type: RelWithDebInfo
  tasks:
  - name: core_tests_group
  distros:
@@ -1232,6 +1233,7 @@ buildvariants:
  llvm_symbolizer: "./clang_binaries/bin/llvm-symbolizer"
  run_with_encryption: On
  enable_asan: On
+ cmake_build_type: RelWithDebInfo
  tasks:
  - name: core_tests_group
  distros:
@@ -1269,6 +1271,7 @@ buildvariants:
  cmake_bindir: "./cmake_binaries/bin"
  fetch_missing_dependencies: On
  enable_asan: On
+ cmake_build_type: RelWithDebInfo
  c_compiler: "./clang_binaries/bin/clang"
  cxx_compiler: "./clang_binaries/bin/clang++"
  llvm_symbolizer: "./clang_binaries/bin/llvm-symbolizer"
@@ -1287,6 +1290,7 @@ buildvariants:
  cmake_bindir: "./cmake_binaries/bin"
  fetch_missing_dependencies: On
  enable_tsan: On
+ cmake_build_type: RelWithDebInfo
  c_compiler: "./clang_binaries/bin/clang"
  cxx_compiler: "./clang_binaries/bin/clang++"
  tasks:
@@ -1301,6 +1305,7 @@ buildvariants:
  cmake_bindir: "./cmake_binaries/bin"
  fetch_missing_dependencies: On
  enable_ubsan: On
+ cmake_build_type: RelWithDebInfo
  c_compiler: "./clang_binaries/bin/clang"
  cxx_compiler: "./clang_binaries/bin/clang++"
  tasks:
@@ -1448,6 +1453,70 @@ buildvariants:
  - name: long-running-tests
  - name: swift-build-and-test
 
+- name: macos-1100-x64-asan
+ display_name: "MacOS 11.0 x86_64 (ASAN)"
+ run_on: macos-1100
+ expansions:
+ cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-macos-universal.tar.gz"
+ cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin"
+ cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake"
+ cmake_generator: Xcode
+ max_jobs: $(sysctl -n hw.logicalcpu)
+ xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer
+ extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=x86_64
+ cmake_build_type: RelWithDebInfo
+ enable_asan: On
+ tasks:
+ - name: compile_test
+
+- name: macos-1100-x64-tsan
+ display_name: "MacOS 11.0 x86_64 (TSAN)"
+ run_on: macos-1100
+ expansions:
+ cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-macos-universal.tar.gz"
+ cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin"
+ cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake"
+ cmake_generator: Xcode
+ max_jobs: $(sysctl -n hw.logicalcpu)
+ xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer
+ extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=x86_64
+ cmake_build_type: RelWithDebInfo
+ enable_tsan: On
+ tasks:
+ - name: compile_test
+
+- name: macos-1100-arm64-asan
+ display_name: "MacOS 11 arm64 (ASAN)"
+ run_on: macos-1100-arm64
+ expansions:
+ cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-macos-universal.tar.gz"
+ cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin"
+ cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake"
+ cmake_generator: Xcode
+ max_jobs: $(sysctl -n hw.logicalcpu)
+ xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer
+ extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64
+ cmake_build_type: RelWithDebInfo
+ enable_asan: On
+ tasks:
+ - name: compile_test
+
+- name: macos-1100-arm64-tsan
+ display_name: "MacOS 11 arm64 (TSAN)"
+ run_on: macos-1100-arm64
+ expansions:
+ cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-macos-universal.tar.gz"
+ cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin"
+ cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake"
+ cmake_generator: Xcode
+ max_jobs: $(sysctl -n hw.logicalcpu)
+ xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer
+ extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64
+ cmake_build_type: RelWithDebInfo
+ enable_tsan: On
+ tasks:
+ - name: compile_test
+
 - name: macos-coverage
  display_name: "MacOS 11 arm64 (Code Coverage)"
  run_on: macos-1100-arm64
@@ -1529,3 +1598,20 @@ buildvariants:
  python3: "/cygdrive/c/python/python37/python.exe"
  tasks:
  - name: compile_test
+
+- name: windows-64-vs2019-asan
+ display_name: "Windows x86_64 (VS 2019 ASAN)"
+ run_on: windows-vsCurrent-large
+ expansions:
+ cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.20.3-windows-x86_64.zip"
+ cmake_bindir: "./cmake-3.20.3-windows-x86_64/bin"
+ cmake_generator: "Visual Studio 16 2019"
+ extra_flags: "-A x64"
+ cmake_build_type: "Debug"
+ enable_asan: On
+ max_jobs: $(($(grep -c proc /proc/cpuinfo) / 2))
+ fetch_missing_dependencies: On
+ curl_base: "/cygdrive/c/curl"
+ python3: "/cygdrive/c/python/python37/python.exe"
+ tasks:
+ - name: compile_test

src/external/s2/CMakeLists.txt

@@ -10,6 +10,7 @@ set(S2_SOURCES
  s1interval.cc
  s2cap.cc
  s2cell.cc
+ s2cellunion.cc
  s2edgeindex.cc
  s2edgeutil.cc
  s2latlngrect.cc

src/realm/CMakeLists.txt

@@ -306,7 +306,6 @@ set_target_properties(Storage PROPERTIES
  OUTPUT_NAME "realm"
 )
 
-target_compile_options(Storage PUBLIC ${REALM_SANITIZER_FLAGS})
 if(REALM_USE_FAST_LINKER)
  target_link_options(Storage PUBLIC ${REALM_USE_FAST_LINKER})
 endif()
@@ -315,9 +314,12 @@ target_compile_definitions(Storage PUBLIC
  $<$<CONFIG:Debug>:REALM_DEBUG=1>
 )
 
-if(NOT MSVC)
+if (REALM_SANITIZER_FLAGS)
  target_compile_options(Storage PUBLIC ${REALM_SANITIZER_FLAGS})
- target_link_options(Storage PUBLIC ${REALM_SANITIZER_FLAGS})
+
+ if (REALM_SANITIZER_LINK_FLAGS)
+ target_link_options(Storage PUBLIC ${REALM_SANITIZER_LINK_FLAGS})
+ endif()
 endif()
 
 target_include_directories(Storage INTERFACE

src/realm/sync/noinst/client_history_impl.cpp

@@ -1194,6 +1194,7 @@ void ClientHistory::update_from_ref_and_version(ref_type ref, version_type versi
  m_arrays->init_from_ref(ref);
  }
  else {
+ REALM_ASSERT_RELEASE(m_group);
  m_arrays.emplace(m_db->get_alloc(), *m_group, ref);
  }
 

src/realm/sync/transform.cpp

@@ -997,16 +997,18 @@ struct MergeUtils {
  bool same_path_element(const Instruction::Path::Element& left,
  const Instruction::Path::Element& right) const noexcept
  {
- const auto& pred = util::overload{
- [&](uint32_t lhs, uint32_t rhs) {
+ auto pred = util::overload{
+ [](uint32_t lhs, uint32_t rhs) {
  return lhs == rhs;
  },
- [&](InternString lhs, InternString rhs) {
+ [this](InternString lhs, InternString rhs) {
  return same_string(lhs, rhs);
  },
- [&](const auto&, const auto&) {
- // FIXME: Paths contain incompatible element types. Should we raise an
- // error here?
+ // FIXME: Paths contain incompatible element types. Should we raise an error here?
+ [](InternString, uint32_t) {
+ return false;
+ },
+ [](uint32_t, InternString) {
  return false;
  },
  };

src/realm/table.hpp

@@ -903,10 +903,10 @@ class ColKeyIterator {
 
 private:
  friend class ColKeys;
- const Table* m_table;
+ ConstTableRef m_table;
  size_t m_pos;
 
- ColKeyIterator(const Table* t, size_t p)
+ ColKeyIterator(const ConstTableRef& t, size_t p)
  : m_table(t)
  , m_pos(p)
  {
@@ -915,8 +915,8 @@ class ColKeyIterator {
 
 class ColKeys {
 public:
- ColKeys(const Table* t)
- : m_table(t)
+ ColKeys(ConstTableRef&& t)
+ : m_table(std::move(t))
  {
  }
 
@@ -947,7 +947,7 @@ class ColKeys {
  }
 
 private:
- const Table* m_table;
+ ConstTableRef m_table;
 };
 
 // Class used to collect a chain of links when building up a Query following links.
@@ -1079,7 +1079,7 @@ class LinkChain {
 
 inline ColKeys Table::get_column_keys() const
 {
- return ColKeys(this);
+ return ColKeys(ConstTableRef(this, m_alloc.get_instance_version()));
 }
 
 inline uint_fast64_t Table::get_content_version() const noexcept

src/realm/timestamp.hpp

@@ -19,6 +19,7 @@
 #ifndef REALM_TIMESTAMP_HPP
 #define REALM_TIMESTAMP_HPP
 
+#include <array>
 #include <cstdint>
 #include <ostream>
 #include <chrono>
@@ -182,8 +183,7 @@ class Timestamp {
  return size_t(m_seconds) ^ size_t(m_nanoseconds);
  }
 
- // Buffer must be at least 32 bytes long
- const char* to_string(char* buffer) const;
+ const char* to_string(std::array<char, 32>& buffer) const;
 
  template <class Ch, class Tr>
  friend std::basic_ostream<Ch, Tr>& operator<<(std::basic_ostream<Ch, Tr>& out, const Timestamp&);
@@ -199,7 +199,7 @@ class Timestamp {
 template <class C, class T>
 inline std::basic_ostream<C, T>& operator<<(std::basic_ostream<C, T>& out, const Timestamp& d)
 {
- char buffer[32];
+ std::array<char, 32> buffer{};
  out << d.to_string(buffer);
  return out;
 }

src/realm/util/logger.cpp

@@ -112,6 +112,8 @@ const std::string_view Logger::level_to_string(Level level) noexcept
 
 void StderrLogger::do_log(Level level, const std::string& message)
 {
+ static Mutex mutex;
+ LockGuard l(mutex);
  // std::cerr is unbuffered, so no need to flush
  std::cerr << get_level_prefix(level) << message << '\n'; // Throws
 }