Merge with upstream rocksdb v7.10.2 (#276)

Merge with upstream tag: v7.10.2. Igor made a PR on this a few months ago, but that branch had conflicts with recent Rocksdb-Cloud changes. So I simply made another one. To resolve conflicts, I made following changes: * remove MaybeVerifyFileChecksums function . This function has been removed in upstream but somehow kept in our repo. Probably we forgot to remove it in previous merge. Removing it since it's causing conflict during merge * Revert a revert commit in v7.8.3. Seems the original commit had a bug and it got reverted in v7.8.3, but the revert commit wasn't included in v7.10.2 (I guess it's fixed forward in master instead of getting reverted). This confuses git merge and I have to manually re-revert the revert commit before merging to get around the issue * Revert a revert commit in v7.8.3. Same reason as 2 * delete cloud_file_cache. we are not using CloudFileCache and it causes conflict during merging. So I simply removed it * Pluggable compaction to include unique id and epoch info. Two new fields are added in VersionEdit and we need to include them to make remote compaction work.
rockset · Aug 17, 2023 · 2ad9544 · 2ad9544
1 parent 0d2b895
commit 2ad9544
Show file tree

Hide file tree

Showing 707 changed files with 24,405 additions and 18,381 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,7 +1,7 @@
 version: 2.1
 
 orbs:
- win: circleci/windows@2.4.0
+ win: circleci/windows@5.0.0
 
 commands:
  install-cmake-on-macos:
@@ -45,6 +45,38 @@ commands:
  echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV
  echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV
 
+ windows-build-steps:
+ steps:
+ - checkout
+ - run:
+ name: "Install thirdparty dependencies"
+ command: |
+ echo "Installing CMake..."
+ choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
+ mkdir $Env:THIRDPARTY_HOME
+ cd $Env:THIRDPARTY_HOME
+ echo "Building Snappy dependency..."
+ curl https://github.com/google/snappy/archive/refs/tags/1.1.8.zip -O snappy-1.1.8.zip
+ unzip -q snappy-1.1.8.zip
+ cd snappy-1.1.8
+ mkdir build
+ cd build
+ & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" ..
+ msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+ - run:
+ name: "Build RocksDB"
+ command: |
+ mkdir build
+ cd build
+ & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 ..
+ cd ..
+ echo "Building with VS version: $Env:CMAKE_GENERATOR"
+ msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+ - run:
+ name: "Test RocksDB"
+ shell: powershell.exe
+ command: |
+ build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
  pre-steps-macos:
  steps:
  - pre-steps
@@ -154,11 +186,6 @@ commands:
  true
 
 executors:
- windows-2xlarge:
- machine:
- image: 'windows-server-2019-vs2019:stable'
- resource_class: windows.2xlarge
- shell: bash.exe
  linux-docker:
  docker:
  # The image configuration is build_tools/ubuntu20_image/Dockerfile
@@ -518,73 +545,35 @@ jobs:
  no_output_timeout: 100m
  - post-steps
 
- build-windows:
- executor: windows-2xlarge
- parameters:
- extra_cmake_opt:
- default: ""
- type: string
- vs_year:
- default: "2019"
- type: string
- cmake_generator:
- default: "Visual Studio 16 2019"
- type: string
+ build-windows-vs2022:
+ executor:
+ name: win/server-2022
+ size: 2xlarge
+ environment:
+ THIRDPARTY_HOME: C:/Users/circleci/thirdparty
+ CMAKE_HOME: C:/Program Files/CMake
+ CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+ SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
+ SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
+ SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
+ CMAKE_GENERATOR: Visual Studio 17 2022
+ steps:
+ - windows-build-steps
+
+ build-windows-vs2019:
+ executor:
+ name: win/server-2019
+ size: 2xlarge
  environment:
  THIRDPARTY_HOME: C:/Users/circleci/thirdparty
- CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
- CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
- SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
- SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
- SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
- VS_YEAR: <<parameters.vs_year>>
- CMAKE_GENERATOR: <<parameters.cmake_generator>>
+ CMAKE_HOME: C:/Program Files/CMake
+ CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+ SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
+ SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
+ SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
+ CMAKE_GENERATOR: Visual Studio 16 2019
  steps:
- - checkout
- - run:
- name: "Setup VS"
- command: |
- if [[ "${VS_YEAR}" == "2019" ]]; then
- echo "VS2019 already present."
- elif [[ "${VS_YEAR}" == "2017" ]]; then
- echo "Installing VS2017..."
- powershell .circleci/vs2017_install.ps1
- elif [[ "${VS_YEAR}" == "2015" ]]; then
- echo "Installing VS2015..."
- powershell .circleci/vs2015_install.ps1
- fi
- - store_artifacts:
- path: \Users\circleci\AppData\Local\Temp\vslogs.zip
- - run:
- name: "Install thirdparty dependencies"
- command: |
- mkdir ${THIRDPARTY_HOME}
- cd ${THIRDPARTY_HOME}
- echo "Installing CMake..."
- curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
- unzip -q cmake-3.16.4-win64-x64.zip
- echo "Building Snappy dependency..."
- curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
- unzip -q snappy-1.1.7.zip
- cd snappy-1.1.7
- mkdir build
- cd build
- ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
- msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
- - run:
- name: "Build RocksDB"
- command: |
- mkdir build
- cd build
- ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
- cd ..
- echo "Building with VS version: ${CMAKE_GENERATOR}"
- msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
- - run:
- name: "Test RocksDB"
- shell: powershell.exe
- command: |
- build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+ - windows-build-steps
 
  build-linux-java:
  executor: linux-docker
@@ -859,15 +848,8 @@ workflows:
  - build-linux-mini-crashtest
  jobs-windows:
  jobs:
- - build-windows:
- name: "build-windows-vs2019"
- - build-windows:
- name: "build-windows-vs2019-cxx20"
- extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
- - build-windows:
- name: "build-windows-vs2017"
- vs_year: "2017"
- cmake_generator: "Visual Studio 15 Win64"
+ - build-windows-vs2022
+ - build-windows-vs2019
  - build-cmake-mingw
  jobs-java:
  jobs:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -253,11 +253,21 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
  endif(HAS_S390X_MARCH_NATIVE)
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
 
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
+ CHECK_C_COMPILER_FLAG("-march=loongarch64" HAS_LOONGARCH64)
+ if(HAS_LOONGARCH64)
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=loongarch64 -mtune=loongarch64")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=loongarch64 -mtune=loongarch64")
+ endif(HAS_LOONGARCH64)
+endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
+
 option(PORTABLE "build a portable binary" OFF)
 option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
 option(FORCE_AVX "force building with AVX, even when PORTABLE=ON" OFF)
 option(FORCE_AVX2 "force building with AVX2, even when PORTABLE=ON" OFF)
 if(PORTABLE)
+ add_definitions(-DROCKSDB_PORTABLE)
+
  # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
  # is available, it is available by default.
  if(FORCE_SSE42 AND NOT MSVC)
@@ -281,6 +291,9 @@ if(PORTABLE)
  if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
  endif()
+ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^loongarch64")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64")
+ endif()
  endif()
 else()
  if(MSVC)
@@ -300,8 +313,7 @@ if(NOT MSVC)
  set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
 endif()
 
-if (NOT PORTABLE OR FORCE_SSE42)
- CHECK_CXX_SOURCE_COMPILES("
+CHECK_CXX_SOURCE_COMPILES("
 #include <cstdint>
 #include <nmmintrin.h>
 #include <wmmintrin.h>
@@ -313,12 +325,11 @@ int main() {
  auto d = _mm_cvtsi128_si64(c);
 }
 " HAVE_SSE42)
- if(HAVE_SSE42)
- add_definitions(-DHAVE_SSE42)
- add_definitions(-DHAVE_PCLMUL)
- elseif(FORCE_SSE42)
- message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
- endif()
+if(HAVE_SSE42)
+ add_definitions(-DHAVE_SSE42)
+ add_definitions(-DHAVE_PCLMUL)
+elseif(FORCE_SSE42)
+ message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
 endif()
 
 # Check if -latomic is required or not
@@ -657,12 +668,13 @@ set(SOURCES
  cache/cache.cc
  cache/cache_entry_roles.cc
  cache/cache_key.cc
+ cache/cache_helpers.cc
  cache/cache_reservation_manager.cc
  cache/charged_cache.cc
  cache/clock_cache.cc
  cache/compressed_secondary_cache.cc
- cache/fast_lru_cache.cc
  cache/lru_cache.cc
+ cache/secondary_cache.cc
  cache/sharded_cache.cc
  db/arena_wrapped_db_iter.cc
  db/blob/blob_contents.cc
@@ -804,6 +816,7 @@ set(SOURCES
  options/options.cc
  options/options_helper.cc
  options/options_parser.cc
+ port/mmap.cc
  port/stack_trace.cc
  table/adaptive/adaptive_table_factory.cc
  table/block_based/binary_search_index_reader.cc
@@ -813,6 +826,7 @@ set(SOURCES
  table/block_based/block_based_table_iterator.cc
  table/block_based/block_based_table_reader.cc
  table/block_based/block_builder.cc
+ table/block_based/block_cache.cc
  table/block_based/block_prefetcher.cc
  table/block_based/block_prefix_index.cc
  table/block_based/data_block_hash_index.cc
@@ -974,7 +988,6 @@ set(SOURCES
  cloud/cloud_manifest.cc
  cloud/cloud_scheduler.cc
  cloud/cloud_storage_provider.cc
- cloud/cloud_file_cache.cc
  cloud/cloud_file_deletion_scheduler.cc
  db/db_impl/db_impl_remote_compaction.cc
  db/db_impl/replication_codec.cc)
@@ -1004,6 +1017,12 @@ if ( ROCKSDB_PLUGINS )
  plugin/${plugin}/${src}
  PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}")
  endforeach()
+ foreach (test ${${plugin}_TESTS})
+ list(APPEND PLUGIN_TESTS plugin/${plugin}/${test})
+ set_source_files_properties(
+ plugin/${plugin}/${test}
+ PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}")
+ endforeach()
  foreach (path ${${plugin}_INCLUDE_PATHS})
  include_directories(${path})
  endforeach()
@@ -1501,6 +1520,7 @@ if(WITH_TESTS)
  utilities/ttl/ttl_test.cc
  utilities/util_merge_operators_test.cc
  utilities/write_batch_with_index/write_batch_with_index_test.cc
+ ${PLUGIN_TESTS}
  )
  endif()
 
@@ -1604,9 +1624,9 @@ endif()
 
 option(WITH_TRACE_TOOLS "build with trace tools" ON)
 if(WITH_TRACE_TOOLS)
- add_executable(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX}
+ add_executable(block_cache_trace_analyzer${ARTIFACT_SUFFIX}
  tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc)
- target_link_libraries(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX}
+ target_link_libraries(block_cache_trace_analyzer${ARTIFACT_SUFFIX}
  ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 
  add_executable(trace_analyzer${ARTIFACT_SUFFIX}