Merge branch 'unstable' into status-impl

.clang-tidy

@@ -1,7 +1,7 @@
 # refer to https://clang.llvm.org/extra/clang-tidy/checks/list.html
-Checks: -*, clang-analyzer-core.*, clang-analyzer-cplusplus.*, clang-analyzer-deadcode.*, clang-analyzer-nullability.*, clang-analyzer-security.*, clang-analyzer-unix.*, clang-analyzer-valist.*, cppcoreguidelines-init-variables, cppcoreguidelines-macro-usage, cppcoreguidelines-interfaces-global-init, cppcoreguidelines-narrowing-conversions, cppcoreguidelines-no-malloc, cppcoreguidelines-prefer-member-initializer, cppcoreguidelines-special-member-functions, cppcoreguidelines-slicing, google-build-explicit-make-pair, google-default-arguments, google-explicit-constructor, modernize-avoid-bind, modernize-loop-convert, modernize-macro-to-enum, modernize-make-shared, modernize-make-unique, modernize-pass-by-value, modernize-redundant-void-arg, modernize-return-braced-init-list, modernize-use-auto, modernize-use-bool-literals, modernize-use-emplace, modernize-use-equals-default, modernize-use-equals-delete, modernize-use-nullptr, modernize-use-override, modernize-use-using, performance-faster-string-find, performance-for-range-copy, performance-implicit-conversion-in-loop, performance-inefficient-algorithm, performance-inefficient-vector-operation, performance-move-const-arg, performance-move-constructor-init, performance-no-automatic-move, performance-trivially-destructible, performance-type-promotion-in-math-fn, performance-unnecessary-copy-initialization, performance-unnecessary-value-param
+Checks: -*, clang-analyzer-core.*, clang-analyzer-cplusplus.*, clang-analyzer-deadcode.*, clang-analyzer-nullability.*, clang-analyzer-security.*, clang-analyzer-unix.*, clang-analyzer-valist.*, cppcoreguidelines-init-variables, cppcoreguidelines-macro-usage, cppcoreguidelines-interfaces-global-init, cppcoreguidelines-narrowing-conversions, cppcoreguidelines-no-malloc, cppcoreguidelines-prefer-member-initializer, cppcoreguidelines-special-member-functions, cppcoreguidelines-slicing, google-build-explicit-make-pair, google-default-arguments, google-explicit-constructor, modernize-avoid-bind, modernize-loop-convert, modernize-macro-to-enum, modernize-make-shared, modernize-make-unique, modernize-pass-by-value, modernize-redundant-void-arg, modernize-return-braced-init-list, modernize-use-auto, modernize-use-bool-literals, modernize-use-emplace, modernize-use-equals-default, modernize-use-equals-delete, modernize-use-nullptr, modernize-use-override, modernize-use-using, performance-faster-string-find, performance-for-range-copy, performance-implicit-conversion-in-loop, performance-inefficient-algorithm, performance-inefficient-vector-operation, performance-move-const-arg, performance-move-constructor-init, performance-no-automatic-move, performance-trivially-destructible, performance-type-promotion-in-math-fn, performance-unnecessary-copy-initialization, performance-unnecessary-value-param, readability-avoid-const-params-in-decls, readability-const-return-type, readability-convert-member-functions-to-static, readability-make-member-function-const, readability-redundant-access-specifiers, readability-redundant-control-flow, readability-redundant-declaration, readability-redundant-member-init, readability-redundant-string-cstr, readability-redundant-string-init, readability-simplify-boolean-expr, readability-simplify-subscript-expr, readability-string-compare
 
-WarningsAsErrors: clang-analyzer-*, -clang-analyzer-security.insecureAPI.rand, google-*, performance-*, cppcoreguidelines-*, modernize-*
+WarningsAsErrors: clang-analyzer-*, -clang-analyzer-security.insecureAPI.rand, google-*, performance-*, cppcoreguidelines-*, modernize-*, readability-*
 
 CheckOptions:
   - key:           cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor

.github/workflows/kvrocks.yaml

@@ -98,6 +98,12 @@ jobs:
           - name: Ubuntu Clang
             os: ubuntu-20.04
             compiler: clang
+          - name: Ubuntu 22 GCC
+            os: ubuntu-22.04
+            compiler: gcc
+          - name: Ubuntu 22 Clang
+            os: ubuntu-22.04
+            compiler: clang
           - name: Ubuntu GCC ASan
             os: ubuntu-20.04
             without_jemalloc: -DDISABLE_JEMALLOC=ON
@@ -140,6 +146,14 @@ jobs:
             os: ubuntu-20.04
             without_luajit: -DUSE_LUAJIT=OFF
             compiler: clang
+          - name: Ubuntu GCC with new encoding
+            os: ubuntu-20.04
+            compiler: gcc
+            new_encoding: -DENABLE_NEW_ENCODING=TRUE
+          - name: Ubuntu Clang with new encoding
+            os: ubuntu-20.04
+            compiler: clang
+            new_encoding: -DENABLE_NEW_ENCODING=TRUE
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -182,7 +196,7 @@ jobs:
       - name: Build Kvrocks
         run: |
           ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.without_jemalloc }} ${{ matrix.without_luajit }} \
-            ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{ matrix.with_openssl }} ${{ env.CMAKE_EXTRA_DEFS }}
+            ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{ matrix.with_openssl }} ${{ matrix.new_encoding }} ${{ env.CMAKE_EXTRA_DEFS }}
 
       - name: Setup Coredump
         if: ${{ startsWith(matrix.os, 'ubuntu') }}

CMakeLists.txt

@@ -30,6 +30,8 @@ option(ENABLE_OPENSSL "enable openssl to support tls connection" OFF)
 option(ENABLE_IPO "enable interprocedural optimization" ON)
 option(ENABLE_UNWIND "enable libunwind in glog" ON)
 option(PORTABLE "build a portable binary (disable arch-specific optimizations)" OFF)
+# TODO: set ENABLE_NEW_ENCODING to ON when we are ready
+option(ENABLE_NEW_ENCODING "enable new encoding (#1033) for storing 64bit size and expire time in milliseconds" OFF)
 
 if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
     cmake_policy(SET CMP0135 NEW)
@@ -120,6 +122,7 @@ include(cmake/tbb.cmake)
 include(cmake/rocksdb.cmake)
 include(cmake/libevent.cmake)
 include(cmake/fmt.cmake)
+include(cmake/jsoncons.cmake)
 
 if (USE_LUAJIT)
 include(cmake/luajit.cmake)
@@ -146,6 +149,7 @@ if (ENABLE_OPENSSL)
 list(APPEND EXTERNAL_LIBS OpenSSL::SSL)
 endif()
 list(APPEND EXTERNAL_LIBS tbb)
+list(APPEND EXTERNAL_LIBS jsoncons)
 list(APPEND EXTERNAL_LIBS Threads::Threads)
 
 # Add git sha to version.h
@@ -194,6 +198,14 @@ target_link_libraries(kvrocks_objs PUBLIC ${EXTERNAL_LIBS})
 if(ENABLE_OPENSSL)
     target_compile_definitions(kvrocks_objs PUBLIC ENABLE_OPENSSL)
 endif()
+if(ENABLE_NEW_ENCODING)
+    target_compile_definitions(kvrocks_objs PUBLIC ENABLE_NEW_ENCODING)
+endif()
+
+# disable LTO on GCC <= 9 due to an ICE
+if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10))
+    set(ENABLE_IPO OFF)
+endif()
 
 if(ENABLE_IPO)
     include(CheckIPOSupported)

NOTICE

@@ -233,65 +233,103 @@ For src/common/rocksdb_crc32c.h and src/storage/batch_debugger.h are modified fr
 We choose the Apache License 2.0 from the dual-licensed RocksDB code, 
 and have added the RocksDB license notice in each file header.
 
-For the library LuaJIT(https://github.com/KvrocksLabs/LuaJIT) -- a Just-In-Time Compiler for Lua. https://luajit.org/
+For the library LuaJIT(https://github.com/KvrocksLabs/LuaJIT):
 
-LuaJIT is free software, released under the MIT license.
+LuaJIT is a Just-In-Time Compiler for Lua, released under the MIT license.
 
-Copyright (C) 2005-2022 Mike Pall. All rights reserved.
+    ===============================================================================
+    LuaJIT -- a Just-In-Time Compiler for Lua. https://luajit.org/
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+    Copyright (C) 2005-2022 Mike Pall. All rights reserved.
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
 
-[ MIT license: https://www.opensource.org/licenses/mit-license.php ]
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
 
-===============================================================================
-[ LuaJIT includes code from Lua 5.1/5.2, which has this license statement: ]
+    [ MIT license: https://www.opensource.org/licenses/mit-license.php ]
 
-Copyright (C) 1994-2012 Lua.org, PUC-Rio.
+    ===============================================================================
+    [ LuaJIT includes code from Lua 5.1/5.2, which has this license statement: ]
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+    Copyright (C) 1994-2012 Lua.org, PUC-Rio.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
 
-===============================================================================
-[ LuaJIT includes code from dlmalloc, which has this license statement: ]
+    ===============================================================================
+    [ LuaJIT includes code from dlmalloc, which has this license statement: ]
 
-This is a version (aka dlmalloc) of malloc/free/realloc written by
-Doug Lea and released to the public domain, as explained at
-https://creativecommons.org/licenses/publicdomain
+    This is a version (aka dlmalloc) of malloc/free/realloc written by
+    Doug Lea and released to the public domain, as explained at
+    https://creativecommons.org/licenses/publicdomain
 
-===============================================================================
+    ===============================================================================
 
 For the library oneTBB (https://github.com/oneapi-src/oneTBB):
 
 oneAPI Threading Building Blocks (oneTBB) is a flexible C++ library that simplifies the work of adding parallelism to complex applications. 
 It is licensed under Apache-2.0 License, refer to LICENSE in the root directory.
+
+
+For the library jsoncons (https://github.com/danielaparker/jsoncons):
+
+jsoncons is a C++, header-only library for constructing JSON and JSON-like data formats such as CBOR.
+It is released under the Boost Software License Version 1.0.
+
+    //          Copyright Daniel Parker 2013 - 2020.
+    // Distributed under the Boost Software License, Version 1.0.
+    //    (See accompanying file LICENSE or copy at
+    //          http://www.boost.org/LICENSE_1_0.txt)
+
+    Boost Software License - Version 1.0 - August 17th, 2003
+
+    Permission is hereby granted, free of charge, to any person or organization
+    obtaining a copy of the software and accompanying documentation covered by
+    this license (the "Software") to use, reproduce, display, distribute,
+    execute, and transmit the Software, and to prepare derivative works of the
+    Software, and to permit third-parties to whom the Software is furnished to
+    do so, all subject to the following:
+
+    The copyright notices in the Software and this entire statement, including
+    the above license grant, this restriction and the following disclaimer,
+    must be included in all copies of the Software, in whole or in part, and
+    all derivative works of the Software, unless such copies or derivative
+    works are solely in the form of machine-executable object code generated by
+    a source language processor.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+    SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+    FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.

cmake/jsoncons.cmake

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include_guard()
+
+include(cmake/utils.cmake)
+
+FetchContent_DeclareGitHubWithMirror(jsoncons
+  danielaparker/jsoncons v0.170.0
+  MD5=02765c8e4ce09eb744f9daacc40ca595
+)
+
+FetchContent_MakeAvailableWithArgs(jsoncons
+  JSONCONS_BUILD_TESTS=OFF
+)

kvrocks.conf

@@ -462,6 +462,18 @@ profiling-sample-record-threshold-ms 100
 # 0-7am every day.
 compaction-checker-range 0-7
 
+# When the compaction checker is triggered, the db will periodically pick the SST file
+# with the highest "deleted percentage" (i.e. the percentage of deleted keys in the SST 
+# file) to compact, in order to free disk space. 
+# However, if a specific SST file was created more than "force-compact-file-age" seconds 
+# ago, and its percentage of deleted keys is higher than 
+# "force-compact-file-min-deleted-percentage", it will be forcely compacted as well.
+
+# Default: 172800 seconds; Range: [60, INT64_MAX];
+# force-compact-file-age 172800
+# Default: 10 %; Range: [1, 100];
+# force-compact-file-min-deleted-percentage 10
+
 # Bgsave scheduler, auto bgsave at scheduled time
 # time expression format is the same as crontab(currently only support * and int)
 # e.g. bgsave-cron 0 3 * * * 0 4 * * *