[pull] unstable from apache:unstable

.github/workflows/kvrocks.yaml

@@ -39,8 +39,8 @@ jobs:
     outputs:
       docs_only: ${{ steps.result.outputs.docs_only }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3.0.0
         id: changes
         with:
           filters: .github/config/changes.yml
@@ -56,9 +56,9 @@ jobs:
     env:
       FORCE_COLOR: 1
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install typos
-        run: curl -LsSf https://github.com/crate-ci/typos/releases/download/v1.16.3/typos-v1.16.3-x86_64-unknown-linux-musl.tar.gz | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+        run: curl -LsSf https://github.com/crate-ci/typos/releases/download/v1.18.2/typos-v1.18.2-x86_64-unknown-linux-musl.tar.gz | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - name: Run typos check
         run: typos --config .github/config/typos.toml
 
@@ -68,15 +68,16 @@ jobs:
     if: ${{ needs.precondition.outputs.docs_only != 'true' }}
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version-file: 'tests/gocase/go.mod'
+          cache: false
       - name: Prepare Dependencies
         run: |
           sudo apt update
           sudo apt install -y clang-format-14 clang-tidy-14
-      - uses: apache/skywalking-eyes/header@v0.4.0
+      - uses: apache/skywalking-eyes/header@v0.5.0
         with:
           config: .github/config/licenserc.yml
       - name: Check with clang-format
@@ -96,7 +97,7 @@ jobs:
           git diff -p > clang-format.patch
           cat clang-format.patch
       - name: Upload format patch
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always() && steps.check-format.outcome != 'success'
         with:
           path: clang-format.patch
@@ -214,7 +215,7 @@ jobs:
 
       - name: Cache redis
         id: cache-redis
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/local/bin/redis-cli
@@ -227,16 +228,17 @@ jobs:
           mkdir -p $HOME/local/bin
           pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version-file: 'tests/gocase/go.mod'
-
+          cache: false
+
       - name: Install gcovr 5.0
         run: pip install gcovr==5.0 # 5.1 is not supported
         if: ${{ matrix.sonarcloud }}
@@ -308,7 +310,7 @@ jobs:
             exit 1
           fi
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ failure() && startsWith(matrix.os, 'ubuntu') }}
         with:
           name: kvrocks-coredumps-${{ matrix.name }}
@@ -330,7 +332,7 @@ jobs:
 
       - name: Upload SonarCloud data
         if: ${{ matrix.sonarcloud }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: sonarcloud-data
           path: ${{ env.SONARCLOUD_OUTPUT_DIR }}
@@ -341,10 +343,10 @@ jobs:
     if: ${{ needs.precondition.outputs.docs_only != 'true' }}
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Get core numbers
         run: echo "NPROC=$(nproc)" >> $GITHUB_ENV
-      - uses: docker/build-push-action@v3
+      - uses: docker/build-push-action@v5
         with:
           context: .
           build-args: MORE_BUILD_ARGS=-j${{ env.NPROC }}
@@ -438,11 +440,12 @@ jobs:
           wget https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.sh
           bash cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
 
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v3 #v4 use Node 20 and not working at CentOS 7
+      - uses: actions/setup-go@v4 #v5 use Node 20 too
         if: ${{ !startsWith(matrix.image, 'opensuse') }}
         with:
           go-version-file: 'tests/gocase/go.mod'
+          cache: false
 
       - name: Build Kvrocks
         run: |

.github/workflows/nightly.yaml

@@ -30,20 +30,20 @@ jobs:
     if: github.repository_owner == 'apache'
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Login Docker Hub
         if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
@@ -53,15 +53,15 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: apache/kvrocks
           flavor: latest=false
           tags: |
             type=sha,prefix=nightly-{{date 'YYYYMMDD'}}-,format=short
             type=raw,value=nightly
 
-      - uses: docker/build-push-action@v3
+      - uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64, linux/arm64

.github/workflows/sonar.yaml

@@ -28,15 +28,15 @@ jobs:
     runs-on: ubuntu-22.04
     if: github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'apache'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: ${{ github.event.workflow_run.head_repository.full_name }}
           ref: ${{ github.event.workflow_run.head_sha }}
           fetch-depth: 0
       - name: Install sonar-scanner and build-wrapper
         uses: SonarSource/sonarcloud-github-c-cpp@v2
       - name: 'Download code coverage'
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           script: |
             let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
@@ -60,7 +60,7 @@ jobs:
           unzip sonarcloud-data.zip -d sonarcloud-data
           ls -a sonarcloud-data
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
       - name: Configure Kvrocks

CMakeLists.txt

@@ -138,6 +138,7 @@ include(cmake/fmt.cmake)
 include(cmake/jsoncons.cmake)
 include(cmake/xxhash.cmake)
 include(cmake/span.cmake)
+include(cmake/trie.cmake)
 
 if (ENABLE_LUAJIT)
     include(cmake/luajit.cmake)
@@ -169,6 +170,7 @@ list(APPEND EXTERNAL_LIBS Threads::Threads)
 list(APPEND EXTERNAL_LIBS ${Backtrace_LIBRARY})
 list(APPEND EXTERNAL_LIBS xxhash)
 list(APPEND EXTERNAL_LIBS span-lite)
+list(APPEND EXTERNAL_LIBS tsl_hat_trie)
 
 # Add git sha to version.h
 find_package(Git REQUIRED)

NOTICE

@@ -65,6 +65,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt
 * fmt(https://github.com/fmtlib/fmt)
 * LuaJIT(https://github.com/KvrocksLabs/LuaJIT)
 * lua(https://github.com/KvrocksLabs/lua, alternative to LuaJIT)
+* hat-trie(https://github.com/Tessil/hat-trie)
 
 ================================================================
 Boost Software License Version 1.0

README.md

@@ -225,6 +225,7 @@ Apache Kvrocks is licensed under the Apache License Version 2.0. See the [LICENS
 ## Social Media
 
 - [Medium](https://kvrocks.medium.com/)
+- [X (Twitter)](https://twitter.com/apache_kvrocks)
 - [Zhihu](https://www.zhihu.com/people/kvrocks) (in Chinese)
 - WeChat Official Account (in Chinese, scan the QR code to follow)
 

cmake/fmt.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(fmt
-  fmtlib/fmt 10.1.1
-  MD5=2a91a7d74be8bfd3a19e7e2abbc7c034
+  fmtlib/fmt 10.2.1
+  MD5=1bba4e8bdd7b0fa98f207559ffa380a3
 )
 
 FetchContent_MakeAvailableWithArgs(fmt)

cmake/jsoncons.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(jsoncons
-  danielaparker/jsoncons v0.172.0
-  MD5=2bc70a1ddc8c5fc96d43c0cb4d10dfa0
+  danielaparker/jsoncons v0.173.4
+  MD5=947254529a8629d001322a78454a23d2
 )
 
 FetchContent_MakeAvailableWithArgs(jsoncons

cmake/rocksdb.cmake

@@ -26,8 +26,8 @@ endif()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(rocksdb
-  facebook/rocksdb v8.10.0
-  MD5=ed06e98fae30c29cceacbfd45a316f06
+  facebook/rocksdb v8.10.2
+  MD5=2155ffb638bfcf42b31818b00d9a3005
 )
 
 FetchContent_GetProperties(jemalloc)

cmake/span.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(span
-  martinmoene/span-lite v0.10.3
-  MD5=ee5c6721d4f4f56a6e6f250c68ad4132
+  martinmoene/span-lite v0.11.0
+  MD5=9786933d03cda33ac7dccbfe85f771dd
 )
 
 FetchContent_MakeAvailableWithArgs(span)

cmake/speedb.cmake

@@ -26,8 +26,8 @@ endif()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(speedb
-  speedb-io/speedb speedb/v2.7.0
-  MD5=9603a0921deb4e3cd9046cf7e9288485
+  speedb-io/speedb speedb/v2.8.0
+  MD5=3da818408057c8c818bfc9adc40d929f
 )
 
 FetchContent_GetProperties(jemalloc)

cmake/trie.cmake

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include_guard()
+
+include(cmake/utils.cmake)
+
+FetchContent_DeclareGitHubWithMirror(trie
+  Tessil/hat-trie 906e6abd1e7063f1dacd3a6b270aa654b525eb0a
+  MD5=a930364e9f6b60371319664bddf78000
+)
+
+FetchContent_MakeAvailableWithArgs(trie)

kvrocks.conf

@@ -309,7 +309,7 @@ max-bitmap-to-string-mb 16
 # If enabled, the cursor will be unsigned 64-bit integers.
 # If disabled, the cursor will be a string.
 # Default: no
-redis-cursor-compatible no
+redis-cursor-compatible yes
 
 # Whether to enable the RESP3 protocol.
 # NOTICE: RESP3 is still under development, don't enable it in production environment.
@@ -536,6 +536,16 @@ compaction-checker-range 0-7
 # rename-command KEYS ""
 
 ################################ MIGRATE #####################################
+# Slot migration supports two ways:
+# - redis-command: Migrate data by redis serialization protocol(RESP).
+# - raw-key-value: Migrate the raw key value data of the storage engine directly.
+#                  This way eliminates the overhead of converting to the redis
+#                  command, reduces resource consumption, improves migration
+#                  efficiency, and can implement a finer rate limit.
+#
+# Default: redis-command
+migrate-type redis-command
+
 # If the network bandwidth is completely consumed by the migration task,
 # it will affect the availability of kvrocks. To avoid this situation,
 # migrate-speed is adopted to limit the migrating speed.
@@ -562,6 +572,18 @@ migrate-pipeline-size 16
 # Default: 10000
 migrate-sequence-gap 10000
 
+# The raw-key-value migration way uses batch for migration. This option sets the batch size
+# for each migration.
+#
+# Default: 16kb
+migrate-batch-size-kb 16
+
+# Rate limit for migration based on raw-key-value, representing the maximum number of data
+# that can be migrated per second. 0 means no limit.
+#
+# Default: 16M
+migrate-batch-rate-limit-mb 16
+
 ################################ ROCKSDB #####################################
 
 # Specify the capacity of column family block cache. A larger block cache
@@ -700,8 +722,7 @@ rocksdb.block_size 16384
 # Default: yes
 rocksdb.cache_index_and_filter_blocks yes
 
-# Specify the compression to use. Only compress level greater
-# than 2 to improve performance.
+# Specify the compression to use. 
 # Accept value: "no", "snappy", "lz4", "zstd", "zlib"
 # default snappy
 rocksdb.compression snappy
@@ -837,7 +858,7 @@ rocksdb.read_options.async_io no
 # buffer cache before the write is considered complete.
 # If this flag is enabled, writes will be slower.
 # If this flag is disabled, and the machine crashes, some recent
-# rites may be lost.  Note that if it is just the process that
+# writes may be lost.  Note that if it is just the process that
 # crashes (i.e., the machine does not reboot), no writes will be
 # lost even if sync==false.
 #

licenses/LICENSE-hat-trie.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Thibaut Goetghebuer-Planchon <tessil@gmx.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.