Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/2.9' into unstable
Browse files Browse the repository at this point in the history
  • Loading branch information
Daniel authored and Daniel committed Jul 18, 2024
2 parents 6d73570 + 57b47a0 commit 6b8e5e4
Show file tree
Hide file tree
Showing 215 changed files with 15,985 additions and 2,252 deletions.
148 changes: 110 additions & 38 deletions .github/workflows/kvrocks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ jobs:
run: |
sudo apt update
sudo apt install -y clang-format-14 clang-tidy-14
- uses: apache/skywalking-eyes/header@v0.5.0
- uses: apache/skywalking-eyes/header@v0.6.0
with:
config: .github/config/licenserc.yml
- name: Check with clang-format
Expand Down Expand Up @@ -110,21 +110,25 @@ jobs:
fail-fast: false
matrix:
include:
- name: Darwin Clang
os: macos-11
# FIXME: update macos-11 to macos-12/13
# - name: Darwin Clang
# os: macos-11
# compiler: auto
- name: Darwin Clang arm64
os: macos-14
compiler: auto
- name: Darwin Clang without Jemalloc
os: macos-11
compiler: auto
without_jemalloc: -DDISABLE_JEMALLOC=ON
- name: Darwin Clang with OpenSSL
os: macos-11
compiler: auto
with_openssl: -DENABLE_OPENSSL=ON
- name: Darwin Clang without luaJIT
os: macos-11
compiler: auto
without_luajit: -DENABLE_LUAJIT=OFF
# - name: Darwin Clang without Jemalloc
# os: macos-11
# compiler: auto
# without_jemalloc: -DDISABLE_JEMALLOC=ON
# - name: Darwin Clang with OpenSSL
# os: macos-11
# compiler: auto
# with_openssl: -DENABLE_OPENSSL=ON
# - name: Darwin Clang without luaJIT
# os: macos-11
# compiler: auto
# without_luajit: -DENABLE_LUAJIT=OFF
- name: Ubuntu GCC
os: ubuntu-20.04
compiler: gcc
Expand Down Expand Up @@ -163,6 +167,11 @@ jobs:
without_jemalloc: -DDISABLE_JEMALLOC=ON
compiler: clang
ignore_when_tsan: -tags="ignore_when_tsan"
- name: Ubuntu Clang UBSAN
os: ubuntu-20.04
with_sanitizer: -DENABLE_UBSAN=ON
without_jemalloc: -DDISABLE_JEMALLOC=ON
compiler: clang
- name: Ubuntu GCC Ninja
os: ubuntu-20.04
with_ninja: --ninja
Expand All @@ -172,7 +181,7 @@ jobs:
compiler: gcc
with_openssl: -DENABLE_OPENSSL=ON
- name: Ubuntu Clang with OpenSSL
os: ubuntu-20.04
os: ubuntu-22.04
compiler: clang
with_openssl: -DENABLE_OPENSSL=ON
- name: Ubuntu GCC without luaJIT
Expand All @@ -183,14 +192,14 @@ jobs:
os: ubuntu-20.04
without_luajit: -DENABLE_LUAJIT=OFF
compiler: clang
- name: Ubuntu GCC with new encoding
- name: Ubuntu GCC with old encoding
os: ubuntu-20.04
compiler: gcc
new_encoding: -DENABLE_NEW_ENCODING=TRUE
- name: Ubuntu Clang with new encoding
os: ubuntu-20.04
new_encoding: -DENABLE_NEW_ENCODING=FALSE
- name: Ubuntu Clang with old encoding
os: ubuntu-22.04
compiler: clang
new_encoding: -DENABLE_NEW_ENCODING=TRUE
new_encoding: -DENABLE_NEW_ENCODING=FALSE
- name: Ubuntu GCC with speedb enabled
os: ubuntu-20.04
compiler: gcc
Expand Down Expand Up @@ -219,14 +228,22 @@ jobs:
with:
path: |
~/local/bin/redis-cli
key: ${{ runner.os }}-redis-cli
key: ${{ runner.os }}-${{ runner.arch }}-redis-cli
- name: Cache redis server
id: cache-redis-server
uses: actions/cache@v4
with:
path: |
~/local/bin/redis-server
key: ${{ runner.os }}-${{ runner.arch }}-redis-server
- name: Install redis
if: steps.cache-redis.outputs.cache-hit != 'true'
if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }}
run: |
curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz
tar -xzvf redis-6.2.7.tar.gz
curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz
tar -xzvf redis-6.2.14.tar.gz
mkdir -p $HOME/local/bin
pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -257,6 +274,8 @@ jobs:
- name: Build Kvrocks (SonarCloud)
if: ${{ matrix.sonarcloud }}
run: |
build-wrapper-linux-x86-64 --out-dir ${{ env.SONARCLOUD_OUTPUT_DIR }} ./x.py build -j$NPROC --compiler ${{ matrix.compiler }} --skip-build
cp -r build _build
build-wrapper-linux-x86-64 --out-dir ${{ env.SONARCLOUD_OUTPUT_DIR }} ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.sonarcloud }}
- name: Setup Coredump
Expand All @@ -281,7 +300,7 @@ jobs:
GOCASE_RUN_ARGS=""
if [[ -n "${{ matrix.with_openssl }}" ]] && [[ "${{ matrix.os }}" == ubuntu* ]]; then
git clone https://github.com/jsha/minica
cd minica && go build && cd ..
cd minica && git checkout 96a5c93723cf3d34b50b3e723a9f05cd3765bc67 && go build && cd ..
./minica/minica --domains localhost
cp localhost/cert.pem tests/gocase/tls/cert/server.crt
cp localhost/key.pem tests/gocase/tls/cert/server.key
Expand All @@ -290,6 +309,29 @@ jobs:
fi
./x.py test go build $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}}
- name: Install redis-py
run: pip3 install redis==4.3.6

- name: Run kvrocks2redis Test
# Currently, when enabling Tsan/Asan or running in macOS 11/14, the value mismatch in destination redis server.
# See https://github.com/apache/kvrocks/issues/2195.
if: ${{ !contains(matrix.name, 'Tsan') && !contains(matrix.name, 'Asan') && !startsWith(matrix.os, 'macos') }}
run: |
ulimit -c unlimited
export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)"
export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)"
$HOME/local/bin/redis-server --daemonize yes
mkdir -p kvrocks2redis-ci-data
./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes
sleep 10s
echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf
cat ./kvrocks2redis-ci.conf
./build/kvrocks2redis -c ./kvrocks2redis-ci.conf
sleep 10s
python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true
sleep 10s
python3 utils/kvrocks2redis/tests/check_consistency.py --src_password=""
- name: Find reports and crashes
if: always()
run: |
Expand Down Expand Up @@ -335,13 +377,15 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: sonarcloud-data
path: ${{ env.SONARCLOUD_OUTPUT_DIR }}
path: |
${{ env.SONARCLOUD_OUTPUT_DIR }}
_build
check-docker:
name: Check Docker image
needs: [precondition, check-and-lint, check-typos]
if: ${{ needs.precondition.outputs.docs_only != 'true' }}
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Get core numbers
Expand Down Expand Up @@ -394,7 +438,7 @@ jobs:
if: ${{ startsWith(matrix.image, 'centos') }}
run: |
yum install -y centos-release-scl-rh
yum install -y devtoolset-11 python3 autoconf automake wget git gcc gcc-c++
yum install -y devtoolset-11 python3 python3-pip autoconf automake wget git gcc gcc-c++
echo "NPROC=$(nproc)" >> $GITHUB_ENV
mv /usr/bin/gcc /usr/bin/gcc-4.8.5
ln -s /opt/rh/devtoolset-11/root/bin/gcc /usr/bin/gcc
Expand All @@ -405,13 +449,13 @@ jobs:
if: ${{ startsWith(matrix.image, 'archlinux') }}
run: |
pacman -Syu --noconfirm
pacman -Sy --noconfirm autoconf automake python3 git wget which cmake make gcc
pacman -Sy --noconfirm autoconf automake python3 python-redis git wget which cmake make gcc
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup openSUSE
if: ${{ startsWith(matrix.image, 'opensuse') }}
run: |
zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 curl tar gzip cmake go
zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 100
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 100
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
Expand All @@ -426,19 +470,29 @@ jobs:
~/local/bin/redis-cli
key: ${{ matrix.image }}-redis-cli

- name: Cache redis server
id: cache-redis-server
uses: actions/cache@v3
with:
path: |
~/local/bin/redis-server
key: ${{ matrix.image }}-redis-server

- name: Install redis
if: steps.cache-redis.outputs.cache-hit != 'true'
if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }}
run: |
curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz
tar -xzvf redis-6.2.7.tar.gz
curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz
tar -xzvf redis-6.2.14.tar.gz
mkdir -p $HOME/local/bin
pushd redis-6.2.7 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd
- name: Install cmake
if: ${{ startsWith(matrix.image, 'centos') }}
run: |
wget https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.sh
bash cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
VERSION=3.26.4
wget https://github.com/Kitware/CMake/releases/download/v$VERSION/cmake-$VERSION-linux-x86_64.sh
bash cmake-$VERSION-linux-x86_64.sh --skip-license --prefix=/usr
- uses: actions/checkout@v3 #v4 use Node 20 and not working at CentOS 7
- uses: actions/setup-go@v4 #v5 use Node 20 too
Expand All @@ -461,6 +515,24 @@ jobs:
GOCASE_RUN_ARGS=""
./x.py test go build $GOCASE_RUN_ARGS
- name: Install redis-py
if: ${{ !startsWith(matrix.image, 'archlinux') }} # already installed
run: pip3 install redis==4.3.6

- name: Run kvrocks2redis Test
run: |
$HOME/local/bin/redis-server --daemonize yes
mkdir -p kvrocks2redis-ci-data
./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes
sleep 10s
echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf
cat ./kvrocks2redis-ci.conf
./build/kvrocks2redis -c ./kvrocks2redis-ci.conf
sleep 10s
python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true
sleep 10s
python3 utils/kvrocks2redis/tests/check_consistency.py --src_password=""
required:
if: always()
name: Required
Expand Down
91 changes: 91 additions & 0 deletions .github/workflows/pr-lint.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: "Lint PR"

on:
pull_request_target:
types:
- opened
- edited
- synchronize

permissions:
pull-requests: read

jobs:
main:
name: Validate PR title
runs-on: ubuntu-latest
steps:
- uses: amannn/action-semantic-pull-request@v5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
# Configure which types are allowed (newline-delimited).
# Default: https://github.com/commitizen/conventional-commit-types
types: |
fix
feat
build
chore
ci
docs
perf
refactor
revert
style
test
# Configure which scopes are allowed (newline-delimited).
# These are regex patterns auto-wrapped in `^ $`.
scopes: |
\S+
# Configure that a scope must always be provided.
requireScope: false
# Configure which scopes are disallowed in PR titles (newline-delimited).
# For instance by setting the value below, `chore(release): ...` (lowercase)
# and `ci(e2e,release): ...` (unknown scope) will be rejected.
# These are regex patterns auto-wrapped in `^ $`.
# disallowScopes: |
# release
# [A-Z]+
# Configure additional validation for the subject based on a regex.
# This example ensures the subject doesn't start with an uppercase character.
# subjectPattern: ^(?![A-Z]).+$
# If `subjectPattern` is configured, you can use this property to override
# the default error message that is shown when the pattern doesn't match.
# The variables `subject` and `title` can be used within the message.
# subjectPatternError: |
# The subject "{subject}" found in the pull request title "{title}"
# didn't match the configured pattern. Please ensure that the subject
# doesn't start with an uppercase character.
# The GitHub base URL will be automatically set to the correct value from the GitHub context variable.
# If you want to override this, you can do so here (not recommended).
# githubBaseUrl: https://github.myorg.com/api/v3
# If the PR contains one of these newline-delimited labels, the
# validation is skipped. If you want to rerun the validation when
# labels change, you might want to use the `labeled` and `unlabeled`
# event triggers in your workflow.
ignoreLabels: |
disable-pr-lint
# If you're using a format for the PR title that differs from the traditional Conventional
# Commits spec, you can use these options to customize the parsing of the type, scope and
# subject. The `headerPattern` should contain a regex where the capturing groups in parentheses
# correspond to the parts listed in `headerPatternCorrespondence`.
# See: https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-commits-parser#headerpattern
# headerPattern: '^(\w*)(?:\(([\w$.\-*/ ]*)\))?: (.*)$'
# headerPatternCorrespondence: type, scope, subject
9 changes: 4 additions & 5 deletions .github/workflows/sonar.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,14 @@ jobs:
fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/sonarcloud-data.zip`, Buffer.from(download.data));
- name: 'Unzip code coverage'
run: |
unzip sonarcloud-data.zip -d sonarcloud-data
ls -a sonarcloud-data
unzip sonarcloud-data.zip
mv _build build
mkdir -p build/CMakeFiles/CMakeTmp
ls -a sonarcloud-data build
- uses: actions/setup-python@v5
with:
python-version: 3.x
- name: Configure Kvrocks
run: |
./x.py build -j$(nproc) --compiler gcc --skip-build

- name: Run sonar-scanner
env:
Expand Down
Loading

0 comments on commit 6b8e5e4

Please sign in to comment.