Skip to content
CI

feat(conn): move time-consuming ops out of ExecuteCommand for context #1288

Sign in to view logs

feat(conn): move time-consuming ops out of ExecuteCommand for context

feat(conn): move time-consuming ops out of ExecuteCommand for context #1288

Workflow file for this run

.github/workflows/kvrocks.yaml at 2e1a3c3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: CI
on: [push, pull_request]
# Concurrency strategy:
# github.workflow: distinguish this workflow from others
# github.event_name: distinguish `push` event from `pull_request` event
# github.event.number: set to the number of the pull request if `pull_request` event
# github.run_id: otherwise, it's a `push` event, only cancel if we rerun the workflow
#
# Reference:
# https://docs.github.com/en/actions/using-jobs/using-concurrency
# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true
jobs:
precondition:
name: Precondition
runs-on: ubuntu-22.04
outputs:
docs_only: ${{ steps.result.outputs.docs_only }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: changes
with:
filters: .github/config/changes.yml
list-files: csv
- name: Calculate changes
id: result
run: |
echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
check-typos:
name: Check typos
runs-on: ubuntu-22.04
env:
FORCE_COLOR: 1
steps:
- uses: actions/checkout@v4
- name: Check typos
uses: crate-ci/typos@v1.25.0
with:
config: .github/config/typos.toml
check-and-lint:
name: Lint and check code
needs: [precondition]
if: ${{ needs.precondition.outputs.docs_only != 'true' }}
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: 'tests/gocase/go.mod'
cache: false
- name: Prepare Dependencies
run: |
sudo apt update
sudo apt install -y clang-format-14 clang-tidy-14
- uses: apache/skywalking-eyes/header@v0.6.0
with:
config: .github/config/licenserc.yml
- name: Check with clang-format
id: check-format
run: ./x.py check format --clang-format-path clang-format-14
- name: Check with clang-tidy
run: |
./x.py build --skip-build
./x.py check tidy -j $(nproc) --clang-tidy-path clang-tidy-14 --run-clang-tidy-path run-clang-tidy-14
- name: Lint with golangci-lint
run: ./x.py check golangci-lint
- name: Prepare format patch
if: always() && steps.check-format.outcome != 'success'
run: |
./x.py format --clang-format-path clang-format-14
git diff -p > clang-format.patch
cat clang-format.patch
- name: Upload format patch
uses: actions/upload-artifact@v4
if: always() && steps.check-format.outcome != 'success'
with:
path: clang-format.patch
build-and-test:
name: Build and test
needs: [precondition, check-and-lint, check-typos]
if: ${{ needs.precondition.outputs.docs_only != 'true' }}
strategy:
fail-fast: false
matrix:
include:
- name: Darwin Clang
os: macos-13
compiler: auto
- name: Darwin Clang arm64
os: macos-14
compiler: auto
- name: Darwin Clang without Jemalloc
os: macos-13
compiler: auto
without_jemalloc: -DDISABLE_JEMALLOC=ON
- name: Darwin Clang with OpenSSL
os: macos-13
compiler: auto
with_openssl: -DENABLE_OPENSSL=ON
- name: Darwin Clang without luaJIT
os: macos-13
compiler: auto
without_luajit: -DENABLE_LUAJIT=OFF
- name: Ubuntu GCC
os: ubuntu-22.04
compiler: gcc
- name: SonarCloud with Coverage
os: ubuntu-22.04
compiler: gcc
sonarcloud: -DCMAKE_CXX_FLAGS=--coverage
- name: Ubuntu Clang
os: ubuntu-22.04
compiler: clang
# FIXME: https://github.com/apache/kvrocks/issues/2411
# - name: Ubuntu 24 GCC
# os: ubuntu-24.04
# compiler: gcc
# - name: Ubuntu 24 Clang
# os: ubuntu-24.04
# compiler: clang
- name: Ubuntu GCC ASan
os: ubuntu-22.04
without_jemalloc: -DDISABLE_JEMALLOC=ON
with_sanitizer: -DENABLE_ASAN=ON
compiler: gcc
- name: Ubuntu Clang ASan
os: ubuntu-22.04
with_sanitizer: -DENABLE_ASAN=ON
without_jemalloc: -DDISABLE_JEMALLOC=ON
compiler: clang
- name: Ubuntu GCC TSan
os: ubuntu-22.04
without_jemalloc: -DDISABLE_JEMALLOC=ON
with_sanitizer: -DENABLE_TSAN=ON
compiler: gcc
ignore_when_tsan: -tags="ignore_when_tsan"
- name: Ubuntu Clang TSan
os: ubuntu-22.04
with_sanitizer: -DENABLE_TSAN=ON
without_jemalloc: -DDISABLE_JEMALLOC=ON
compiler: clang
ignore_when_tsan: -tags="ignore_when_tsan"
- name: Ubuntu Clang UBSAN
os: ubuntu-22.04
with_sanitizer: -DENABLE_UBSAN=ON
without_jemalloc: -DDISABLE_JEMALLOC=ON
compiler: clang
- name: Ubuntu GCC Ninja
os: ubuntu-22.04
with_ninja: --ninja
compiler: gcc
- name: Ubuntu GCC with OpenSSL
os: ubuntu-22.04
compiler: gcc
with_openssl: -DENABLE_OPENSSL=ON
- name: Ubuntu Clang with OpenSSL
os: ubuntu-22.04
compiler: clang
with_openssl: -DENABLE_OPENSSL=ON
- name: Ubuntu GCC without luaJIT
os: ubuntu-22.04
without_luajit: -DENABLE_LUAJIT=OFF
compiler: gcc
- name: Ubuntu Clang without luaJIT
os: ubuntu-22.04
without_luajit: -DENABLE_LUAJIT=OFF
compiler: clang
- name: Ubuntu GCC with old encoding
os: ubuntu-22.04
compiler: gcc
new_encoding: -DENABLE_NEW_ENCODING=FALSE
- name: Ubuntu Clang with old encoding
os: ubuntu-22.04
compiler: clang
new_encoding: -DENABLE_NEW_ENCODING=FALSE
runs-on: ${{ matrix.os }}
env:
SONARCLOUD_OUTPUT_DIR: sonarcloud-data
FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true
steps:
- name: Setup macOS
if: ${{ startsWith(matrix.os, 'macos') }}
run: |
brew install --quiet --formula cmake gcc autoconf automake libtool openssl coreutils
echo "NPROC=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV
echo "CMAKE_EXTRA_DEFS=-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl" >> $GITHUB_ENV
- name: Setup Linux
if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: |
sudo apt update
sudo apt install -y ninja-build
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Cache redis
id: cache-redis
uses: actions/cache@v4
with:
path: |
~/local/bin/redis-cli
key: ${{ runner.os }}-${{ runner.arch }}-redis-cli
- name: Cache redis server
id: cache-redis-server
uses: actions/cache@v4
with:
path: |
~/local/bin/redis-server
key: ${{ runner.os }}-${{ runner.arch }}-redis-server
- name: Install redis
if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }}
run: |
curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz
tar -xzvf redis-6.2.14.tar.gz
mkdir -p $HOME/local/bin
pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: 3.x
- uses: actions/setup-go@v5
with:
go-version-file: 'tests/gocase/go.mod'
cache: false
- name: Install gcovr 5.0
run: pip install gcovr==5.0 # 5.1 is not supported
if: ${{ matrix.sonarcloud }}
- name: Install sonar-scanner and build-wrapper
uses: SonarSource/sonarcloud-github-c-cpp@v2
if: ${{ matrix.sonarcloud }}
- name: Build Kvrocks
if: ${{ !matrix.sonarcloud }}
run: |
./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.without_jemalloc }} \
${{ matrix.without_luajit }} ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{ matrix.with_openssl }} \
${{ matrix.new_encoding }} ${{ env.CMAKE_EXTRA_DEFS }}
- name: Build Kvrocks (SonarCloud)
if: ${{ matrix.sonarcloud }}
run: |
build-wrapper-linux-x86-64 --out-dir ${{ env.SONARCLOUD_OUTPUT_DIR }} ./x.py build -j$NPROC --compiler ${{ matrix.compiler }} --skip-build
cp -r build _build
build-wrapper-linux-x86-64 --out-dir ${{ env.SONARCLOUD_OUTPUT_DIR }} ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.sonarcloud }}
- name: Setup Coredump
if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: |
echo "$(pwd)/coredumps/corefile-%e-%p-%t" | sudo tee /proc/sys/kernel/core_pattern
mkdir coredumps
- name: Run Unit Test
run: |
ulimit -c unlimited
export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)"
export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)"
./x.py test cpp
- name: Run Go Integration Cases
run: |
ulimit -c unlimited
export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)"
export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)"
export PATH=$PATH:$HOME/local/bin/
GOCASE_RUN_ARGS=""
if [[ -n "${{ matrix.with_openssl }}" ]] && [[ "${{ matrix.os }}" == ubuntu* ]]; then
git clone https://github.com/jsha/minica
cd minica && git checkout 96a5c93723cf3d34b50b3e723a9f05cd3765bc67 && go build && cd ..
./minica/minica --domains localhost
cp localhost/cert.pem tests/gocase/tls/cert/server.crt
cp localhost/key.pem tests/gocase/tls/cert/server.key
cp minica.pem tests/gocase/tls/cert/ca.crt
GOCASE_RUN_ARGS="-tlsEnable"
fi
./x.py test go build $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}}
- name: Install redis-py
run: pip3 install redis==4.3.6
- name: Run kvrocks2redis Test
# Currently, when enabling Tsan/Asan or running in macOS 11/14, the value mismatch in destination redis server.
# See https://github.com/apache/kvrocks/issues/2195.
if: ${{ !contains(matrix.name, 'Tsan') && !contains(matrix.name, 'Asan') && !startsWith(matrix.os, 'macos') }}
run: |
ulimit -c unlimited
export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)"
export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)"
$HOME/local/bin/redis-server --daemonize yes
mkdir -p kvrocks2redis-ci-data
./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes
sleep 10s
echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf
cat ./kvrocks2redis-ci.conf
./build/kvrocks2redis -c ./kvrocks2redis-ci.conf
sleep 10s
python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true
sleep 10s
python3 utils/kvrocks2redis/tests/check_consistency.py --src_password=""
- name: Find reports and crashes
if: always()
run: |
SANITIZER_OUTPUT=$(grep "Sanitizer:" tests/gocase/workspace -r || true)
if [[ $SANITIZER_OUTPUT ]]; then
echo "found sanitizer reports:"
echo "$SANITIZER_OUTPUT"
echo "detail log:"
cat $(echo "$SANITIZER_OUTPUT" | awk -F ':' '{print $1}')
exit 1
fi
CRASHES=$(grep "Ooops!" tests/gocase/workspace -r || true)
if [[ $CRASHES ]]; then
echo "found crashes:"
echo "$CRASHES"
echo "detail log:"
cat $(echo "$CRASHES" | awk -F ':' '{print $1}')
exit 1
fi
- uses: actions/upload-artifact@v4
if: ${{ failure() && startsWith(matrix.os, 'ubuntu') }}
with:
name: kvrocks-coredumps-${{ matrix.name }}
path: |
./build/kvrocks
./coredumps/*
- name: Collect coverage into one XML report
if: ${{ matrix.sonarcloud }}
run: |
gcovr --sonarqube > ${{ env.SONARCLOUD_OUTPUT_DIR }}/coverage.xml
- name: Add event information
if: ${{ matrix.sonarcloud }}
env:
GITHUB_EVENT_JSON: ${{ toJson(github.event) }}
run: |
echo "$GITHUB_EVENT_JSON" | tee ${{ env.SONARCLOUD_OUTPUT_DIR }}/github-event.json
- name: Upload SonarCloud data
if: ${{ matrix.sonarcloud }}
uses: actions/upload-artifact@v4
with:
name: sonarcloud-data
path: |
${{ env.SONARCLOUD_OUTPUT_DIR }}
_build
check-docker:
name: Check Docker image
needs: [precondition, check-and-lint, check-typos]
if: ${{ needs.precondition.outputs.docs_only != 'true' }}
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Get core numbers
run: echo "NPROC=$(nproc)" >> $GITHUB_ENV
- uses: docker/build-push-action@v6
with:
context: .
build-args: MORE_BUILD_ARGS=-j${{ env.NPROC }}
push: false
tags: kvrocks:ci
outputs: type=docker
- name: Test built image
run: |
docker run --rm kvrocks:ci -v
ID="$(docker run --rm -d -p 6666:6666 kvrocks:ci)"
sleep 1m
if [ "$(docker inspect --format='{{.State.Health.Status}}' $ID)" != "healthy" ]; then
echo "The container is not healthy."
exit 1
fi
if [ "$(ss --listening --no-header --tcp '( sport = :6666 )')" == "" ]; then
echo "The container listening port can not be accessed from outside."
exit 1
fi
docker stop $ID
build-and-test-in-container:
name: Build and test in container
needs: [precondition, check-and-lint, check-typos]
if: ${{ needs.precondition.outputs.docs_only != 'true' }}
strategy:
fail-fast: false
matrix:
include:
- name: openSUSE Leap 15
image: opensuse/leap:15
compiler: gcc
- name: ArchLinux
image: archlinux:base
compiler: gcc
- name: Rocky Linux 8
image: rockylinux:8
compiler: gcc
- name: Rocky Linux 9
image: rockylinux:9
compiler: gcc
- name: Debian 12
image: debian:12
compiler: gcc
runs-on: ubuntu-22.04
container:
image: ${{ matrix.image }}
steps:
- name: Setup ArchLinux
if: ${{ startsWith(matrix.image, 'archlinux') }}
run: |
pacman -Syu --noconfirm
pacman -Sy --noconfirm autoconf automake python3 python-redis git wget which cmake make gcc
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup openSUSE
if: ${{ startsWith(matrix.image, 'opensuse') }}
run: |
zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 100
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 100
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup Rocky Linux 8
if: ${{ startsWith(matrix.image, 'rockylinux:8') }}
run: |
dnf install -y epel-release
dnf config-manager --set-enabled powertools
dnf install -y git gcc-toolset-12 autoconf automake libtool libstdc++-static python3 python3-pip openssl-devel which cmake
source /opt/rh/gcc-toolset-12/enable
update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
update-alternatives --install /usr/bin/cc cc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100
update-alternatives --install /usr/bin/c++ c++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup Rocky Linux 9
if: ${{ startsWith(matrix.image, 'rockylinux:9') }}
run: |
dnf install -y epel-release
dnf config-manager --set-enabled crb
dnf install -y git gcc-toolset-12 autoconf automake libtool libstdc++-static python3 python3-pip openssl-devel which cmake
source /opt/rh/gcc-toolset-12/enable
update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
update-alternatives --install /usr/bin/cc cc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100
update-alternatives --install /usr/bin/c++ c++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup Debian 12
if: ${{ startsWith(matrix.image, 'debian') }}
run: |
apt update
apt install -y bash build-essential cmake curl git libssl-dev libtool python3 python3-pip wget
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Cache redis
id: cache-redis
uses: actions/cache@v4
with:
path: |
~/local/bin/redis-cli
key: ${{ matrix.image }}-redis-cli
- name: Cache redis server
id: cache-redis-server
uses: actions/cache@v4
with:
path: |
~/local/bin/redis-server
key: ${{ matrix.image }}-redis-server
- name: Install redis
if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }}
shell: bash
run: |
curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz
tar -xzvf redis-6.2.14.tar.gz
mkdir -p $HOME/local/bin
pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
if: ${{ !startsWith(matrix.image, 'opensuse') }}
with:
go-version-file: 'tests/gocase/go.mod'
cache: false
- name: Build Kvrocks
run: |
./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }}
- name: Run Unit Test
run: |
./x.py test cpp
- name: Run Go Integration Cases
run: |
export PATH=$PATH:$HOME/local/bin/
GOCASE_RUN_ARGS=""
./x.py test go build $GOCASE_RUN_ARGS
- name: Install redis-py for openSUSE and Rocky
if: ${{ !startsWith(matrix.image, 'archlinux') && !startsWith(matrix.image, 'debian') }}
run: pip3 install redis==4.3.6
- name: Install redis-py for Debian
if: ${{ startsWith(matrix.image, 'debian') }}
run: apt install -y python3-redis
- name: Run kvrocks2redis Test
# FIXME: https://github.com/apache/kvrocks/issues/2574
if: ${{ !startsWith(matrix.image, 'archlinux') }}
shell: bash
run: |
$HOME/local/bin/redis-server --daemonize yes
mkdir -p kvrocks2redis-ci-data
./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes
sleep 10s
echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf
cat ./kvrocks2redis-ci.conf
./build/kvrocks2redis -c ./kvrocks2redis-ci.conf
sleep 10s
python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true
sleep 10s
python3 utils/kvrocks2redis/tests/check_consistency.py --src_password=""
required:
if: always()
name: Required
runs-on: ubuntu-latest
needs:
- precondition
- build-and-test
- build-and-test-in-container
- check-docker
steps:
- name: Merge requirement checking
if: ${{ needs.precondition.outputs.docs_only != 'true' }}
run: |
if [[ ! ( \
"${{ needs.build-and-test.result }}" == "success" \
&& "${{ needs.build-and-test-in-container.result }}" == "success" \
&& "${{ needs.check-docker.result }}" == "success" \
) ]]; then
echo "Required jobs haven't been completed successfully."
exit 1
fi
- name: Sentinel
run: true