From cbc4f18d186eca4a772d64d6e8ad978a7a0d8cb0 Mon Sep 17 00:00:00 2001 From: Zakelly Date: Wed, 20 Mar 2024 00:11:51 +0800 Subject: [PATCH 1/7] Enable ci for kvrocks2redis --- .github/workflows/kvrocks.yaml | 94 +++++++++++++++++-- utils/kvrocks2redis/tests/populate-kvrocks.py | 11 ++- 2 files changed, 97 insertions(+), 8 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index 675a23b6f25..f207fc5c9ea 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -218,18 +218,33 @@ jobs: - name: Cache redis id: cache-redis + if: ${{ !startsWith(matrix.os, 'macos') }} uses: actions/cache@v4 with: path: | ~/local/bin/redis-cli key: ${{ runner.os }}-${{ runner.arch }}-redis-cli + - name: Cache redis server + id: cache-redis-server + if: ${{ !startsWith(matrix.os, 'macos') }} + uses: actions/cache@v4 + with: + path: | + ~/local/bin/redis-server + key: ${{ matrix.os }}-redis-server - name: Install redis - if: steps.cache-redis.outputs.cache-hit != 'true' + if: ${{ !startsWith(matrix.os, 'macos') && (steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true') }} run: | curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz tar -xzvf redis-6.2.7.tar.gz mkdir -p $HOME/local/bin pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd + + - name: Setup redis (macOS) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + brew install redis@6.2 - uses: actions/checkout@v4 with: @@ -256,7 +271,7 @@ jobs: ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.without_jemalloc }} \ ${{ matrix.without_luajit }} ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{ matrix.with_openssl }} \ ${{ matrix.new_encoding }} ${{ matrix.with_speedb }} ${{ env.CMAKE_EXTRA_DEFS }} - + - name: Build Kvrocks (SonarCloud) if: ${{ matrix.sonarcloud }} run: | @@ -293,6 +308,46 @@ jobs: fi ./x.py test go build $GOCASE_RUN_ARGS ${{ matrix.ignore_when_tsan}} + - name: Install redis-py + run: pip3 install redis==4.3.6 + + - name: Run kvrocks2redis Test + if: ${{ !startsWith(matrix.os, 'macos') }} + run: | + ulimit -c unlimited + export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)" + export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)" + $HOME/local/bin/redis-server --daemonize yes + mkdir -p data + ./build/kvrocks --dir `pwd`/data --pidfile `pwd`/kvrocks.pid --daemonize yes + sleep 15s + echo -en "data-dir `pwd`/data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf + cat ./kvrocks2redis-ci.conf + ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf + sleep 15s + python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true + sleep 15s + ps aux + python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" + + - name: Run kvrocks2redis Test (macOS) + if: ${{ startsWith(matrix.os, 'macos') }} + run: | + ulimit -c unlimited + brew services start redis@6.2 + mkdir -p data + ./build/kvrocks --dir `pwd`/data --pidfile `pwd`/kvrocks.pid --daemonize yes + sleep 15 + echo -en "data-dir `pwd`/data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf + cat ./kvrocks2redis-ci.conf + ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf + sleep 15 + python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true + sleep 15 + ps aux + python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" + + - name: Find reports and crashes if: always() run: | @@ -397,7 +452,7 @@ jobs: if: ${{ startsWith(matrix.image, 'centos') }} run: | yum install -y centos-release-scl-rh - yum install -y devtoolset-11 python3 autoconf automake wget git gcc gcc-c++ + yum install -y devtoolset-11 python3 python3-pip autoconf automake wget git gcc gcc-c++ echo "NPROC=$(nproc)" >> $GITHUB_ENV mv /usr/bin/gcc /usr/bin/gcc-4.8.5 ln -s /opt/rh/devtoolset-11/root/bin/gcc /usr/bin/gcc @@ -408,13 +463,13 @@ jobs: if: ${{ startsWith(matrix.image, 'archlinux') }} run: | pacman -Syu --noconfirm - pacman -Sy --noconfirm autoconf automake python3 git wget which cmake make gcc + pacman -Sy --noconfirm autoconf automake python3 python-redis git wget which cmake make gcc echo "NPROC=$(nproc)" >> $GITHUB_ENV - name: Setup openSUSE if: ${{ startsWith(matrix.image, 'opensuse') }} run: | - zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 curl tar gzip cmake go + zypper install -y gcc11 gcc11-c++ make wget git autoconf automake python3 python3-pip curl tar gzip cmake go update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 100 update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 100 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 @@ -429,13 +484,22 @@ jobs: ~/local/bin/redis-cli key: ${{ matrix.image }}-redis-cli + - name: Cache redis server + id: cache-redis-server + uses: actions/cache@v3 + with: + path: | + ~/local/bin/redis-server + key: ${{ matrix.image }}-redis-server + - name: Install redis - if: steps.cache-redis.outputs.cache-hit != 'true' + if: steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' run: | curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz tar -xzvf redis-6.2.7.tar.gz mkdir -p $HOME/local/bin pushd redis-6.2.7 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.7 && USE_JEMALLOC=no make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd - name: Install cmake if: ${{ startsWith(matrix.image, 'centos') }} @@ -465,6 +529,24 @@ jobs: GOCASE_RUN_ARGS="" ./x.py test go build $GOCASE_RUN_ARGS + - name: Install redis-py + if: ${{ !startsWith(matrix.image, 'archlinux') }} # already installed + run: pip3 install redis==4.3.6 + + - name: Run kvrocks2redis Test + run: | + $HOME/local/bin/redis-server --daemonize yes + mkdir -p data + ./build/kvrocks --dir `pwd`/data --pidfile `pwd`/kvrocks.pid --daemonize yes + sleep 15s + echo -en "data-dir `pwd`/data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf + cat ./kvrocks2redis-ci.conf + ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf + sleep 15s + python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true + sleep 15s + python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" + required: if: always() name: Required diff --git a/utils/kvrocks2redis/tests/populate-kvrocks.py b/utils/kvrocks2redis/tests/populate-kvrocks.py index 8883a0dfd36..fefae86a0fe 100644 --- a/utils/kvrocks2redis/tests/populate-kvrocks.py +++ b/utils/kvrocks2redis/tests/populate-kvrocks.py @@ -146,8 +146,10 @@ def run_test(client, cases : list): print('******* Some case test fail *******') for cmd in fails: print(cmd) + return False else: print('All case passed.') + return True if __name__ == '__main__': @@ -155,5 +157,10 @@ def run_test(client, cases : list): client = redis.Redis(host=args.host, port=args.port, decode_responses=True, password=args.password) if args.flushdb: client.flushdb() - run_test(client, PopulateCases) - run_test(client, AppendCases) + succ = True + if not run_test(client, PopulateCases): + succ = False + if not run_test(client, AppendCases): + succ = False + if not succ: + raise AssertionError("Test failed. See details above.") From 47ec2998b1153146afff53d7f0f761df807f5e0a Mon Sep 17 00:00:00 2001 From: Zakelly Date: Sat, 23 Mar 2024 23:59:31 +0800 Subject: [PATCH 2/7] Disable kvrocks2redis_ci for tsan/asan and macos --- .github/workflows/kvrocks.yaml | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index f207fc5c9ea..7840953c780 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -312,7 +312,9 @@ jobs: run: pip3 install redis==4.3.6 - name: Run kvrocks2redis Test - if: ${{ !startsWith(matrix.os, 'macos') }} + # Currently, when enabling Tsan/Asan or runing in macos 11/14, the value mismatch in destination redis server. + # See https://github.com/apache/kvrocks/issues/2195 + if: ${{ !contains(matrix.name, 'Tsan') && !contains(matrix.name, 'Asan') && !startsWith(matrix.os, 'macos') }} run: | ulimit -c unlimited export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)" @@ -330,24 +332,6 @@ jobs: ps aux python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" - - name: Run kvrocks2redis Test (macOS) - if: ${{ startsWith(matrix.os, 'macos') }} - run: | - ulimit -c unlimited - brew services start redis@6.2 - mkdir -p data - ./build/kvrocks --dir `pwd`/data --pidfile `pwd`/kvrocks.pid --daemonize yes - sleep 15 - echo -en "data-dir `pwd`/data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf - cat ./kvrocks2redis-ci.conf - ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf - sleep 15 - python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true - sleep 15 - ps aux - python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" - - - name: Find reports and crashes if: always() run: | From 6b8d67807b7de58c326b12a5c07cc2389f548b44 Mon Sep 17 00:00:00 2001 From: Zakelly Date: Sun, 24 Mar 2024 13:40:21 +0800 Subject: [PATCH 3/7] Fix typos --- .github/workflows/kvrocks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index 7840953c780..b10ecb3c4f0 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -312,8 +312,8 @@ jobs: run: pip3 install redis==4.3.6 - name: Run kvrocks2redis Test - # Currently, when enabling Tsan/Asan or runing in macos 11/14, the value mismatch in destination redis server. - # See https://github.com/apache/kvrocks/issues/2195 + # Currently, when enabling Tsan/Asan or running in macOS 11/14, the value mismatch in destination redis server. + # See https://github.com/apache/kvrocks/issues/2195. if: ${{ !contains(matrix.name, 'Tsan') && !contains(matrix.name, 'Asan') && !startsWith(matrix.os, 'macos') }} run: | ulimit -c unlimited From 739e9fe2734cd2dfb16e50d799125ce541401ff8 Mon Sep 17 00:00:00 2001 From: Zakelly Date: Sun, 24 Mar 2024 13:48:12 +0800 Subject: [PATCH 4/7] No special rule for macos when building redis --- .github/workflows/kvrocks.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index b10ecb3c4f0..ae252253a61 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -218,7 +218,6 @@ jobs: - name: Cache redis id: cache-redis - if: ${{ !startsWith(matrix.os, 'macos') }} uses: actions/cache@v4 with: path: | @@ -226,14 +225,13 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-redis-cli - name: Cache redis server id: cache-redis-server - if: ${{ !startsWith(matrix.os, 'macos') }} uses: actions/cache@v4 with: path: | ~/local/bin/redis-server key: ${{ matrix.os }}-redis-server - name: Install redis - if: ${{ !startsWith(matrix.os, 'macos') && (steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true') }} + if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }} run: | curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz tar -xzvf redis-6.2.7.tar.gz @@ -241,11 +239,6 @@ jobs: pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd - - name: Setup redis (macOS) - if: ${{ startsWith(matrix.os, 'macos') }} - run: | - brew install redis@6.2 - - uses: actions/checkout@v4 with: fetch-depth: 0 From d216837fc68969828d9230564e14c250025ababf Mon Sep 17 00:00:00 2001 From: Zakelly Date: Sun, 24 Mar 2024 19:51:10 +0800 Subject: [PATCH 5/7] Address comments --- .github/workflows/kvrocks.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index ae252253a61..3245fc5355c 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -229,15 +229,15 @@ jobs: with: path: | ~/local/bin/redis-server - key: ${{ matrix.os }}-redis-server + key: ${{ runner.os }}-${{ runner.arch }}-redis-server - name: Install redis - if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }} + if: ${{ steps.cache-redis.outputs.cache-hit != 'true' }} run: | - curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz - tar -xzvf redis-6.2.7.tar.gz + curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz + tar -xzvf redis-6.2.14.tar.gz mkdir -p $HOME/local/bin - pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd - pushd redis-6.2.7 && BUILD_TLS=yes make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd + pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.14 && BUILD_TLS=yes make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd - uses: actions/checkout@v4 with: @@ -313,10 +313,10 @@ jobs: export LSAN_OPTIONS="suppressions=$(realpath ./tests/lsan-suppressions)" export TSAN_OPTIONS="suppressions=$(realpath ./tests/tsan-suppressions)" $HOME/local/bin/redis-server --daemonize yes - mkdir -p data - ./build/kvrocks --dir `pwd`/data --pidfile `pwd`/kvrocks.pid --daemonize yes + mkdir -p kvrocks2redis-ci-data + ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes sleep 15s - echo -en "data-dir `pwd`/data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf + echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf cat ./kvrocks2redis-ci.conf ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf sleep 15s @@ -513,10 +513,10 @@ jobs: - name: Run kvrocks2redis Test run: | $HOME/local/bin/redis-server --daemonize yes - mkdir -p data - ./build/kvrocks --dir `pwd`/data --pidfile `pwd`/kvrocks.pid --daemonize yes + mkdir -p kvrocks2redis-ci-data + ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes sleep 15s - echo -en "data-dir `pwd`/data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf + echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf cat ./kvrocks2redis-ci.conf ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf sleep 15s From 14fa11e773d9c0d902581419c85984d225cb2cbe Mon Sep 17 00:00:00 2001 From: Zakelly Date: Tue, 26 Mar 2024 22:52:35 +0800 Subject: [PATCH 6/7] Address comments --- .github/workflows/kvrocks.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index 3245fc5355c..ad830626e25 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -231,7 +231,7 @@ jobs: ~/local/bin/redis-server key: ${{ runner.os }}-${{ runner.arch }}-redis-server - name: Install redis - if: ${{ steps.cache-redis.outputs.cache-hit != 'true' }} + if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }} run: | curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz tar -xzvf redis-6.2.14.tar.gz @@ -470,13 +470,13 @@ jobs: key: ${{ matrix.image }}-redis-server - name: Install redis - if: steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' + if: ${{ steps.cache-redis.outputs.cache-hit != 'true' || steps.cache-redis-server.outputs.cache-hit != 'true' }} run: | - curl -O https://download.redis.io/releases/redis-6.2.7.tar.gz - tar -xzvf redis-6.2.7.tar.gz + curl -O https://download.redis.io/releases/redis-6.2.14.tar.gz + tar -xzvf redis-6.2.14.tar.gz mkdir -p $HOME/local/bin - pushd redis-6.2.7 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd - pushd redis-6.2.7 && USE_JEMALLOC=no make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd + pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd + pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd - name: Install cmake if: ${{ startsWith(matrix.image, 'centos') }} From a1a085f6d66767fe81ac7091e9285b95575d4ce7 Mon Sep 17 00:00:00 2001 From: Zakelly Date: Wed, 10 Apr 2024 23:04:13 +0800 Subject: [PATCH 7/7] Make it sleep 10s in kvrocks2redis ci --- .github/workflows/kvrocks.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index d0f191bed9b..d3fc3a54f25 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -315,13 +315,13 @@ jobs: $HOME/local/bin/redis-server --daemonize yes mkdir -p kvrocks2redis-ci-data ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes - sleep 15s + sleep 10s echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf cat ./kvrocks2redis-ci.conf ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf - sleep 15s + sleep 10s python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true - sleep 15s + sleep 10s ps aux python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" @@ -515,13 +515,13 @@ jobs: $HOME/local/bin/redis-server --daemonize yes mkdir -p kvrocks2redis-ci-data ./build/kvrocks --dir `pwd`/kvrocks2redis-ci-data --pidfile `pwd`/kvrocks.pid --daemonize yes - sleep 15s + sleep 10s echo -en "data-dir `pwd`/kvrocks2redis-ci-data\ndaemonize yes\noutput-dir ./\nnamespace.__namespace 127.0.0.1 6379\n" >> ./kvrocks2redis-ci.conf cat ./kvrocks2redis-ci.conf ./build/kvrocks2redis -c ./kvrocks2redis-ci.conf - sleep 15s + sleep 10s python3 utils/kvrocks2redis/tests/populate-kvrocks.py --password="" --flushdb=true - sleep 15s + sleep 10s python3 utils/kvrocks2redis/tests/check_consistency.py --src_password="" required: