CI: initial setup

speedb-io · Nov 23, 2022 · f262e3d · f262e3d
1 parent ebf4dcc
commit f262e3d
Show file tree

Hide file tree

Showing 7 changed files with 550 additions and 6 deletions.
diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml
@@ -0,0 +1,209 @@
+name: CI
+
+on: 
+  #push:
+  workflow_dispatch:
+  workflow_call:
+  pull_request_review:
+    types: [submitted]
+
+
+permissions: write-all
+
+jobs:
+  #Sanity:
+    #uses: speedb-io/speedb/.github/workflows/sanity_check.yml@main
+
+  Build:
+    #needs: [Sanity]
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    runs-on: [self-hosted, ubuntu, asrunner]
+    strategy:
+      matrix:
+        include:
+          - name: verify build
+            command: cmake ..  -GNinja
+          - name: optimized build
+            command: cmake .. -DCMAKE_BUILD_TYPE=Release  -GNinja
+          - mame: clang build
+            command: CC=clang CXX=clang++ cmake ..  -GNinja
+    container:
+      image: alpine:3.14
+
+    steps:
+    - name: Pre-build
+      run: |
+        env
+        rm -rf /usr/share/dotnet || echo ""
+        df -h
+        apk add git
+        echo "nameserver 8.8.8.8" > /etc/resolv.conf
+        apk add bash python3 py3-pip clang clang-extra-tools shellcheck gcc g++ cmake ninja ccache \
+        openjdk10 gflags-dev snappy-dev lz4-dev bzip2-dev zstd-dev zlib-dev linux-headers openssh-client tar
+        python3 -m pip install lint-diffs flake8
+        
+        
+    - name: Checkout
+      uses: actions/checkout@v3 
+
+
+    - name: Prepare ccache timestamp
+      id: ccache_cache_timestamp
+      shell: cmake -P {0}
+      run: |
+        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+        message("::set-output name=timestamp::${current_date}")
+        
+        
+    - name: ccache cache files
+      uses: actions/cache@v2
+      with:
+        path: ~/.ccache
+        key: ${{runner.os}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
+        restore-keys: |
+          ${{runner.os}}-ccache-
+        
+      
+    - name: ${{ matrix.name }}
+      run: |
+        if [ -d "$GITHUB_WORKSPACE/build" ]; then
+          echo >&2 "error: the build directory should not exist" && false NIK 
+        fi
+        if [ -d "~/.ccache" ]; then
+          echo "Already exists"
+        else
+          mkdir -p ~/.ccache
+          ls ~ | grep cache || echo ""
+          touch ~/.ccache/ccache.txt
+          echo "aaa" > ~/.ccache/ccache.txt
+          ls ~/.ccache
+          cat ~/.ccache/ccache.txt
+        fi
+        mkdir -p "$GITHUB_WORKSPACE/build"
+        cd "$GITHUB_WORKSPACE/build"
+        export "CCACHE_BASEDIR=$HOME"
+        export "CCACHE_DIR=$HOME/.ccache"
+        export "CCACHE_COMPILERCHECK=content"
+        ${{ matrix.command }} -DPORTABLE=1 -DWITH_GFLAGS=1 \
+        -DWITH_ZLIB=1 -DWITH_SNAPPY=1 -DWITH_BZ2=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 \
+        -DWITH_JNI=1 -DJAVA_HOME=/usr/lib/jvm/default-jvm \
+        -DWITH_BENCHMARK_TOOLS=1 -DWITH_CORE_TOOLS=1 -DWITH_TOOLS=1 \
+        -DWITH_TESTS=1 -DWITH_ALL_TESTS=1 -DWITH_EXAMPLES=1
+        ninja
+        
+  #Performance:
+    #if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    #needs: [Build]
+    #uses: speedb-io/speedb/.github/workflows/perf-test.yml@main
+
+  QA-Tests:
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Build]
+    uses: speedb-io/speedb/.github/workflows/qa-tests.yml@main
+
+  Fuzz:
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Build]
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: ubuntu:18.04
+    strategy:
+      matrix:
+        include:
+          - name: db_fuzzer
+          - name: db_map_fuzzer
+
+    steps:
+      - name: Pre-build
+        run: |
+          apt update && apt install -y sudo python3 git clang-tools cmake make automake ucommon-utils libtool gettext pkg-config build-essential clang-10 zlib1g-dev libbz2-dev ninja-build liblzma-dev autoconf libsnappy-dev libzstd-dev liblz4-dev binutils m4 g++-10 unzip
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+          echo "nameserver 8.8.8.8" > /etc/resolv.conf
+      
+      - uses: actions/checkout@v3
+
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: ${{ matrix.name }}
+        run: |
+           echo 'git clone https://github.com/google/libprotobuf-mutator.git \n
+             cd libprotobuf-mutator \n
+             git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f \n
+             cd .. \n
+             export CC=clang && export CXX=clang++ && mkdir LPM && cd LPM \n
+             ln -s /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so \n
+             ln -s /usr/bin/clang-10 /usr/bin/clang \n
+             ln -s /usr/bin/clang++-10 /usr/bin/clang++ \n
+             cmake ../libprotobuf-mutator -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON -DLIB_PROTO_MUTATOR_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \n
+             ninja \n
+             ninja install \n
+             export PKG_CONFIG_PATH=$PWD:$PWD/external.protobuf/lib/pkgconfig/ \n
+             export PATH=$PWD/external.protobuf/bin:$PATH \n
+             cd $GITHUB_WORKSPACE \n
+             COMPILE_WITH_ASAN=1 PORTABLE=1 make -j2 static_lib \n
+             cd $GITHUB_WORKSPACE/fuzz \n
+             make ${{ matrix.name }} \n
+             ls -alFh $GITHUB_WORKSPACE/fuzz/ \n
+             echo ASAN_OPTIONS=detect_leaks=0 ./db_fuzzer \n' > prepfuz.sh
+           chmod +x prepfuz.sh
+           bash -xv prepfuz.sh
+           mkdir -p $GITHUB_WORKSPACE/out/
+           ASAN_OPTIONS=detect_odr_violation=0 $GITHUB_WORKSPACE/fuzz/${{ matrix.name }} 2>&1 | sudo tee $GITHUB_WORKSPACE/out/${{ matrix.name }}.log
+           tail -20 $GITHUB_WORKSPACE/out/${{ matrix.name }}.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "${{ matrix.name }} failed!"
+           
+      - name: Copy ${{ matrix.name }} logs to S3
+        run: |
+          aws s3 cp $GITHUB_WORKSPACE/out/${{ matrix.name }}.log s3://spdb-github-ci/
+    
+  fuzzer_sum:
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: ubuntu:18.04
+    needs: [ Fuzz ]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: pre
+        run: |
+          apt update && apt install -y sudo unzip curl
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+          
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: download db_fuzzer.log from s3
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/out
+          aws s3 cp s3://spdb-github-ci/db_fuzzer.log $GITHUB_WORKSPACE/out/db_fuzzer.log || echo "db_fuzzer.log not found"
+          ls -alFh $GITHUB_WORKSPACE/out
+          
+      - name: download db_map_fuzzer.log from s3
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/out
+          aws s3 cp s3://spdb-github-ci/db_map_fuzzer.log $GITHUB_WORKSPACE/out/db_map_fuzzer.log || echo "db_map_fuzzer.log not found"
+          ls -alFh $GITHUB_WORKSPACE/out
+      
+      - name: fuzzersum
+        run: |
+          echo "~~~~ db_map_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_map_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "db_map_fuzzer failed!"
+          echo "~~~~ db_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "db_fuzzer failed!"
+          
+      - name: clear s3 bucket
+        run: |
+          aws s3 rm s3://spdb-github-ci/ --recursive --include "*"
diff --git a/.github/workflows/new_release_line.yml b/.github/workflows/new_release_line.yml
@@ -0,0 +1,80 @@
+name: New Release Line
+
+on:
+  workflow_dispatch:
+    inputs:
+      new_branch_major:
+        description: "Next release Major version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+      new_branch_minor:
+        description: "Next release Minor version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+
+
+jobs:
+  build:
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: alpine:3.14
+
+    steps:
+      - name: pre-run
+        run: apk update && apk add git
+      - uses: actions/checkout@v3
+
+      - name: Calculate tag
+        run: |
+          git config --global --add safe.directory /__w/rocksdb/rocksdb
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git pull
+          major=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MAJOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
+          minor=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MINOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
+          patch=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_PATCH' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
+          new_minor=$(echo $minor + 1 | bc)
+          echo $minor
+          echo "major=$major" >> $GITHUB_ENV
+          echo "minor=$minor" >> $GITHUB_ENV
+          echo "new_minor=$new_minor" >> $GITHUB_ENV
+          current_version="$major.$minor.99"
+          echo $current_version
+      
+      - name: Create release branch
+        run: |
+          if [ -z "${{ inputs.new_branch_major }}" ]; then
+            echo "empty"
+            new_branch="release/${{ env.major }}.${{ env.minor }}"
+            echo $new_branch
+            git checkout -b $new_branch
+            git push -u origin $new_branch
+          else
+            echo "minor=$new_minor" >> $GITHUB_ENV
+            echo "not empty"
+            new_branch="release/${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}"
+            echo $new_branch
+            git checkout -b $new_branch
+            git push -u origin $new_branch
+          fi
+      - name: Update version.h
+        run: |
+          if [ -z "${{ inputs.new_branch_major }}" ]; then
+            git checkout main
+            git tag "v${{ env.major }}.${{ env.minor }}"
+            git push --tags
+            sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ env.major }}/' include/rocksdb/version.h
+            sed -i -e 's/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR ${{ env.new_minor }}/' include/rocksdb/version.h
+            sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h
+            cat include/rocksdb/version.h
+            git commit -am "updated version.h version"
+            git push
+          else
+            git checkout main
+            new_minor=$(echo ${{ inputs.new_branch_minor }} + 1 | bc)
+            echo $new_minor
+            git tag "v${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}"
+            git push --tags
+            sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ inputs.new_branch_major }}/' include/rocksdb/version.h
+            sed -i -e "s/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR $new_minor/g" include/rocksdb/version.h
+            sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h
+            cat include/rocksdb/version.h
+            git commit -am "updated version.h version"
+            git push
+          fi
diff --git a/.github/workflows/perf-test.yml b/.github/workflows/perf-test.yml
@@ -0,0 +1,21 @@
+name: Performance Test
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+
+jobs:
+  perf_test:    
+    runs-on: perftest
+
+    steps:
+
+      - name: Run autoperf script via remotnic
+        run: |
+          echo Run auto perf test
+          #echo  ${{ github.sender.login }} 
+          #ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.event.pusher.name }}  
+          ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.actor }} run_db_bench_large_obj
+          #ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.event.pusher.name }} run_db_bench_small_obj
+          #ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.event.pusher.name }} run_db_bench_huge_memtable
diff --git a/.github/workflows/qa-tests.yml b/.github/workflows/qa-tests.yml
@@ -0,0 +1,62 @@
+name: QA Tests
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+env:
+  GTEST_THROW_ON_FAILURE: 0
+  GTEST_OUTPUT: \"xml:/tmp/test-results/\"
+  SKIP_FORMAT_BUCK_CHECKS: 1
+  GTEST_COLOR: 1
+  CTEST_OUTPUT_ON_FAILURE: 1
+  CTEST_TEST_TIMEOUT: 300
+  ZLIB_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib"
+  BZIP2_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2"
+  SNAPPY_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy"
+  LZ4_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4"
+  ZSTD_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd"
+
+jobs:
+  test:
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: ubuntu:18.04
+    strategy:
+      matrix:
+        include:
+          - name: unit-test
+            short_test: TMPD=\home\ubuntu make -j2 check
+            long_test: TMPD=\home\ubuntu make -j2 check
+          - name: black-box
+            short_test: CRASH_TEST_EXT_ARGS="--duration=500" make -j2 blackbox_asan_crash_test
+            long_test: CRASH_TEST_EXT_ARGS="--duration=10000" make -j2 blackbox_asan_crash_test
+          - name: white-box
+            short_test: CRASH_TEST_EXT_ARGS="--duration=500" make -j2 whitebox_asan_crash_test
+            long_test: CRASH_TEST_EXT_ARGS="--duration=10000" make -j2 whitebox_asan_crash_test
+
+    steps:
+      - name: Network hotfix
+        run: echo "nameserver 8.8.8.8" > /etc/resolv.conf
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Pre
+        run: |
+          df -H
+          apt-get update -y && apt-get install -y sudo default-jre default-jdk clang-format parallel moreutils g++ libgflags-dev software-properties-common golang-go gccgo binutils liblz4-dev libsnappy1v5 mlocate sysstat nvme-cli python3 python3-pip nodejs npm liblz4-dev libsnappy-dev libzstd-dev gdb dstat unzip apt-transport-https ca-certificates curl gnupg lsb-release || echo ""
+          sudo mv /usr/bin/parallel /usr/bin/gnu_parallel
+          sudo rm -rf /etc/parallel/config || echo ""
+          
+      - name: ${{ matrix.name }}
+        run: |
+          echo $GITHUB_REF_NAME
+          if echo $GITHUB_REF_NAME | grep -Eq 'release/'; then
+            echo long
+            make clean && ${{ matrix.long_test }}
+          else
+            echo short
+            make clean && ${{ matrix.short_test }}
+          fi
+          df -H