CI: initial setup

Devops ci/add build tools (#104)

new release line network hotfix (#109)

.github/workflows/artifact-release.yml

@@ -0,0 +1,73 @@
+name: Create release artifact
+
+on:
+  workflow_dispatch:
+
+jobs:
+
+  cleanup:
+    runs-on: [self-hosted, linux, x64, myrunner]
+    container:
+      image: ubuntu:latest
+    steps:
+      - name: Cleaning up the $GITHUB_WORKSPACE as root from a Docker image
+        run: find /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/. -name . -o -prune -exec rm -rf -- {} + || true
+
+  build:
+    needs: cleanup
+    runs-on: [self-hosted, linux, x64, myrunner]
+
+    steps:
+
+      - uses: actions/checkout@v3
+
+      - name: install dependencies
+        run: |
+          echo ~ && sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y docker-ce docker-ce-cli containerd.io unzip curl zip unzip
+          
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: docker run
+        run: |
+          export BUILD_OUTPUT_DIR=~/build
+          echo $BUILD_OUTPUT_DIR
+          rm -rf $BUILD_OUTPUT_DIR
+          mkdir -p $BUILD_OUTPUT_DIR
+          docker build -f ./build_tools/Dockerfile -t spdb-build .
+          docker run --rm -v $(readlink -f ${BUILD_OUTPUT_DIR}):/out -v $GITHUB_WORKSPACE:/speedb -e GITHUB_WORKSPACE=$GITHUB_WORKSPACE spdb-build
+          ls -alFh $BUILD_OUTPUT_DIR
+          zip --junk-paths "Release-$GITHUB_REF_NAME.zip" $BUILD_OUTPUT_DIR/*
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref_name }}
+          draft: false
+          prerelease: false
+      - name: Upload Release Asset
+        id: upload-release-asset 
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
+          asset_path: ./Release-${{ github.ref_name}}.zip
+          asset_name: Release-${{ github.ref_name}}.zip
+          asset_content_type: application/zip
+
+      - name: Upload artifacts to S3
+        run: |
+          export BUILD_OUTPUT_DIR=~/build
+          echo $BUILD_OUTPUT_DIR
+          aws s3 cp $BUILD_OUTPUT_DIR s3://spdb-github-artifacts/"release-$GITHUB_REF_NAME" --recursive
+          aws s3 ls s3://spdb-github-artifacts/
+          rm -rf $BUILD_OUTPUT_DIR

.github/workflows/ci_pipeline.yml

@@ -0,0 +1,209 @@
+name: CI
+
+on: 
+  #push:
+  workflow_dispatch:
+  workflow_call:
+  pull_request_review:
+    types: [submitted]
+
+
+permissions: write-all
+
+jobs:
+  #Sanity:
+    #uses: speedb-io/speedb/.github/workflows/sanity_check.yml@main
+
+  Build:
+    #needs: [Sanity]
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    runs-on: [self-hosted, ubuntu, asrunner]
+    strategy:
+      matrix:
+        include:
+          - name: verify build
+            command: cmake ..  -GNinja
+          - name: optimized build
+            command: cmake .. -DCMAKE_BUILD_TYPE=Release  -GNinja
+          - mame: clang build
+            command: CC=clang CXX=clang++ cmake ..  -GNinja
+    container:
+      image: alpine:3.14
+
+    steps:
+    - name: Pre-build
+      run: |
+        env
+        rm -rf /usr/share/dotnet || echo ""
+        df -h
+        apk add git
+        echo "nameserver 8.8.8.8" > /etc/resolv.conf
+        apk add bash python3 py3-pip clang clang-extra-tools shellcheck gcc g++ cmake ninja ccache \
+        openjdk10 gflags-dev snappy-dev lz4-dev bzip2-dev zstd-dev zlib-dev linux-headers openssh-client tar
+        python3 -m pip install lint-diffs flake8
+        
+        
+    - name: Checkout
+      uses: actions/checkout@v3 
+
+
+    - name: Prepare ccache timestamp
+      id: ccache_cache_timestamp
+      shell: cmake -P {0}
+      run: |
+        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+        message("::set-output name=timestamp::${current_date}")
+        
+        
+    - name: ccache cache files
+      uses: actions/cache@v2
+      with:
+        path: ~/.ccache
+        key: ${{runner.os}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
+        restore-keys: |
+          ${{runner.os}}-ccache-
+        
+      
+    - name: ${{ matrix.name }}
+      run: |
+        if [ -d "$GITHUB_WORKSPACE/build" ]; then
+          echo >&2 "error: the build directory should not exist" && false NIK 
+        fi
+        if [ -d "~/.ccache" ]; then
+          echo "Already exists"
+        else
+          mkdir -p ~/.ccache
+          ls ~ | grep cache || echo ""
+          touch ~/.ccache/ccache.txt
+          echo "aaa" > ~/.ccache/ccache.txt
+          ls ~/.ccache
+          cat ~/.ccache/ccache.txt
+        fi
+        mkdir -p "$GITHUB_WORKSPACE/build"
+        cd "$GITHUB_WORKSPACE/build"
+        export "CCACHE_BASEDIR=$HOME"
+        export "CCACHE_DIR=$HOME/.ccache"
+        export "CCACHE_COMPILERCHECK=content"
+        ${{ matrix.command }} -DPORTABLE=1 -DWITH_GFLAGS=1 \
+        -DWITH_ZLIB=1 -DWITH_SNAPPY=1 -DWITH_BZ2=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 \
+        -DWITH_JNI=1 -DJAVA_HOME=/usr/lib/jvm/default-jvm \
+        -DWITH_BENCHMARK_TOOLS=1 -DWITH_CORE_TOOLS=1 -DWITH_TOOLS=1 \
+        -DWITH_TESTS=1 -DWITH_ALL_TESTS=1 -DWITH_EXAMPLES=1
+        ninja
+        
+  #Performance:
+    #if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    #needs: [Build]
+    #uses: speedb-io/speedb/.github/workflows/perf-test.yml@main
+
+  QA-Tests:
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Build]
+    uses: speedb-io/speedb/.github/workflows/qa-tests.yml@main
+
+  Fuzz:
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Build]
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: ubuntu:18.04
+    strategy:
+      matrix:
+        include:
+          - name: db_fuzzer
+          - name: db_map_fuzzer
+
+    steps:
+      - name: Pre-build
+        run: |
+          apt update && apt install -y sudo python3 git clang-tools cmake make automake ucommon-utils libtool gettext pkg-config build-essential clang-10 zlib1g-dev libbz2-dev ninja-build liblzma-dev autoconf libsnappy-dev libzstd-dev liblz4-dev binutils m4 g++-10 unzip
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+          echo "nameserver 8.8.8.8" > /etc/resolv.conf
+      
+      - uses: actions/checkout@v3
+
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: ${{ matrix.name }}
+        run: |
+           echo 'git clone https://github.com/google/libprotobuf-mutator.git \n
+             cd libprotobuf-mutator \n
+             git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f \n
+             cd .. \n
+             export CC=clang && export CXX=clang++ && mkdir LPM && cd LPM \n
+             ln -s /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so \n
+             ln -s /usr/bin/clang-10 /usr/bin/clang \n
+             ln -s /usr/bin/clang++-10 /usr/bin/clang++ \n
+             cmake ../libprotobuf-mutator -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON -DLIB_PROTO_MUTATOR_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \n
+             ninja \n
+             ninja install \n
+             export PKG_CONFIG_PATH=$PWD:$PWD/external.protobuf/lib/pkgconfig/ \n
+             export PATH=$PWD/external.protobuf/bin:$PATH \n
+             cd $GITHUB_WORKSPACE \n
+             COMPILE_WITH_ASAN=1 PORTABLE=1 make -j2 static_lib \n
+             cd $GITHUB_WORKSPACE/fuzz \n
+             make ${{ matrix.name }} \n
+             ls -alFh $GITHUB_WORKSPACE/fuzz/ \n
+             echo ASAN_OPTIONS=detect_leaks=0 ./db_fuzzer \n' > prepfuz.sh
+           chmod +x prepfuz.sh
+           bash -xv prepfuz.sh
+           mkdir -p $GITHUB_WORKSPACE/out/
+           ASAN_OPTIONS=detect_odr_violation=0 $GITHUB_WORKSPACE/fuzz/${{ matrix.name }} 2>&1 | sudo tee $GITHUB_WORKSPACE/out/${{ matrix.name }}.log
+           tail -20 $GITHUB_WORKSPACE/out/${{ matrix.name }}.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "${{ matrix.name }} failed!"
+           
+      - name: Copy ${{ matrix.name }} logs to S3
+        run: |
+          aws s3 cp $GITHUB_WORKSPACE/out/${{ matrix.name }}.log s3://spdb-github-ci/
+    
+  fuzzer_sum:
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: ubuntu:18.04
+    needs: [ Fuzz ]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: pre
+        run: |
+          apt update && apt install -y sudo unzip curl
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+          
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: download db_fuzzer.log from s3
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/out
+          aws s3 cp s3://spdb-github-ci/db_fuzzer.log $GITHUB_WORKSPACE/out/db_fuzzer.log || echo "db_fuzzer.log not found"
+          ls -alFh $GITHUB_WORKSPACE/out
+          
+      - name: download db_map_fuzzer.log from s3
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/out
+          aws s3 cp s3://spdb-github-ci/db_map_fuzzer.log $GITHUB_WORKSPACE/out/db_map_fuzzer.log || echo "db_map_fuzzer.log not found"
+          ls -alFh $GITHUB_WORKSPACE/out
+      
+      - name: fuzzersum
+        run: |
+          echo "~~~~ db_map_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_map_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "db_map_fuzzer failed!"
+          echo "~~~~ db_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "db_fuzzer failed!"
+          
+      - name: clear s3 bucket
+        run: |
+          aws s3 rm s3://spdb-github-ci/ --recursive --include "*"

.github/workflows/new_release_line.yml

@@ -0,0 +1,83 @@
+name: New Release Line
+
+on:
+  workflow_dispatch:
+    inputs:
+      new_branch_major:
+        description: "Next release Major version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+      new_branch_minor:
+        description: "Next release Minor version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+
+
+jobs:
+  build:
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: alpine:3.14
+
+    steps:
+      - name: pre-run
+        run: |
+          echo "nameserver 8.8.8.8" > /etc/resolv.conf
+          apk update && apk add git
+          
+      - uses: actions/checkout@v3
+
+      - name: Calculate tag
+        run: |
+          git config --global --add safe.directory /__w/rocksdb/rocksdb
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git pull
+          major=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MAJOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
+          minor=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MINOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
+          patch=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_PATCH' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
+          new_minor=$(echo $minor + 1 | bc)
+          echo $minor
+          echo "major=$major" >> $GITHUB_ENV
+          echo "minor=$minor" >> $GITHUB_ENV
+          echo "new_minor=$new_minor" >> $GITHUB_ENV
+          current_version="$major.$minor.99"
+          echo $current_version
+      
+      - name: Create release branch
+        run: |
+          if [ -z "${{ inputs.new_branch_major }}" ]; then
+            echo "empty"
+            new_branch="release/${{ env.major }}.${{ env.minor }}"
+            echo $new_branch
+            git checkout -b $new_branch
+            git push -u origin $new_branch
+          else
+            echo "minor=$new_minor" >> $GITHUB_ENV
+            echo "not empty"
+            new_branch="release/${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}"
+            echo $new_branch
+            git checkout -b $new_branch
+            git push -u origin $new_branch
+          fi
+      - name: Update version.h
+        run: |
+          if [ -z "${{ inputs.new_branch_major }}" ]; then
+            git checkout main
+            git tag "v${{ env.major }}.${{ env.minor }}"
+            git push --tags
+            sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ env.major }}/' include/rocksdb/version.h
+            sed -i -e 's/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR ${{ env.new_minor }}/' include/rocksdb/version.h
+            sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h
+            cat include/rocksdb/version.h
+            git commit -am "updated version.h version"
+            git push
+          else
+            git checkout main
+            new_minor=$(echo ${{ inputs.new_branch_minor }} + 1 | bc)
+            echo $new_minor
+            git tag "v${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}"
+            git push --tags
+            sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ inputs.new_branch_major }}/' include/rocksdb/version.h
+            sed -i -e "s/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR $new_minor/g" include/rocksdb/version.h
+            sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h
+            cat include/rocksdb/version.h
+            git commit -am "updated version.h version"
+            git push
+          fi

.github/workflows/perf-test.yml

@@ -0,0 +1,21 @@
+name: Performance Test
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+
+jobs:
+  perf_test:    
+    runs-on: perftest
+
+    steps:
+
+      - name: Run autoperf script via remotnic
+        run: |
+          echo Run auto perf test
+          #echo  ${{ github.sender.login }} 
+          #ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.event.pusher.name }}  
+          ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.actor }} run_db_bench_large_obj
+          #ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.event.pusher.name }} run_db_bench_small_obj
+          #ssh -i ~/remo.k remo@9.148.1.183 /home/remo/tremotnic.sh main HG_auto_T1 ${{ github.event.pusher.name }} run_db_bench_huge_memtable