initial CI

.github/workflows/aws-test.yml

@@ -0,0 +1,65 @@
+name: aws-test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  fuzzer_sum:
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:18.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: pre
+        run: |
+          apt update && apt install -y sudo unzip curl
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+          
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: download db_fuzzer.log from s3
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/out
+          aws s3 cp s3://spdb-github-ci/db_fuzzer.log $GITHUB_WORKSPACE/out/db_fuzzer.log
+          ls -alFh $GITHUB_WORKSPACE/out
+          
+      - name: download db_map_fuzzer.log from s3
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/out
+          aws s3 cp s3://spdb-github-ci/db_map_fuzzer.log $GITHUB_WORKSPACE/out/db_map_fuzzer.log
+          ls -alFh $GITHUB_WORKSPACE/out
+      
+#       - name: Download a Build Artifact
+#         uses: actions/download-artifact@v3.0.0
+#         with:
+#           path: |
+#             ${{ github.workspace }}/out/db_fuzzer.log
+
+#       - name: Download a Build Artifact
+#         uses: actions/download-artifact@v3.0.0
+#         with:
+#           path: |
+#             ${{ github.workspace }}/out/db_map_fuzzer.log
+
+      - name: fuzzersum
+        run: |
+          echo "~~~~ db_map_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_map_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "skoot"
+          echo "~~~~ db_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "skid"
+          
+      - name: clear s3 bucket
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/down
+          aws s3 rm s3://spdb-github-ci/ --recursive --include "*"
+
+

.github/workflows/ci_pipeline.yml

@@ -0,0 +1,83 @@
+name: CI
+
+on: [pull_request, workflow_dispatch]
+# on: 
+#   workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: verify build
+            command: cmake ..  -GNinja
+          - name: optimized build
+            command: cmake .. -DCMAKE_BUILD_TYPE=Release  -GNinja
+          - mame: clang build
+            command: CC=clang CXX=clang++ cmake ..  -GNinja
+    container:
+      image: alpine:3.14
+
+    steps:
+    - name: Pre-build
+      run: |
+        rm -rf /usr/share/dotnet || echo ""
+        df -h
+        apk add git bash python3 py3-pip clang clang-extra-tools shellcheck gcc g++ cmake ninja ccache \
+        openjdk10 gflags-dev snappy-dev lz4-dev bzip2-dev zstd-dev zlib-dev linux-headers openssh-client tar
+        python3 -m pip install lint-diffs flake8
+    - name: Checkout
+      uses: actions/checkout@v2 
+
+
+    - name: Prepare ccache timestamp
+      id: ccache_cache_timestamp
+      shell: cmake -P {0}
+      run: |
+        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+        message("::set-output name=timestamp::${current_date}")
+        
+        
+    - name: ccache cache files
+      uses: actions/cache@v2
+      with:
+        path: ~/.ccache
+        key: ${{runner.os}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
+        restore-keys: |
+          ${{runner.os}}-ccache-
+        
+      
+    - name: ${{ matrix.name }}
+      run: |
+        df -h
+        ls /etc/apk/world
+        git config --global --add safe.directory $GITHUB_WORKSPACE
+        if [ -d "$GITHUB_WORKSPACE/build" ]; then
+          echo >&2 "error: the build directory should not exist" && false NIK 
+        fi
+        if [ -d "~/.ccache" ]; then
+          echo "Already exists"
+        else
+          mkdir -p ~/.ccache
+          ls ~ | grep cache || echo ""
+          touch ~/.ccache/ccache.txt
+          echo "aaa" > ~/.ccache/ccache.txt
+          ls ~/.ccache
+          cat ~/.ccache/ccache.txt
+        fi
+        mkdir -p "$GITHUB_WORKSPACE/build"
+        cd "$GITHUB_WORKSPACE/build"
+        export "CCACHE_BASEDIR=$HOME"
+        export "CCACHE_DIR=$HOME/.ccache"
+        export "CCACHE_COMPILERCHECK=content"
+        ${{ matrix.command }} -DWITH_GFLAGS=1 \
+        -DWITH_ZLIB=1 -DWITH_SNAPPY=1 -DWITH_BZ2=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 \
+        -DWITH_JNI=1 -DJAVA_HOME=/usr/lib/jvm/default-jvm \
+        -DWITH_BENCHMARK_TOOLS=1 -DWITH_CORE_TOOLS=1 -DWITH_TOOLS=1 \
+        -DWITH_TESTS=1 -DWITH_ALL_TESTS=1 -DWITH_EXAMPLES=1
+        ninja
+        
+  sanity:
+    needs: [build]
+    uses: speedb-io/speedb/.github/workflows/sanity_check.yml@main

.github/workflows/full-tests.yml

@@ -0,0 +1,45 @@
+name: Tests
+
+on:
+  workflow_dispatch:
+
+env:
+  GTEST_THROW_ON_FAILURE: 0
+  GTEST_OUTPUT: \"xml:/tmp/test-results/\"
+  SKIP_FORMAT_BUCK_CHECKS: 1
+  GTEST_COLOR: 1
+  CTEST_OUTPUT_ON_FAILURE: 1
+  CTEST_TEST_TIMEOUT: 300
+  ZLIB_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib"
+  BZIP2_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2"
+  SNAPPY_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy"
+  LZ4_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4"
+  ZSTD_DOWNLOAD_BASE: "https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: unit-test
+            command: make clean && make -j2 check && echo $? || echo $?
+          - mame: black-box
+            command: make clean && CRASH_TEST_EXT_ARGS=" --duration=500" make -j2 blackbox_asan_crash_test && echo $? || echo $?
+          - mame: white-box
+            command: make clean && CRASH_TEST_EXT_ARGS="--duration=500" make -j2 whitebox_asan_crash_test && echo $? || echo $?
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Pre
+        run: |
+          df -H
+          sudo apt-get update -y && sudo apt-get install -y clang-format parallel g++ libgflags-dev software-properties-common golang-go gccgo binutils liblz4-dev libsnappy1v5 mlocate sysstat nvme-cli python3 python3-pip nodejs npm liblz4-dev libsnappy-dev libzstd-dev gdb dstat unzip apt-transport-https ca-certificates curl gnupg lsb-release || echo ""
+          sudo mv /usr/bin/parallel /usr/bin/gnu_parallel
+          sudo rm -rf /etc/parallel/config || echo ""
+          sudo apt-get install moreutils
+      - name: ${{ matrix.name }}
+        run:  ${{ matrix.command }}

.github/workflows/fuzz.yml

@@ -0,0 +1,171 @@
+name: fuzz-test
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  fuzz-build:
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:18.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Pre-build
+        run: apt update && apt install -y sudo python3 git clang-tools cmake make automake libtool gettext pkg-config build-essential clang-10 zlib1g-dev libbz2-dev ninja-build liblzma-dev autoconf libsnappy-dev libzstd-dev liblz4-dev binutils m4 g++-10
+
+      - name: fuzbuild
+        run: |
+           #repo="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}"
+           #echo repo $repo
+           #git remote add origin $repo
+           echo "test"
+           pwd
+           echo 'git clone https://github.com/google/libprotobuf-mutator.git \n
+             cd libprotobuf-mutator \n
+             git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f \n
+             cd .. \n
+             export CC=clang && export CXX=clang++ && mkdir LPM && cd LPM \n
+             ln -s /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so \n
+             ln -s /usr/bin/clang-10 /usr/bin/clang \n
+             ln -s /usr/bin/clang++-10 /usr/bin/clang++ \n
+             cmake ../libprotobuf-mutator -GNinja -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON -DLIB_PROTO_MUTATOR_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \n
+             ninja \n
+             ninja install \n
+             export PKG_CONFIG_PATH=$PWD:$PWD/external.protobuf/lib/pkgconfig/ \n
+             export PATH=$PWD/external.protobuf/bin:$PATH \n
+             cd $GITHUB_WORKSPACE \n
+             git fetch \n
+             git checkout master \n
+             COMPILE_WITH_ASAN=1 make -j2 static_lib \n
+             cd $GITHUB_WORKSPACE/fuzz \n
+             make db_map_fuzzer \n
+             make db_fuzzer \n
+             ls -alFh $GITHUB_WORKSPACE/fuzz/ \n
+             echo ASAN_OPTIONS=detect_leaks=0 ./db_fuzzer \n' > prepfuz.sh
+           cat prepfuz.sh     
+           chmod +x prepfuz.sh
+           bash -xv prepfuz.sh
+           
+      - name: Upload db_map_fuzzer Artifact
+        uses: actions/upload-artifact@v3.1.0
+        with:
+          name: db_map_fuzzer
+          path: |
+             ${{ github.workspace }}/fuzz/db_map_fuzzer
+          retention-days: 1
+
+      - name: Upload db_fuzzer Artifact
+        uses: actions/upload-artifact@v3.1.0
+        with:
+          name: db_fuzzer
+          path: |
+             ${{ github.workspace }}/fuzz/db_fuzzer
+          retention-days: 1
+
+
+  db_fuzzer:
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:18.04
+    continue-on-error: true
+    needs: [fuzz-build]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download a Build Artifact
+        uses: actions/download-artifact@v3.0.0
+        with:
+          name: db_fuzzer
+          path: ${{ github.workspace }}/fuzz/db_fuzzer
+
+      - name: Pre-build
+        run: |
+          apt update && apt install -y libtool sudo gettext pkg-config libsnappy-dev libzstd-dev liblz4-dev binutils m4
+          mkdir -p $GITHUB_WORKSPACE/out
+
+      - name: dbfuzzer
+        run: |
+          ls -l $GITHUB_WORKSPACE/out
+          ls -alFh $GITHUB_WORKSPACE/fuzz
+          sudo chown -R $USER:$USER "$GITHUB_WORKSPACE"
+          sudo chown $USER:$USER "$GITHUB_WORKSPACE/fuzz/db_fuzzer/db_fuzzer"
+          chmod 777 $GITHUB_WORKSPACE/fuzz/db_fuzzer/db_fuzzer
+          whoami
+          ls -alFh $GITHUB_WORKSPACE/fuzz/db_fuzzer
+          ASAN_OPTIONS=detect_odr_violation=0 $GITHUB_WORKSPACE/fuzz/db_fuzzer/db_fuzzer 2>&1 | sudo tee $GITHUB_WORKSPACE/out/db_fuzzer.log 
+          ls -l $GITHUB_WORKSPACE/out/
+          
+      - name: Upload a Build Artifact
+        uses: actions/upload-artifact@v3.1.0
+        with:
+          path: ${{ github.workspace }}/out/db_fuzzer.log
+          retention-days: 1
+
+  db_map_fuzzer:
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:18.04
+    continue-on-error: true
+    needs:  [fuzz-build]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download a Build Artifact
+        uses: actions/download-artifact@v3.0.0
+        with:
+          name: db_map_fuzzer
+          path: ${{ github.workspace }}/fuzz/db_map_fuzzer
+
+      - name: Pre-build
+        run: |
+          apt update && apt install -y sudo libtool gettext pkg-config libsnappy-dev libzstd-dev liblz4-dev binutils m4 
+          mkdir -p $GITHUB_WORKSPACE/out/
+          
+      - name: dbmapfuzzer
+        run: |
+          ls -l $GITHUB_WORKSPACE/out/
+          sudo chown -R $USER:$USER "$GITHUB_WORKSPACE"
+          sudo chown $USER:$USER "$GITHUB_WORKSPACE/fuzz/db_map_fuzzer/db_map_fuzzer"
+          chmod 777 $GITHUB_WORKSPACE/fuzz/db_map_fuzzer/db_map_fuzzer
+          whoami
+          ls -alFh $GITHUB_WORKSPACE/fuzz/db_map_fuzzer
+          ASAN_OPTIONS=detect_odr_violation=0 $GITHUB_WORKSPACE/fuzz/db_map_fuzzer/db_map_fuzzer 2>&1 | tee $GITHUB_WORKSPACE/out/db_map_fuzzer.log 
+      
+      - name: Upload a Build Artifact
+        uses: actions/upload-artifact@v3.1.0
+        with:
+          path: ${{ github.workspace }}/out/db_map_fuzzer.log
+          retention-days: 1
+
+  fuzzer_sum:
+    runs-on: ubuntu-18.04
+    container:
+      image:  alpine:3.14
+    needs: [db_map_fuzzer, db_fuzzer]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download a Build Artifact
+        uses: actions/download-artifact@v3.0.0
+        with:
+          path: |
+            ${{ github.workspace }}/out/db_fuzzer.log
+            
+      - name: Download a Build Artifact
+        uses: actions/download-artifact@v3.0.0
+        with:
+          path: |
+            ${{ github.workspace }}/out/db_map_fuzzer.log
+      
+      - name: fuzzersum
+        run: |
+          echo "~~~~ db_map_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_map_fuzzer.log/artifact/db_map_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "skoot"
+          echo "~~~~ db_fuzzer LOGS ~~~~~"
+          tail -20 $GITHUB_WORKSPACE/out/db_fuzzer.log/artifact/db_fuzzer.log | grep "==AddressSanitizer. Thread limit (4194304 threads) exceeded\. Dying\." || echo "skid"