Slowdown writes based on WBM's dirty memory usage

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,28 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. 
+2. 
+3. 
+4. 
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**System (please complete the following information):**
+ - OS: [e.g. RHEL8.6]
+ - Hardware [e.g. Intel Xeon Ice Lake, 64GB, NVMe]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Owner: 
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/artifact-release.yml

@@ -0,0 +1,112 @@
+name: Create release artifacts
+
+on:
+  push:
+    tags:
+      - 'speedb/v*'
+
+permissions:
+  contents: write # Needed for release assets upload
+  id-token: write # Needed for AWS credentials setting
+
+jobs:
+  build:
+    runs-on: [self-hosted, ubuntu, asrunner]
+
+    container:
+      image: centos:7.9.2009
+
+    steps:
+      - name: pre
+        run: |
+          yum install -y centos-release-scl epel-release
+          yum install -y make devtoolset-11-gcc-c++ \
+            coreutils wget unzip which git python3 openssl openssl-devel \
+            libzstd-devel lz4-devel snappy-devel zlib-devel \
+            java-1.8.0-openjdk-devel
+          echo "PATH=/opt/rh/devtoolset-11/root/usr/bin:${PATH}" >> $GITHUB_ENV
+          echo "RELEASE_VERSION=${GITHUB_REF_NAME#speedb/v}" >> $GITHUB_ENV
+
+      - name: Install CMake
+        run: |
+          CMAKE_RELEASE=3.20.1
+          wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_RELEASE}/cmake-${CMAKE_RELEASE}.tar.gz
+          tar xf cmake-${CMAKE_RELEASE}.tar.gz
+          cd cmake-${CMAKE_RELEASE}
+          ./bootstrap
+          make -j$(nproc) && make install
+          cd .. && rm -rf cmake-${CMAKE_RELEASE}*
+
+      - name: Install awscli
+        run: |
+          wget "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -O "awscliv2.zip"
+          unzip awscliv2.zip
+          ./aws/install
+          rm -rf aws awscliv2.zip
+
+      - uses: actions/checkout@v3
+
+      - run: mkdir "$GITHUB_WORKSPACE/out"
+
+      - name: Build and package release libraries
+        run: |
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DWITH_GFLAGS=0 -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1
+          mkdir -p "$GITHUB_WORKSPACE/out/root"
+          DESTDIR="$GITHUB_WORKSPACE/out/root" make -j$(nproc) install
+          ( cd "$GITHUB_WORKSPACE/out/root" && tar czf ../speedb-${RELEASE_VERSION}.tar.gz . )
+          rm -rf "$GITHUB_WORKSPACE/out/root"
+          cd .. && rm -rf build
+
+      - name: Build release Jar
+        run: |
+          make clean
+          LIB_MODE=static DEBUG_LEVEL=0 PORTABLE=1 JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) rocksdbjavastatic
+          cp "java/target/speedbjni-${RELEASE_VERSION}-linux64.jar" "$GITHUB_WORKSPACE/out"
+
+      - name: Build db_bench
+        run: |
+          yum install -y gflags-devel
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DWITH_GFLAGS=1 \
+            -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 \
+            -DWITH_BENCHMARK_TOOLS=1 -DROCKSDB_BUILD_SHARED=1
+          make -j$(nproc) db_bench
+          tar czf "$GITHUB_WORKSPACE/out/db_bench-speedb-${RELEASE_VERSION}.tar.gz" db_bench
+          cd .. && rm -rf build
+
+      - name: Generate checksums
+        run: |
+          for f in $GITHUB_WORKSPACE/out/*; do
+            sha256sum "$f" > "$f.sha256"
+          done
+
+      - name: Get release date
+        run: |
+          echo "RELEASE_DATE=$(git for-each-ref "--format=%(creatordate:short)" "refs/tags/${GITHUB_REF_NAME}")" >> $GITHUB_ENV
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          generate_release_notes: false
+          name: Speedb ${{ env.RELEASE_VERSION }} (${{ env.RELEASE_DATE }})
+          files: |
+            out/db_bench-speedb-${{ env.RELEASE_VERSION }}.tar.gz
+            out/db_bench-speedb-${{ env.RELEASE_VERSION }}.tar.gz.sha256
+            out/speedb-${{ env.RELEASE_VERSION }}.tar.gz
+            out/speedb-${{ env.RELEASE_VERSION }}.tar.gz.sha256
+            out/speedbjni-${{ env.RELEASE_VERSION }}-linux64.jar
+            out/speedbjni-${{ env.RELEASE_VERSION }}-linux64.jar.sha256
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: Upload artifacts to S3
+        run: |
+          aws s3 cp "$GITHUB_WORKSPACE/out" "s3://spdb-github-artifacts/release-${RELEASE_VERSION}" --recursive
+          rm -rf "$GITHUB_WORKSPACE/out"

.github/workflows/ci_pipeline.yml

@@ -0,0 +1,110 @@
+name: CI
+
+on: 
+  #push:
+  workflow_dispatch:
+  workflow_call:
+  pull_request_review:
+    types: [submitted]
+
+
+permissions: write-all
+
+jobs:
+  #Sanity:
+    #uses: speedb-io/speedb/.github/workflows/sanity_check.yml@main
+
+  Build:
+    #needs: [Sanity]
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    runs-on: [self-hosted, ubuntu, asrunner]
+    strategy:
+      matrix:
+        include:
+          - name: verify build
+            command: cmake ..  -GNinja
+          - name: optimized build
+            command: cmake .. -DCMAKE_BUILD_TYPE=Release  -GNinja
+          - mame: clang build
+            command: CC=clang CXX=clang++ cmake ..  -GNinja
+    container:
+      image: alpine:3.14
+
+    steps:
+    - name: Pre-build
+      run: |
+        env
+        rm -rf /usr/share/dotnet || echo ""
+        df -h
+        apk add git
+        echo "nameserver 8.8.8.8" > /etc/resolv.conf
+        apk add bash python3 py3-pip clang clang-extra-tools shellcheck gcc g++ cmake ninja ccache \
+        openjdk10 gflags-dev snappy-dev lz4-dev bzip2-dev zstd-dev zlib-dev linux-headers openssh-client tar
+        python3 -m pip install lint-diffs flake8
+
+
+    - name: Checkout
+      uses: actions/checkout@v3 
+
+
+    - name: Prepare ccache timestamp
+      id: ccache_cache_timestamp
+      shell: cmake -P {0}
+      run: |
+        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+        message("::set-output name=timestamp::${current_date}")
+
+
+    - name: ccache cache files
+      uses: actions/cache@v2
+      with:
+        path: ~/.ccache
+        key: ${{runner.os}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
+        restore-keys: |
+          ${{runner.os}}-ccache-
+
+
+    - name: ${{ matrix.name }}
+      run: |
+        if [ -d "$GITHUB_WORKSPACE/build" ]; then
+          echo >&2 "error: the build directory should not exist" && false NIK 
+        fi
+        if [ -d "~/.ccache" ]; then
+          echo "Already exists"
+        else
+          mkdir -p ~/.ccache
+          ls ~ | grep cache || echo ""
+          touch ~/.ccache/ccache.txt
+          echo "aaa" > ~/.ccache/ccache.txt
+          ls ~/.ccache
+          cat ~/.ccache/ccache.txt
+        fi
+        mkdir -p "$GITHUB_WORKSPACE/build"
+        cd "$GITHUB_WORKSPACE/build"
+        export "CCACHE_BASEDIR=$HOME"
+        export "CCACHE_DIR=$HOME/.ccache"
+        export "CCACHE_COMPILERCHECK=content"
+        ${{ matrix.command }} -DPORTABLE=1 -DWITH_GFLAGS=1 \
+        -DWITH_ZLIB=1 -DWITH_SNAPPY=1 -DWITH_BZ2=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 \
+        -DWITH_JNI=1 -DJAVA_HOME=/usr/lib/jvm/default-jvm \
+        -DWITH_BENCHMARK_TOOLS=1 -DWITH_CORE_TOOLS=1 -DWITH_TOOLS=1 \
+        -DWITH_TESTS=1 -DWITH_ALL_TESTS=1 -DWITH_EXAMPLES=1
+        ninja
+
+  #Performance:
+    #if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    #needs: [Build]
+    #uses: speedb-io/speedb/.github/workflows/perf-test.yml@main
+
+  QA-Tests:
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Build]
+    uses: speedb-io/speedb/.github/workflows/qa-tests.yml@main
+
+  Fuzz:
+    if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
+    needs: [Build]
+    uses: ./.github/workflows/test_fuzz.yml
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/new_release_line.yml

@@ -0,0 +1,106 @@
+name: New Release Line
+
+on:
+  workflow_dispatch:
+    inputs:
+      new_branch_major:
+        description: "Next release Major version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+        required: false
+      new_branch_minor:
+        description: "Next release Minor version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+        required: false
+    branches:
+      - main
+      - 'release/*'
+
+permissions:
+  contents: read
+
+jobs:
+  tag_version:
+    runs-on: [self-hosted, ubuntu, asrunner]
+    container:
+      image: alpine:3.14
+
+    env:
+      VERSION_FILE: speedb/version.h
+
+    steps:
+      - name: pre
+        run: |
+          echo "nameserver 8.8.8.8" > /etc/resolv.conf
+          apk add git openssh-client
+
+      - name: Verify chosen version
+        run: |
+          if ! echo "${{ inputs.new_branch_major }}" | grep -q "^[0-9]*$"; then
+            echo >&2 "error: major version must be a positive number" && false
+          fi
+          if ! echo "${{ inputs.new_branch_minor }}" | grep -q "^[0-9]*$"; then
+            echo >&2 "error: minor version must be a positive number" && false
+          fi
+
+          if [ "${{ inputs.new_branch_major }}${{ inputs.new_branch_minor }}" != "" ] && [ "$GITHUB_REF" != "refs/heads/main" ]; then
+            echo >&2 "error: cannot cut a major or a minor release from a branch that isn't main" && false
+          elif [ "$GITHUB_REF" != "refs/heads/main" ] && ! echo "$GITHUB_REF" | grep -q "^refs/heads/release/"; then
+            echo "error: cannot cut a patch release from a non-release branch" && false
+          fi
+
+      - uses: actions/checkout@v3
+        with:
+          ssh-key: ${{ secrets.RELEASE_SSH_KEY }}
+
+      - name: Calculate new version
+        run: |
+          major=$(grep '_MAJOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//')
+          minor=$(grep '_MINOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//')
+
+          if [ -n "${{ inputs.new_branch_major }}" ] && [ "${{ inputs.new_branch_major }}" -lt "$major" ]; then
+            echo >&2 "error: the chosen major version is lower than current one" && false
+          elif [ -n "${{ inputs.new_branch_major }}" ] && [ "${{ inputs.new_branch_major }}" -gt "$major" ]; then
+            major=${{ inputs.new_branch_major }}
+            if [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -ne 0 ]; then
+              echo >&2 "error: cannot bump minor version when bumping major version" && false
+            fi
+            minor=0
+            patch=0
+          elif [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -lt "$minor" ]; then
+            echo >&2 "error: the chosen minor version is lower than current one" && false
+          elif [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -gt "$minor" ]; then
+            minor=${{ inputs.new_branch_minor }}
+            patch=0
+          elif [ "$GITHUB_REF" = "refs/heads/main" ]; then
+            minor=$(( $minor + 1 ))
+            patch=0
+          else
+            patch=$(( $(grep '_PATCH\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') + 1 ))
+          fi
+
+          echo "major=$major" >> $GITHUB_ENV
+          echo "minor=$minor" >> $GITHUB_ENV
+          echo "patch=$patch" >> $GITHUB_ENV
+
+      - name: Update version.h
+        run: |
+          git config user.name "GitHub Runner Bot"
+          git config user.email "<>"
+
+          sed -i -e "s/\(#define [^\s]\+_MAJOR\s\+\)[0-9]\+/\1${major}/" "$VERSION_FILE"
+          sed -i -e "s/\(#define [^\s]\+_MINOR\s\+\)[0-9]\+/\1${minor}/" "$VERSION_FILE"
+          sed -i -e "s/\(#define [^\s]\+_PATCH\s\+\)[0-9]\+/\1${patch}/" "$VERSION_FILE"
+
+          git add "$VERSION_FILE"
+          git commit -m "release: publish version ${major}.${minor}.${patch}"
+          git push origin ${GITHUB_REF#refs/heads/}
+
+      - name: Tag and release
+        run: |
+          # Create a branch if it's a major or a minor release
+          if [ "$patch" -eq 0 ]; then
+            git checkout -b "release/${major}.${minor}"
+            git push -u origin "release/${major}.${minor}"
+          fi
+
+          # Create a tag for the release
+          git tag "speedb/v${major}.${minor}.${patch}"
+          git push origin "speedb/v${major}.${minor}.${patch}"