build.yaml

name: Tests

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  build_dependencies:
    name: Build deps [ubuntu]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Get dependencies hash
        id: get-hash
        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
      - name: Cache lookup
        uses: actions/cache/restore@v4
        id: cache-lookup
        with:
          path: deps
          key: deps-${{ steps.get-hash.outputs.deps_hash }}
          lookup-only: true
      - name: Set up dependencies
        if: steps.cache-lookup.outputs.cache-hit != 'true'
        run: |
          sudo apt-get update
          sudo apt-get install -qy build-essential \
            gdb \
            curl \
            python3.10 \
            python3-pip \
            cmake \
            ninja-build \
            pkg-config \
            bison \
            libfl-dev \
            libbenchmark-dev \
            libgmock-dev \
            libz-dev
      - name: Fetch & Build non packaged dependencies
        if: steps.cache-lookup.outputs.cache-hit != 'true'
        run: |
          mkdir -p deps
          cd deps
          ../docker/build_deps.sh
      - name: Cache save
        if: steps.cache-lookup.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: deps
          key: deps-${{ steps.get-hash.outputs.deps_hash }}

  build_ubuntu:
    name: Build [ubuntu]
    runs-on: ubuntu-latest
    needs: build_dependencies
    steps:
      - uses: actions/checkout@v4
      - name: Get dependencies hash
        id: get-hash
        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
      - uses: actions/cache/restore@v4
        with:
          path: deps
          key: deps-${{ steps.get-hash.outputs.deps_hash }}
      - name: Set up dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -qy build-essential \
            gdb \
            curl \
            python3.10 \
            python3-pip \
            cmake \
            ninja-build \
            pkg-config \
            bison \
            libfl-dev \
            libbenchmark-dev \
            libgmock-dev \
            libz-dev
      - name: Install cached non packaged dependencies
        working-directory: deps
        run: ../docker/build_deps.sh
      - name: Build BlazingMQ
        env:
          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
        run: |
          cmake -S . -B build/blazingmq -G Ninja \
            -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
            -DCMAKE_BUILD_TYPE=Debug \
            -DBDE_BUILD_TARGET_SAFE=ON \
            -DBDE_BUILD_TARGET_64=ON \
            -DBDE_BUILD_TARGET_CPP17=ON \
            -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
            -DCMAKE_INSTALL_LIBDIR=lib64
          cmake --build build/blazingmq --parallel 8 --target all
      - name: Clean-up build directories before caching
        run: |
          find . -name "*.o" -type f -delete
          find . -name "*.a" -type f -delete
      - uses: actions/cache@v4
        with:
          path: |
            build/blazingmq
            deps
            /opt/bb/include
          key: cache-${{ github.sha }}

  unit_tests_cxx:
    name: UT [c++]
    runs-on: ubuntu-latest
    needs: build_dependencies
    steps:
      - uses: actions/checkout@v4
      - name: Get dependencies hash
        id: get-hash
        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
      - uses: actions/cache/restore@v4
        with:
          path: deps
          key: deps-${{ steps.get-hash.outputs.deps_hash }}
      - name: Set up dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -qy build-essential \
            gdb \
            curl \
            python3.10 \
            python3-pip \
            cmake \
            ninja-build \
            pkg-config \
            bison \
            libfl-dev \
            libbenchmark-dev \
            libgmock-dev \
            libz-dev
      - name: Install cached non packaged dependencies
        working-directory: deps
        run: ../docker/build_deps.sh
      - name: Build BlazingMQ UTs
        env:
          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
        run: |
          cmake -S . -B build/blazingmq -G Ninja \
            -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
            -DCMAKE_BUILD_TYPE=Debug \
            -DBDE_BUILD_TARGET_SAFE=ON \
            -DBDE_BUILD_TARGET_64=ON \
            -DBDE_BUILD_TARGET_CPP17=ON \
            -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
            -DCMAKE_INSTALL_LIBDIR=lib64
          cmake --build build/blazingmq --parallel 8 --target all.t
      - name: Run C++ Unit Tests
        run: |
          cd ${{ github.workspace }}/build/blazingmq
          ctest -E mwcsys_executil.t --output-on-failure

  unit_tests_python:
    name: UT [python]
    runs-on: ubuntu-latest
    needs: build_ubuntu
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache/@v4
        with:
          path: |
            build/blazingmq
            deps
            /opt/bb/include
          key: cache-${{ github.sha }}
      - name: Run Python Unit Tests
        env:
          PYTHONPATH: ${{ github.workspace }}/src/python
        run: |
          pip install -r ${{ github.workspace }}/src/python/requirements.txt
          pip install "xsdata[cli]"
          cd ${{ github.workspace }}
          src/python/bin/schemagen
          src/python/bin/tweakgen
          pytest src/python

  integration_tests_ubuntu:
    name: IT [${{ matrix.cluster }}/${{ matrix.mode }}]
    strategy:
      matrix:
        mode: ["legacy_mode", "fsm_mode"]
        cluster: ["single", "multi"]
      fail-fast: false
    runs-on: ubuntu-latest
    needs: build_ubuntu
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache/@v4
        with:
          path: |
            build/blazingmq
            deps
            /opt/bb/include
          key: cache-${{ github.sha }}
      - name: Run Integration Tests
        run: |
          pip install -r ${{ github.workspace }}/src/python/requirements.txt
          ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \
            --log-level ERROR                   \
            --log-file-level=info               \
            --bmq-tolerate-dirty-shutdown       \
            --bmq-log-dir=failure-logs          \
            --bmq-log-level=INFO                \
            --junitxml=integration-tests.xml    \
            --tb long                           \
            --reruns=3                          \
            -n 4 -v

  fuzz_tests_ubuntu:
    name: Fuzz test [${{ matrix.request }}]
    strategy:
      matrix:
        request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"]
      fail-fast: false
    runs-on: ubuntu-latest
    needs: build_ubuntu
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache/@v4
        with:
          path: |
            build/blazingmq
            deps
            /opt/bb/include
          key: cache-${{ github.sha }}
      - name: Run Fuzz Test
        run: |
          pip install -r ${{ github.workspace }}/src/python/requirements.txt
          cd src/python
          python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }}

  build_macosx:
    name: Build [macosx_${{ matrix.arch }}]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - os: macos-14
            arch: arm64
    steps:
      - uses: actions/checkout@v4
      - name: Set up dependencies
        run: |
          brew install \
            curl \
            python@3.10 \
            ninja \
            bison \
            flex \
            google-benchmark
      - name: Build BlazingMQ
        run: bin/build-darwin.sh

  build_and_test_bmqprometheus_plugin:
    name: "Build Prometheus plugin [ubuntu]"
    runs-on: ubuntu-latest
    needs: build_ubuntu
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache/@v4
        with:
          path: |
            build/blazingmq
            deps
            /opt/bb/include
          key: cache-${{ github.sha }}
      - name: Set up plugins dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -qy build-essential \
            gdb \
            curl \
            python3.10 \
            cmake \
            ninja-build \
            pkg-config \
            bison \
            libfl-dev \
            libbenchmark-dev \
            libgmock-dev \
            libz-dev \
            autoconf \
            libtool
      - name: Create dependency fetcher working directory
        run: mkdir -p deps
      - name: Fetch & Build non packaged plugins dependencies
        working-directory: deps
        run:  ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh
      - name: Build plugins
        env:
          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
        run: |
          cmake -S . -B build/blazingmq -G Ninja \
            -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
            -DCMAKE_BUILD_TYPE=Debug \
            -DBDE_BUILD_TARGET_SAFE=ON \
            -DBDE_BUILD_TARGET_64=ON \
            -DBDE_BUILD_TARGET_CPP17=ON \
            -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
            -DCMAKE_INSTALL_LIBDIR=lib64 \
            -DINSTALL_TARGETS=prometheus
          cmake --build build/blazingmq --parallel 8 --target all
      - name: Create prometheus dir
        run:  mkdir -p prometheus_dir
      - name: Download Prometheus
        run:  curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/
      - name: Run Prometheus
        run:  ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
      - name: Run Pushgateway
        run: |
          docker pull prom/pushgateway
          docker run -d -p 9091:9091 prom/pushgateway
      - name: Run BMQPrometheus plugin integration test in "pull" mode
        run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker
      - name: Clear Prometheus database and restart
        run: |
          curl -X POST "http://localhost:9090/-/quit"
          rm -rf prometheus_dir/data
          ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
      - name: Run Prometheus plugin integration test in "push" mode
        run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker

  documentation:
    name: "Documentation"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -qy doxygen
      - name: Build docs
        run: |
          doxygen Doxyfile

  shellcheck:
    name: Shellcheck
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run ShellCheck
        uses: ludeeus/action-shellcheck@master

  docker_build_ubuntu:
    name: "Docker [ubuntu]"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Docker Single-Node Workflow
        run: docker compose -f docker/single-node/docker-compose.yaml up --build -d
      - name: Docker Cluster Workflow
        run: docker compose -f docker/cluster/docker-compose.yaml up --build -d