ci-coverage.yaml

name: ci-coverage

on:
  push:
    branches: [main]
    paths:
        - "KubeArmor/**"
        - ".github/workflows/ci-coverage.yaml"
        - "tests/**"
  pull_request:
    branches: [main]
    paths:
        - "KubeArmor/**"
        - ".github/workflows/ci-coverage.yaml"
        - "tests/**"

jobs:
  calculate-k8s-coverage:
    name: Auto-testing Framework / ${{ matrix.os }} / ${{ matrix.runtime }}
    runs-on: ${{ matrix.os }}
    env:
      RUNTIME: ${{ matrix.runtime }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-20.04, bpflsm]
        runtime: ["containerd", "crio"]
        exclude:
          - os: bpflsm
            runtime: crio
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true

      - uses: actions/setup-go@v5
        with:
          go-version-file: 'KubeArmor/go.mod'

      - name: Check what paths were updated
        uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            controller:
              - 'pkg/KubeArmorController/**'

      - name: Install the latest LLVM toolchain
        run: ./.github/workflows/install-llvm.sh

      - name: Compile libbpf
        run: ./.github/workflows/install-libbpf.sh

      - name: Setup a Kubernetes environment
        run: ./.github/workflows/install-k3s.sh

      - name: Generate KubeArmor artifacts
        run: |
          #set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation
          export IS_COVERAGE=true
          GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh

      - name: Build Kubearmor-Operator
        working-directory: pkg/KubeArmorOperator
        run: |
          make docker-build

      - name: Build KubeArmorController
        if: steps.filter.outputs.controller == 'true'
        run: make -C pkg/KubeArmorController/ docker-build TAG=latest

      - name: deploy pre existing pod 
        run: |
          kubectl apply -f ./tests/k8s_env/ksp/pre-run-pod.yaml
          sleep 60
          kubectl get pods -A

      - name: Run KubeArmor
        run: |
          if [ ${{ matrix.runtime }} == "containerd" ]; then
            docker save kubearmor/kubearmor-test-init:latest | sudo k3s ctr images import -
            docker save kubearmor/kubearmor-test:latest | sudo k3s ctr images import -
            docker save kubearmor/kubearmor-operator:latest | sudo k3s ctr images import -
            docker save kubearmor/kubearmor-snitch:latest | sudo k3s ctr images import -
            
            if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then 
              docker save kubearmor/kubearmor-controller:latest | sudo k3s ctr images import -
            fi
          else
            if [ ${{ matrix.runtime }} == "crio" ]; then
              docker save kubearmor/kubearmor-test-init:latest | sudo podman load
              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test-init:latest
              docker save kubearmor/kubearmor-test:latest | sudo podman load
              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test:latest
              docker save kubearmor/kubearmor-operator:latest | sudo podman load
              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-operator:latest
              docker save kubearmor/kubearmor-snitch:latest | sudo podman load
              sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-snitch:latest
              if [ ${{ steps.filter.outputs.controller }} == 'true' ]; then
                docker save kubearmor/kubearmor-controller:latest | sudo podman load
                sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-controller:latest
              fi
            fi
          fi
          docker system prune -a -f 
          docker buildx prune -a -f
          helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest
          kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
          kubectl get pods -A
          if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then
            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml --dry-run=client -o json | \
            jq '.spec.kubearmorControllerImage.imagePullPolicy = "Never"' | \
            kubectl apply -f -
          else 
            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml
          fi
          # kubectl wait -n kubearmor --timeout=5m --for=jsonpath='{.status.phase}'=Running kubearmorconfigs/kubearmorconfig-test
          # kubectl wait --timeout=7m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch,kubearmor-app!=kubearmor-controller -n kubearmor
          # kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor
          sleep 60
          kubectl get pods -A
          DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}')
          echo "DaemonSet: $DAEMONSET_NAME"

          kubectl patch daemonset $DAEMONSET_NAME -n kubearmor --type='json' -p='[
            {
              "op": "add",
              "path": "/spec/template/spec/volumes/-",
              "value": {
                "name": "coverage-storage",
                "hostPath": {
                  "path": "/coverage",
                  "type": "DirectoryOrCreate"
                }
              }
            },
            {
              "op": "add",
              "path": "/spec/template/spec/containers/0/volumeMounts/-",
              "value": {
                "mountPath": "/coverage",
                "name": "coverage-storage"
              }
            },
            {
              "op": "add",
              "path": "/spec/template/spec/containers/0/args/-",
              "value": "-test.coverprofile=/coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out"
            }
          ]'

          sleep 15

      - name: Add KubeArmor host visibility
        run: |
          ./.github/workflows/host-visibility.sh
          DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}')
          # normal approach didn't worked for cri-o pod
          LABEL_SELECTOR=$(kubectl get daemonset $DAEMONSET_NAME -n kubearmor -o jsonpath='{.spec.selector.matchLabels}' | jq -r 'to_entries[] | "\(.key)=\(.value)"' | paste -sd, -)
          POD_NAME=$(kubectl get pods -n kubearmor -l "$LABEL_SELECTOR" -o jsonpath='{.items[*].metadata.name}')
          echo "Pod: $POD_NAME"
          echo "POD_NAME=$POD_NAME" >> $GITHUB_ENV
        
      - name: Test KubeArmor using Ginkgo
        run: |
          kubectl logs $POD_NAME -n kubearmor
          go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
          make
        working-directory: ./tests/k8s_env
        timeout-minutes: 30
        env:
          POD_NAME: ${{ env.POD_NAME }}
      
      - name: Kill KubeArmor prcoess in the pod
        run: |
          KUBEARMOR_PID=$(kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "ps aux | grep '[K]ubeArmor/kubearmor-test' | awk '{print \$1}'")
          kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "kill -s SIGINT $KUBEARMOR_PID"
          sleep 10
        env:
          POD_NAME: ${{ env.POD_NAME }}

      - name: Extract coverage file
        run: |
          for i in {1..24}; do
            if [ -f /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out ]; then
              cp /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out
              break
            fi
            sleep 5
          done
          ls -l
        working-directory: KubeArmor
        env:
          POD_NAME: ${{ env.POD_NAME }}

      - name: Get karmor sysdump
        if: ${{ failure() }}
        run: |
          kubectl describe pod -n kubearmor -l kubearmor-app=kubearmor
          curl -sfL http://get.kubearmor.io/ | sudo sh -s -- -b /usr/local/bin
          mkdir -p /tmp/kubearmor/ && cd /tmp/kubearmor && karmor sysdump

      - name: Archive log artifacts
        if: ${{ failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: kubearmor.logs
          path: |
            /tmp/kubearmor/
            /tmp/kubearmor.*

      - name: Measure code coverage
        if: ${{ always() }}
        run: |
          ls -l
          go tool cover -func coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out
          # go install github.com/modocache/gover@latest
          # gover
        working-directory: KubeArmor
        env:
          GOPATH: ${{ matrix.os == 'bpflsm' && '/home/vagrant/go' || '/home/runner/go' }}

      - name: Upload coverage file
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: coverage-k8s-${{ matrix.os }}-${{ matrix.runtime }}
          path: KubeArmor/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out

      - name: Run cleanup
        if: ${{ always() && matrix.os == 'bpflsm' }}
        run: ./.github/workflows/cleanup.sh


  docker-compose-coverage:
    name: Calculate coverage in docker mode / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, bpflsm]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true

      - uses: actions/setup-go@v5
        with:
          go-version-file: 'KubeArmor/go.mod'

      - name: Install the latest LLVM toolchain
        run: ./.github/workflows/install-llvm.sh

      - name: Compile libbpf
        run: ./.github/workflows/install-libbpf.sh

      - name: Generate KubeArmor artifacts
        run: |
          # set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation
          export IS_COVERAGE=true
          GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh

      - name: Run KubeArmor init container
        run: |
          docker run --name kubearmor-test-init -v /tmp/:/opt/kubearmor/BPF kubearmor/kubearmor-test-init:latest

      - name: Run KubeArmor test container
        run: |
          sudo mkdir -p /coverage
          docker run -d --name kubearmor-test --privileged --pid host -p 32767:32767 \
          -v /tmp/:/opt/kubearmor/BPF \
          -v /sys/fs/bpf:/sys/fs/bpf \
          -v /sys/kernel/security:/sys/kernel/security \
          -v /sys/kernel/debug:/sys/kernel/debug \
          -v /var/run/docker.sock:/var/run/docker.sock \
          -v /var/lib/docker:/var/lib/docker \
          -v /etc/apparmor.d:/etc/apparmor.d \
          -v /coverage/:/coverage/ \
          kubearmor/kubearmor-test:latest -k8s=false -enableKubeArmorHostPolicy -coverageTest=false -test.coverprofile=/coverage/coverage_docker.out

      - name: Test KubeArmor using Ginkgo
        run: |
          go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
          make
        working-directory: ./tests/nonk8s_env
        timeout-minutes: 30

      - name: Copy coverage report
        run: |
          CONTAINER_ID=$(docker ps -qf "name=kubearmor")
          sleep 2
          KUBEARMOR_PID=$(docker exec $CONTAINER_ID sh -c "pgrep -o -f 'kubearmor'")
          sleep 2
          docker exec $CONTAINER_ID sh -c "kill -s SIGINT $KUBEARMOR_PID"
          sleep 15
          for i in {1..24}; do
            if [ -f /coverage/coverage_docker.out ]; then
              cp /coverage/coverage_docker.out coverage_docker_${{ matrix.os }}.out
              break
            fi
            sleep 5
          done
          sleep 2
        working-directory: KubeArmor

      - name: Archive log artifacts
        if: ${{ failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: kubearmor.logs
          path: |
            /tmp/kubearmor/
            /tmp/kubearmor.*

      - name: Measure code coverage
        if: ${{ always() }}
        run: |
          ls -l
          go tool cover -func coverage_docker_${{ matrix.os }}.out
        working-directory: KubeArmor
        env:
          GOPATH: ${{ matrix.os == 'bpflsm' && '/home/vagrant/go' || '/home/runner/go' }}

      - name: Save coverage file
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: coverage-docker-${{ matrix.os }}
          path: KubeArmor/coverage_docker_${{ matrix.os }}.out

      - name: Run cleanup
        if: ${{ always() && matrix.os == 'bpflsm' }}
        run: ./.github/workflows/cleanup.sh


  calculate-systemd-coverage:
    name: Calculate coverage in systemd mode / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, bpflsm]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true

      - uses: actions/setup-go@v5
        with:
          go-version-file: 'KubeArmor/go.mod'
    
      - name: Install the latest LLVM toolchain
        run: ./.github/workflows/install-llvm.sh

      - name: Compile libbpf
        run: ./.github/workflows/install-libbpf.sh

      - name: Install GoReleaser
        uses: goreleaser/goreleaser-action@v2
        with:
          install-only: true
          version: v1.25.0

      - name: Install protoc
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler
          protoc --version

      - name: Install protoc-gen-go
        if: ${{ matrix.os == 'bpflsm' }}
        run: |
          go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
        env:
          GOPATH: /home/vagrant/go

      - name: Build Systemd Release
        run: make local-release
        working-directory: KubeArmor

      - name: Install KubeArmor
        run: sudo apt install -y ./dist/kubearmor*amd64.deb
        working-directory: KubeArmor

      - name: Compile test binary
        run: go test -covermode=atomic -coverpkg=./... -c . -o kubearmor-test
        working-directory: KubeArmor

      - name: Replace with test binary
        run: |
          sudo mkdir -p /coverage
          sudo rm /opt/kubearmor/kubearmor
          sudo cp kubearmor-test /opt/kubearmor/
          ls -l /opt/kubearmor/
          sudo sed -i 's|ExecStart=/opt/kubearmor/kubearmor|ExecStart=/opt/kubearmor/kubearmor-test -test.coverprofile=/coverage/coverage_systemd.out|' /lib/systemd/system/kubearmor.service
          sudo systemctl daemon-reload
          sudo systemctl restart kubearmor.service
        working-directory: KubeArmor
      
      - name: Check journalctl
        run: sudo journalctl -u kubearmor --no-pager || true

      - name: Test kubearmor using ginkgo 
        run: |
          go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
          make
        working-directory: ./tests/nonk8s_env
        timeout-minutes: 30
      
      - name: Kill kubearmor process and copy coverage file
        run: |
          sudo systemctl stop kubearmor
          sleep 15
          for i in {1..24}; do
            if [ -f /coverage/coverage_systemd.out ]; then
              sudo cp /coverage/coverage_systemd.out coverage_systemd_${{ matrix.os }}.out
              break
            fi
            sleep 5
          done
        working-directory: KubeArmor
      
      - name: Measure code coverage
        if: ${{ always() }}
        run: |
          ls -l
          go tool cover -func coverage_systemd_${{ matrix.os }}.out
        working-directory: KubeArmor
        env:
          GOPATH: ${{ matrix.os == 'bpflsm' && '/home/vagrant/go' || '/home/runner/go' }}

      - name: Save coverage file
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: coverage-systemd-${{ matrix.os }}
          path: KubeArmor/coverage_systemd_${{ matrix.os }}.out

      - name: Run cleanup
        if: ${{ always() && matrix.os == 'bpflsm' }}
        run: |
          sudo systemctl disable kubearmor.service
          sudo rm -rf /opt/kubearmor/
          sudo apt-get --purge remove -y kubearmor
          sudo systemctl daemon-reload

  merge-and-upload-coverage:
    name: Merge and Upload Coverage
    runs-on: ubuntu-latest
    needs: [calculate-k8s-coverage, docker-compose-coverage, calculate-systemd-coverage]
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true

      - uses: actions/setup-go@v5
        with:
          go-version-file: 'KubeArmor/go.mod'

      - name: Download all coverage files
        uses: actions/download-artifact@v4
        with:
          path: KubeArmor/
          pattern: coverage*
          merge-multiple: true

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}