tests.yaml

name: tests
run-name: tests, branch:${{ github.ref_name }}, triggered by @${{ github.actor }}

concurrency:
  # Run only for most recent commit in PRs but for all tags and commits on main
  # Ref: https://docs.github.com/en/actions/using-jobs/using-concurrency
  group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
  cancel-in-progress: true

on:
  pull_request:
    branches:
      - '*'
  push:
    branches:
      - 'main'
      - 'release/*'
    tags:
      - '*'
  workflow_dispatch: {}

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Setup go
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run lint
      env:
        # Our .golangci.yaml has fix: true, but we don't want that in CI therefore the below override.
        GOLANGCI_LINT_FLAGS: "--fix=false"
      run: make lint

  verify:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Setup go
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: Verify manifests consistency
      run: make verify.manifests

    - name: Verify generators consistency
      uses: nick-fields/retry@v3
      with:
        timeout_minutes: 3
        max_attempts: 3
        command: make verify.generators

  samples:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Setup go
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - name: Create k8s KinD Cluster
      uses: helm/kind-action@v1.10.0

    - uses: jdx/mise-action@v2
      with:
        install: false

    # We use install.all to install all CRDs and resources also the ones that are not bundled
    # in base kustomization (e.g. currently AIGateway) but which have samples defined.
    - name: Verify installing CRDs via kustomize works
      run: make install.all

    - name: Install and delete each sample one by one
      run: make test.samples

    - name: Verify that uninstalling operator CRDs via kustomize works
      run: make uninstall.all

  install-with-kustomize:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Setup go
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - name: Create k8s KinD Cluster
      uses: helm/kind-action@v1.10.0

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: Verify installing CRDs via kustomize works
      run: make install

    - name: Verify deploying operator via kustomize works
      run: make deploy

    - name: Verify that undeploying operator via kustomize works
      run: make undeploy

    - name: Verify that uninstalling operator CRDs via kustomize works
      run: make uninstall

  build:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - run: make build.operator

    - name: Test the binary by running it with -version flag
      run: ./bin/manager -version | ./scripts/verify-version.sh ${{ github.repository }}

  unit-tests:
    runs-on: ubuntu-latest
    steps:
    - name: checkout repository
      uses: actions/checkout@v4

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run unit tests
      run: make test.unit
      env:
        KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS: ${{ secrets.KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS }}
        GOTESTSUM_JUNITFILE: "unit-tests.xml"

    - name: collect test coverage
      uses: actions/upload-artifact@v3
      with:
        name: coverage
        path: coverage.unit.out

    - name: collect test report
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
        name: tests-report
        path: unit-tests.xml
  
  envtest-tests:
    runs-on: ubuntu-latest
    steps:
    - name: checkout repository
      uses: actions/checkout@v4

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run envtest tests
      run: make test.envtest
      env:
        GOTESTSUM_JUNITFILE: "envtest-tests.xml"

    - name: collect test coverage
      uses: actions/upload-artifact@v3
      with:
        name: coverage-envtest
        path: coverage.envtest.out

    - name: collect test report
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
        name: tests-report
        path: envtest-tests.xml  

  conformance-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
        - router-flavor: traditional_compatible
        - router-flavor: expressions
    steps:
    - name: checkout repository
      uses: actions/checkout@v4
      with:
        # Fetch all history so that we can get the correct commit hash for the conformance tests.
        fetch-depth: 0

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run conformance tests
      run: make test.conformance
      env:
        GOTESTSUM_JUNITFILE: conformance-tests-${{ matrix.router-flavor }}.xml
        TEST_KONG_ROUTER_FLAVOR: ${{ matrix.router-flavor }}

    - name: upload diagnostics
      if: ${{ always() }}
      uses: actions/upload-artifact@v4
      with:
        name: diagnostics-conformance
        path: /tmp/ktf-diag*
        if-no-files-found: ignore

    - name: collect test report
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
        name: tests-report
        path: conformance-tests-${{ matrix.router-flavor }}.xml

    - name: collect conformance report
      uses: actions/upload-artifact@v4
      with:
        name: conformance-report-${{ matrix.router-flavor }}
        path: standard-*-report.yaml

  integration-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        webhook-enabled:
          - "true"
          - "false"
    name: "integration-tests (WEBHOOK_ENABLED=${{ matrix.webhook-enabled }})"
    steps:
    - name: checkout repository
      uses: actions/checkout@v4

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run integration tests
      run: make test.integration
      env:
        KONG_TEST_DISABLE_CERTMANAGER: "true"
        KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS: ${{ secrets.KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS }}
        WEBHOOK_ENABLED: ${{ matrix.webhook-enabled }}
        KONG_CONTROLLER_OUT: stdout
        GOTESTSUM_JUNITFILE: integration-tests-webhook-enabled-${{ matrix.webhook-enabled }}.xml
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        KONG_TEST_KONNECT_ACCESS_TOKEN: ${{ secrets.KONG_TEST_KONNECT_ACCESS_TOKEN }}
        KONG_TEST_KONNECT_SERVER_URL: us.api.konghq.tech

    - name: upload diagnostics
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: diagnostics-integration-webhook-enabled-${{ matrix.webhook-enabled }}
        path: /tmp/ktf-diag*
        if-no-files-found: ignore

    - name: collect test coverage
      uses: actions/upload-artifact@v3
      with:
        name: coverage
        path: coverage.integration.out

    - name: collect test report
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
        name: tests-report
        path: integration-tests-webhook-enabled-${{ matrix.webhook-enabled }}.xml

  integration-tests-bluegreen:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        webhook-enabled:
          - "true"
          - "false"
    name: "integration-tests-bluegreen (WEBHOOK_ENABLED=${{ matrix.webhook-enabled }})"
    steps:
    - name: checkout repository
      uses: actions/checkout@v4

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run integration tests
      run: make test.integration_bluegreen
      env:
        KONG_TEST_DISABLE_CERTMANAGER: "true"
        WEBHOOK_ENABLED: ${{ matrix.webhook-enabled }}
        KONG_CONTROLLER_OUT: stdout
        GOTESTSUM_JUNITFILE: integration-tests-bluegreen-webhook-enabled-${{ matrix.webhook-enabled }}.xml
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: upload diagnostics
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: diagnostics-integration-bluegreen-webhook-enabled-${{ matrix.webhook-enabled }}
        path: /tmp/ktf-diag*
        if-no-files-found: ignore

    - name: collect test coverage
      uses: actions/upload-artifact@v3
      with:
        name: coverage
        path: coverage.integration-bluegreen.out

    - name: collect test report
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: tests-report
        path: integration-tests-bluegreen-webhook-enabled-${{ matrix.webhook-enabled }}.xml
  
  # Test reconciling Gateway with provisioning DataPlane failures.
  # This test introduces a wrong gateway that will have errors on validation all `DataPlane`s, so it should run isolated.
  integration-tests-provision-fail:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
    name: integration-tests-provision-dataplane-fail
    steps:
    - name: checkout repository
      uses: actions/checkout@v4

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run integration tests
      run: make test.integration_provision_dataplane_fail
      env:
        KONG_TEST_DISABLE_CERTMANAGER: "true"
        KONG_CONTROLLER_OUT: stdout
        KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS: ${{ secrets.KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS }}
        GOTESTSUM_JUNITFILE: integration-tests-provision-dataplane-fail.xml
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    
    - name: upload diagnostics
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: diagnostics-integration-provision-fail-webhook-enabled-${{ matrix.webhook-enabled }}
        path: /tmp/ktf-diag*
        if-no-files-found: ignore

    - name: collect test coverage
      uses: actions/upload-artifact@v3
      with:
        name: coverage
        path: coverage.integration-provision-dataplane-fail.out

    - name: collect test report
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: tests-report
        path: integration-tests-provision-dataplane-fai.xml
  
  e2e-tests:
    runs-on: ubuntu-latest
    steps:
    - name: checkout repository
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: setup golang
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod

    - name: build docker image
      env:
        IMG: gateway-operator
        TAG: e2e-${{ github.sha }}
      run: make docker.build

    - uses: jdx/mise-action@v2
      with:
        install: false

    - name: run e2e tests
      run: make test.e2e
      env:
        KONG_TEST_DISABLE_CERTMANAGER: "true"
        KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD: gateway-operator:e2e-${{ github.sha }}
        GOTESTSUM_JUNITFILE: "e2e-tests.xml"
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: upload diagnostics
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: diagnostics-e2e
        path: /tmp/ktf-diag*
        if-no-files-found: ignore

    - name: collect test report
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
        name: tests-report
        path: e2e-tests.xml

  buildpulse-report:
    needs:
      - unit-tests
      - integration-tests
      # https://github.com/Kong/gateway-operator/issues/11#issuecomment-1514712925
      # - conformance-tests
      - e2e-tests
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:

      - name: checkout repository
        uses: actions/checkout@v4

      - name: download tests report
        id: download-coverage
        uses: actions/download-artifact@v3
        with:
          name: tests-report
          path: report

      - name: Upload test results to BuildPulse for flaky test detection
        if: ${{ !cancelled() }}
        uses: buildpulse/buildpulse-action@v0.11.0
        with:
          account: 962416
          repository: 477814940
          path: report/*.xml
          key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
          secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}

  # We need this step to fail the workflow if any of the previous steps failed or were cancelled.
  # It allows to use this particular job as a required check for PRs.
  # Ref: https://github.com/orgs/community/discussions/26822#discussioncomment-3305794
  passed:
    runs-on: ubuntu-latest
    needs:
      - lint
      - verify
      - install-with-kustomize
      - build
      - unit-tests
      - samples
      # - conformance-tests
      - integration-tests
      - integration-tests-bluegreen
      - integration-tests-provision-fail
      - e2e-tests
      - buildpulse-report
    if: always()
    steps:
      - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: |
          echo "Some jobs failed or were cancelled."
          exit 1