From 9526b6482a138aefbf3254c0dacc7e944d08e359 Mon Sep 17 00:00:00 2001 From: Tim Flannagan Date: Tue, 14 Jan 2025 15:46:31 -0500 Subject: [PATCH] .github: Disable permafailing jobs (#10455) Signed-off-by: timflannagan --- .github/workflows/pr-kubernetes-tests.yaml | 247 ++++++++++----------- .github/workflows/regression-tests.yaml | 60 ++--- 2 files changed, 153 insertions(+), 154 deletions(-) diff --git a/.github/workflows/pr-kubernetes-tests.yaml b/.github/workflows/pr-kubernetes-tests.yaml index 79ddaf794fe..1051ad57872 100644 --- a/.github/workflows/pr-kubernetes-tests.yaml +++ b/.github/workflows/pr-kubernetes-tests.yaml @@ -1,135 +1,134 @@ -name: Kubernetes Tests -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] +# name: Kubernetes Tests +# on: +# pull_request: +# types: [opened, synchronize, reopened, ready_for_review] -env: - VERSION: '1.0.0-ci1' - GITHUB_TOKEN: ${{ github.token }} +# env: +# VERSION: '1.0.0-ci1' +# GITHUB_TOKEN: ${{ github.token }} -jobs: - prepare_env: - name: Prepare Environment - runs-on: ubuntu-22.04 - timeout-minutes: 5 - outputs: - should-auto-succeed-kube-tests: ${{ steps.run-strategy.outputs.auto_succeed }} - steps: - - name: Cancel Previous Actions - uses: styfle/cancel-workflow-action@0.12.1 - with: - access_token: ${{ github.token }} - - id: checkout-code - uses: actions/checkout@v4 - with: - # We require gathering the branch and tag history since we rely on a `git diff` - # which compares the state of two branches - fetch-depth: 0 - - id: process-skip-directives - uses: ./.github/workflows/composite-actions/process-skip-directives - with: - base-ref: ${{ github.base_ref }} - - id: run-strategy - name: Determine Test Run Strategy - run: | - skip_kube_tests=${{ steps.process-skip-directives.outputs.skip-kube-tests }} - if [[ ! -z $skip_kube_tests && $skip_kube_tests = true ]] ; then - echo "auto_succeed=true" >> $GITHUB_OUTPUT - fi +# jobs: +# prepare_env: +# name: Prepare Environment +# runs-on: ubuntu-22.04 +# timeout-minutes: 5 +# outputs: +# should-auto-succeed-kube-tests: ${{ steps.run-strategy.outputs.auto_succeed }} +# steps: +# - name: Cancel Previous Actions +# uses: styfle/cancel-workflow-action@0.12.1 +# with: +# access_token: ${{ github.token }} +# - id: checkout-code +# uses: actions/checkout@v4 +# with: +# # We require gathering the branch and tag history since we rely on a `git diff` +# # which compares the state of two branches +# fetch-depth: 0 +# - id: process-skip-directives +# uses: ./.github/workflows/composite-actions/process-skip-directives +# with: +# base-ref: ${{ github.base_ref }} +# - id: run-strategy +# name: Determine Test Run Strategy +# run: | +# skip_kube_tests=${{ steps.process-skip-directives.outputs.skip-kube-tests }} +# if [[ ! -z $skip_kube_tests && $skip_kube_tests = true ]] ; then +# echo "auto_succeed=true" >> $GITHUB_OUTPUT +# fi - end_to_end_tests: - name: End-to-End (${{ matrix.test.cluster-name }}) - needs: prepare_env - runs-on: ubuntu-22.04 - timeout-minutes: 60 - if: ${{ !github.event.pull_request.draft }} - strategy: - fail-fast: false - matrix: - # We intentionally set an upper threshold of 30 minutes for our End-to-End tests - # Our goal is to load balance tests in a way that allows quick iteration on PRs - # If tests are exceeding the 30-minute limit, please see: - # /test/kubernetes/e2e/load_balancing_tests.md - # - # Above each test below, we document the latest date/time for the GitHub action step to run - # NOTE: We use the GitHub action step time (as opposed to the `go test` time), because it is easier to capture +# end_to_end_tests: +# name: End-to-End (${{ matrix.test.cluster-name }}) +# needs: prepare_env +# runs-on: ubuntu-22.04 +# timeout-minutes: 60 +# if: ${{ !github.event.pull_request.draft }} +# strategy: +# fail-fast: false +# matrix: +# # We intentionally set an upper threshold of 30 minutes for our End-to-End tests +# # Our goal is to load balance tests in a way that allows quick iteration on PRs +# # If tests are exceeding the 30-minute limit, please see: +# # /test/kubernetes/e2e/load_balancing_tests.md +# # +# # Above each test below, we document the latest date/time for the GitHub action step to run +# # NOTE: We use the GitHub action step time (as opposed to the `go test` time), because it is easier to capture - test: - # Dec 4, 2024: 22 minutes - - cluster-name: 'cluster-one' - go-test-args: '-v -timeout=25m' - go-test-run-regex: '^TestK8sGateway$$/^RouteDelegation$$|^TestGlooctlGlooGatewayEdgeGateway$$|^TestGlooctlK8sGateway$$' +# test: +# # Dec 4, 2024: 22 minutes +# - cluster-name: 'cluster-one' +# go-test-args: '-v -timeout=25m' +# go-test-run-regex: '^TestK8sGateway$$/^RouteDelegation$$|^TestGlooctlGlooGatewayEdgeGateway$$|^TestGlooctlK8sGateway$$' - # Dec 4, 2024: 23 minutes - - cluster-name: 'cluster-two' - go-test-args: '-v -timeout=25m' - go-test-run-regex: '^TestK8sGatewayIstioRevision$$|^TestRevisionIstioRegression$$|^TestK8sGateway$$/^Deployer$$|^TestK8sGateway$$/^RouteOptions$$|^TestK8sGateway$$/^VirtualHostOptions$$|^TestK8sGateway$$/^Upstreams$$|^TestK8sGateway$$/^HeadlessSvc$$|^TestK8sGateway$$/^PortRouting$$|^TestK8sGatewayMinimalDefaultGatewayParameters$$|^TestK8sGateway$$/^DirectResponse$$|^TestK8sGateway$$/^HttpListenerOptions$$|^TestK8sGateway$$/^ListenerOptions$$|^TestK8sGateway$$/^GlooAdminServer$$' +# # Dec 4, 2024: 23 minutes +# - cluster-name: 'cluster-two' +# go-test-args: '-v -timeout=25m' +# go-test-run-regex: '^TestK8sGatewayIstioRevision$$|^TestRevisionIstioRegression$$|^TestK8sGateway$$/^Deployer$$|^TestK8sGateway$$/^RouteOptions$$|^TestK8sGateway$$/^VirtualHostOptions$$|^TestK8sGateway$$/^Upstreams$$|^TestK8sGateway$$/^HeadlessSvc$$|^TestK8sGateway$$/^PortRouting$$|^TestK8sGatewayMinimalDefaultGatewayParameters$$|^TestK8sGateway$$/^DirectResponse$$|^TestK8sGateway$$/^HttpListenerOptions$$|^TestK8sGateway$$/^ListenerOptions$$|^TestK8sGateway$$/^GlooAdminServer$$' - # Dec 4, 2024: 24 minutes - - cluster-name: 'cluster-three' - go-test-args: '-v -timeout=30m' - go-test-run-regex: '(^TestK8sGatewayIstioAutoMtls$$|^TestAutomtlsIstioEdgeApisGateway$$|^TestIstioEdgeApiGateway$$|^TestIstioRegression$$)' +# # Dec 4, 2024: 24 minutes +# - cluster-name: 'cluster-three' +# go-test-args: '-v -timeout=30m' +# go-test-run-regex: '(^TestK8sGatewayIstioAutoMtls$$|^TestAutomtlsIstioEdgeApisGateway$$|^TestIstioEdgeApiGateway$$|^TestIstioRegression$$)' - # Dec 4, 2024: 21 minutes - - cluster-name: 'cluster-four' - go-test-args: '-v -timeout=30m' - go-test-run-regex: '(^TestK8sGatewayIstio$$|^TestGlooGatewayEdgeGateway$$|^TestGlooctlIstioInjectEdgeApiGateway$$)' +# # Dec 4, 2024: 21 minutes +# - cluster-name: 'cluster-four' +# go-test-args: '-v -timeout=30m' +# go-test-run-regex: '(^TestK8sGatewayIstio$$|^TestGlooGatewayEdgeGateway$$|^TestGlooctlIstioInjectEdgeApiGateway$$)' - # Dec 4, 2024: 24 minutes - - cluster-name: 'cluster-five' - go-test-args: '-v -timeout=30m' - go-test-run-regex: '^TestFullEnvoyValidation$$|^TestValidationStrict$$|^TestValidationAlwaysAccept$$|^TestTransformationValidationDisabled$$' +# # Dec 4, 2024: 24 minutes +# - cluster-name: 'cluster-five' +# go-test-args: '-v -timeout=30m' +# go-test-run-regex: '^TestFullEnvoyValidation$$|^TestValidationStrict$$|^TestValidationAlwaysAccept$$|^TestTransformationValidationDisabled$$' - # Dec 4, 2024: 26 minutes - - cluster-name: 'cluster-six' - go-test-args: '-v -timeout=30m' - go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestK8sGatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestK8sGatewayAws$$|^TestK8sGateway$$/^HTTPRouteServices$$|^TestK8sGateway$$/^TCPRouteServices$$|^TestZeroDowntimeRollout$$' +# # Dec 4, 2024: 26 minutes +# - cluster-name: 'cluster-six' +# go-test-args: '-v -timeout=30m' +# go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestK8sGatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestK8sGatewayAws$$|^TestK8sGateway$$/^HTTPRouteServices$$|^TestK8sGateway$$/^TCPRouteServices$$|^TestZeroDowntimeRollout$$' - # Dec 4, 2024: 13 minutes - - cluster-name: 'cluster-seven' - go-test-args: '-v -timeout=25m' - go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestWatchNamespaceSelector$$' +# # Dec 4, 2024: 13 minutes +# - cluster-name: 'cluster-seven' +# go-test-args: '-v -timeout=25m' +# go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestWatchNamespaceSelector$$' - # In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support - # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/ - version-files: - - file: './.github/workflows/.env/pr-tests/versions.env' - - steps: - - id: auto-succeed-tests - if: needs.prepare_env.outputs.should-auto-succeed-kube-tests == 'true' - run: | - echo "Regression tests auto-succeeded" - - uses: actions/checkout@v4 - - name: Prep Go Runner - uses: ./.github/workflows/composite-actions/prep-go-runner - if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' - # The dotenv action is used to load key-value pairs from files. - # In this case, the file is specified in the matrix and will contain the versions of the tools to use - - name: Dotenv Action - uses: falti/dotenv-action@v1.1.4 - if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' - id: dotenv - with: - path: ${{ matrix.version-files.file }} - log-variables: true - - id: setup-kind-cluster - name: Setup KinD Cluster - if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' - uses: ./.github/workflows/composite-actions/setup-kind-cluster - with: - cluster-name: ${{ matrix.test.cluster-name }} - kind-node-version: ${{ steps.dotenv.outputs.node_version }} - kind-version: ${{ steps.dotenv.outputs.kind_version }} - kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} - helm-version: ${{ steps.dotenv.outputs.helm_version }} - istio-version: ${{ steps.dotenv.outputs.istio_version }} - - id: run-tests - if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' - uses: ./.github/workflows/composite-actions/kubernetes-e2e-tests - with: - cluster-name: ${{ matrix.test.cluster-name }} - test-args: ${{ matrix.test.go-test-args }} - run-regex: ${{ matrix.test.go-test-run-regex }} - istio-version: ${{ steps.dotenv.outputs.istio_version }} - matrix-label: "pr" +# # In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support +# # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/ +# version-files: +# - file: './.github/workflows/.env/pr-tests/versions.env' +# steps: +# - id: auto-succeed-tests +# if: needs.prepare_env.outputs.should-auto-succeed-kube-tests == 'true' +# run: | +# echo "Regression tests auto-succeeded" +# - uses: actions/checkout@v4 +# - name: Prep Go Runner +# uses: ./.github/workflows/composite-actions/prep-go-runner +# if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' +# # The dotenv action is used to load key-value pairs from files. +# # In this case, the file is specified in the matrix and will contain the versions of the tools to use +# - name: Dotenv Action +# uses: falti/dotenv-action@v1.1.4 +# if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' +# id: dotenv +# with: +# path: ${{ matrix.version-files.file }} +# log-variables: true +# - id: setup-kind-cluster +# name: Setup KinD Cluster +# if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' +# uses: ./.github/workflows/composite-actions/setup-kind-cluster +# with: +# cluster-name: ${{ matrix.test.cluster-name }} +# kind-node-version: ${{ steps.dotenv.outputs.node_version }} +# kind-version: ${{ steps.dotenv.outputs.kind_version }} +# kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} +# helm-version: ${{ steps.dotenv.outputs.helm_version }} +# istio-version: ${{ steps.dotenv.outputs.istio_version }} +# - id: run-tests +# if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true' +# uses: ./.github/workflows/composite-actions/kubernetes-e2e-tests +# with: +# cluster-name: ${{ matrix.test.cluster-name }} +# test-args: ${{ matrix.test.go-test-args }} +# run-regex: ${{ matrix.test.go-test-run-regex }} +# istio-version: ${{ steps.dotenv.outputs.istio_version }} +# matrix-label: "pr" diff --git a/.github/workflows/regression-tests.yaml b/.github/workflows/regression-tests.yaml index 170675c2270..1b452a9651b 100644 --- a/.github/workflows/regression-tests.yaml +++ b/.github/workflows/regression-tests.yaml @@ -39,36 +39,36 @@ jobs: echo "auto_succeed=true" >> $GITHUB_OUTPUT fi - regression_tests: - name: k8s regression tests (${{matrix.kube-e2e-test-type}}) - needs: prepare_env - runs-on: ubuntu-22.04 - timeout-minutes: 60 - if: ${{ !github.event.pull_request.draft }} - strategy: - fail-fast: false - matrix: - # upgrade tests are run on LTS but not on main branch, for main they are run nightly - # ingress will be deprecated from 1.17. Ref: https://solo-io-corp.slack.com/archives/G01EERAK3KJ/p1716389614777799 - # this is the github action version of ternary op - kube-e2e-test-type: [ 'gateway', 'gloo', 'upgrade' ] - kube-version: [ { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' } ] - image-variant: - - distroless - merge-to-main: - - ${{ github.event.pull_request.base.ref == 'main' }} - exclude: - - merge-to-main: true - kube-e2e-test-type: upgrade - steps: - - uses: actions/checkout@v4 - - id: run-tests - if: needs.prepare_env.outputs.should-auto-succeed-regression-tests != 'true' - uses: ./.github/workflows/composite-actions/regression-tests - - id: auto-succeed-tests - if: needs.prepare_env.outputs.should-auto-succeed-regression-tests == 'true' - run: | - echo "Regression tests auto-succeeded" + # regression_tests: + # name: k8s regression tests (${{matrix.kube-e2e-test-type}}) + # needs: prepare_env + # runs-on: ubuntu-22.04 + # timeout-minutes: 60 + # if: ${{ !github.event.pull_request.draft }} + # strategy: + # fail-fast: false + # matrix: + # # upgrade tests are run on LTS but not on main branch, for main they are run nightly + # # ingress will be deprecated from 1.17. Ref: https://solo-io-corp.slack.com/archives/G01EERAK3KJ/p1716389614777799 + # # this is the github action version of ternary op + # kube-e2e-test-type: [ 'gateway', 'gloo', 'upgrade' ] + # kube-version: [ { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' } ] + # image-variant: + # - distroless + # merge-to-main: + # - ${{ github.event.pull_request.base.ref == 'main' }} + # exclude: + # - merge-to-main: true + # kube-e2e-test-type: upgrade + # steps: + # - uses: actions/checkout@v4 + # - id: run-tests + # if: needs.prepare_env.outputs.should-auto-succeed-regression-tests != 'true' + # uses: ./.github/workflows/composite-actions/regression-tests + # - id: auto-succeed-tests + # if: needs.prepare_env.outputs.should-auto-succeed-regression-tests == 'true' + # run: | + # echo "Regression tests auto-succeeded" kube_gateway_api_conformance_tests: name: kubernetes gateway api conformance tests (${{matrix.image-variant}})