From 44b046bdf019bfb41aae7ec4c0676479db932d7a Mon Sep 17 00:00:00 2001 From: Nahshon Unna Tsameret <60659093+nunnatsa@users.noreply.github.com> Date: Mon, 2 Sep 2024 03:50:02 -0400 Subject: [PATCH] kubevirtci lanes: run in a pod (#3077) * kubevirtci lanes: run in a pod Run the functional test from a pod, as done in openshift-ci lanes, to get the same tests. Enable the namespace webhook in K8s, and enable the delete namespace functional test. Signed-off-by: Nahshon Unna-Tsameret * Allow setting KUBEVIRTCI_TAG for functional tests Signed-off-by: Nahshon Unna-Tsameret --------- Signed-off-by: Nahshon Unna-Tsameret --- automation/test.sh | 10 ++----- build/Dockerfile | 3 +- build/Dockerfile.artifacts | 3 +- build/Dockerfile.functest | 3 +- build/Dockerfile.webhook | 3 +- cluster/kubevirtci.sh | 3 +- deploy/webhooks.yaml | 53 ++++++++++++++++++++++++++++++++++ hack/common.sh | 2 +- hack/run-tests-in-container.sh | 5 ++-- hack/test_delete_ns.sh | 52 +++++++++++++++------------------ 10 files changed, 93 insertions(+), 44 deletions(-) diff --git a/automation/test.sh b/automation/test.sh index aa5b6da92c..31351f054f 100755 --- a/automation/test.sh +++ b/automation/test.sh @@ -14,11 +14,5 @@ make cluster-up trap '{ make cluster-down; }' EXIT SIGINT SIGTERM SIGSTOP make cluster-sync -make ci-functest - -# Upgrade test requires OLM which is currently -# only available with okd providers -if [[ $TARGET =~ okd-.* || $TARGET =~ ocp-.* ]]; then - make upgrade-test - make ci-functest -fi +export KUBECONFIG=$(_kubevirtci/cluster-up/kubeconfig.sh) +JOB_TYPE="stdci" GINKGO_LABELS=${GINKGO_LABELS} make functest diff --git a/build/Dockerfile b/build/Dockerfile index ce5de5fe37..2cdd378d6c 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -28,4 +28,5 @@ ARG git_url=https://github.com/kubevirt/hyperconverged-cluster-operator.git ARG git_sha=NONE LABEL multi.GIT_URL=${git_url} \ - multi.GIT_SHA=${git_sha} + multi.GIT_SHA=${git_sha} \ + app=hyperconverged-cluster-operator diff --git a/build/Dockerfile.artifacts b/build/Dockerfile.artifacts index 0a7ee43ca2..47b1215290 100644 --- a/build/Dockerfile.artifacts +++ b/build/Dockerfile.artifacts @@ -46,7 +46,8 @@ ARG git_url=https://github.com/kubevirt/hyperconverged-cluster-operator.git ARG git_sha=NONE LABEL multi.GIT_URL=${git_url} \ - multi.GIT_SHA=${git_sha} + multi.GIT_SHA=${git_sha} \ + app=virt-artifacts-server CMD if [[ -d "/proc/sys/net/ipv4" && -d "/proc/sys/net/ipv6" ]]; \ then \ diff --git a/build/Dockerfile.functest b/build/Dockerfile.functest index 5da8a658e4..1acde93705 100644 --- a/build/Dockerfile.functest +++ b/build/Dockerfile.functest @@ -29,4 +29,5 @@ ARG git_url=https://github.com/kubevirt/hyperconverged-cluster-operator.git ARG git_sha=NONE LABEL multi.GIT_URL=${git_url} \ - multi.GIT_SHA=${git_sha} + multi.GIT_SHA=${git_sha} \ + app=hyperconverged-cluster-functest diff --git a/build/Dockerfile.webhook b/build/Dockerfile.webhook index 4da04d5993..35006b6a6d 100644 --- a/build/Dockerfile.webhook +++ b/build/Dockerfile.webhook @@ -26,4 +26,5 @@ ARG git_url=https://github.com/kubevirt/hyperconverged-cluster-operator.git ARG git_sha=NONE LABEL multi.GIT_URL=${git_url} \ - multi.GIT_SHA=${git_sha} + multi.GIT_SHA=${git_sha} \ + app=hyperconverged-cluster-webhook diff --git a/cluster/kubevirtci.sh b/cluster/kubevirtci.sh index 0da40904a5..2905257d3e 100644 --- a/cluster/kubevirtci.sh +++ b/cluster/kubevirtci.sh @@ -13,7 +13,8 @@ # limitations under the License. export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-'k8s-1.30'} -export KUBEVIRTCI_TAG=$(curl -L -Ss https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirtci/latest) +export LATEST_KUBEVIRTCI_TAG=$(curl -L -Ss https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirtci/latest) +export KUBEVIRTCI_TAG=${KUBEVIRTCI_TAG:-${LATEST_KUBEVIRTCI_TAG}} KUBEVIRTCI_PATH="${PWD}/_kubevirtci" KUBEVIRTCI_REPO='https://github.com/kubevirt/kubevirtci.git' diff --git a/deploy/webhooks.yaml b/deploy/webhooks.yaml index 229e43d1cb..a60524287a 100644 --- a/deploy/webhooks.yaml +++ b/deploy/webhooks.yaml @@ -56,6 +56,59 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: 30 +- admissionReviewVersions: + - v1beta1 + - v1 + clientConfig: + # caBundle: WILL BE INJECTED BY CERT-MANAGER BECAUSE OF THE ANNOTATION + service: + name: hyperconverged-cluster-webhook-service + namespace: kubevirt-hyperconverged + path: /mutate-ns-hco-kubevirt-io + port: 4343 + failurePolicy: Fail + name: mutate-ns-hco.kubevirt.io + objectSelector: + matchLabels: + kubernetes.io/metadata.name: kubevirt-hyperconverged + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: + - namespaces + sideEffects: NoneOnDryRun + timeoutSeconds: 10 + #type: MutatingAdmissionWebhook +- admissionReviewVersions: + - v1beta1 + - v1 + clientConfig: + # caBundle: WILL BE INJECTED BY CERT-MANAGER BECAUSE OF THE ANNOTATION + service: + name: hyperconverged-cluster-webhook-service + namespace: kubevirt-hyperconverged + path: /mutate-hco-kubevirt-io-v1beta1-hyperconverged + port: 4343 + failurePolicy: Fail + name: mutate-hyperconverged-hco.kubevirt.io + rules: + - apiGroups: + - hco.kubevirt.io + apiVersions: + - v1alpha1 + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - hyperconvergeds + sideEffects: NoneOnDryRun + timeoutSeconds: 10 + #type: MutatingAdmissionWebhook --- apiVersion: cert-manager.io/v1 kind: Certificate diff --git a/hack/common.sh b/hack/common.sh index 46286f4e7d..1977998f62 100644 --- a/hack/common.sh +++ b/hack/common.sh @@ -31,7 +31,7 @@ CNA_URL_PREFIX="https://github.com/kubevirt/cluster-network-addons-operator/rele mem_size=${KUBEVIRT_MEMORY_SIZE:-5120M} num_nodes=${KUBEVIRT_NUM_NODES:-1} -KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.17} +KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.30} BASE_PATH=${KUBEVIRTCI_CONFIG_PATH:-$PWD} KUBEVIRTCI_PATH=$(kubevirtci::path) CMD=${CMD:-} diff --git a/hack/run-tests-in-container.sh b/hack/run-tests-in-container.sh index 490c9b76fe..a35897877f 100755 --- a/hack/run-tests-in-container.sh +++ b/hack/run-tests-in-container.sh @@ -5,6 +5,7 @@ set -exuo pipefail INSTALLED_NAMESPACE=${INSTALLED_NAMESPACE:-"kubevirt-hyperconverged"} OUTPUT_DIR=${ARTIFACT_DIR:-"$(pwd)/_out"} +FUNCTEST_IMAGE=${FUNCTEST_IMAGE:-} source hack/common.sh source cluster/kubevirtci.sh @@ -17,7 +18,7 @@ if [ "${JOB_TYPE}" == "stdci" ]; then KUBECTL_BINARY="cluster/kubectl.sh" fi -if [[ ${JOB_TYPE} = "prow" ]]; then +if [[ ${JOB_TYPE} = "prow" && -n ${FUNCTEST_IMAGE} ]]; then KUBECTL_BINARY="oc" computed_test_image=${FUNCTEST_IMAGE} else @@ -126,4 +127,4 @@ echo "Exiting... Exit code: $exitCode" # Brutally delete HCO removing the namespace where it's running" source hack/test_delete_ns.sh -test_delete_ns +CMD=${KUBECTL_BINARY} test_delete_ns diff --git a/hack/test_delete_ns.sh b/hack/test_delete_ns.sh index 26dc1d55a4..faddf97d30 100755 --- a/hack/test_delete_ns.sh +++ b/hack/test_delete_ns.sh @@ -17,40 +17,36 @@ # Copyright 2020 Red Hat, Inc. # +INSTALLED_NAMESPACE=${INSTALLED_NAMESPACE:-"kubevirt-hyperconverged"} function test_delete_ns(){ set -ex - if [ "${CMD}" == "oc" ]; then - echo "Trying to delete kubevirt-hyperconverged namespace when the hyperconverged CR is still there" - # this should fail with a clear error message - DELETE_ERROR_TEXT="$(${CMD} delete namespace kubevirt-hyperconverged 2>&1 || true)" - - # try to mitigate CI flakiness when we randomly get - # "x509: certificate signed by unknown authority" errors - if [[ $DELETE_ERROR_TEXT == *"x509: certificate signed by unknown authority"* ]]; then - # gave it time to recovery - sleep 300 - DELETE_ERROR_TEXT="$(${CMD} delete namespace kubevirt-hyperconverged 2>&1 || true)" - fi - # and eventually try again... - if [[ $DELETE_ERROR_TEXT == *"x509: certificate signed by unknown authority"* ]]; then - sleep 300 - DELETE_ERROR_TEXT="$(${CMD} delete namespace kubevirt-hyperconverged 2>&1 || true)" - fi - - echo "${DELETE_ERROR_TEXT}" | grep "denied the request: HyperConverged CR is still present, please remove it before deleting the containing hcoNamespace" - - echo "kubevirt-hyperconverged namespace should be still there" - ${CMD} get namespace kubevirt-hyperconverged -o yaml - - else - echo "Ignoring webhook on k8s where we don't have OLM based validating webhooks" + echo "Trying to delete ${INSTALLED_NAMESPACE} namespace when the hyperconverged CR is still there" + # this should fail with a clear error message + DELETE_ERROR_TEXT="$(${CMD} delete namespace ${INSTALLED_NAMESPACE} 2>&1 || true)" + + # try to mitigate CI flakiness when we randomly get + # "x509: certificate signed by unknown authority" errors + if [[ $DELETE_ERROR_TEXT == *"x509: certificate signed by unknown authority"* ]]; then + # gave it time to recovery + sleep 300 + DELETE_ERROR_TEXT="$(${CMD} delete namespace ${INSTALLED_NAMESPACE} 2>&1 || true)" fi + # and eventually try again... + if [[ $DELETE_ERROR_TEXT == *"x509: certificate signed by unknown authority"* ]]; then + sleep 300 + DELETE_ERROR_TEXT="$(${CMD} delete namespace ${INSTALLED_NAMESPACE} 2>&1 || true)" + fi + + echo "${DELETE_ERROR_TEXT}" | grep "denied the request: HyperConverged CR is still present, please remove it before deleting the containing hcoNamespace" + + echo "${INSTALLED_NAMESPACE} namespace should be still there" + ${CMD} get namespace ${INSTALLED_NAMESPACE} -o yaml echo "Delete the hyperconverged CR to remove the product" - timeout 10m ${CMD} delete hyperconverged -n kubevirt-hyperconverged kubevirt-hyperconverged + timeout 10m ${CMD} delete hyperconverged -n ${INSTALLED_NAMESPACE} kubevirt-hyperconverged - echo "Finally delete kubevirt-hyperconverged namespace" - timeout 10m ${CMD} delete namespace kubevirt-hyperconverged + echo "Finally delete ${INSTALLED_NAMESPACE} namespace" + timeout 10m ${CMD} delete namespace ${INSTALLED_NAMESPACE} }