From 73d9428760ac2cdcda7979bc871e8e90639be23a Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 14 Jan 2021 08:47:57 +0100 Subject: [PATCH] e2e: fetch volume metrics from Kubelet Test if metrics are available at all. The actual values are a little difficult to validate. Block-mode volumes are not supported by Kubernetes <= 1.20 yet. See-also: kubernetes/kubernetes#97972 Signed-off-by: Niels de Vos --- e2e/node.go | 19 +++++++++++++++++++ e2e/pvc.go | 42 ++++++++++++++++++++++++++++++++++++++++++ e2e/utils.go | 6 ++++++ scripts/minikube.sh | 11 +++++++---- 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/e2e/node.go b/e2e/node.go index 9a4546af226f..992c6c371a4f 100644 --- a/e2e/node.go +++ b/e2e/node.go @@ -2,7 +2,9 @@ package e2e import ( "context" + "errors" + core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -42,3 +44,20 @@ func checkNodeHasLabel(c kubernetes.Interface, labelKey, labelValue string) erro } return nil } + +// List all nodes in the cluster (we have one), and return the IP-address. +// Possibly need to add a selector, pick the node where a Pod is running? +func getKubeletIP(c kubernetes.Interface) (string, error) { + nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return "", err + } + + for _, address := range nodes.Items[0].Status.Addresses { + if address.Type == core.NodeInternalIP { + return address.Address, nil + } + } + + return "", errors.New("could not find internal IP for node") +} diff --git a/e2e/pvc.go b/e2e/pvc.go index 7ec4264c5e14..063c8d103582 100644 --- a/e2e/pvc.go +++ b/e2e/pvc.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" v1 "k8s.io/api/core/v1" @@ -235,3 +236,44 @@ func checkPVSelectorValuesForPVC(f *framework.Framework, pvc *v1.PersistentVolum } return nil } + +func getMetricsForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, t int) error { + kubelet, err := getKubeletIP(f.ClientSet) + if err != nil { + return err + } + + // kubelet needs to be started with --read-only-port=10255 + cmd := fmt.Sprintf("curl --silent 'http://%s:10255/metrics'", kubelet) + + // retry as kubelet does not immediately have the metrics available + timeout := time.Duration(t) * time.Minute + return wait.PollImmediate(poll, timeout, func() (bool, error) { + stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + e2elog.Logf("failed to get metrics for pvc %q (%v): %v", pvc.Name, err, stdErr) + return false, nil + } + if stdOut == "" { + e2elog.Logf("no metrics received from kublet on IP %s", kubelet) + return false, nil + } + + namespace := fmt.Sprintf("namespace=%q", pvc.Namespace) + name := fmt.Sprintf("persistentvolumeclaim=%q", pvc.Name) + + for _, line := range strings.Split(stdOut, "\n") { + if !strings.HasPrefix(line, "kubelet_volume_stats_") { + continue + } + if strings.Contains(line, namespace) && strings.Contains(line, name) { + // TODO: validate metrics if possible + e2elog.Logf("found metrics for pvc %s/%s: %s", pvc.Namespace, pvc.Name, line) + return true, nil + } + } + + e2elog.Logf("no metrics found for pvc %s/%s", pvc.Namespace, pvc.Name) + return false, nil + }) +} diff --git a/e2e/utils.go b/e2e/utils.go index c45d074678d9..aeb8c75fe1d8 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -322,6 +322,12 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error { if stdErr != "" { return fmt.Errorf("failed to touch a file as non-root user %v", stdErr) } + + err = getMetricsForPVC(f, pvc, deployTimeout) + if err != nil { + return err + } + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) if err != nil { return err diff --git a/scripts/minikube.sh b/scripts/minikube.sh index 4ab46ace501d..c1b478764705 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -175,7 +175,7 @@ fi K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,ExpandCSIVolumes=true"} #extra-config for kube https://minikube.sigs.k8s.io/docs/reference/configuration/kubernetes/ -EXTRA_CONFIG=${EXTRA_CONFIG:-"--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy"} +EXTRA_CONFIG_PSP="--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy" # kubelet.resolv-conf needs to point to a file, not a symlink # the default minikube VM has /etc/resolv.conf -> /run/systemd/resolve/resolv.conf @@ -192,6 +192,9 @@ EXTRA_CONFIG="${EXTRA_CONFIG} --extra-config=kubelet.resolv-conf=${RESOLV_CONF}" #extra Rook configuration ROOK_BLOCK_POOL_NAME=${ROOK_BLOCK_POOL_NAME:-"newrbdpool"} +# enable read-only anonymous access to kubelet metrics +EXTRA_CONFIG="${EXTRA_CONFIG} --extra-config=kubelet.read-only-port=10255" + if [[ "${KUBE_VERSION}" == "latest" ]]; then # update the version string from latest with the real version KUBE_VERSION=$(curl -L https://storage.googleapis.com/kubernetes-release/release/stable.txt 2> /dev/null) @@ -215,16 +218,16 @@ up) if minikube_supports_psp; then enable_psp # shellcheck disable=SC2086 - ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" ${EXTRA_CONFIG} + ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" ${EXTRA_CONFIG} ${EXTRA_CONFIG_PSP} else # This is a workaround to fix psp issues in minikube >1.6.2 and <1.11.0 # shellcheck disable=SC2086 - ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" + ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" ${EXTRA_CONFIG} DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" ${minikube} kubectl -- apply -f "$DIR"/psp.yaml ${minikube} stop # shellcheck disable=SC2086 - ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" ${EXTRA_CONFIG} + ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" ${EXTRA_CONFIG} ${EXTRA_CONFIG_PSP} fi # create a link so the default dataDirHostPath will work for this