From 4da779c44c47f6d9225b02db1e4d476beb05e535 Mon Sep 17 00:00:00 2001 From: dthomson25 Date: Tue, 28 Aug 2018 16:40:32 -0700 Subject: [PATCH] Add PVC healthcheck to controller (#501) (#537) --- util/health/health.go | 26 ++++++++++++++++++++ util/health/health_test.go | 24 +++++++++++++++++++ util/health/testdata/pvc-bound.yaml | 34 +++++++++++++++++++++++++++ util/health/testdata/pvc-pending.yaml | 26 ++++++++++++++++++++ util/kube/kube.go | 15 ++++++------ 5 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 util/health/testdata/pvc-bound.yaml create mode 100644 util/health/testdata/pvc-pending.yaml diff --git a/util/health/health.go b/util/health/health.go index a3db39f88154c..733f5fb820d01 100644 --- a/util/health/health.go +++ b/util/health/health.go @@ -33,6 +33,8 @@ func GetAppHealth(obj *unstructured.Unstructured) (*appv1.HealthStatus, error) { health, err = getReplicaSetHealth(obj) case kube.DaemonSetKind: health, err = getDaemonSetHealth(obj) + case kube.PersistentVolumeClaimKind: + health, err = getPvcHealth(obj) default: health = &appv1.HealthStatus{Status: appv1.HealthStatusHealthy} } @@ -68,6 +70,29 @@ func IsWorse(current, new appv1.HealthStatusCode) bool { return newIndex > currentIndex } +func getPvcHealth(obj *unstructured.Unstructured) (*appv1.HealthStatus, error) { + obj, err := kube.ConvertToVersion(obj, "", "v1") + if err != nil { + return nil, err + } + var pvc coreV1.PersistentVolumeClaim + err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &pvc) + if err != nil { + return nil, err + } + + switch pvc.Status.Phase { + case coreV1.ClaimLost: + return &appv1.HealthStatus{Status: appv1.HealthStatusDegraded}, nil + case coreV1.ClaimPending: + return &appv1.HealthStatus{Status: appv1.HealthStatusProgressing}, nil + case coreV1.ClaimBound: + return &appv1.HealthStatus{Status: appv1.HealthStatusHealthy}, nil + default: + return &appv1.HealthStatus{Status: appv1.HealthStatusUnknown}, nil + } +} + func getIngressHealth(obj *unstructured.Unstructured) (*appv1.HealthStatus, error) { obj, err := kube.ConvertToVersion(obj, "extensions", "v1beta1") if err != nil { @@ -122,6 +147,7 @@ func getDeploymentHealth(obj *unstructured.Unstructured) (*appv1.HealthStatus, e if err != nil { return nil, err } + // Borrowed at kubernetes/kubectl/rollout_status.go https://github.com/kubernetes/kubernetes/blob/5232ad4a00ec93942d0b2c6359ee6cd1201b46bc/pkg/kubectl/rollout_status.go#L80 if deployment.Generation <= deployment.Status.ObservedGeneration { cond := getDeploymentCondition(deployment.Status, v1.DeploymentProgressing) diff --git a/util/health/health_test.go b/util/health/health_test.go index 634a20a3f576e..ef8df843bb4b7 100644 --- a/util/health/health_test.go +++ b/util/health/health_test.go @@ -58,3 +58,27 @@ func TestStatefulSetHealth(t *testing.T) { assert.NotNil(t, health) assert.Equal(t, appv1.HealthStatusHealthy, health.Status) } + +func TestPvcHealthy(t *testing.T) { + yamlBytes, err := ioutil.ReadFile("./testdata/pvc-bound.yaml") + assert.Nil(t, err) + var obj unstructured.Unstructured + err = yaml.Unmarshal(yamlBytes, &obj) + assert.Nil(t, err) + health, err := GetAppHealth(&obj) + assert.Nil(t, err) + assert.NotNil(t, health) + assert.Equal(t, appv1.HealthStatusHealthy, health.Status) +} + +func TestPvcPending(t *testing.T) { + yamlBytes, err := ioutil.ReadFile("./testdata/pvc-pending.yaml") + assert.Nil(t, err) + var obj unstructured.Unstructured + err = yaml.Unmarshal(yamlBytes, &obj) + assert.Nil(t, err) + health, err := GetAppHealth(&obj) + assert.Nil(t, err) + assert.NotNil(t, health) + assert.Equal(t, appv1.HealthStatusProgressing, health.Status) +} diff --git a/util/health/testdata/pvc-bound.yaml b/util/health/testdata/pvc-bound.yaml new file mode 100644 index 0000000000000..d728c8166d88f --- /dev/null +++ b/util/health/testdata/pvc-bound.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"e57a9040-a984-11e8-836b-c4b301c4d0d1","leaseDurationSeconds":15,"acquireTime":"2018-08-27T23:00:54Z","renewTime":"2018-08-27T23:00:56Z","leaderTransitions":0}' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"labels":{"applications.argoproj.io/app-name":"working-pvc"},"name":"testpvc","namespace":"argocd"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"2Gi"}}}} + pv.kubernetes.io/bind-completed: "yes" + pv.kubernetes.io/bound-by-controller: "yes" + volume.beta.kubernetes.io/storage-provisioner: docker.io/hostpath + creationTimestamp: 2018-08-27T23:00:54Z + finalizers: + - kubernetes.io/pvc-protection + labels: + applications.argoproj.io/app-name: working-pvc + name: testpvc + namespace: argocd + resourceVersion: "323170" + selfLink: /api/v1/namespaces/argocd/persistentvolumeclaims/testpvc + uid: 0cedda2c-aa4d-11e8-a271-025000000001 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + storageClassName: hostpath + volumeName: pvc-0cedda2c-aa4d-11e8-a271-025000000001 +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 2Gi + phase: Bound diff --git a/util/health/testdata/pvc-pending.yaml b/util/health/testdata/pvc-pending.yaml new file mode 100644 index 0000000000000..6748289e99c3d --- /dev/null +++ b/util/health/testdata/pvc-pending.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"labels":{"applications.argoproj.io/app-name":"working-pvc"},"name":"testpvc-2","namespace":"argocd"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"2Gi"}},"storageClassName":"slow"}} + volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/aws-ebs + creationTimestamp: 2018-08-27T23:00:54Z + finalizers: + - kubernetes.io/pvc-protection + labels: + applications.argoproj.io/app-name: working-pvc + name: testpvc-2 + namespace: argocd + resourceVersion: "323141" + selfLink: /api/v1/namespaces/argocd/persistentvolumeclaims/testpvc-2 + uid: 0cedfc44-aa4d-11e8-a271-025000000001 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + storageClassName: slow +status: + phase: Pending diff --git a/util/kube/kube.go b/util/kube/kube.go index 02a0d3b95af87..6a040b1124a3e 100644 --- a/util/kube/kube.go +++ b/util/kube/kube.go @@ -49,13 +49,14 @@ const ( ) const ( - ServiceKind = "Service" - EndpointsKind = "Endpoints" - DeploymentKind = "Deployment" - ReplicaSetKind = "ReplicaSet" - StatefulSetKind = "StatefulSet" - DaemonSetKind = "DaemonSet" - IngressKind = "Ingress" + ServiceKind = "Service" + EndpointsKind = "Endpoints" + DeploymentKind = "Deployment" + ReplicaSetKind = "ReplicaSet" + StatefulSetKind = "StatefulSet" + DaemonSetKind = "DaemonSet" + IngressKind = "Ingress" + PersistentVolumeClaimKind = "PersistentVolumeClaim" ) const (