From 7f4bd75ad5590ef44b3dfa34ad6afb7dc1392312 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Tue, 25 May 2021 16:12:58 -0700 Subject: [PATCH 1/9] plugin/k8s: Report on deploy and release status This commit implements reporting on a deploy status, and leaves the release implementation to a future commit. Some refactoring needs to be put in place to reduce code duplication across the Platform and Releaser components. This will be handled in the next commit --- builtin/k8s/platform.go | 141 ++++++++++++++++++++++++++++++++++++++++ builtin/k8s/release.go | 4 +- builtin/k8s/releaser.go | 29 +++++++++ go.mod | 2 +- go.sum | 4 ++ 5 files changed, 178 insertions(+), 2 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 09d57d8771d..da563cadbbc 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/golang/protobuf/ptypes" "github.com/hashicorp/go-hclog" "github.com/mitchellh/mapstructure" corev1 "k8s.io/api/core/v1" @@ -20,8 +21,10 @@ import ( "github.com/hashicorp/waypoint-plugin-sdk/component" "github.com/hashicorp/waypoint-plugin-sdk/docs" "github.com/hashicorp/waypoint-plugin-sdk/framework/resource" + sdk "github.com/hashicorp/waypoint-plugin-sdk/proto/gen" "github.com/hashicorp/waypoint-plugin-sdk/terminal" "github.com/hashicorp/waypoint/builtin/docker" + "github.com/hashicorp/waypoint/internal/clierrors" ) const ( @@ -70,6 +73,10 @@ func (p *Platform) ValidateAuth() error { return nil } +func (p *Platform) StatusFunc() interface{} { + return p.Status +} + // DefaultReleaserFunc implements component.PlatformReleaser func (p *Platform) DefaultReleaserFunc() interface{} { var rc ReleaserConfig @@ -641,6 +648,139 @@ func (p *Platform) Destroy( return rm.DestroyAll(ctx, log, sg, ui) } +func (p *Platform) Status( + ctx context.Context, + log hclog.Logger, + deployment *Deployment, + ui terminal.UI, +) (*sdk.StatusReport, error) { + sg := ui.StepGroup() + defer sg.Wait() + + step := sg.Add("Gathering health report for Kubernetes platform...") + defer step.Abort() + + csInfo, err := p.getClientset() + if err != nil { + return nil, err + } + clientSet := csInfo.Clientset + namespace := csInfo.Namespace + if p.config.Namespace != "" { + namespace = p.config.Namespace + } + + podClient := clientSet.CoreV1().Pods(namespace) + podLabelId := fmt.Sprintf("app=%s", deployment.Name) + podList, err := podClient.List(ctx, metav1.ListOptions{LabelSelector: podLabelId}) + if err != nil { + ui.Output( + "Error listing pods to determine application health: %s", clierrors.Humanize(err), + terminal.WithErrorStyle(), + ) + return nil, err + } + + // Create our status report + step.Update("Building status report for running pods...") + result := p.buildStatusReport(podList) + + result.TimeGenerated = ptypes.TimestampNow() + log.Debug("status report complete") + + // update output based on main health state + step.Update("Finished building report for Kubernetes platform") + step.Done() + + st := ui.Status() + defer st.Close() + + st.Update("Determining overall container health...") + if result.Health == sdk.StatusReport_READY { + st.Step(terminal.StatusOK, fmt.Sprintf("Deployment %q is reporting ready!", deployment.Name)) + } else if result.Health == sdk.StatusReport_PARTIAL { + st.Step(terminal.StatusWarn, fmt.Sprintf("Deployment %q is reporting partially available!", deployment.Name)) + } else { + st.Step(terminal.StatusError, fmt.Sprintf("Deployment %q is reporting not ready!", deployment.Name)) + } + + return &result, nil +} + +// Translate a K8S container status into a Waypoint Health Status +func (p *Platform) containerStatusToHealth( + containerStatus corev1.ContainerStatus, +) *sdk.StatusReport_Resource { + var resourceHealth sdk.StatusReport_Resource + resourceHealth.Name = containerStatus.Name + + // ContainerStatus.State is a struct of possible container states. If one + // is set, that is the current state of the container + if containerStatus.State.Running != nil || containerStatus.Ready == true { + resourceHealth.Health = sdk.StatusReport_READY + resourceHealth.HealthMessage = "container is reporting running" // no message defined by k8s api + } else if containerStatus.State.Waiting != nil { + resourceHealth.Health = sdk.StatusReport_PARTIAL + resourceHealth.HealthMessage = containerStatus.State.Waiting.Message + } else if containerStatus.State.Terminated != nil { + resourceHealth.Health = sdk.StatusReport_DOWN + resourceHealth.HealthMessage = containerStatus.State.Terminated.Message + } else { + resourceHealth.Health = sdk.StatusReport_UNKNOWN + resourceHealth.HealthMessage = "container health could not be determined" + } + + return &resourceHealth +} + +// Translate a Pod Phase into a Waypoint Health Status +func (p *Platform) podPhaseToHealth( + phase corev1.PodPhase, +) sdk.StatusReport_Health { + var healthResult sdk.StatusReport_Health + + // TODO: what about alive? + switch phase { + case corev1.PodPending: + healthResult = sdk.StatusReport_PARTIAL + case corev1.PodRunning: + healthResult = sdk.StatusReport_READY // ALIVE? READY? + case corev1.PodSucceeded: + healthResult = sdk.StatusReport_PARTIAL + case corev1.PodFailed: + healthResult = sdk.StatusReport_DOWN + case corev1.PodUnknown: + healthResult = sdk.StatusReport_UNKNOWN + default: + healthResult = sdk.StatusReport_UNKNOWN + } + + return healthResult +} + +func (p *Platform) buildStatusReport( + podList *corev1.PodList, +) sdk.StatusReport { + var result sdk.StatusReport + result.External = true + resources := make([]*sdk.StatusReport_Resource, len(podList.Items)) + + // Report on most recently observed status of a deployments pod + for _, pod := range podList.Items { + // Overall Pod Health + podStatus := pod.Status + result.HealthMessage = podStatus.Message + result.Health = p.podPhaseToHealth(podStatus.Phase) + + // Pod containers health + for i, containerStatus := range podStatus.ContainerStatuses { + resources[i] = p.containerStatusToHealth(containerStatus) + } + } + + return result +} + // Config is the configuration structure for the Platform. type Config struct { // Annotations are added to the pod spec of the deployed application. This is @@ -961,4 +1101,5 @@ var ( _ component.Configurable = (*Platform)(nil) _ component.Documented = (*Platform)(nil) _ component.Destroyer = (*Platform)(nil) + _ component.Status = (*Platform)(nil) ) diff --git a/builtin/k8s/release.go b/builtin/k8s/release.go index 1248abef27b..d6760b6b5c2 100644 --- a/builtin/k8s/release.go +++ b/builtin/k8s/release.go @@ -24,4 +24,6 @@ func (*Release) newService(name string) *corev1.Service { func (r *Release) URL() string { return r.Url } -var _ component.Release = (*Release)(nil) +var ( + _ component.Release = (*Release)(nil) +) diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index c545ee73323..5f4992a79b5 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/waypoint-plugin-sdk/component" "github.com/hashicorp/waypoint-plugin-sdk/docs" + sdk "github.com/hashicorp/waypoint-plugin-sdk/proto/gen" "github.com/hashicorp/waypoint-plugin-sdk/terminal" ) @@ -41,6 +42,11 @@ func (r *Releaser) DestroyFunc() interface{} { return r.Destroy } +// StatusFunc implements component.Status +func (r *Releaser) StatusFunc() interface{} { + return r.Status +} + // Release creates a Kubernetes service configured for the deployment func (r *Releaser) Release( ctx context.Context, @@ -268,6 +274,28 @@ func (r *Releaser) Destroy( return nil } +func (r *Releaser) Status( + ctx context.Context, + log hclog.Logger, + release *Release, + ui terminal.UI, +) (*sdk.StatusReport, error) { + sg := ui.StepGroup() + defer sg.Wait() + + step := sg.Add("Gathering health report for Kubernetes platform...") + defer step.Abort() + + // update output based on main health state + step.Update("Finished building report for Kubernetes platform") + step.Done() + + st := ui.Status() + defer st.Close() + + return &sdk.StatusReport{}, nil +} + // ReleaserConfig is the configuration structure for the Releaser. type ReleaserConfig struct { // Annotations to be applied to the kube service. @@ -377,4 +405,5 @@ var ( _ component.Destroyer = (*Releaser)(nil) _ component.Configurable = (*Releaser)(nil) _ component.Documented = (*Releaser)(nil) + _ component.Status = (*Releaser)(nil) ) diff --git a/go.mod b/go.mod index d8f95973317..c63bef21abf 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/hashicorp/vault/api v1.0.5-0.20200519221902-385fac77e20f github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9 - github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210518161041-53694fdd7d98 + github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526204726-417a273d6412 github.com/imdario/mergo v0.3.11 github.com/improbable-eng/grpc-web v0.13.0 github.com/kevinburke/go-bindata v3.22.0+incompatible diff --git a/go.sum b/go.sum index 134d801eb04..c7ca6fa57ea 100644 --- a/go.sum +++ b/go.sum @@ -686,6 +686,10 @@ github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9 h1:i9hzlv2S github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9/go.mod h1:ObgQSWSX9rsNofh16kctm6XxLW2QW1Ay6/9ris6T6DU= github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210518161041-53694fdd7d98 h1:9xcGgccOCi9PL1yj2pnhs8VN2mQYuC+Ym2RQMmR2zWA= github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210518161041-53694fdd7d98/go.mod h1:3EzMFm8svUyyR35eop57AZmFBqFbIJVx6/MkALuNEjo= +github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526195711-3299c82307ed h1:fp63GiII/QMZ+g62lxiPr8IydgmuWDRh4KboXBp5rWY= +github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526195711-3299c82307ed/go.mod h1:+E+oNFwlph6aW16gCCsk9HfElWuF70h5TrGjcvSmD9A= +github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526204726-417a273d6412 h1:qw6H1ef5BJsze++0GoPrFNXw0lV57mk2ADvdau0l2Jw= +github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526204726-417a273d6412/go.mod h1:3EzMFm8svUyyR35eop57AZmFBqFbIJVx6/MkALuNEjo= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= From 7951c6c72e7bd2d966cc574da9b93305699f3249 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 26 May 2021 15:12:35 -0700 Subject: [PATCH 2/9] Move status report funcs into shared util This commit moves some of the basic status report building funcs into a utils file so that they can be used by both the Platform and Releaser status funcs --- builtin/k8s/platform.go | 76 +---------------------------- builtin/k8s/releaser.go | 47 +++++++++++++++++- builtin/k8s/status_report_util.go | 80 +++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 76 deletions(-) create mode 100644 builtin/k8s/status_report_util.go diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index da563cadbbc..b7787f04c23 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -683,7 +683,7 @@ func (p *Platform) Status( // Create our status report step.Update("Building status report for running pods...") - result := p.buildStatusReport(podList) + result := buildStatusReport(podList) result.TimeGenerated = ptypes.TimestampNow() log.Debug("status report complete") @@ -707,80 +707,6 @@ func (p *Platform) Status( return &result, nil } -// Translate a K8S container status into a Waypoint Health Status -func (p *Platform) containerStatusToHealth( - containerStatus corev1.ContainerStatus, -) *sdk.StatusReport_Resource { - var resourceHealth sdk.StatusReport_Resource - resourceHealth.Name = containerStatus.Name - - // ContainerStatus.State is a struct of possible container states. If one - // is set, that is the current state of the container - if containerStatus.State.Running != nil || containerStatus.Ready == true { - resourceHealth.Health = sdk.StatusReport_READY - resourceHealth.HealthMessage = "container is reporting running" // no message defined by k8s api - } else if containerStatus.State.Waiting != nil { - resourceHealth.Health = sdk.StatusReport_PARTIAL - resourceHealth.HealthMessage = containerStatus.State.Waiting.Message - } else if containerStatus.State.Terminated != nil { - resourceHealth.Health = sdk.StatusReport_DOWN - resourceHealth.HealthMessage = containerStatus.State.Terminated.Message - } else { - resourceHealth.Health = sdk.StatusReport_UNKNOWN - resourceHealth.HealthMessage = "container health could not be determined" - } - - return &resourceHealth -} - -// Translate a Pod Phase into a Waypoint Health Status -func (p *Platform) podPhaseToHealth( - phase corev1.PodPhase, -) sdk.StatusReport_Health { - var healthResult sdk.StatusReport_Health - - // TODO: what about alive? - switch phase { - case corev1.PodPending: - healthResult = sdk.StatusReport_PARTIAL - case corev1.PodRunning: - healthResult = sdk.StatusReport_READY // ALIVE? READY? - case corev1.PodSucceeded: - healthResult = sdk.StatusReport_PARTIAL - case corev1.PodFailed: - healthResult = sdk.StatusReport_DOWN - case corev1.PodUnknown: - healthResult = sdk.StatusReport_UNKNOWN - default: - healthResult = sdk.StatusReport_UNKNOWN - } - - return healthResult -} - -func (p *Platform) buildStatusReport( - podList *corev1.PodList, -) sdk.StatusReport { - var result sdk.StatusReport - result.External = true - resources := make([]*sdk.StatusReport_Resource, len(podList.Items)) - - // Report on most recently observed status of a deployments pod - for _, pod := range podList.Items { - // Overall Pod Health - podStatus := pod.Status - result.HealthMessage = podStatus.Message - result.Health = p.podPhaseToHealth(podStatus.Phase) - - // Pod containers health - for i, containerStatus := range podStatus.ContainerStatuses { - resources[i] = p.containerStatusToHealth(containerStatus) - } - } - - return result -} - // Config is the configuration structure for the Platform. type Config struct { // Annotations are added to the pod spec of the deployed application. This is diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 5f4992a79b5..0147a794246 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -6,7 +6,9 @@ import ( "strconv" "time" + "github.com/golang/protobuf/ptypes" "github.com/hashicorp/go-hclog" + "github.com/hashicorp/waypoint/internal/clierrors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -286,6 +288,40 @@ func (r *Releaser) Status( step := sg.Add("Gathering health report for Kubernetes platform...") defer step.Abort() + // Get our client + clientset, namespace, _, err := clientset(r.config.KubeconfigPath, r.config.Context) + if err != nil { + return nil, err + } + if r.config.Namespace != "" { + namespace = r.config.Namespace + } + + serviceclient := clientset.CoreV1().Services(namespace) + service, err := serviceclient.Get(ctx, release.ServiceName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + appName := service.Spec.Selector["name"] + podClient := clientset.CoreV1().Pods(namespace) + podLabelId := fmt.Sprintf("app=%s", appName) + podList, err := podClient.List(ctx, metav1.ListOptions{LabelSelector: podLabelId}) + if err != nil { + ui.Output( + "Error listing pods to determine application health: %s", clierrors.Humanize(err), + terminal.WithErrorStyle(), + ) + return nil, err + } + + // Create our status report + step.Update("Building status report for running pods...") + result := buildStatusReport(podList) + + result.TimeGenerated = ptypes.TimestampNow() + log.Debug("status report complete") + // update output based on main health state step.Update("Finished building report for Kubernetes platform") step.Done() @@ -293,7 +329,16 @@ func (r *Releaser) Status( st := ui.Status() defer st.Close() - return &sdk.StatusReport{}, nil + st.Update("Determining overall container health...") + if result.Health == sdk.StatusReport_READY { + st.Step(terminal.StatusOK, fmt.Sprintf("Release %q is reporting ready!", appName)) + } else if result.Health == sdk.StatusReport_PARTIAL { + st.Step(terminal.StatusWarn, fmt.Sprintf("Release %q is reporting partially available!", appName)) + } else { + st.Step(terminal.StatusError, fmt.Sprintf("Release %q is reporting not ready!", appName)) + } + + return &result, nil } // ReleaserConfig is the configuration structure for the Releaser. diff --git a/builtin/k8s/status_report_util.go b/builtin/k8s/status_report_util.go new file mode 100644 index 00000000000..82124de2f4e --- /dev/null +++ b/builtin/k8s/status_report_util.go @@ -0,0 +1,80 @@ +package k8s + +import ( + corev1 "k8s.io/api/core/v1" + + sdk "github.com/hashicorp/waypoint-plugin-sdk/proto/gen" +) + +// Translate a K8S container status into a Waypoint Health Status +func containerStatusToHealth( + containerStatus corev1.ContainerStatus, +) *sdk.StatusReport_Resource { + var resourceHealth sdk.StatusReport_Resource + resourceHealth.Name = containerStatus.Name + + // ContainerStatus.State is a struct of possible container states. If one + // is set, that is the current state of the container + if containerStatus.State.Running != nil || containerStatus.Ready == true { + resourceHealth.Health = sdk.StatusReport_READY + resourceHealth.HealthMessage = "container is reporting running" // no message defined by k8s api + } else if containerStatus.State.Waiting != nil { + resourceHealth.Health = sdk.StatusReport_PARTIAL + resourceHealth.HealthMessage = containerStatus.State.Waiting.Message + } else if containerStatus.State.Terminated != nil { + resourceHealth.Health = sdk.StatusReport_DOWN + resourceHealth.HealthMessage = containerStatus.State.Terminated.Message + } else { + resourceHealth.Health = sdk.StatusReport_UNKNOWN + resourceHealth.HealthMessage = "container health could not be determined" + } + + return &resourceHealth +} + +// Translate a Pod Phase into a Waypoint Health Status +func podPhaseToHealth( + phase corev1.PodPhase, +) sdk.StatusReport_Health { + var healthResult sdk.StatusReport_Health + + switch phase { + case corev1.PodPending: + healthResult = sdk.StatusReport_PARTIAL + case corev1.PodRunning: + healthResult = sdk.StatusReport_READY + case corev1.PodSucceeded: + healthResult = sdk.StatusReport_PARTIAL + case corev1.PodFailed: + healthResult = sdk.StatusReport_DOWN + case corev1.PodUnknown: + healthResult = sdk.StatusReport_UNKNOWN + default: + healthResult = sdk.StatusReport_UNKNOWN + } + + return healthResult +} + +func buildStatusReport( + podList *corev1.PodList, +) sdk.StatusReport { + var result sdk.StatusReport + result.External = true + resources := make([]*sdk.StatusReport_Resource, len(podList.Items)) + + // Report on most recently observed status of a deployments pod + for _, pod := range podList.Items { + // Overall Pod Health + podStatus := pod.Status + result.HealthMessage = podStatus.Message + result.Health = podPhaseToHealth(podStatus.Phase) + + // Pod containers health + for i, containerStatus := range podStatus.ContainerStatuses { + resources[i] = containerStatusToHealth(containerStatus) + } + } + + return result +} From 174c40104f6c8716ae442c8730053048ad657ca0 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 26 May 2021 15:13:16 -0700 Subject: [PATCH 3/9] go mod tidy --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index c7ca6fa57ea..d5be47c461e 100644 --- a/go.sum +++ b/go.sum @@ -684,10 +684,6 @@ github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef h1:YKouRHFf github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef/go.mod h1:cAGI4nVnEfAyMeqt9oB+Mase8DNn3qA/LDNHURiwssY= github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9 h1:i9hzlv2SpmaNcQ8ZLGn01fp2Gqyejj0juVs7rYDgecE= github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9/go.mod h1:ObgQSWSX9rsNofh16kctm6XxLW2QW1Ay6/9ris6T6DU= -github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210518161041-53694fdd7d98 h1:9xcGgccOCi9PL1yj2pnhs8VN2mQYuC+Ym2RQMmR2zWA= -github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210518161041-53694fdd7d98/go.mod h1:3EzMFm8svUyyR35eop57AZmFBqFbIJVx6/MkALuNEjo= -github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526195711-3299c82307ed h1:fp63GiII/QMZ+g62lxiPr8IydgmuWDRh4KboXBp5rWY= -github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526195711-3299c82307ed/go.mod h1:+E+oNFwlph6aW16gCCsk9HfElWuF70h5TrGjcvSmD9A= github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526204726-417a273d6412 h1:qw6H1ef5BJsze++0GoPrFNXw0lV57mk2ADvdau0l2Jw= github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210526204726-417a273d6412/go.mod h1:3EzMFm8svUyyR35eop57AZmFBqFbIJVx6/MkALuNEjo= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= From a08b219a333f2659271ce0e2106976bce50f0918 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 26 May 2021 15:22:09 -0700 Subject: [PATCH 4/9] Adjust status for PodPending phase --- builtin/k8s/status_report_util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/k8s/status_report_util.go b/builtin/k8s/status_report_util.go index 82124de2f4e..afb97d2dabb 100644 --- a/builtin/k8s/status_report_util.go +++ b/builtin/k8s/status_report_util.go @@ -40,7 +40,7 @@ func podPhaseToHealth( switch phase { case corev1.PodPending: - healthResult = sdk.StatusReport_PARTIAL + healthResult = sdk.StatusReport_ALIVE case corev1.PodRunning: healthResult = sdk.StatusReport_READY case corev1.PodSucceeded: @@ -56,6 +56,7 @@ func podPhaseToHealth( return healthResult } +// Take a list of Pods and build a Waypoint Status Report based on their reported health func buildStatusReport( podList *corev1.PodList, ) sdk.StatusReport { From 9a90b3cfce263e6f5599f59708a5b3bb3f31266e Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Wed, 26 May 2021 15:32:49 -0700 Subject: [PATCH 5/9] Print resource health if overall status is not ready --- builtin/k8s/platform.go | 7 +++++++ builtin/k8s/releaser.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index b7787f04c23..0c48a4c8727 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -704,6 +704,13 @@ func (p *Platform) Status( st.Step(terminal.StatusError, fmt.Sprintf("Deployment %q is reporting not ready!", deployment.Name)) } + // More UI detail for non-ready resources + for _, resource := range result.Resources { + if resource.Health != sdk.StatusReport_READY { + st.Step(terminal.StatusWarn, fmt.Sprintf("Resource %q is reporting %q", resource.Name, resource.Health.String())) + } + } + return &result, nil } diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 0147a794246..6d4f73a93ca 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -338,6 +338,13 @@ func (r *Releaser) Status( st.Step(terminal.StatusError, fmt.Sprintf("Release %q is reporting not ready!", appName)) } + // More UI detail for non-ready resources + for _, resource := range result.Resources { + if resource.Health != sdk.StatusReport_READY { + st.Step(terminal.StatusWarn, fmt.Sprintf("Resource %q is reporting %q", resource.Name, resource.Health.String())) + } + } + return &result, nil } From c1c895d540e79ef4b27148d88136ed4422e44f02 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Thu, 27 May 2021 08:48:52 -0700 Subject: [PATCH 6/9] Report on all possible pod status --- builtin/k8s/status_report_util.go | 53 ++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/builtin/k8s/status_report_util.go b/builtin/k8s/status_report_util.go index afb97d2dabb..997f5dec3b8 100644 --- a/builtin/k8s/status_report_util.go +++ b/builtin/k8s/status_report_util.go @@ -62,20 +62,63 @@ func buildStatusReport( ) sdk.StatusReport { var result sdk.StatusReport result.External = true - resources := make([]*sdk.StatusReport_Resource, len(podList.Items)) + var resources []*sdk.StatusReport_Resource + + // Build health for every possible pod for overal health report + var ready, alive, down, unknown int // Report on most recently observed status of a deployments pod + // Pod resources and its containers will be in order inside Resources for _, pod := range podList.Items { // Overall Pod Health podStatus := pod.Status - result.HealthMessage = podStatus.Message - result.Health = podPhaseToHealth(podStatus.Phase) + + // Determine overall health report based on all pods + switch podStatus.Phase { + case corev1.PodPending: + alive++ + case corev1.PodRunning: + ready++ + case corev1.PodSucceeded: + alive++ + case corev1.PodFailed: + down++ + case corev1.PodUnknown: + unknown++ + default: + unknown++ + } + + podHealth := podPhaseToHealth(podStatus.Phase) + resources = append(resources, &sdk.StatusReport_Resource{ + Health: podHealth, + HealthMessage: podStatus.Message, + Name: pod.ObjectMeta.Name, + }) // Pod containers health - for i, containerStatus := range podStatus.ContainerStatuses { - resources[i] = containerStatusToHealth(containerStatus) + for _, containerStatus := range podStatus.ContainerStatuses { + resources = append(resources, containerStatusToHealth(containerStatus)) } } + // Overall health status for report + if ready == len(podList.Items) { + result.Health = sdk.StatusReport_READY + result.HealthMessage = "all pods are reporting ready" + } else if down == len(podList.Items) { + result.Health = sdk.StatusReport_DOWN + result.HealthMessage = "all pods are reporting down" + } else if unknown == len(podList.Items) { + result.Health = sdk.StatusReport_UNKNOWN + result.HealthMessage = "status of all pods cannot be determined" + } else if alive == len(podList.Items) { + result.Health = sdk.StatusReport_ALIVE + result.HealthMessage = "all pods are reporting alive" + } else { + result.Health = sdk.StatusReport_PARTIAL + result.HealthMessage = "all pods are reporting a mixed statusu" + } + return result } From 8214ecee2c60df28d6e0f9552b7bdc39a3ee0d15 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Thu, 27 May 2021 08:55:51 -0700 Subject: [PATCH 7/9] Add note about ui.Status --- builtin/k8s/platform.go | 2 ++ builtin/k8s/releaser.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 0c48a4c8727..09d2dbf3389 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -692,6 +692,8 @@ func (p *Platform) Status( step.Update("Finished building report for Kubernetes platform") step.Done() + // NOTE(briancain): Replace ui.Status with StepGroups once this bug + // has been fixed: https://github.com/hashicorp/waypoint/issues/1536 st := ui.Status() defer st.Close() diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 6d4f73a93ca..8fdcc726bd4 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -326,6 +326,8 @@ func (r *Releaser) Status( step.Update("Finished building report for Kubernetes platform") step.Done() + // NOTE(briancain): Replace ui.Status with StepGroups once this bug + // has been fixed: https://github.com/hashicorp/waypoint/issues/1536 st := ui.Status() defer st.Close() From e12e41a9db74b35d838dc17f509bb0d9fd940eb0 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Thu, 27 May 2021 09:46:08 -0700 Subject: [PATCH 8/9] Defer in func for handling future step additions --- builtin/k8s/platform.go | 2 +- builtin/k8s/releaser.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 09d2dbf3389..26b3bf7b2c1 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -658,7 +658,7 @@ func (p *Platform) Status( defer sg.Wait() step := sg.Add("Gathering health report for Kubernetes platform...") - defer step.Abort() + defer func() { step.Abort() }() // Defer in func in case more steps are added to this func in the future csInfo, err := p.getClientset() if err != nil { diff --git a/builtin/k8s/releaser.go b/builtin/k8s/releaser.go index 8fdcc726bd4..4fdcb30e3c8 100644 --- a/builtin/k8s/releaser.go +++ b/builtin/k8s/releaser.go @@ -62,7 +62,7 @@ func (r *Releaser) Release( sg := ui.StepGroup() step := sg.Add("Initializing Kubernetes client...") - defer step.Abort() + defer func() { step.Abort() }() // Defer in func in case more steps are added to this func in the future // Get our clientset clientset, ns, config, err := clientset(r.config.KubeconfigPath, r.config.Context) From d489e3267837b88837135da74e575be6bef8fb8c Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Thu, 27 May 2021 10:22:12 -0700 Subject: [PATCH 9/9] Additional checks on Pod condition for determining overall report health This commit adds some extra checks on the Pods condition when its phase is running to give us some extra detail about if the pod is actually ready, or just running and alive --- builtin/k8s/status_report_util.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/builtin/k8s/status_report_util.go b/builtin/k8s/status_report_util.go index 997f5dec3b8..acfb508f590 100644 --- a/builtin/k8s/status_report_util.go +++ b/builtin/k8s/status_report_util.go @@ -78,7 +78,15 @@ func buildStatusReport( case corev1.PodPending: alive++ case corev1.PodRunning: - ready++ + // Extra checks on the latest condition to ensure pod is reporting ready and running + for _, c := range podStatus.Conditions { + if c.Status == corev1.ConditionTrue && c.Type == corev1.PodReady { + ready++ + break + } + } + + alive++ case corev1.PodSucceeded: alive++ case corev1.PodFailed: