From 5664436e20cfa37672f4ce8544575d0ae90fcde3 Mon Sep 17 00:00:00 2001
From: Robert Jacob
string
alias)
+PodStatus is a short description of the status a Pod can be in.
+Value | +Description | +
---|---|
"Failed" |
+PodFailed means that all containers in the pod have terminated, and at least one container has +terminated in a failure (exited with a non-zero exit code or was stopped by the system). + |
+
"Pending" |
+PodPending means the pod has been accepted by the system, but one or more of the containers +has not been started. This includes time before being bound to a node, as well as time spent +pulling images onto the host. + |
+
"Ready" |
+PodReady means the pod has been started and the readiness probe reports a successful status. + |
+
"Running" |
+PodRunning means the pod has been bound to a node and all of the containers have been started. +At least one container is still running or is in the process of being restarted. + |
+
"Unknown" |
+PodStatusUnknown is used when none of the other statuses apply or the information is not ready yet. + |
+
map[k8s.io/api/core/v1.PodPhase][]string
alias)
+(map[github.com/grafana/loki/operator/apis/loki/v1.PodStatus][]string
alias)
(Appears on:LokiStackComponentStatus)
diff --git a/operator/internal/status/components.go b/operator/internal/status/components.go index f86cd3a65df73..2d3061ecfc500 100644 --- a/operator/internal/status/components.go +++ b/operator/internal/status/components.go @@ -70,8 +70,29 @@ func appendPodStatus(ctx context.Context, k k8s.Client, component, stack, ns str return nil, kverrors.Wrap(err, "failed to list pods for LokiStack component", "name", stack, "component", component) } for _, pod := range pods.Items { - phase := pod.Status.Phase - psm[phase] = append(psm[phase], pod.Name) + status := podStatus(&pod) + psm[status] = append(psm[status], pod.Name) } return psm, nil } + +func podStatus(pod *corev1.Pod) lokiv1.PodStatus { + status := pod.Status + switch status.Phase { + case corev1.PodFailed: + return lokiv1.PodFailed + case corev1.PodPending: + return lokiv1.PodPending + case corev1.PodRunning: + default: + return lokiv1.PodStatusUnknown + } + + for _, c := range status.ContainerStatuses { + if !c.Ready { + return lokiv1.PodRunning + } + } + + return lokiv1.PodReady +} diff --git a/operator/internal/status/components_test.go b/operator/internal/status/components_test.go index 698b6a536d421..111074f880031 100644 --- a/operator/internal/status/components_test.go +++ b/operator/internal/status/components_test.go @@ -15,7 +15,7 @@ import ( "github.com/grafana/loki/operator/internal/manifests" ) -func createPodList(baseName string, phases ...corev1.PodPhase) *corev1.PodList { +func createPodList(baseName string, ready bool, phases ...corev1.PodPhase) *corev1.PodList { items := []corev1.Pod{} for i, p := range phases { items = append(items, corev1.Pod{ @@ -24,6 +24,11 @@ func createPodList(baseName string, phases ...corev1.PodPhase) *corev1.PodList { }, Status: corev1.PodStatus{ Phase: p, + ContainerStatuses: []corev1.ContainerStatus{ + { + Ready: ready, + }, + }, }, }) } @@ -78,37 +83,37 @@ func TestGenerateComponentStatus(t *testing.T) { manifests.LabelGatewayComponent: {}, }, wantComponentStatus: &lokiv1.LokiStackComponentStatus{ - Compactor: map[corev1.PodPhase][]string{}, - Distributor: map[corev1.PodPhase][]string{}, - IndexGateway: map[corev1.PodPhase][]string{}, - Ingester: map[corev1.PodPhase][]string{}, - Querier: map[corev1.PodPhase][]string{}, - QueryFrontend: map[corev1.PodPhase][]string{}, - Gateway: map[corev1.PodPhase][]string{}, - Ruler: map[corev1.PodPhase][]string{}, + Compactor: lokiv1.PodStatusMap{}, + Distributor: lokiv1.PodStatusMap{}, + IndexGateway: lokiv1.PodStatusMap{}, + Ingester: lokiv1.PodStatusMap{}, + Querier: lokiv1.PodStatusMap{}, + QueryFrontend: lokiv1.PodStatusMap{}, + Gateway: lokiv1.PodStatusMap{}, + Ruler: lokiv1.PodStatusMap{}, }, }, { desc: "all one pod running", componentPods: map[string]*corev1.PodList{ - manifests.LabelCompactorComponent: createPodList(manifests.LabelCompactorComponent, corev1.PodRunning), - manifests.LabelDistributorComponent: createPodList(manifests.LabelDistributorComponent, corev1.PodRunning), - manifests.LabelIngesterComponent: createPodList(manifests.LabelIngesterComponent, corev1.PodRunning), - manifests.LabelQuerierComponent: createPodList(manifests.LabelQuerierComponent, corev1.PodRunning), - manifests.LabelQueryFrontendComponent: createPodList(manifests.LabelQueryFrontendComponent, corev1.PodRunning), - manifests.LabelIndexGatewayComponent: createPodList(manifests.LabelIndexGatewayComponent, corev1.PodRunning), - manifests.LabelRulerComponent: createPodList(manifests.LabelRulerComponent, corev1.PodRunning), - manifests.LabelGatewayComponent: createPodList(manifests.LabelGatewayComponent, corev1.PodRunning), + manifests.LabelCompactorComponent: createPodList(manifests.LabelCompactorComponent, false, corev1.PodRunning), + manifests.LabelDistributorComponent: createPodList(manifests.LabelDistributorComponent, false, corev1.PodRunning), + manifests.LabelIngesterComponent: createPodList(manifests.LabelIngesterComponent, false, corev1.PodRunning), + manifests.LabelQuerierComponent: createPodList(manifests.LabelQuerierComponent, false, corev1.PodRunning), + manifests.LabelQueryFrontendComponent: createPodList(manifests.LabelQueryFrontendComponent, false, corev1.PodRunning), + manifests.LabelIndexGatewayComponent: createPodList(manifests.LabelIndexGatewayComponent, false, corev1.PodRunning), + manifests.LabelRulerComponent: createPodList(manifests.LabelRulerComponent, false, corev1.PodRunning), + manifests.LabelGatewayComponent: createPodList(manifests.LabelGatewayComponent, false, corev1.PodRunning), }, wantComponentStatus: &lokiv1.LokiStackComponentStatus{ - Compactor: map[corev1.PodPhase][]string{corev1.PodRunning: {"compactor-pod-0"}}, - Distributor: map[corev1.PodPhase][]string{corev1.PodRunning: {"distributor-pod-0"}}, - IndexGateway: map[corev1.PodPhase][]string{corev1.PodRunning: {"index-gateway-pod-0"}}, - Ingester: map[corev1.PodPhase][]string{corev1.PodRunning: {"ingester-pod-0"}}, - Querier: map[corev1.PodPhase][]string{corev1.PodRunning: {"querier-pod-0"}}, - QueryFrontend: map[corev1.PodPhase][]string{corev1.PodRunning: {"query-frontend-pod-0"}}, - Gateway: map[corev1.PodPhase][]string{corev1.PodRunning: {"lokistack-gateway-pod-0"}}, - Ruler: map[corev1.PodPhase][]string{corev1.PodRunning: {"ruler-pod-0"}}, + Compactor: lokiv1.PodStatusMap{lokiv1.PodRunning: {"compactor-pod-0"}}, + Distributor: lokiv1.PodStatusMap{lokiv1.PodRunning: {"distributor-pod-0"}}, + IndexGateway: lokiv1.PodStatusMap{lokiv1.PodRunning: {"index-gateway-pod-0"}}, + Ingester: lokiv1.PodStatusMap{lokiv1.PodRunning: {"ingester-pod-0"}}, + Querier: lokiv1.PodStatusMap{lokiv1.PodRunning: {"querier-pod-0"}}, + QueryFrontend: lokiv1.PodStatusMap{lokiv1.PodRunning: {"query-frontend-pod-0"}}, + Gateway: lokiv1.PodStatusMap{lokiv1.PodRunning: {"lokistack-gateway-pod-0"}}, + Ruler: lokiv1.PodStatusMap{lokiv1.PodRunning: {"ruler-pod-0"}}, }, }, } diff --git a/operator/internal/status/lokistack.go b/operator/internal/status/lokistack.go index 1212e5c2f7e92..1ec2f596739df 100644 --- a/operator/internal/status/lokistack.go +++ b/operator/internal/status/lokistack.go @@ -18,8 +18,9 @@ import ( const ( messageReady = "All components ready" - messageFailed = "Some LokiStack components failed" - messagePending = "Some LokiStack components pending on dependencies" + messageFailed = "One or more LokiStack components failed" + messagePending = "One or more LokiStack components pending on dependencies" + messageRunning = "All components are running, but some readiness checks are failing" messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness" messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness" ) @@ -35,6 +36,11 @@ var ( Message: messagePending, Reason: string(lokiv1.ReasonPendingComponents), } + conditionRunning = metav1.Condition{ + Type: string(lokiv1.ConditionPending), + Message: messageRunning, + Reason: string(lokiv1.ReasonPendingComponents), + } conditionReady = metav1.Condition{ Type: string(lokiv1.ConditionReady), Message: messageReady, @@ -76,28 +82,28 @@ func SetDegradedCondition(ctx context.Context, k k8s.Client, req ctrl.Request, m func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, k k8s.Client, req ctrl.Request, stack *lokiv1.LokiStack) (metav1.Condition, error) { // Check for failed pods first - failed := len(cs.Compactor[corev1.PodFailed]) + - len(cs.Distributor[corev1.PodFailed]) + - len(cs.Ingester[corev1.PodFailed]) + - len(cs.Querier[corev1.PodFailed]) + - len(cs.QueryFrontend[corev1.PodFailed]) + - len(cs.Gateway[corev1.PodFailed]) + - len(cs.IndexGateway[corev1.PodFailed]) + - len(cs.Ruler[corev1.PodFailed]) + failed := len(cs.Compactor[lokiv1.PodFailed]) + + len(cs.Distributor[lokiv1.PodFailed]) + + len(cs.Ingester[lokiv1.PodFailed]) + + len(cs.Querier[lokiv1.PodFailed]) + + len(cs.QueryFrontend[lokiv1.PodFailed]) + + len(cs.Gateway[lokiv1.PodFailed]) + + len(cs.IndexGateway[lokiv1.PodFailed]) + + len(cs.Ruler[lokiv1.PodFailed]) if failed != 0 { return conditionFailed, nil } // Check for pending pods - pending := len(cs.Compactor[corev1.PodPending]) + - len(cs.Distributor[corev1.PodPending]) + - len(cs.Ingester[corev1.PodPending]) + - len(cs.Querier[corev1.PodPending]) + - len(cs.QueryFrontend[corev1.PodPending]) + - len(cs.Gateway[corev1.PodPending]) + - len(cs.IndexGateway[corev1.PodPending]) + - len(cs.Ruler[corev1.PodPending]) + pending := len(cs.Compactor[lokiv1.PodPending]) + + len(cs.Distributor[lokiv1.PodPending]) + + len(cs.Ingester[lokiv1.PodPending]) + + len(cs.Querier[lokiv1.PodPending]) + + len(cs.QueryFrontend[lokiv1.PodPending]) + + len(cs.Gateway[lokiv1.PodPending]) + + len(cs.IndexGateway[lokiv1.PodPending]) + + len(cs.Ruler[lokiv1.PodPending]) if pending != 0 { if stack.Spec.Replication != nil && len(stack.Spec.Replication.Zones) > 0 { @@ -120,6 +126,20 @@ func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus, return conditionPending, nil } + // Check if there are pods that are running but not ready + running := len(cs.Compactor[lokiv1.PodRunning]) + + len(cs.Distributor[lokiv1.PodRunning]) + + len(cs.Ingester[lokiv1.PodRunning]) + + len(cs.Querier[lokiv1.PodRunning]) + + len(cs.QueryFrontend[lokiv1.PodRunning]) + + len(cs.Gateway[lokiv1.PodRunning]) + + len(cs.IndexGateway[lokiv1.PodRunning]) + + len(cs.Ruler[lokiv1.PodRunning]) + + if running > 0 { + return conditionRunning, nil + } + return conditionReady, nil } diff --git a/operator/internal/status/lokistack_test.go b/operator/internal/status/lokistack_test.go index 81473c046e43a..6eb91a770d331 100644 --- a/operator/internal/status/lokistack_test.go +++ b/operator/internal/status/lokistack_test.go @@ -184,8 +184,8 @@ func TestGenerateCondition(t *testing.T) { { desc: "container pending", componentStatus: &lokiv1.LokiStackComponentStatus{ - Ingester: map[corev1.PodPhase][]string{ - corev1.PodPending: { + Ingester: lokiv1.PodStatusMap{ + lokiv1.PodPending: { "pod-0", }, }, @@ -195,8 +195,8 @@ func TestGenerateCondition(t *testing.T) { { desc: "container failed", componentStatus: &lokiv1.LokiStackComponentStatus{ - Ingester: map[corev1.PodPhase][]string{ - corev1.PodFailed: { + Ingester: lokiv1.PodStatusMap{ + lokiv1.PodFailed: { "pod-0", }, }, @@ -267,8 +267,8 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) { }, } componentStatus := &lokiv1.LokiStackComponentStatus{ - Ingester: map[corev1.PodPhase][]string{ - corev1.PodPending: { + Ingester: lokiv1.PodStatusMap{ + lokiv1.PodPending: { "pod-0", }, }, diff --git a/operator/internal/status/status_test.go b/operator/internal/status/status_test.go index 89cc0d1ab76b5..997caa5ff2849 100644 --- a/operator/internal/status/status_test.go +++ b/operator/internal/status/status_test.go @@ -33,26 +33,26 @@ func TestRefreshSuccess(t *testing.T) { } componentPods := map[string]*corev1.PodList{ - manifests.LabelCompactorComponent: createPodList(manifests.LabelCompactorComponent, corev1.PodRunning), - manifests.LabelDistributorComponent: createPodList(manifests.LabelDistributorComponent, corev1.PodRunning), - manifests.LabelIngesterComponent: createPodList(manifests.LabelIngesterComponent, corev1.PodRunning), - manifests.LabelQuerierComponent: createPodList(manifests.LabelQuerierComponent, corev1.PodRunning), - manifests.LabelQueryFrontendComponent: createPodList(manifests.LabelQueryFrontendComponent, corev1.PodRunning), - manifests.LabelIndexGatewayComponent: createPodList(manifests.LabelIndexGatewayComponent, corev1.PodRunning), - manifests.LabelRulerComponent: createPodList(manifests.LabelRulerComponent, corev1.PodRunning), - manifests.LabelGatewayComponent: createPodList(manifests.LabelGatewayComponent, corev1.PodRunning), + manifests.LabelCompactorComponent: createPodList(manifests.LabelCompactorComponent, true, corev1.PodRunning), + manifests.LabelDistributorComponent: createPodList(manifests.LabelDistributorComponent, true, corev1.PodRunning), + manifests.LabelIngesterComponent: createPodList(manifests.LabelIngesterComponent, true, corev1.PodRunning), + manifests.LabelQuerierComponent: createPodList(manifests.LabelQuerierComponent, true, corev1.PodRunning), + manifests.LabelQueryFrontendComponent: createPodList(manifests.LabelQueryFrontendComponent, true, corev1.PodRunning), + manifests.LabelIndexGatewayComponent: createPodList(manifests.LabelIndexGatewayComponent, true, corev1.PodRunning), + manifests.LabelRulerComponent: createPodList(manifests.LabelRulerComponent, true, corev1.PodRunning), + manifests.LabelGatewayComponent: createPodList(manifests.LabelGatewayComponent, true, corev1.PodRunning), } wantStatus := lokiv1.LokiStackStatus{ Components: lokiv1.LokiStackComponentStatus{ - Compactor: map[corev1.PodPhase][]string{corev1.PodRunning: {"compactor-pod-0"}}, - Distributor: map[corev1.PodPhase][]string{corev1.PodRunning: {"distributor-pod-0"}}, - IndexGateway: map[corev1.PodPhase][]string{corev1.PodRunning: {"index-gateway-pod-0"}}, - Ingester: map[corev1.PodPhase][]string{corev1.PodRunning: {"ingester-pod-0"}}, - Querier: map[corev1.PodPhase][]string{corev1.PodRunning: {"querier-pod-0"}}, - QueryFrontend: map[corev1.PodPhase][]string{corev1.PodRunning: {"query-frontend-pod-0"}}, - Gateway: map[corev1.PodPhase][]string{corev1.PodRunning: {"lokistack-gateway-pod-0"}}, - Ruler: map[corev1.PodPhase][]string{corev1.PodRunning: {"ruler-pod-0"}}, + Compactor: lokiv1.PodStatusMap{lokiv1.PodReady: {"compactor-pod-0"}}, + Distributor: lokiv1.PodStatusMap{lokiv1.PodReady: {"distributor-pod-0"}}, + IndexGateway: lokiv1.PodStatusMap{lokiv1.PodReady: {"index-gateway-pod-0"}}, + Ingester: lokiv1.PodStatusMap{lokiv1.PodReady: {"ingester-pod-0"}}, + Querier: lokiv1.PodStatusMap{lokiv1.PodReady: {"querier-pod-0"}}, + QueryFrontend: lokiv1.PodStatusMap{lokiv1.PodReady: {"query-frontend-pod-0"}}, + Gateway: lokiv1.PodStatusMap{lokiv1.PodReady: {"lokistack-gateway-pod-0"}}, + Ruler: lokiv1.PodStatusMap{lokiv1.PodReady: {"ruler-pod-0"}}, }, Storage: lokiv1.LokiStackStorageStatus{}, Conditions: []metav1.Condition{