Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pkg/ansible - update status to include failure message on the status. #639

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 65 additions & 54 deletions pkg/ansible/controller/reconcile.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,17 @@ import (
"encoding/json"
"errors"
"os"
"strings"
"time"

ansiblestatus "github.com/operator-framework/operator-sdk/pkg/ansible/controller/status"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This name is not super great but status seems to be overused.

"github.com/operator-framework/operator-sdk/pkg/ansible/events"
"github.com/operator-framework/operator-sdk/pkg/ansible/proxy/kubeconfig"
"github.com/operator-framework/operator-sdk/pkg/ansible/runner"
"github.com/operator-framework/operator-sdk/pkg/ansible/runner/eventapi"

"github.com/sirupsen/logrus"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand Down Expand Up @@ -80,7 +83,9 @@ func (r *AnsibleOperatorReconciler) Reconcile(request reconcile.Request) (reconc
finalizers := append(pendingFinalizers, finalizer)
u.SetFinalizers(finalizers)
err := r.Client.Update(context.TODO(), u)
return reconcileResult, err
if err != nil {
return reconcileResult, err
}
}
if !contains(pendingFinalizers, finalizer) && deleted {
logrus.Info("Resource is terminated, skipping reconcilation")
Expand All @@ -96,34 +101,32 @@ func (r *AnsibleOperatorReconciler) Reconcile(request reconcile.Request) (reconc
if err != nil {
return reconcileResult, err
}
reconcileResult.Requeue = true
return reconcileResult, nil
}
status := u.Object["status"]
_, ok = status.(map[string]interface{})
if !ok {
logrus.Debugf("status was not found")
u.Object["status"] = map[string]interface{}{}
statusInterface := u.Object["status"]
statusMap, _ := statusInterface.(map[string]interface{})
Copy link
Contributor

@hasbro17 hasbro17 Oct 28, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We no longer need to check if Object["status"] exists or is of type map[string]interface{}?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The below func CreateFromMap will check this and handle the edge cases.

crStatus := ansiblestatus.CreateFromMap(statusMap)

// If there is no current status add that we are working on this resource.
errCond := ansiblestatus.GetCondition(crStatus, ansiblestatus.FailureConditionType)
succCond := ansiblestatus.GetCondition(crStatus, ansiblestatus.RunningConditionType)

// If the condition is currently running, making sure that the values are correct.
// If they are the same a no-op, if they are different then it is a good thing we
// are updating it.
if (errCond == nil && succCond == nil) || (succCond != nil && succCond.Reason != ansiblestatus.SuccessfulReason) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you just walk through what would need to happen to make the expression after || be true? I'm not seeing what that scenario would be in practical terms.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

c := ansiblestatus.NewCondition(
ansiblestatus.RunningConditionType,
v1.ConditionTrue,
nil,
ansiblestatus.RunningReason,
ansiblestatus.RunningMessage,
)
ansiblestatus.SetCondition(&crStatus, *c)
u.Object["status"] = crStatus
err = r.Client.Update(context.TODO(), u)
if err != nil {
return reconcileResult, err
}
reconcileResult.Requeue = true
return reconcileResult, nil
}

// If status is an empty map we can assume CR was just created
if len(u.Object["status"].(map[string]interface{})) == 0 {
logrus.Debugf("Setting phase status to %v", StatusPhaseCreating)
u.Object["status"] = ResourceStatus{
Phase: StatusPhaseCreating,
}
err = r.Client.Update(context.TODO(), u)
if err != nil {
return reconcileResult, err
}
reconcileResult.Requeue = true
return reconcileResult, nil
}

ownerRef := metav1.OwnerReference{
Expand All @@ -145,11 +148,12 @@ func (r *AnsibleOperatorReconciler) Reconcile(request reconcile.Request) (reconc

// iterate events from ansible, looking for the final one
statusEvent := eventapi.StatusJobEvent{}
failureMessages := eventapi.FailureMessages{}
for event := range eventChan {
for _, eHandler := range r.EventHandlers {
go eHandler.Handle(u, event)
}
if event.Event == "playbook_on_stats" {
if event.Event == eventapi.EventPlaybookOnStats {
// convert to StatusJobEvent; would love a better way to do this
data, err := json.Marshal(event)
if err != nil {
Expand All @@ -160,6 +164,9 @@ func (r *AnsibleOperatorReconciler) Reconcile(request reconcile.Request) (reconc
return reconcile.Result{}, err
}
}
if event.Event == eventapi.EventRunnerOnFailed {
failureMessages = append(failureMessages, event.GetFailedPlaybookMessage())
}
}
if statusEvent.Event == "" {
err := errors.New("did not receive playbook_on_stats event")
Expand All @@ -168,14 +175,7 @@ func (r *AnsibleOperatorReconciler) Reconcile(request reconcile.Request) (reconc
}

// We only want to update the CustomResource once, so we'll track changes and do it at the end
var needsUpdate bool
runSuccessful := true
for _, count := range statusEvent.EventData.Failures {
if count > 0 {
runSuccessful = false
break
}
}
runSuccessful := len(failureMessages) == 0
// The finalizer has run successfully, time to remove it
if deleted && finalizerExists && runSuccessful {
finalizers := []string{}
Expand All @@ -185,31 +185,42 @@ func (r *AnsibleOperatorReconciler) Reconcile(request reconcile.Request) (reconc
}
}
u.SetFinalizers(finalizers)
needsUpdate = true
}

statusMap, ok := u.Object["status"].(map[string]interface{})
if !ok {
u.Object["status"] = ResourceStatus{
Status: NewStatusFromStatusJobEvent(statusEvent),
}
logrus.Infof("adding status for the first time")
needsUpdate = true
} else {
// Need to conver the map[string]interface into a resource status.
if update, status := UpdateResourceStatus(statusMap, statusEvent); update {
u.Object["status"] = status
needsUpdate = true
err := r.Client.Update(context.TODO(), u)
if err != nil {
return reconcileResult, err
}
}
if needsUpdate {
err = r.Client.Update(context.TODO(), u)
}
ansibleStatus := ansiblestatus.NewAnsibleResultFromStatusJobEvent(statusEvent)

if !runSuccessful {
reconcileResult.Requeue = true
return reconcileResult, err
}
sc := ansiblestatus.GetCondition(crStatus, ansiblestatus.RunningConditionType)
sc.Status = v1.ConditionFalse
ansiblestatus.SetCondition(&crStatus, *sc)
c := ansiblestatus.NewCondition(
ansiblestatus.FailureConditionType,
v1.ConditionTrue,
ansibleStatus,
ansiblestatus.FailedReason,
strings.Join(failureMessages, "\n"),
)
ansiblestatus.SetCondition(&crStatus, *c)
} else {
c := ansiblestatus.NewCondition(
ansiblestatus.RunningConditionType,
v1.ConditionTrue,
ansibleStatus,
ansiblestatus.SuccessfulReason,
ansiblestatus.SuccessfulMessage,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be useful to record what you did to some degree in the message? Or perhaps you just want that to be in the logs/events?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking that events would be more useful for this.

)
// Remove the failure condition if set, because this completed successfully.
ansiblestatus.RemoveCondition(&crStatus, ansiblestatus.FailureConditionType)
ansiblestatus.SetCondition(&crStatus, *c)
}
// This needs the status subresource to be enabled by default.
u.Object["status"] = crStatus
err = r.Client.Update(context.TODO(), u)
return reconcileResult, err

}

func contains(l []string, s string) bool {
Expand Down
167 changes: 167 additions & 0 deletions pkg/ansible/controller/status/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
// Copyright 2018 The Operator-SDK Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package status

import (
"time"

"github.com/operator-framework/operator-sdk/pkg/ansible/runner/eventapi"
"github.com/sirupsen/logrus"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
host = "localhost"
)

// AnsibleResult - encapsulation of the ansible result.
type AnsibleResult struct {
Ok int `json:"ok"`
Changed int `json:"changed"`
Skipped int `json:"skipped"`
Failures int `json:"failures"`
TimeOfCompletion eventapi.EventTime `json:"completion"`
}

// NewAnsibleResultFromStatusJobEvent - creates a Ansible status from job event.
func NewAnsibleResultFromStatusJobEvent(je eventapi.StatusJobEvent) *AnsibleResult {
// ok events.
a := &AnsibleResult{TimeOfCompletion: je.Created}
if v, ok := je.EventData.Changed[host]; ok {
a.Changed = v
}
if v, ok := je.EventData.Ok[host]; ok {
a.Ok = v
}
if v, ok := je.EventData.Skipped[host]; ok {
a.Skipped = v
}
if v, ok := je.EventData.Failures[host]; ok {
a.Failures = v
}
return a
}

// NewAnsibleResultFromMap - creates a Ansible status from a job event.
func NewAnsibleResultFromMap(sm map[string]interface{}) *AnsibleResult {
//Create Old top level status
// ok events.
a := &AnsibleResult{}
if v, ok := sm["changed"]; ok {
a.Changed = int(v.(int64))
}
if v, ok := sm["ok"]; ok {
a.Ok = int(v.(int64))
}
if v, ok := sm["skipped"]; ok {
a.Skipped = int(v.(int64))
}
if v, ok := sm["failures"]; ok {
a.Failures = int(v.(int64))
}
if v, ok := sm["completion"]; ok {
s := v.(string)
a.TimeOfCompletion.UnmarshalJSON([]byte(s))
}
return a
}

// ConditionType - type of condition
type ConditionType string

const (
// RunningConditionType - condition type of running.
RunningConditionType ConditionType = "Running"
// FailureConditionType - condition type of failure.
FailureConditionType ConditionType = "Failure"
)

// Condition - the condition for the ansible operator.
type Condition struct {
Type ConditionType `json:"type"`
Status v1.ConditionStatus `json:"status"`
LastTransitionTime metav1.Time `json:"lastTransitionTime"`
AnsibleResult *AnsibleResult `json:"ansibleResult,omitempty"`
Reason string `json:"reason"`
Message string `json:"message"`
}

func createConditionFromMap(cm map[string]interface{}) Condition {
ct, ok := cm["type"].(string)
if !ok {
//If we do not find the string we are defaulting
// to make sure we can at least update the status.
ct = string(RunningConditionType)
}
status, ok := cm["status"].(string)
if !ok {
status = string(v1.ConditionTrue)
}
reason, ok := cm["reason"].(string)
if !ok {
reason = RunningReason
}
message, ok := cm["message"].(string)
if !ok {
message = RunningMessage
}
asm, ok := cm["ansibleStatus"].(map[string]interface{})
var ansibleResult *AnsibleResult
if ok {
ansibleResult = NewAnsibleResultFromMap(asm)
}
ltts, ok := cm["lastTransitionTime"].(string)
ltt := metav1.Now()
if ok {
t, err := time.Parse("2006-01-02T15:04:05Z", ltts)
if err != nil {
logrus.Warningf("unable to parse time for status condition: %v", ltts)
} else {
ltt = metav1.NewTime(t)
}
}
return Condition{
Type: ConditionType(ct),
Status: v1.ConditionStatus(status),
LastTransitionTime: ltt,
Reason: reason,
Message: message,
AnsibleResult: ansibleResult,
}
}

// Status - The status for custom resources managed by the operator-sdk.
type Status struct {
Conditions []Condition `json:"conditions"`
}

// CreateFromMap - create a status from the map
func CreateFromMap(statusMap map[string]interface{}) Status {
conditionsInterface, ok := statusMap["conditions"].([]interface{})
if !ok {
return Status{Conditions: []Condition{}}
}
conditions := []Condition{}
for _, ci := range conditionsInterface {
cm, ok := ci.(map[string]interface{})
if !ok {
logrus.Warningf("unknown condition, removing condition: %v", ci)
continue
}
conditions = append(conditions, createConditionFromMap(cm))
}
return Status{Conditions: conditions}
}
Loading