helper: Add ContinuousTargetOccurence to work around inconsistency

helper/resource/state.go

@@ -29,6 +29,9 @@ type StateChangeConf struct {
 	Timeout        time.Duration    // The amount of time to wait before timeout
 	MinTimeout     time.Duration    // Smallest time to wait before refreshes
 	NotFoundChecks int              // Number of times to allow not found
+
+	// This is to work around inconsistent APIs
+	ContinuousTargetOccurence int // Number of times the Target state has to occur continuously
 }
 
 // WaitForState watches an object and waits for it to achieve the state
@@ -49,12 +52,17 @@ func (conf *StateChangeConf) WaitForState() (interface{}, error) {
 	log.Printf("[DEBUG] Waiting for state to become: %s", conf.Target)
 
 	notfoundTick := 0
+	targetOccurence := 0
 
 	// Set a default for times to check for not found
 	if conf.NotFoundChecks == 0 {
 		conf.NotFoundChecks = 20
 	}
 
+	if conf.ContinuousTargetOccurence == 0 {
+		conf.ContinuousTargetOccurence = 1
+	}
+
 	var result interface{}
 	var resulterr error
 
@@ -88,7 +96,12 @@ func (conf *StateChangeConf) WaitForState() (interface{}, error) {
 
 			// If we're waiting for the absence of a thing, then return
 			if result == nil && len(conf.Target) == 0 {
-				return
+				targetOccurence += 1
+				if conf.ContinuousTargetOccurence == targetOccurence {
+					return
+				} else {
+					continue
+				}
 			}
 
 			if result == nil {
@@ -102,17 +115,24 @@ func (conf *StateChangeConf) WaitForState() (interface{}, error) {
 			} else {
 				// Reset the counter for when a resource isn't found
 				notfoundTick = 0
+				found := false
 
 				for _, allowed := range conf.Target {
 					if currentState == allowed {
-						return
+						found = true
+						targetOccurence += 1
+						if conf.ContinuousTargetOccurence == targetOccurence {
+							return
+						} else {
+							continue
+						}
 					}
 				}
 
-				found := false
 				for _, allowed := range conf.Pending {
 					if currentState == allowed {
 						found = true
+						targetOccurence = 0
 						break
 					}
 				}

helper/resource/state_test.go

@@ -25,6 +25,87 @@ func SuccessfulStateRefreshFunc() StateRefreshFunc {
 	}
 }
 
+type StateGenerator struct {
+	position      int
+	stateSequence []string
+}
+
+func (r *StateGenerator) NextState() (int, string, error) {
+	p, v := r.position, ""
+	if len(r.stateSequence)-1 >= p {
+		v = r.stateSequence[p]
+	} else {
+		return -1, "", errors.New("No more states available")
+	}
+
+	r.position += 1
+
+	return p, v, nil
+}
+
+func NewStateGenerator(sequence []string) *StateGenerator {
+	r := &StateGenerator{}
+	r.stateSequence = sequence
+
+	return r
+}
+
+func InconsistentStateRefreshFunc() StateRefreshFunc {
+	sequence := []string{
+		"done", "replicating",
+		"done", "done", "done",
+		"replicating",
+		"done", "done", "done",
+	}
+
+	r := NewStateGenerator(sequence)
+
+	return func() (interface{}, string, error) {
+		idx, s, err := r.NextState()
+		if err != nil {
+			return nil, "", err
+		}
+
+		return idx, s, nil
+	}
+}
+
+func TestWaitForState_inconsistent_positive(t *testing.T) {
+	conf := &StateChangeConf{
+		Pending:                   []string{"replicating"},
+		Target:                    []string{"done"},
+		Refresh:                   InconsistentStateRefreshFunc(),
+		Timeout:                   10 * time.Second,
+		ContinuousTargetOccurence: 3,
+	}
+
+	idx, err := conf.WaitForState()
+
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if idx != 4 {
+		t.Fatalf("Expected index 4, given %d", idx.(int))
+	}
+}
+
+func TestWaitForState_inconsistent_negative(t *testing.T) {
+	conf := &StateChangeConf{
+		Pending:                   []string{"replicating"},
+		Target:                    []string{"done"},
+		Refresh:                   InconsistentStateRefreshFunc(),
+		Timeout:                   10 * time.Second,
+		ContinuousTargetOccurence: 4,
+	}
+
+	_, err := conf.WaitForState()
+
+	if err == nil && err.Error() != "timeout while waiting for state to become 'done'" {
+		t.Fatalf("err: %s", err)
+	}
+}
+
 func TestWaitForState_timeout(t *testing.T) {
 	conf := &StateChangeConf{
 		Pending: []string{"pending", "incomplete"},