Wait for newly created services to stabilize.

remind101 · Jun 14, 2016 · 26969f4 · 26969f4
1 parent 5439056
commit 26969f4
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 1 deletion.
diff --git a/server/cloudformation/ecs.go b/server/cloudformation/ecs.go
@@ -15,6 +15,7 @@ type ecsClient interface {
 	CreateService(*ecs.CreateServiceInput) (*ecs.CreateServiceOutput, error)
 	DeleteService(*ecs.DeleteServiceInput) (*ecs.DeleteServiceOutput, error)
 	UpdateService(*ecs.UpdateServiceInput) (*ecs.UpdateServiceOutput, error)
+	WaitUntilServicesStable(*ecs.DescribeServicesInput) error
 }
 
 type LoadBalancer struct {
@@ -117,7 +118,17 @@ func (p *ECSServiceResource) create(properties *ECSServiceProperties) (string, e
 		return "", fmt.Errorf("error creating service: %v", err)
 	}
 
-	return *resp.Service.ServiceArn, nil
+	arn := resp.Service.ServiceArn
+	if err := p.ecs.WaitUntilServicesStable(&ecs.DescribeServicesInput{
+		Cluster:  properties.Cluster,
+		Services: []*string{arn},
+	}); err != nil {
+		// We're ignoring this error, because the service was created,
+		// and if the service doesn't stabilize, it's better to just let
+		// the stack finish creating than rolling back.
+	}
+
+	return *arn, nil
 }
 
 func (p *ECSServiceResource) delete(service, cluster *string) error {

diff --git a/server/cloudformation/ecs_test.go b/server/cloudformation/ecs_test.go
@@ -28,6 +28,11 @@ func TestECSServiceResource_Create(t *testing.T) {
 		},
 	}, nil)
 
+	e.On("WaitUntilServicesStable", &ecs.DescribeServicesInput{
+		Cluster:  aws.String("cluster"),
+		Services: []*string{aws.String("arn:aws:ecs:us-east-1:012345678901:service/acme-inc-web-A")},
+	}).Return(nil)
+
 	id, data, err := p.Provision(Request{
 		RequestType: Create,
 		ResourceProperties: &ECSServiceProperties{
@@ -98,6 +103,11 @@ func TestECSServiceResource_Update_RequiresReplacement(t *testing.T) {
 		},
 	}, nil)
 
+	e.On("WaitUntilServicesStable", &ecs.DescribeServicesInput{
+		Cluster:  aws.String("clusterB"),
+		Services: []*string{aws.String("arn:aws:ecs:us-east-1:012345678901:service/acme-inc-web-B")},
+	}).Return(nil)
+
 	id, data, err := p.Provision(Request{
 		RequestType:        Update,
 		PhysicalResourceId: "arn:aws:ecs:us-east-1:012345678901:service/acme-inc-web-A",
@@ -263,3 +273,8 @@ func (m *mockECS) DeleteService(input *ecs.DeleteServiceInput) (*ecs.DeleteServi
 	args := m.Called(input)
 	return args.Get(0).(*ecs.DeleteServiceOutput), args.Error(1)
 }
+
+func (m *mockECS) WaitUntilServicesStable(input *ecs.DescribeServicesInput) error {
+	args := m.Called(input)
+	return args.Error(0)
+}