Lifecycle hook support

api/v1alpha1/instancegroup_types.go

@@ -17,6 +17,7 @@ package v1alpha1
 
 import (
 	"fmt"
+	"reflect"
 	"strings"
 
 	"github.com/keikoproj/instance-manager/controllers/common"
@@ -70,6 +71,12 @@ const (
 
 	FileSystemTypeXFS  = "xfs"
 	FileSystemTypeEXT4 = "ext4"
+
+	LifecycleHookResultAbandon           = "ABANDON"
+	LifecycleHookResultContinue          = "CONTINUE"
+	LifecycleHookTransitionLaunch        = "Launch"
+	LifecycleHookTransitionTerminate     = "Terminate"
+	LifecycleHookDefaultHeartbeatTimeout = 300
 )
 
 var (
@@ -87,9 +94,10 @@ var (
 		},
 	}
 
-	AllowedFileSystemTypes = []string{FileSystemTypeXFS, FileSystemTypeEXT4}
-
-	log = ctrl.Log.WithName("v1alpha1")
+	AllowedFileSystemTypes            = []string{FileSystemTypeXFS, FileSystemTypeEXT4}
+	LifecycleHookAllowedTransitions   = []string{LifecycleHookTransitionLaunch, LifecycleHookTransitionTerminate}
+	LifecycleHookAllowedDefaultResult = []string{LifecycleHookResultAbandon, LifecycleHookResultContinue}
+	log                               = ctrl.Log.WithName("v1alpha1")
 )
 
 // InstanceGroup is the Schema for the instancegroups API
@@ -188,6 +196,17 @@ type EKSConfiguration struct {
 	ExistingInstanceProfileName string              `json:"instanceProfileName,omitempty"`
 	ManagedPolicies             []string            `json:"managedPolicies,omitempty"`
 	MetricsCollection           []string            `json:"metricsCollection,omitempty"`
+	LifecycleHooks              []LifecycleHookSpec `json:"lifecycleHooks,omitempty"`
+}
+
+type LifecycleHookSpec struct {
+	Name             string `json:"name"`
+	Lifecycle        string `json:"lifecycle"`
+	DefaultResult    string `json:"defaultResult,omitempty"`
+	HeartbeatTimeout int64  `json:"heartbeatTimeout,omitempty"`
+	NotificationArn  string `json:"notificationArn"`
+	Metadata         string `json:"metadata,omitempty"`
+	RoleArn          string `json:"roleArn"`
 }
 
 type UserDataStage struct {
@@ -323,6 +342,40 @@ func (c *EKSConfiguration) Validate() error {
 		c.SuspendedProcesses = processes
 	}
 
+	hooks := []LifecycleHookSpec{}
+	for _, h := range c.LifecycleHooks {
+		if h.HeartbeatTimeout == 0 {
+			h.HeartbeatTimeout = LifecycleHookDefaultHeartbeatTimeout
+		}
+		if common.StringEmpty(h.DefaultResult) {
+			h.DefaultResult = LifecycleHookResultAbandon
+		}
+		if common.ContainsEqualFold(LifecycleHookAllowedDefaultResult, h.DefaultResult) {
+			h.DefaultResult = strings.ToUpper(h.DefaultResult)
+		} else {
+			h.DefaultResult = LifecycleHookResultAbandon
+		}
+		if !common.ContainsEqualFold(LifecycleHookAllowedTransitions, h.Lifecycle) {
+			return errors.Errorf("validation failed, 'lifecycle' is a required parameter and must be in %+v", LifecycleHookAllowedTransitions)
+		}
+		if strings.EqualFold(h.Lifecycle, LifecycleHookTransitionLaunch) {
+			h.Lifecycle = awsprovider.LifecycleHookTransitionLaunch
+		} else if strings.EqualFold(h.Lifecycle, LifecycleHookTransitionTerminate) {
+			h.Lifecycle = awsprovider.LifecycleHookTransitionTerminate
+		}
+		if common.StringEmpty(h.Name) {
+			return errors.Errorf("validation failed, 'name' is a required parameter")
+		}
+		if common.StringEmpty(h.NotificationArn) || !strings.HasPrefix(h.NotificationArn, awsprovider.ARNPrefix) {
+			return errors.Errorf("validation failed, 'notificationArn' is a required parameter and must be a valid IAM role ARN")
+		}
+		if common.StringEmpty(h.RoleArn) || !strings.HasPrefix(h.NotificationArn, awsprovider.ARNPrefix) {
+			return errors.Errorf("validation failed, 'roleArn' is a required parameter and must be a valid IAM role ARN")
+		}
+		hooks = append(hooks, h)
+	}
+	c.SetLifecycleHooks(hooks)
+
 	if common.StringEmpty(c.Image) {
 		return errors.Errorf("validation failed, 'image' is a required parameter")
 	}
@@ -413,6 +466,20 @@ func (ig *InstanceGroup) Validate() error {
 func (c *EKSConfiguration) GetRoleName() string {
 	return c.ExistingRoleName
 }
+func (c *EKSConfiguration) GetLifecycleHooks() []LifecycleHookSpec {
+	return c.LifecycleHooks
+}
+func (c *EKSConfiguration) SetLifecycleHooks(hooks []LifecycleHookSpec) {
+	c.LifecycleHooks = hooks
+}
+func (h LifecycleHookSpec) ExistInSlice(hooks []LifecycleHookSpec) bool {
+	for _, hook := range hooks {
+		if reflect.DeepEqual(hook, h) {
+			return true
+		}
+	}
+	return false
+}
 func (c *EKSConfiguration) GetInstanceProfileName() string {
 	return c.ExistingInstanceProfileName
 }

config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml

@@ -91,6 +91,31 @@ spec:
                       additionalProperties:
                         type: string
                       type: object
+                    lifecycleHooks:
+                      items:
+                        properties:
+                          defaultResult:
+                            type: string
+                          heartbeatTimeout:
+                            format: int64
+                            type: integer
+                          lifecycle:
+                            type: string
+                          metadata:
+                            type: string
+                          name:
+                            type: string
+                          notificationArn:
+                            type: string
+                          roleArn:
+                            type: string
+                        required:
+                        - lifecycle
+                        - name
+                        - notificationArn
+                        - roleArn
+                        type: object
+                      type: array
                     managedPolicies:
                       items:
                         type: string

controllers/providers/aws/aws.go

@@ -51,6 +51,7 @@ const (
 	GetRoleTTL                      time.Duration = 60 * time.Second
 	GetInstanceProfileTTL           time.Duration = 60 * time.Second
 	DescribeNodegroupTTL            time.Duration = 60 * time.Second
+	DescribeLifecycleHooksTTL       time.Duration = 180 * time.Second
 	DescribeClusterTTL              time.Duration = 180 * time.Second
 	DescribeSecurityGroupsTTL       time.Duration = 180 * time.Second
 	DescribeSubnetsTTL              time.Duration = 180 * time.Second
@@ -99,16 +100,47 @@ var (
 		"GroupTotalCapacity",
 	}
 
-	AllowedVolumeTypes = []string{"gp2", "io1", "sc1", "st1"}
+	AllowedVolumeTypes               = []string{"gp2", "io1", "sc1", "st1"}
+	LifecycleHookTransitionLaunch    = "autoscaling:EC2_INSTANCE_LAUNCHING"
+	LifecycleHookTransitionTerminate = "autoscaling:EC2_INSTANCE_TERMINATING"
 )
 
 const (
-	IAMPolicyPrefix = "arn:aws:iam::aws:policy"
-	IAMARNPrefix    = "arn:aws:iam::"
-
+	IAMPolicyPrefix                         = "arn:aws:iam::aws:policy"
+	IAMARNPrefix                            = "arn:aws:iam::"
+	ARNPrefix                               = "arn:aws:"
 	LaunchConfigurationNotFoundErrorMessage = "Launch configuration name not found"
 )
 
+func (w *AwsWorker) CreateLifecycleHook(input *autoscaling.PutLifecycleHookInput) error {
+	_, err := w.AsgClient.PutLifecycleHook(input)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (w *AwsWorker) DeleteLifecycleHook(asgName, hookName string) error {
+	_, err := w.AsgClient.DeleteLifecycleHook(&autoscaling.DeleteLifecycleHookInput{
+		AutoScalingGroupName: aws.String(asgName),
+		LifecycleHookName:    aws.String(hookName),
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (w *AwsWorker) DescribeLifecycleHooks(asgName string) ([]*autoscaling.LifecycleHook, error) {
+	out, err := w.AsgClient.DescribeLifecycleHooks(&autoscaling.DescribeLifecycleHooksInput{
+		AutoScalingGroupName: aws.String(asgName),
+	})
+	if err != nil {
+		return []*autoscaling.LifecycleHook{}, err
+	}
+	return out.LifecycleHooks, nil
+}
+
 func (w *AwsWorker) RoleExist(name string) (*iam.Role, bool) {
 	out, err := w.GetRole(name)
 	if err != nil {
@@ -794,6 +826,7 @@ func GetAwsAsgClient(region string, cacheCfg *cache.Config, maxRetries int) auto
 	cache.AddCaching(sess, cacheCfg)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeAutoScalingGroups", DescribeAutoScalingGroupsTTL)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeLaunchConfigurations", DescribeLaunchConfigurationsTTL)
+	cacheCfg.SetCacheTTL("autoscaling", "DescribeLifecycleHooks", DescribeLifecycleHooksTTL)
 	sess.Handlers.Complete.PushFront(func(r *request.Request) {
 		ctx := r.HTTPRequest.Context()
 		log.V(1).Info("AWS API call",

controllers/provisioners/config.go

@@ -30,14 +30,16 @@ import (
 )
 
 var (
-	EKSConfigurationPath = "spec.eks.configuration"
-	EKSTagsPath          = fmt.Sprintf("%v.tags", EKSConfigurationPath)
-	EKSVolumesPath       = fmt.Sprintf("%v.volumes", EKSConfigurationPath)
+	EKSConfigurationPath  = "spec.eks.configuration"
+	EKSTagsPath           = fmt.Sprintf("%v.tags", EKSConfigurationPath)
+	EKSVolumesPath        = fmt.Sprintf("%v.volumes", EKSConfigurationPath)
+	EKSLifecycleHooksPath = fmt.Sprintf("%v.lifecycleHooks", EKSConfigurationPath)
 
 	// MergeSchema defines the key to merge by
 	MergeSchema = map[string]string{
-		EKSTagsPath:    "key",
-		EKSVolumesPath: "name",
+		EKSTagsPath:           "key",
+		EKSVolumesPath:        "name",
+		EKSLifecycleHooksPath: "name",
 	}
 )
 

controllers/provisioners/eks/cloud.go

@@ -37,6 +37,7 @@ type DiscoveredState struct {
 	ClusterNodes         *corev1.NodeList
 	OwnedScalingGroups   []*autoscaling.Group
 	ScalingGroup         *autoscaling.Group
+	LifecycleHooks       []*autoscaling.LifecycleHook
 	ScalingConfiguration scaling.Configuration
 	IAMRole              *iam.Role
 	AttachedPolicies     []*iam.AttachedPolicy
@@ -128,9 +129,14 @@ func (ctx *EksInstanceGroupContext) CloudDiscovery() error {
 
 	state.SetProvisioned(true)
 	state.SetScalingGroup(targetScalingGroup)
+	asgName := aws.StringValue(targetScalingGroup.AutoScalingGroupName)
 
+	state.LifecycleHooks, err = ctx.AwsWorker.DescribeLifecycleHooks(asgName)
+	if err != nil {
+		return errors.Wrap(err, "failed to describe lifecycle hooks")
+	}
 	// update status with scaling group info
-	status.SetActiveScalingGroupName(aws.StringValue(targetScalingGroup.AutoScalingGroupName))
+	status.SetActiveScalingGroupName(asgName)
 	status.SetCurrentMin(int(aws.Int64Value(targetScalingGroup.MinSize)))
 	status.SetCurrentMax(int(aws.Int64Value(targetScalingGroup.MaxSize)))
 

controllers/provisioners/eks/create.go

@@ -122,6 +122,10 @@ func (ctx *EksInstanceGroupContext) CreateScalingGroup(lcName string) error {
 		return err
 	}
 
+	if err := ctx.UpdateLifecycleHooks(asgName); err != nil {
+		return err
+	}
+
 	state.Publisher.Publish(kubeprovider.InstanceGroupCreatedEvent, "instancegroup", instanceGroup.GetName(), "scalinggroup", asgName)
 	return nil
 }

controllers/provisioners/eks/eks_test.go

@@ -340,11 +340,17 @@ type MockAutoScalingClient struct {
 	EnableMetricsCollectionErr             error
 	DisableMetricsCollectionErr            error
 	UpdateSuspendProcessesErr              error
+	DescribeLifecycleHooksErr              error
+	PutLifecycleHookErr                    error
+	DeleteLifecycleHookErr                 error
 	DeleteLaunchConfigurationCallCount     int
+	PutLifecycleHookCallCount              int
+	DeleteLifecycleHookCallCount           int
 	LaunchConfiguration                    *autoscaling.LaunchConfiguration
 	LaunchConfigurations                   []*autoscaling.LaunchConfiguration
 	AutoScalingGroup                       *autoscaling.Group
 	AutoScalingGroups                      []*autoscaling.Group
+	LifecycleHooks                         []*autoscaling.LifecycleHook
 }
 
 func (a *MockAutoScalingClient) EnableMetricsCollection(input *autoscaling.EnableMetricsCollectionInput) (*autoscaling.EnableMetricsCollectionOutput, error) {
@@ -422,6 +428,20 @@ func (a *MockAutoScalingClient) ResumeProcesses(input *autoscaling.ScalingProces
 	return &autoscaling.ResumeProcessesOutput{}, a.UpdateSuspendProcessesErr
 }
 
+func (a *MockAutoScalingClient) DescribeLifecycleHooks(input *autoscaling.DescribeLifecycleHooksInput) (*autoscaling.DescribeLifecycleHooksOutput, error) {
+	return &autoscaling.DescribeLifecycleHooksOutput{LifecycleHooks: a.LifecycleHooks}, a.DescribeLifecycleHooksErr
+}
+
+func (a *MockAutoScalingClient) DeleteLifecycleHook(input *autoscaling.DeleteLifecycleHookInput) (*autoscaling.DeleteLifecycleHookOutput, error) {
+	a.DeleteLifecycleHookCallCount++
+	return &autoscaling.DeleteLifecycleHookOutput{}, a.DeleteLifecycleHookErr
+}
+
+func (a *MockAutoScalingClient) PutLifecycleHook(input *autoscaling.PutLifecycleHookInput) (*autoscaling.PutLifecycleHookOutput, error) {
+	a.PutLifecycleHookCallCount++
+	return &autoscaling.PutLifecycleHookOutput{}, a.PutLifecycleHookErr
+}
+
 type MockEc2Client struct {
 	ec2iface.EC2API
 	DescribeSubnetsErr        error