diff --git a/modules/graphdb/main.tf b/modules/graphdb/main.tf index 507c74a..4bf85f9 100644 --- a/modules/graphdb/main.tf +++ b/modules/graphdb/main.tf @@ -87,6 +87,11 @@ resource "aws_autoscaling_group" "graphdb_auto_scaling_group" { target_group_arns = var.graphdb_target_group_arns + instance_maintenance_policy { + min_healthy_percentage = var.instance_maintenance_policy_min_healthy_percentage + max_healthy_percentage = var.instance_maintenance_policy_max_healthy_percentage + } + launch_template { id = aws_launch_template.graphdb.id version = aws_launch_template.graphdb.latest_version @@ -119,3 +124,4 @@ resource "aws_autoscaling_group" "graphdb_auto_scaling_group" { } } } + diff --git a/modules/graphdb/templates/00_functions.sh b/modules/graphdb/templates/00_functions.sh index 01d10cf..3e8c3b9 100644 --- a/modules/graphdb/templates/00_functions.sh +++ b/modules/graphdb/templates/00_functions.sh @@ -1,12 +1,79 @@ #!/usr/bin/env bash -# Generic helper functions - -# Function to print messages with timestamps +# Function to log messages with a timestamp log_with_timestamp() { echo "$(date '+%Y-%m-%d %H:%M:%S'): $1" } +# Function to check ASG node counts +wait_for_asg_nodes() { + local ASG_NAME="$1" + local RETRY_DELAY=10 + # Get the desired capacity of the ASG + local NODE_COUNT=$(aws autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-names "$ASG_NAME" \ + --query "AutoScalingGroups[0].DesiredCapacity" \ + --output text) + + if [ "$NODE_COUNT" == "None" ]; then + log_with_timestamp "Error: Unable to retrieve Desired Capacity for ASG: $ASG_NAME." + exit 1 + fi + + log_with_timestamp "Checking ASG node count for $ASG_NAME with desired node count: $NODE_COUNT" + + while true; do + # Check InService and Terminating states via ASG + local IN_SERVICE_NODE_COUNT + IN_SERVICE_NODE_COUNT=$(aws autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-names "$ASG_NAME" \ + --query "AutoScalingGroups[0].Instances[?LifecycleState=='InService'] | length(@)" \ + --output text) + + local TERMINATING_NODE_COUNT + TERMINATING_NODE_COUNT=$(aws autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-names "$ASG_NAME" \ + --query "AutoScalingGroups[0].Instances[?LifecycleState=='Terminating'] | length(@)" \ + --output text) + + # Check for instances in EC2 shutting-down state + local SHUTTING_DOWN_NODE_COUNT + SHUTTING_DOWN_NODE_COUNT=$(aws ec2 describe-instances \ + --filters "Name=instance-state-name,Values=shutting-down" \ + --query "Reservations[].Instances[].InstanceId | length(@)" \ + --output text) + + log_with_timestamp "InService: $IN_SERVICE_NODE_COUNT, Terminating: $TERMINATING_NODE_COUNT, Shutting-down: $SHUTTING_DOWN_NODE_COUNT, Desired: $NODE_COUNT" + + if [[ -z "$IN_SERVICE_NODE_COUNT" || "$IN_SERVICE_NODE_COUNT" -le "$NODE_COUNT" ]] \ + && [[ "$TERMINATING_NODE_COUNT" -eq 0 ]] \ + && [[ "$SHUTTING_DOWN_NODE_COUNT" -eq 0 ]]; then + log_with_timestamp "Conditions met: InService <= $NODE_COUNT, no Terminating, no Shutting-down. Proceeding..." + break + else + log_with_timestamp "Conditions not met. Waiting... (InService: $IN_SERVICE_NODE_COUNT, Terminating: $TERMINATING_NODE_COUNT, Shutting-down: $SHUTTING_DOWN_NODE_COUNT)" + sleep "$RETRY_DELAY" + fi + done +} + +# Wait for the desired nodes count +wait_desired_node_count() { + local ASG_NAME="$1" + + if [ -z "$ASG_NAME" ]; then + log_with_timestamp "Error: ASG name not provided." + exit 1 + fi + + log_with_timestamp "Starting script execution for ASG: $ASG_NAME" + wait_for_asg_nodes "$ASG_NAME" + log_with_timestamp "Proceeding with the rest of the script after ASG check." +} + +# Execute the main function with the provided ASG name +wait_desired_node_count "${name}" + # Function which waits for all DNS records to be created wait_dns_records() { local ZONE_ID="$1" diff --git a/modules/graphdb/variables.tf b/modules/graphdb/variables.tf index c6f894c..8c90835 100644 --- a/modules/graphdb/variables.tf +++ b/modules/graphdb/variables.tf @@ -394,3 +394,15 @@ variable "ebs_default_kms_key" { description = "Define default KMS key" type = string } + +variable "instance_maintenance_policy_min_healthy_percentage" { + description = "Define minimum healthy percentage for the Instance Maintenance Policy" + type = number + default = 66 +} + +variable "instance_maintenance_policy_max_healthy_percentage" { + description = "Define maximum healthy percentage for the Instance Maintenance Policy" + type = number + default = 100 +}