Use EC2 Root Volume Replacement to replace macOS hosts

lib/aws-semaphore-agent-stack.js

@@ -94,6 +94,23 @@ class AwsSemaphoreAgentStack extends Stack {
         new PolicyStatement({
           effect: Effect.ALLOW,
           actions: [
+            "ec2:CreateReplaceRootVolumeTask"
+          ],
+          resources: [
+            `arn:aws:ec2:*:${this.account}:instance/*`,
+            `arn:aws:ec2:${this.region}::image/*`
+          ],
+          conditions: {
+            "StringLike": {
+              "aws:userid": "*:${ec2:InstanceID}"
+            }
+          }
+        }),
+        new PolicyStatement({
+          effect: Effect.ALLOW,
+          actions: [
+            "autoscaling:EnterStandby",
+            "autoscaling:ExitStandby",
             "autoscaling:SetInstanceHealth",
             "autoscaling:TerminateInstanceInAutoScalingGroup"
           ],
@@ -323,6 +340,7 @@ class AwsSemaphoreAgentStack extends Stack {
       blockDevices: this.createBlockDevices(),
       securityGroup: securityGroup,
       instanceInitiatedShutdownBehavior: InstanceInitiatedShutdownBehavior.TERMINATE,
+      instanceMetadataTags: true,
       keyName: keyName != "" ? keyName : undefined
     });
 

packer/macos/files/start-agent.sh

@@ -204,6 +204,24 @@ region=$(curl \
   --show-error \
   --location "http://169.254.169.254/latest/meta-data/placement/region"
 )
+instance_id=$(curl \
+  -H "X-aws-ec2-metadata-token: $token" \
+  --fail \
+  --silent \
+  --show-error \
+  --location "http://169.254.169.254/latest/meta-data/instance-id"
+)
+asg_name=$(curl \
+  -H "X-aws-ec2-metadata-token: $token" \
+  --fail \
+  --silent \
+  --show-error \
+  --location "http://169.254.169.254/latest/meta-data/tags/instance/aws:autoscaling:groupName"
+)
+
+# If the instance is in the "Standby" LifecycleState (after root volume replacement) we want to exit so the health-check runs
+# If it's the first boot the instance may already be in "InService" LifecycleState so we ignore any errors from this command
+aws autoscaling exit-standby --region "$region" --instance-ids "$instance_id" --auto-scaling-group-name "$asg_name" || true
 
 # The parameters required for the agent configuration are stored in an SSM parameter.
 # We need to fetch them before proceeding with anything else.

packer/macos/files/terminate-instance.sh

@@ -27,8 +27,20 @@ if [[ $SEMAPHORE_AGENT_SHUTDOWN_REASON == "IDLE" ]]; then
     --instance-id "$instance_id" \
     --should-decrement-desired-capacity
 else
-  aws autoscaling terminate-instance-in-auto-scaling-group \
+  # Enter standby LifecycleState because the EC2 health check will fail while we're replacing the root volume
+  # We also decrement desired capacity so the ASG doesn't create a new replacement instance in the meantime
+  # The instance will exit standby automatically in start-agent.sh after reboot
+  asg_name=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/tags/instance/aws:autoscaling:groupName")
+  aws autoscaling enter-standby \
+    --region "$region" \
+    --instance-ids "$instance_id" \
+    --auto-scaling-group-name "$asg_name" \
+    --should-decrement-desired-capacity
+  # https://aws.amazon.com/blogs/compute/new-reset-amazon-ec2-mac-instances-to-a-known-state-using-replace-root-volume-capability/
+  ami_id=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/ami-id")
+  aws ec2 create-replace-root-volume-task \
     --region "$region" \
     --instance-id "$instance_id" \
-    --no-should-decrement-desired-capacity
+    --image-id "$ami_id" \
+    --delete-replaced-root-volume
 fi

packer/macos/macos.pkr.hcl

@@ -76,7 +76,7 @@ source "amazon-ebs" "macos" {
     owners = ["amazon"]
 
     filters = {
-      name                = "amzn-ec2-macos-12.6*"
+      name                = "amzn-ec2-macos-13.7*"
       architecture        = "${var.arch}*"
       root-device-type    = "ebs"
       virtualization-type = "hvm"