Upgrade Base Image to Amazon Linux 2023

Makefile

@@ -3,7 +3,7 @@
 VERSION = $(shell git describe --tags --candidates=1)
 SHELL = /bin/bash -o pipefail
 
-PACKER_VERSION ?= 1.6.2
+PACKER_VERSION ?= 1.8.6
 PACKER_LINUX_FILES = $(exec find packer/linux)
 PACKER_WINDOWS_FILES = $(exec find packer/windows)
 

goss.yaml

@@ -109,10 +109,16 @@ command:
   "git-lfs --version":
     exit-status: 0
 
-  "/etc/cron.hourly/docker-low-disk-gc":
+  "systemctl is-enabled docker-gc.timer":
     exit-status: 0
 
-  "/etc/cron.hourly/docker-gc":
+  /usr/local/bin/docker-gc:
+    exit-status: 0
+
+  "systemctl is-enabled docker-low-disk-gc.timer":
+    exit-status: 0
+
+  /usr/local/bin/docker-low-disk-gc:
     exit-status: 0
 
   # Checks that docker is running

packer/linux/buildkite-ami.json

@@ -11,7 +11,7 @@
       "region": "{{user `region`}}",
       "source_ami_filter": {
         "filters": {
-          "name": "amzn2-ami-kernel-5.10-hvm-2.0.*-gp2",
+          "name": "al2023-ami-minimal-2023.0.*.*-kernel-*",
           "architecture": "{{user `arch`}}",
           "virtualization-type": "hvm"
         },
@@ -21,7 +21,7 @@
       "instance_type": "{{user `instance_type`}}",
       "ssh_username": "ec2-user",
       "ami_name": "buildkite-stack-linux-{{user `arch`}}-{{isotime | clean_resource_name}}",
-      "ami_description": "Buildkite Elastic Stack (Amazon Linux 2 LTS w/ docker)",
+      "ami_description": "Buildkite Elastic Stack (Amazon Linux 2023 w/ docker)",
       "ami_groups": ["all"]
     }
   ],
@@ -74,13 +74,7 @@
     },
     {
       "type": "shell",
-      "script": "scripts/upgrade-kernel.sh"
-    },
-    {
-      "type": "shell",
-      "inline": [
-        "rm /home/ec2-user/.ssh/authorized_keys"
-      ]
+      "inline": ["rm /home/ec2-user/.ssh/authorized_keys"]
     }
   ]
 }

packer/linux/conf/docker/systemd/docker-gc.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Clean files used by docker except for the build cache
+Wants=docker-gc.timer
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/docker-gc
+
+[Install]
+WantedBy=multi-user.target

packer/linux/conf/docker/systemd/docker-gc.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Clean files used by docker except for the build cache
+Requires=docker-gc.service
+
+[Timer]
+Unit=docker-gc.service
+OnCalendar=hourly
+
+[Install]
+WantedBy=timers.target

packer/linux/conf/docker/systemd/docker-low-disk-gc.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Clean files used by docker including the build cache when disk space is low
+Wants=docker-low-disk-gc.timer
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/docker-low-disk-gc
+
+[Install]
+WantedBy=multi-user.target

packer/linux/conf/docker/systemd/docker-low-disk-gc.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Clean files used by docker including the build cache when disk space is low
+Requires=docker-low-disk-gc.service
+
+[Timer]
+Unit=docker-low-disk-gc.service
+OnCalendar=hourly
+
+[Install]
+WantedBy=timers.target

packer/linux/scripts/install-docker.sh

@@ -3,7 +3,6 @@ set -eu -o pipefail
 
 DOCKER_VERSION=20.10.23
 DOCKER_RELEASE="stable"
-DOCKER_COMPOSE_VERSION=1.29.2
 DOCKER_COMPOSE_V2_VERSION=2.16.0
 DOCKER_BUILDX_VERSION="0.10.5"
 MACHINE=$(uname -m)
@@ -33,42 +32,18 @@ sudo curl -Lfs -o /etc/systemd/system/docker.socket "https://raw.githubuserconte
 sudo systemctl daemon-reload
 sudo systemctl enable docker.service
 
-if [ "${MACHINE}" == "x86_64" ]; then
-	echo "Downloading docker-compose..."
-	sudo curl -Lsf -o /usr/bin/docker-compose https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64
-	sudo chmod +x /usr/bin/docker-compose
-	docker-compose --version
-elif [[ "${MACHINE}" == "aarch64" ]]; then
-  sudo yum install -y gcc-c++ libffi-devel openssl11 openssl11-devel python3-devel
-
-  # docker-compose depends on the cryptography package, v3.4 of which
-  # introduces a build dependency on rust; let's avoid that for now.
-  # https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#34---2021-02-07
-  # This should be unpinned ASAP; hopefully docker-compose will offer binary
-  # download for arm64 at some point:
-  # https://github.com/docker/compose/issues/7472
-  CONSTRAINT_FILE="/tmp/docker-compose-pip-constraint"
-  echo 'cryptography<3.4' >"$CONSTRAINT_FILE"
-  echo 'urllib3<2' >"$CONSTRAINT_FILE"
-  sudo pip3 install --constraint "$CONSTRAINT_FILE" "docker-compose==${DOCKER_COMPOSE_VERSION}"
-
-	docker-compose version
-else
-  echo "No docker compose option configured for arch ${MACHINE}"
-  exit 1
-fi
-
-echo "Adding docker cron tasks..."
-sudo cp /tmp/conf/docker/cron.hourly/docker-gc /etc/cron.hourly/docker-gc
-sudo cp /tmp/conf/docker/cron.hourly/docker-low-disk-gc /etc/cron.hourly/docker-low-disk-gc
-sudo chmod +x /etc/cron.hourly/docker-*
+echo "Adding docker systemd timers..."
+sudo cp /tmp/conf/docker/scripts/* /usr/local/bin
+sudo cp /tmp/conf/docker/systemd/docker-* /etc/systemd/system
+sudo chmod +x /usr/local/bin/docker-*
+sudo systemctl daemon-reload
+sudo systemctl enable docker-gc.timer docker-low-disk-gc.timer
 
 echo "Installing jq..."
 sudo yum install -y -q jq
 jq --version
 
 echo "Installing docker buildx..."
-
 DOCKER_CLI_DIR=/usr/libexec/docker/cli-plugins
 sudo mkdir -p "${DOCKER_CLI_DIR}"
 
@@ -82,7 +57,6 @@ case "${MACHINE}" in
     ;;
 esac
 
-
 sudo curl --location --fail --silent --output "${DOCKER_CLI_DIR}/docker-buildx" "https://github.com/docker/buildx/releases/download/v${DOCKER_BUILDX_VERSION}/buildx-v${DOCKER_BUILDX_VERSION}.linux-${BUILDX_ARCH}"
 sudo chmod +x "${DOCKER_CLI_DIR}/docker-buildx"
 docker buildx version
@@ -91,6 +65,9 @@ sudo curl --location --fail --silent --output "${DOCKER_CLI_DIR}/docker-compose"
 sudo chmod +x "${DOCKER_CLI_DIR}/docker-compose"
 docker compose version
 
+sudo ln -s "${DOCKER_CLI_DIR}/docker-compose" /usr/bin/docker-compose
+docker-compose version
+
 echo "Installing qemu..."
 sudo yum install -y qemu qemu-user-static
 

packer/linux/scripts/install-utils.sh

@@ -4,16 +4,15 @@ set -eu -o pipefail
 echo "Updating core packages"
 sudo yum update -y
 
-echo "Updating awscli..."
-sudo yum install -y python2-pip
+echo "Installing python..."
 sudo yum install -y python3-pip python3 python3-setuptools
-sudo pip install --upgrade awscli
-sudo pip install future
-sudo pip3 install future
 
 echo "Installing zip utils..."
 sudo yum install -y zip unzip git pigz
 
+echo "Installing aws utils..."
+sudo yum install -y awscli-2 aws-cfn-bootstrap
+
 echo "Installing bk elastic stack bin files..."
 sudo chmod +x /tmp/conf/bin/bk-*
 sudo mv /tmp/conf/bin/bk-* /usr/local/bin