DevOps: update ubuntu and debian images

.circleci/config.yml

@@ -12,7 +12,7 @@ orbs:
 parameters:
   ubuntu_image:
     type: string
-    default: "ubuntu-2004:2023.04.2"
+    default: "ubuntu-2404:2024.11.1"
   build_dir:
     type: string
     default: "/opt/cibuild"

.github/workflows/codegen_verification.yml

@@ -6,7 +6,7 @@ on:
   pull_request:
 jobs:
   codegen_verification:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     services:
       converter:
         image: swaggerapi/swagger-converter@sha256:dcfd1c2537f5f271cb4ec942d08aa59ca41b9a24078040061a772afca7e548ae # v1.0.4

.github/workflows/container.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   build-and-push:
     name: Build and Push to DockerHub
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04 as builder
+FROM ubuntu:24.04 as builder
 
 ARG GO_VERSION="1.23.3"
 
@@ -41,7 +41,7 @@ RUN /dist/files/build/install.sh \
     -b "${BRANCH}" \
     -s "${SHA}"
 
-FROM debian:bookworm-20240311-slim as final
+FROM debian:bookworm-20250407-slim AS final
 
 ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" KMD_PORT="7833" ALGORAND_DATA="/algod/data"
 

Makefile

@@ -8,13 +8,14 @@ else
 export GOPATH := $(shell go env GOPATH)
 GOPATH1 := $(firstword $(subst :, ,$(GOPATH)))
 endif
+GOBIN       := $(if $(shell go env GOBIN),$(shell go env GOBIN),$(GOPATH1)/bin)
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
 OS_TYPE     := $(shell ./scripts/ostype.sh)
 # overrides for cross-compiling platform-specific binaries
 ifdef CROSS_COMPILE_ARCH
   ARCH := $(CROSS_COMPILE_ARCH)
-  GO_INSTALL := CGO_ENABLED=1 GOOS=$(OS_TYPE) GOARCH=$(ARCH) go build -o $(GOPATH1)/bin-$(OS_TYPE)-$(ARCH)
+  GO_INSTALL := CGO_ENABLED=1 GOOS=$(OS_TYPE) GOARCH=$(ARCH) go build -o $(GOBIN)-$(OS_TYPE)-$(ARCH)
 else
   GO_INSTALL := go install
 endif
@@ -105,10 +106,10 @@ fmt:
 	./scripts/check_license.sh -i
 
 fix: build
-	$(GOPATH1)/bin/algofix */
+	$(GOBIN)/algofix */
 
 lint: deps
-	$(GOPATH1)/bin/golangci-lint run -c .golangci.yml
+	$(GOBIN)/golangci-lint run -c .golangci.yml
 
 expectlint:
 	cd test/e2e-go/cli/goal/expect && python3 expect_linter.py *.exp
@@ -139,7 +140,7 @@ prof:
 	cd node && go test $(GOTAGS) -cpuprofile=cpu.out -memprofile=mem.out -mutexprofile=mutex.out
 
 generate: deps
-	PATH=$(GOPATH1)/bin:$$PATH go generate ./...
+	PATH=$(GOBIN):$$PATH go generate ./...
 
 msgp: $(patsubst %,%/msgp_gen.go,$(MSGP_GENERATE))
 
@@ -153,9 +154,9 @@ logic:
 %/msgp_gen.go: deps ALWAYS
 		@set +e; \
 		printf "msgp: $(@D)..."; \
-		$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
+		$(GOBIN)/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
 		if [ "$$?" != "0" ]; then \
-			printf "failed:\n$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
+			printf "failed:\n$(GOBIN)/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
 			cat ./$@.out; \
 			rm ./$@.out; \
 			exit 1; \
@@ -178,25 +179,25 @@ crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a:
 universal:
 ifeq ($(OS_TYPE),darwin)
 	# build amd64 Mac binaries
-	mkdir -p $(GOPATH1)/bin-darwin-amd64
-	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
+	mkdir -p $(GOBIN)-darwin-amd64
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOBIN)-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
 
 	# build arm64 Mac binaries
-	mkdir -p $(GOPATH1)/bin-darwin-arm64
-	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
+	mkdir -p $(GOBIN)-darwin-arm64
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOBIN)-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
 
 	# same for buildsrc-special
 	cd tools/block-generator && \
-	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
-	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOBIN)-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOBIN)-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
 
 	# lipo together
-	mkdir -p $(GOPATH1)/bin
-	for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \
-		if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \
-			lipo -create -output $(GOPATH1)/bin/$$binary \
-			$(GOPATH1)/bin-darwin-arm64/$$binary \
-			$(GOPATH1)/bin-darwin-amd64/$$binary; \
+	mkdir -p $(GOBIN)
+	for binary in $$(ls $(GOBIN)-darwin-arm64); do \
+		if [ -f $(GOBIN)-darwin-amd64/$$binary ]; then \
+			lipo -create -output $(GOBIN)/$$binary \
+			$(GOBIN)-darwin-arm64/$$binary \
+			$(GOBIN)-darwin-amd64/$$binary; \
 		else \
 			echo "Warning: Binary $$binary exists in arm64 but not in amd64"; \
 		fi \
@@ -220,7 +221,7 @@ $(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/lib
 	cd daemon/kmd/lib/kmdapi && \
 		python3 genSwaggerWrappers.py $(KMD_API_SWAGGER_WRAPPER)
 	cd daemon/kmd && \
-		PATH=$(GOPATH1)/bin:$$PATH \
+		PATH=$(GOBIN):$$PATH \
 		go generate ./...
 	rm daemon/kmd/lib/kmdapi/$(KMD_API_SWAGGER_WRAPPER)
 
@@ -274,23 +275,23 @@ check-go-version:
 ## We overwrite bin-race/kmd with a non -race version due to
 ## the incredible performance impact of -race on Scrypt.
 build-race: build
-	@mkdir -p $(GOPATH1)/bin-race
-	GOBIN=$(GOPATH1)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -race -ldflags="$(GOLDFLAGS)" ./...
-	cp $(GOPATH1)/bin/kmd $(GOPATH1)/bin-race
+	@mkdir -p $(GOBIN)-race
+	GOBIN=$(GOBIN)-race go install $(GOTRIMPATH) $(GOTAGS) -race -ldflags="$(GOLDFLAGS)" ./...
+	cp $(GOBIN)/kmd $(GOBIN)-race
 
-NONGO_BIN_FILES=$(GOPATH1)/bin/find-nodes.sh $(GOPATH1)/bin/update.sh $(GOPATH1)/bin/COPYING $(GOPATH1)/bin/ddconfig.sh
+NONGO_BIN_FILES=$(GOBIN)/find-nodes.sh $(GOBIN)/update.sh $(GOBIN)/COPYING $(GOBIN)/ddconfig.sh
 
 NONGO_BIN: $(NONGO_BIN_FILES)
 
-$(GOPATH1)/bin/find-nodes.sh: scripts/find-nodes.sh
+$(GOBIN)/find-nodes.sh: scripts/find-nodes.sh
 
-$(GOPATH1)/bin/update.sh: cmd/updater/update.sh
+$(GOBIN)/update.sh: cmd/updater/update.sh
 
-$(GOPATH1)/bin/COPYING: COPYING
+$(GOBIN)/COPYING: COPYING
 
-$(GOPATH1)/bin/ddconfig.sh: scripts/ddconfig.sh
+$(GOBIN)/ddconfig.sh: scripts/ddconfig.sh
 
-$(GOPATH1)/bin/%:
+$(GOBIN)/%:
 	cp -f $< $@
 
 test: build
@@ -315,7 +316,7 @@ testall: fulltest integration
 
 clean:
 	go clean -i ./...
-	rm -f $(GOPATH1)/bin/node_exporter
+	rm -f $(GOBIN)/bin/node_exporter
 	cd crypto/libsodium-fork && \
 		test ! -e Makefile || make clean
 	rm -rf crypto/lib
@@ -326,17 +327,17 @@ clean:
 # clean without crypto
 cleango:
 	go clean -i ./...
-	rm -f $(GOPATH1)/bin/node_exporter
+	rm -f $(GOBIN)/node_exporter
 
 # assign the phony target node_exporter the dependency of the actual executable.
-node_exporter: $(GOPATH1)/bin/node_exporter
+node_exporter: $(GOBIN)/node_exporter
 
 # The recipe for making the node_exporter is by extracting it from the gzipped&tar file.
 # The file is was taken from the S3 cloud and it traditionally stored at
 # /travis-build-artifacts-us-ea-1.algorand.network/algorand/node_exporter/latest/node_exporter-stable-linux-x86_64.tar.gz
-$(GOPATH1)/bin/node_exporter:
-	mkdir -p $(GOPATH1)/bin && \
-	cd $(GOPATH1)/bin && \
+$(GOBIN)/node_exporter:
+	mkdir -p $(GOBIN) && \
+	cd $(GOBIN) && \
 	if [ -z "$(CROSS_COMPILE_ARCH)" ]; then \
 		tar -xzvf $(SRCPATH)/installer/external/node_exporter-stable-$(shell ./scripts/ostype.sh)-$(shell uname -m | tr '[:upper:]' '[:lower:]').tar.gz; \
 	else \
@@ -358,7 +359,7 @@ gen/%/genesis.dump: gen/%/genesis.json
 	./scripts/dump_genesis.sh $< > $@
 
 gen/%/genesis.json: gen/%.json gen/generate.go buildsrc
-	$(GOPATH1)/bin/genesis -q $(SHORT_PART_PERIOD_FLAG) -n $(shell basename $(shell dirname $@)) -c $< -d $(subst .json,,$<)
+	$(GOBIN)/genesis -q $(SHORT_PART_PERIOD_FLAG) -n $(shell basename $(shell dirname $@)) -c $< -d $(subst .json,,$<)
 
 gen: $(addsuffix gen, $(NETWORKS)) mainnetgen
 
@@ -376,15 +377,15 @@ mainnetgen: gen/mainnet/genesis.dump
 # This target is preserved as part of the history on how mainnet genesis.json was generated from the CSV file.
 gen/mainnet/genesis.json: gen/pregen/mainnet/genesis.csv buildsrc
 	mkdir -p gen/mainnet
-	cat gen/pregen/mainnet/genesis.csv | $(GOPATH1)/bin/incorporate -m gen/pregen/mainnet/metadata.json > gen/mainnet/genesis.json
+	cat gen/pregen/mainnet/genesis.csv | $(GOBIN)/incorporate -m gen/pregen/mainnet/metadata.json > gen/mainnet/genesis.json
 
 capabilities: build
-	sudo setcap cap_ipc_lock+ep $(GOPATH1)/bin/kmd
+	sudo setcap cap_ipc_lock+ep $(GOBIN)/kmd
 
 dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 
 install: build
-	scripts/dev_install.sh -p $(GOPATH1)/bin
+	scripts/dev_install.sh -p $(GOBIN)
 
 .PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger universal
 

cmd/updater/update.sh

@@ -492,7 +492,7 @@ function run_systemd_action() {
 function backup_binaries() {
     echo "Backing up current binary files..."
     mkdir -p "${BINDIR}/backup"
-    BACKUPFILES="algod kmd carpenter doberman goal update.sh updater diagcfg"
+    BACKUPFILES="algotmpl algod kmd carpenter doberman goal update.sh updater diagcfg"
     # add node_exporter to the files list we're going to backup, but only we if had it previously deployed.
     [ -f "${BINDIR}/node_exporter" ] && BACKUPFILES="${BACKUPFILES} node_exporter"
     tar -zcf "${BINDIR}/backup/bin-v${CURRENTVER}.tar.gz" -C "${BINDIR}" ${BACKUPFILES} >/dev/null 2>&1

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 ARG GOLANG_VERSION
 ENV DEBIAN_FRONTEND noninteractive
 

docker/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 ARG GOLANG_VERSION
 
 RUN apt-get update && apt-get install -y git wget autoconf build-essential shellcheck

docker/build/Dockerfile-deploy

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 ubuntu:20.04
+FROM --platform=linux/amd64 ubuntu:24.04
 ARG GOLANG_VERSION
 
 RUN apt-get update && apt-get install -y git wget autoconf jq bsdmainutils shellcheck make python3 libtool g++

docker/build/aptly.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ARG ARCH=amd64
 

docker/build/cicd.ubuntu.Dockerfile

@@ -1,26 +1,29 @@
 ARG ARCH="amd64"
 
-FROM ${ARCH}/ubuntu:20.04
+FROM ${ARCH}/ubuntu:24.04
 ARG GOLANG_VERSION
 ARG ARCH="amd64"
 ARG GOARCH="amd64"
-ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -y build-essential git wget autoconf jq bsdmainutils shellcheck awscli libtool
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y update
+RUN apt-get install -y build-essential git wget autoconf jq bsdmainutils shellcheck libtool
+RUN apt-get install -y curl unzip
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip
+RUN ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update
 WORKDIR /root
 RUN wget https://dl.google.com/go/go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz \
     && tar -xvf go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz && \
     mv go /usr/local
+RUN mkdir -p /app/go
 ENV GOROOT=/usr/local/go \
-    GOPATH=$HOME/go \
+    GOPATH=/app/go \
     ARCH_TYPE=${ARCH}
-RUN mkdir -p $GOPATH/src/github.com/algorand
-COPY . $GOPATH/src/github.com/algorand/go-algorand
+WORKDIR /app
+COPY . /app
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH \
     GOPROXY=https://proxy.golang.org,https://pkg.go.dev,https://goproxy.io,direct
-WORKDIR $GOPATH/src/github.com/algorand/go-algorand
 RUN git config --global --add safe.directory '*'
 RUN make clean
-RUN rm -rf $GOPATH/src/github.com/algorand/go-algorand && \
-    mkdir -p $GOPATH/src/github.com/algorand/go-algorand
 RUN echo "vm.max_map_count = 262144" >> /etc/sysctl.conf
 CMD ["/bin/bash"]

docker/build/docker.ubuntu.Dockerfile

@@ -1,6 +1,6 @@
 ARG ARCH="amd64"
 
-FROM ${ARCH}/ubuntu:20.04
+FROM ${ARCH}/ubuntu:24.04
 ARG GOLANG_VERSION
 ARG ARCH="amd64"
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install curl python python3.7 python3-pip build-essential apt-transport-https ca-certificates software-properties-common -y && \

docker/build/releases-page.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install git python3 python3-pip -y && \

scripts/build_package.sh

@@ -37,8 +37,22 @@ if [[ "${UNAME}" == *"MINGW"* ]]; then
 	GOPATH1=$HOME/go
 else
 	export GOPATH=$(go env GOPATH)
+    GOPATH1=${GOPATH%%:*}
 fi
-export GOPATHBIN=${GOPATH%%:*}/bin
+
+# Setup GOBIN
+# If env var GOBIN is set, use it
+# Otherwise, use go env GOBIN if set
+# Otherwise default to $GOPATH1/bin
+
+if [[ -n "${GOBIN}" ]]; then
+    export GOBIN=${GOBIN}
+elif [[ -n "$(go env GOBIN)" ]]; then
+    export GOBIN=$(go env GOBIN)
+else
+    export GOBIN=${GOPATH1}/bin
+fi
+
 REPO_DIR=$(pwd)
 
 echo "Building package for '${OS} - ${ARCH}'"
@@ -63,9 +77,9 @@ DEFAULT_RELEASE_NETWORK=$(./scripts/compute_branch_release_network.sh "${DEFAULT
 mkdir ${PKG_ROOT}/bin
 
 # If you modify this list, also update this list in ./cmd/updater/update.sh backup_binaries()
-bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "find-nodes.sh" "goal" "kmd" "msgpacktool" "node_exporter" "tealcut" "tealdbg" "update.sh" "updater" "COPYING")
+bin_files=("algocfg" "algotmpl" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "find-nodes.sh" "goal" "kmd" "msgpacktool" "node_exporter" "tealcut" "tealdbg" "update.sh" "updater" "COPYING")
 for bin in "${bin_files[@]}"; do
-    cp ${GOPATHBIN}/${bin} ${PKG_ROOT}/bin
+    cp ${GOBIN}/${bin} ${PKG_ROOT}/bin
     if [ $? -ne 0 ]; then exit 1; fi
 done
 
@@ -109,7 +123,7 @@ echo "Staging tools package files"
 bin_files=("algons" "coroner" "dispenser" "netgoal" "nodecfg" "pingpong" "cc_service" "cc_agent" "cc_client" "loadgenerator" "COPYING" "dsign" "catchpointdump" "block-generator")
 mkdir -p ${TOOLS_ROOT}
 for bin in "${bin_files[@]}"; do
-    cp ${GOPATHBIN}/${bin} ${TOOLS_ROOT}
+    cp ${GOBIN}/${bin} ${TOOLS_ROOT}
     if [ $? -ne 0 ]; then exit 1; fi
 done
 
@@ -118,7 +132,7 @@ TEST_UTILS_ROOT=${PKG_ROOT}/test-utils
 bin_files=("algotmpl" "COPYING")
 mkdir -p ${TEST_UTILS_ROOT}
 for bin in "${bin_files[@]}"; do
-    cp ${GOPATHBIN}/${bin} ${TEST_UTILS_ROOT}
+    cp ${GOBIN}/${bin} ${TEST_UTILS_ROOT}
     if [ $? -ne 0 ]; then exit 1; fi
 done
 

scripts/configure_dev.sh

@@ -73,7 +73,7 @@ function install_windows_shellcheck() {
 
 if [ "${OS}" = "linux" ]; then
     if ! which sudo >/dev/null; then
-        "$SCRIPTPATH/install_linux_deps.sh"
+        DEBIAN_FRONTEND="$DEBIAN_FRONTEND" "$SCRIPTPATH/install_linux_deps.sh"
     else
         sudo "$SCRIPTPATH/install_linux_deps.sh"
     fi

scripts/travis/configure_dev.sh

@@ -6,7 +6,7 @@ set +e
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
 OS=$("${SCRIPTPATH}/../ostype.sh")
 ARCH=$("${SCRIPTPATH}/../archtype.sh")
-
+export DEBIAN_FRONTEND="noninteractive"
 if [[ "${OS}" == "linux" ]]; then
     if [[ "${ARCH}" == "arm64" ]]; then
         set -e