Ops: Bump ubuntu and centos versions for building and testing (#6378)

Co-authored-by: John Lee <john@onetechnical.com>
Co-authored-by: John Lee <john.lee@algorand.com>

Author: gmalouf

.circleci/config.yml

@@ -6,7 +6,7 @@ on:
   pull_request:
 jobs:
   codegen_verification:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     services:
       converter:
         image: swaggerapi/swagger-converter@sha256:dcfd1c2537f5f271cb4ec942d08aa59ca41b9a24078040061a772afca7e548ae # v1.0.4

.github/workflows/codegen_verification.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   build-and-push:
     name: Build and Push to DockerHub
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4

.github/workflows/container.yml

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04 as builder
+FROM ubuntu:24.04 AS builder
 
 ARG GO_VERSION="1.23.9"
 
@@ -41,7 +41,7 @@ RUN /dist/files/build/install.sh \
     -b "${BRANCH}" \
     -s "${SHA}"
 
-FROM debian:bookworm-20240311-slim as final
+FROM debian:bookworm-20250630-slim AS final
 
 ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" KMD_PORT="7833" ALGORAND_DATA="/algod/data"
 

Dockerfile

@@ -8,13 +8,14 @@ else
 export GOPATH := $(shell go env GOPATH)
 GOPATH1 := $(firstword $(subst :, ,$(GOPATH)))
 endif
+GOBIN := $(if $(shell go env GOBIN),$(shell go env GOBIN),$(GOPATH1)/bin)
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
 OS_TYPE     := $(shell ./scripts/ostype.sh)
 # overrides for cross-compiling platform-specific binaries
 ifdef CROSS_COMPILE_ARCH
   ARCH := $(CROSS_COMPILE_ARCH)
-  GO_INSTALL := CGO_ENABLED=1 GOOS=$(OS_TYPE) GOARCH=$(ARCH) go build -o $(GOPATH1)/bin-$(OS_TYPE)-$(ARCH)
+  GO_INSTALL := CGO_ENABLED=1 GOOS=$(OS_TYPE) GOARCH=$(ARCH) go build -o $(GOBIN)
 else
   GO_INSTALL := go install
 endif
@@ -105,10 +106,10 @@ fmt:
 	./scripts/check_license.sh -i
 
 fix: build
-	$(GOPATH1)/bin/algofix */
+	$(GOBIN)/algofix */
 
 lint: deps
-	$(GOPATH1)/bin/golangci-lint run -c .golangci.yml
+	$(GOBIN)/golangci-lint run -c .golangci.yml
 
 expectlint:
 	cd test/e2e-go/cli/goal/expect && python3 expect_linter.py *.exp
@@ -139,7 +140,7 @@ prof:
 	cd node && go test $(GOTAGS) -cpuprofile=cpu.out -memprofile=mem.out -mutexprofile=mutex.out
 
 generate: deps
-	PATH=$(GOPATH1)/bin:$$PATH go generate ./...
+	PATH=$(GOBIN):$$PATH go generate ./...
 
 msgp: $(patsubst %,%/msgp_gen.go,$(MSGP_GENERATE))
 
@@ -153,9 +154,9 @@ logic:
 %/msgp_gen.go: deps ALWAYS
 		@set +e; \
 		printf "msgp: $(@D)..."; \
-		$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
+		$(GOBIN)/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
 		if [ "$$?" != "0" ]; then \
-			printf "failed:\n$(GOPATH1)/bin/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
+			printf "failed:\n$(GOBIN)/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
 			cat ./$@.out; \
 			rm ./$@.out; \
 			exit 1; \
@@ -178,29 +179,42 @@ crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a:
 universal:
 ifeq ($(OS_TYPE),darwin)
 	# build amd64 Mac binaries
-	mkdir -p $(GOPATH1)/bin-darwin-amd64
-	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
+	mkdir -p $(GOBIN)-darwin-amd64
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOBIN)-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
 
 	# build arm64 Mac binaries
-	mkdir -p $(GOPATH1)/bin-darwin-arm64
-	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
+	mkdir -p $(GOBIN)-darwin-arm64
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOBIN)-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
 
 	# same for buildsrc-special
 	cd tools/block-generator && \
-	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
-	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOBIN)-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=13.0" --host=x86_64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOBIN)-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=13.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=13.0" --host=aarch64-apple-darwin' $(MAKE)
 
 	# lipo together
-	mkdir -p $(GOPATH1)/bin
-	for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \
-		if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \
-			lipo -create -output $(GOPATH1)/bin/$$binary \
-			$(GOPATH1)/bin-darwin-arm64/$$binary \
-			$(GOPATH1)/bin-darwin-amd64/$$binary; \
+	mkdir -p $(GOBIN)
+	for binary in $$(ls $(GOBIN)-darwin-arm64); do \
+		skip=false; \
+		for nongo_file in $(NONGO_BIN_FILES) $(GOBIN)/update.sh ; do \
+			if [ "$(GOBIN)/$$binary" = "$$nongo_file" ]; then \
+				echo "Skipping non-binary file: $$binary"; \
+				skip=true; \
+				break; \
+			fi; \
+		done; \
+		if [ "$$skip" = "true" ]; then \
+			continue; \
+		fi; \
+		if [ -f $(GOBIN)-darwin-amd64/$$binary ]; then \
+			lipo -create -output $(GOBIN)/$$binary \
+			$(GOBIN)-darwin-arm64/$$binary \
+			$(GOBIN)-darwin-amd64/$$binary; \
 		else \
 			echo "Warning: Binary $$binary exists in arm64 but not in amd64"; \
-		fi \
+		fi; \
 	done
+		# for node_exporter cross-compilation is using universal binary already
+		cp -f $(GOBIN)-darwin-arm64/node_exporter $(GOBIN)/node_exporter
 else
 	echo "OS_TYPE must be darwin for universal builds, skipping"
 endif
@@ -220,7 +234,7 @@ $(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/lib
 	cd daemon/kmd/lib/kmdapi && \
 		python3 genSwaggerWrappers.py $(KMD_API_SWAGGER_WRAPPER)
 	cd daemon/kmd && \
-		PATH=$(GOPATH1)/bin:$$PATH \
+		PATH=$(GOBIN):$$PATH \
 		go generate ./...
 	rm daemon/kmd/lib/kmdapi/$(KMD_API_SWAGGER_WRAPPER)
 
@@ -274,23 +288,23 @@ check-go-version:
 ## We overwrite bin-race/kmd with a non -race version due to
 ## the incredible performance impact of -race on Scrypt.
 build-race: build
-	@mkdir -p $(GOPATH1)/bin-race
-	GOBIN=$(GOPATH1)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -race -ldflags="$(GOLDFLAGS)" ./...
-	cp $(GOPATH1)/bin/kmd $(GOPATH1)/bin-race
+	@mkdir -p $(GOBIN)-race
+	GOBIN=$(GOBIN)-race go install $(GOTRIMPATH) $(GOTAGS) -race -ldflags="$(GOLDFLAGS)" ./...
+	cp $(GOBIN)/kmd $(GOBIN)-race
 
-NONGO_BIN_FILES=$(GOPATH1)/bin/find-nodes.sh $(GOPATH1)/bin/update.sh $(GOPATH1)/bin/COPYING $(GOPATH1)/bin/ddconfig.sh
+NONGO_BIN_FILES=$(GOBIN)/find-nodes.sh $(GOBIN)/update.sh $(GOBIN)/COPYING $(GOBIN)/ddconfig.sh
 
 NONGO_BIN: $(NONGO_BIN_FILES)
 
-$(GOPATH1)/bin/find-nodes.sh: scripts/find-nodes.sh
+$(GOBIN)/find-nodes.sh: scripts/find-nodes.sh
 
-$(GOPATH1)/bin/update.sh: cmd/updater/update.sh
+$(GOBIN)/update.sh: cmd/updater/update.sh
 
-$(GOPATH1)/bin/COPYING: COPYING
+$(GOBIN)/COPYING: COPYING
 
-$(GOPATH1)/bin/ddconfig.sh: scripts/ddconfig.sh
+$(GOBIN)/ddconfig.sh: scripts/ddconfig.sh
 
-$(GOPATH1)/bin/%:
+$(GOBIN)/%:
 	cp -f $< $@
 
 test: build
@@ -315,7 +329,7 @@ testall: fulltest integration
 
 clean:
 	go clean -i ./...
-	rm -f $(GOPATH1)/bin/node_exporter
+	rm -f $(GOBIN)/node_exporter
 	cd crypto/libsodium-fork && \
 		test ! -e Makefile || make clean
 	rm -rf crypto/lib
@@ -326,17 +340,17 @@ clean:
 # clean without crypto
 cleango:
 	go clean -i ./...
-	rm -f $(GOPATH1)/bin/node_exporter
+	rm -f $(GOBIN)/node_exporter
 
 # assign the phony target node_exporter the dependency of the actual executable.
-node_exporter: $(GOPATH1)/bin/node_exporter
+node_exporter: $(GOBIN)/node_exporter
 
 # The recipe for making the node_exporter is by extracting it from the gzipped&tar file.
 # The file is was taken from the S3 cloud and it traditionally stored at
 # /travis-build-artifacts-us-ea-1.algorand.network/algorand/node_exporter/latest/node_exporter-stable-linux-x86_64.tar.gz
-$(GOPATH1)/bin/node_exporter:
-	mkdir -p $(GOPATH1)/bin && \
-	cd $(GOPATH1)/bin && \
+$(GOBIN)/node_exporter:
+	mkdir -p $(GOBIN) && \
+	cd $(GOBIN) && \
 	if [ -z "$(CROSS_COMPILE_ARCH)" ]; then \
 		tar -xzvf $(SRCPATH)/installer/external/node_exporter-stable-$(shell ./scripts/ostype.sh)-$(shell uname -m | tr '[:upper:]' '[:lower:]').tar.gz; \
 	else \
@@ -358,7 +372,7 @@ gen/%/genesis.dump: gen/%/genesis.json
 	./scripts/dump_genesis.sh $< > $@
 
 gen/%/genesis.json: gen/%.json gen/generate.go buildsrc
-	$(GOPATH1)/bin/genesis -q $(SHORT_PART_PERIOD_FLAG) -n $(shell basename $(shell dirname $@)) -c $< -d $(subst .json,,$<)
+	$(GOBIN)/genesis -q $(SHORT_PART_PERIOD_FLAG) -n $(shell basename $(shell dirname $@)) -c $< -d $(subst .json,,$<)
 
 gen: $(addsuffix gen, $(NETWORKS)) mainnetgen
 
@@ -376,15 +390,15 @@ mainnetgen: gen/mainnet/genesis.dump
 # This target is preserved as part of the history on how mainnet genesis.json was generated from the CSV file.
 gen/mainnet/genesis.json: gen/pregen/mainnet/genesis.csv buildsrc
 	mkdir -p gen/mainnet
-	cat gen/pregen/mainnet/genesis.csv | $(GOPATH1)/bin/incorporate -m gen/pregen/mainnet/metadata.json > gen/mainnet/genesis.json
+	cat gen/pregen/mainnet/genesis.csv | $(GOBIN)/incorporate -m gen/pregen/mainnet/metadata.json > gen/mainnet/genesis.json
 
 capabilities: build
-	sudo setcap cap_ipc_lock+ep $(GOPATH1)/bin/kmd
+	sudo setcap cap_ipc_lock+ep $(GOBIN)/kmd
 
 dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 
 install: build
-	scripts/dev_install.sh -p $(GOPATH1)/bin
+	scripts/dev_install.sh -p $(GOBIN)
 
 .PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger universal
 

Makefile

@@ -21,7 +21,7 @@ Development is done using the Go Programming Language. The Go version is specifi
 
 ### Linux / OSX
 
-We currently strive to support Debian-based distributions, with Ubuntu 20.04 as our official release target. Building on Arch Linux also works well. Our core engineering team uses Linux and OSX, so both environments are well-supported for development.
+We currently strive to support Debian-based distributions, with Ubuntu 24.04 as our official release target. Building on Arch Linux also works well. Our core engineering team uses Linux and OSX, so both environments are well-supported for development.
 
 **OSX Only**: [Homebrew (brew)](https://brew.sh) must be installed before continuing. [Here](https://docs.brew.sh/Installation) are the installation requirements.
 

README.md

@@ -495,6 +495,8 @@ function backup_binaries() {
     BACKUPFILES="algod kmd carpenter doberman goal update.sh updater diagcfg"
     # add node_exporter to the files list we're going to backup, but only we if had it previously deployed.
     [ -f "${BINDIR}/node_exporter" ] && BACKUPFILES="${BACKUPFILES} node_exporter"
+    # If we have algotmpl, we should back it up too
+    [ -f "${BINDIR}/algotmpl" ] && BACKUPFILES="${BACKUPFILES} algotmpl"
     tar -zcf "${BINDIR}/backup/bin-v${CURRENTVER}.tar.gz" -C "${BINDIR}" ${BACKUPFILES} >/dev/null 2>&1
 }
 

cmd/updater/update.sh

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 ARG GOLANG_VERSION
 ENV DEBIAN_FRONTEND noninteractive
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 ARG GOLANG_VERSION
 
 RUN apt-get update && apt-get install -y git wget autoconf build-essential shellcheck

docker/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 ubuntu:20.04
+FROM --platform=linux/amd64 ubuntu:24.04
 ARG GOLANG_VERSION
 
 RUN apt-get update && apt-get install -y git wget autoconf jq bsdmainutils shellcheck make python3 libtool g++