Build: Use go tool and separate module file, go run to remove some build scripts (#6491)

Author: jannotti

.github/actions/setup-test/action.yml

@@ -28,13 +28,3 @@ runs:
         sudo dpkg-reconfigure -f noninteractive man-db
         sudo apt-get install -y --no-upgrade expect
       shell: bash
-    - name: Cache gotestsum
-      id: cache-gotestsum
-      uses: actions/cache@v4
-      with:
-        path: ~/go/bin/gotestsum
-        key: gotestsum-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('scripts/buildtools/versions') }}
-    - name: Install gotestsum
-      if: steps.cache-gotestsum.outputs.cache-hit != 'true'
-      run: ./scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
-      shell: bash

.github/workflows/ci-nightly.yml

@@ -102,11 +102,10 @@ jobs:
       - name: Run tests
         run: |
           ./scripts/configure_dev.sh
-          ./scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
           PACKAGES="$(go list ./... | grep -v /go-algorand/test/)"
           export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')          
           mkdir -p test_results/${{ matrix.platform }}_test_nightly/${PARTITION_ID}
-          gotestsum --format standard-quiet \
+          go tool -modfile=tool.mod gotestsum --format standard-quiet \
             --junitfile ~/test_results/${{ matrix.platform }}_test_nightly/${PARTITION_ID}/results.xml \
             --jsonfile ~/test_results/${{ matrix.platform }}_test_nightly/${PARTITION_ID}/testresults.json \
             -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" \
@@ -184,7 +183,6 @@ jobs:
       - name: Run integration tests
         run: |
           ./scripts/configure_dev.sh
-          ./scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
           mkdir -p ~/test_results/${{ matrix.platform }}_integration_nightly/${PARTITION_ID}
           TEST_RESULTS=~/test_results/${{ matrix.platform }}_integration_nightly/${PARTITION_ID} \
           test/scripts/run_integration_tests.sh
@@ -241,7 +239,6 @@ jobs:
       - name: Run E2E expect tests
         run: |
           scripts/configure_dev.sh
-          scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
           mkdir -p ~/test_results/${{ matrix.platform }}_e2e_expect_nightly/${PARTITION_ID}
           TEST_RESULTS=~/test_results/${{ matrix.platform }}_e2e_expect_nightly/${PARTITION_ID} \
           test/scripts/run_integration_tests.sh
@@ -308,7 +305,6 @@ jobs:
       - name: Run E2E subs tests
         run: |
           scripts/configure_dev.sh
-          scripts/buildtools/install_buildtools.sh -o "gotest.tools/gotestsum"
           mkdir -p ~/test_results/${{ matrix.platform }}_e2e_subs_nightly
           TEST_RESULTS=~/test_results/${{ matrix.platform }}_e2e_subs_nightly \
           test/scripts/run_integration_tests.sh

.github/workflows/ci-pr.yml

@@ -51,7 +51,7 @@ jobs:
           PACKAGES="$(go list ./... | grep -v /go-algorand/test/)"
           export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')          
           mkdir -p test_results/${{ matrix.platform }}_test/${PARTITION_ID}
-          gotestsum --format standard-quiet \
+          go tool -modfile=tool.mod gotestsum --format standard-quiet \
             --junitfile ~/test_results/${{ matrix.platform }}_test/${PARTITION_ID}/results.xml \
             --jsonfile ~/test_results/${{ matrix.platform }}_test/${PARTITION_ID}/testresults.json \
             -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" $SHORTTEST \

AGENTS.md

@@ -50,8 +50,6 @@ To verify that this wasn't missed, we run verification steps, which can be found
 ### Development Setup
 ```bash
 ./scripts/configure_dev.sh                    # Initial environment setup
-./scripts/buildtools/install_buildtools.sh   # Install build tools
-make deps                                     # Check/install dependencies
 ```
 
 ### Single Test Execution

Makefile

@@ -43,12 +43,7 @@ GOTAGSLIST          := sqlite_unlock_notify sqlite_omit_load_extension
 # e.g. make GOTAGSCUSTOM=msgtrace
 GOTAGSLIST += ${GOTAGSCUSTOM}
 
-# If available, use gotestsum instead of 'go test'.
-ifeq (, $(shell which gotestsum))
-export GOTESTCOMMAND=go test
-else
-export GOTESTCOMMAND=gotestsum --format pkgname --jsonfile testresults.json --
-endif
+GOTESTCOMMAND := go tool -modfile=tool.mod gotestsum --format pkgname --jsonfile testresults.json --
 
 ifeq ($(OS_TYPE), darwin)
 # M1 Mac--homebrew install location in /opt/homebrew
@@ -109,8 +104,8 @@ fix: build
 modernize:
 	GOTOOLCHAIN=auto go run golang.org/x/tools/go/analysis/passes/modernize/cmd/modernize@latest -any=false -bloop=false -rangeint=false -fmtappendf=false -waitgroup=false -stringsbuilder=false -omitzero=false -fix ./...
 
-lint: deps
-	$(GOBIN)/golangci-lint run -c .golangci.yml
+lint:
+	go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.6.1 run -c .golangci.yml
 
 warninglint: custom-golangci-lint
 	./custom-golangci-lint run -c .golangci-warnings.yml
@@ -143,7 +138,7 @@ cover:
 prof:
 	cd node && go test $(GOTAGS) -cpuprofile=cpu.out -memprofile=mem.out -mutexprofile=mutex.out
 
-generate: deps
+generate:
 	PATH=$(GOBIN):$$PATH go generate ./...
 
 msgp: $(patsubst %,%/msgp_gen.go,$(MSGP_GENERATE))
@@ -154,13 +149,13 @@ api:
 logic:
 	make -C data/transactions/logic
 
-
-%/msgp_gen.go: deps ALWAYS
+MSGP := go run github.com/algorand/msgp@v1.1.61
+%/msgp_gen.go: ALWAYS
 		@set +e; \
-		printf "msgp: $(@D)..."; \
-		$(GOBIN)/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
+		printf "$(MSGP) $(@D)..."; \
+		$(MSGP) -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand > ./$@.out 2>&1; \
 		if [ "$$?" != "0" ]; then \
-			printf "failed:\n$(GOBIN)/msgp -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
+			printf "failed:\n$(MSGP) -file ./$(@D) -o $@ -warnmask github.com/algorand/go-algorand\n"; \
 			cat ./$@.out; \
 			rm ./$@.out; \
 			exit 1; \
@@ -234,9 +229,6 @@ else
 	echo "OS_TYPE must be darwin for universal builds, skipping"
 endif
 
-deps:
-	./scripts/check_deps.sh
-
 # artifacts
 
 # Regenerate kmd swagger spec files
@@ -263,15 +255,15 @@ $(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/lib
 		touch $@; \
 	fi
 
-$(KMD_API_SWAGGER_INJECT): deps $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
+$(KMD_API_SWAGGER_INJECT): $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
 	./daemon/kmd/lib/kmdapi/bundle_swagger_json.sh
 
 # generated files we should make sure we clean
 GENERATED_FILES := \
 	$(KMD_API_SWAGGER_INJECT) \
 	$(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
 
-rebuild_kmd_swagger: deps
+rebuild_kmd_swagger:
 	rm -f $(GENERATED_FILES)
 	# we need to invoke the make here since we want to ensure that the deletion and re-creating are sequential
 	make $(KMD_API_SWAGGER_INJECT)
@@ -418,7 +410,7 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 install: build
 	scripts/dev_install.sh -p $(GOBIN)
 
-.PHONY: default fmt lint check_shell sanity cover prof deps build build-race build-e2e test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger universal libsodium modernize
+.PHONY: default fmt lint check_shell sanity cover prof build build-race build-e2e test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger universal libsodium modernize
 
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule

README.md

@@ -31,7 +31,6 @@ We currently strive to support Debian-based distributions, with Ubuntu 24.04 as
 git clone https://github.com/algorand/go-algorand
 cd go-algorand
 ./scripts/configure_dev.sh
-./scripts/buildtools/install_buildtools.sh
 ```
 
 At this point, you are ready to build go-algorand. We use `make` and have several targets to automate common tasks.

agreement/actions.go

@@ -26,7 +26,7 @@ import (
 	"github.com/algorand/go-algorand/protocol"
 )
 
-//go:generate stringer -type=actionType
+//go:generate go tool -modfile=../tool.mod stringer -type=actionType
 type actionType uint8
 
 const (

agreement/actiontype_string.go

@@ -22,7 +22,7 @@ import (
 	"github.com/algorand/go-algorand/logging"
 )
 
-//go:generate stringer -type=coserviceType
+//go:generate go tool -modfile=../tool.mod stringer -type=coserviceType
 const (
 	demuxCoserviceType coserviceType = iota
 	tokenizerCoserviceType