feat: Better lint and test for devs and Makefile cleanup.

This change does several things all towards the end of making
it easier for developers to run tests and lint on go code.

Now a developer can run either 'make go-test' or 'make lint'
without having lots of C dependencies or running in a specific
container or stacker.

Things changed:

 * Add a build tag 'skipembed' that builds without needing
   cmd/stacker/lxc-wrapper/lxc-wrapper . This allows you to
   build (or test) without that file.  In order to ensure that
   you do not get very far trying to run a 'stacker' binary,
   I've added a 'panic' if stacker binary is built with skipembed.

 * Add download-tools target to makefile to ease in downloading of
   regctl and zot.

 * Move the downloading of golangci-lint from the github workflow
   to Makefile.  This makes it easier for developer to get it.
   It is also added to the 'download-tools' target.

 * be specific about the version of golangci-lint that is used.
   1.54.2.  golangci-lint's docs specifically say to do this:

    > IMPORTANT: It's highly recommended installing a specific version
    > of golangci-lint available on the releases page.

   changing it is simply a matter of changing the value in the makefile.

 * Move running of 'go test' out of the lint target to its own
   'go-test' target.  go-test target will now write a coverage.html
   so you can view coverage easily.

 * add 'dlbin' makefile "function" for downloading binaries, and use
   that for regctl and zot from their rules.

 * rename TOOLS_DIR to TOOLS_D (consistency with BUILD_D, and just
   shorter)

 * move coverage.txt output from test to hack/coverage.txt for easier
   git-ignoring.

Signed-off-by: Scott Moser <smoser@brickies.net>

.github/workflows/build.yaml

@@ -65,9 +65,9 @@ jobs:
           sudo add-apt-repository -y ppa:project-machine/squashfuse
           sudo apt-get update
           sudo apt-get install -yy lxc-utils lxc-dev libacl1-dev jq libcap-dev libseccomp-dev libpam-dev bats parallel libzstd-dev
-          GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin
           sudo apt-get install -yy autoconf automake make autogen autoconf libtool binutils git squashfs-tools libcryptsetup-dev libdevmapper-dev cryptsetup-bin squashfuse
+          GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci
+          make download-tools
           echo "running kernel is: $(uname -a)"
       - name: Go-download
         run: |

.gitignore

@@ -4,6 +4,8 @@
 /stacker-dynamic
 .build
 
+hack/
+
 # IDEs
 .vscode
 .idea

Makefile

@@ -24,14 +24,18 @@ STACKER_BUILD_UBUNTU_IMAGE?=$(STACKER_DOCKER_BASE)ubuntu:latest
 LXC_CLONE_URL?=https://github.com/lxc/lxc
 LXC_BRANCH?=stable-5.0
 
+HACK_D := $(TOP_LEVEL)/hack
 # helper tools
-TOOLSDIR := $(shell pwd)/hack/tools
-REGCLIENT := $(TOOLSDIR)/bin/regctl
+TOOLS_D := $(HACK_D)/tools
+REGCLIENT := $(TOOLS_D)/bin/regctl
 REGCLIENT_VERSION := v0.5.1
 # OCI registry
-ZOT := $(TOOLSDIR)/bin/zot
+ZOT := $(TOOLS_D)/bin/zot
 ZOT_VERSION := 2.0.0-rc6
 
+GOLANGCI_LINT_VERSION = v1.54.2
+GOLANGCI_LINT = $(TOOLS_D)/golangci-lint/$(GOLANGCI_LINT_VERSION)/golangci-lint
+
 STAGE1_STACKER ?= ./stacker-dynamic
 
 STACKER_DEPS = $(GO_SRC) go.mod go.sum
@@ -61,30 +65,44 @@ go-download:
 	go mod download
 
 .PHONY: lint
-lint: cmd/stacker/lxc-wrapper/lxc-wrapper $(GO_SRC)
+lint: $(GO_SRC) $(GOLANGCI_LINT)
 	go mod tidy
 	go fmt ./... && ([ -z $(CI) ] || git diff --exit-code)
 	bash test/static-analysis.sh
-	go test -v -trimpath -cover -coverpkg stackerbuild.io/stacker/./... -coverprofile=coverage.txt -covermode=atomic -tags "$(BUILD_TAGS)" stackerbuild.io/stacker/./...
-	$(shell go env GOPATH)/bin/golangci-lint run --build-tags "$(BUILD_TAGS)"
+	$(GOLANGCI_LINT) run --build-tags "$(BUILD_TAGS) skipembed"
+
+.PHONY: go-test
+go-test:
+	go test -v -trimpath -cover -coverprofile=$(HACK_D)/coverage.txt -covermode=atomic -tags "exclude_graphdriver_btrfs exclude_graphdriver_devicemapper containers_image_openpgp osusergo netgo skipembed"  ./...
+	go tool cover -html $(HACK_D)/coverage.txt  -o $(HACK_D)/coverage.html
+
+.PHONY: download-tools
+download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT)
+
+$(GOLANGCI_LINT):
+	@mkdir -p $(dir $@)
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(dir $@)"
+	@mkdir -p "$(TOOLS_D)/bin"
+	ln -sf "$@" "$(TOOLS_D)/bin/"
+
+# dlbin is used with $(call dlbin,path,url)
+# it downloads a url to path and makes it executable.
+# it creates dest dir and atomically moves into place. t gets <name>.pid
+dlbin = set -x; mkdir -p $(dir $1) && t=$1.$$$$ && curl -Lo "$$t" "$2" && chmod +x "$$t" && mv "$$t" "$1"
 
 $(REGCLIENT):
-	mkdir -p $(TOOLSDIR)/bin
-	curl -Lo $(REGCLIENT) https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64
-	chmod +x $(REGCLIENT)
+	$(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64)
 
 $(ZOT):
-	mkdir -p $(TOOLSDIR)/bin
-	curl -Lo $(ZOT) https://github.com/project-zot/zot/releases/download/v$(ZOT_VERSION)/zot-linux-amd64-minimal
-	chmod +x $(ZOT)
+	$(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64)
 
 TEST?=$(patsubst test/%.bats,%,$(wildcard test/*.bats))
 PRIVILEGE_LEVEL?=
 
 # make check TEST=basic will run only the basic test
 # make check PRIVILEGE_LEVEL=unpriv will run only unprivileged tests
 .PHONY: check
-check: lint test
+check: lint test go-test
 
 .PHONY: test
 test: stacker $(REGCLIENT) $(ZOT)

cmd/stacker/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"embed"
 	"fmt"
 	"os"
 	"os/exec"
@@ -28,9 +27,6 @@ var (
 	lxc_version = ""
 )
 
-//go:embed lxc-wrapper/lxc-wrapper
-var embeddedFS embed.FS
-
 func shouldShowProgress(ctx *cli.Context) bool {
 	/* if the user provided explicit recommendations, follow those */
 	if ctx.Bool("no-progress") {
@@ -84,6 +80,9 @@ func shouldSkipInternalUserns(ctx *cli.Context) bool {
 }
 
 func main() {
+	if !hasEmbedded {
+		panic("stacker was built without embedded binaries.")
+	}
 	sigquits := make(chan os.Signal, 1)
 	go func() {
 		for range sigquits {

cmd/stacker/main_embed.go

@@ -0,0 +1,10 @@
+//go:build !skipembed
+
+package main
+
+import "embed"
+
+//go:embed lxc-wrapper/lxc-wrapper
+var embeddedFS embed.FS
+
+const hasEmbedded = true

cmd/stacker/main_noembed.go

@@ -0,0 +1,9 @@
+//go:build skipembed
+
+package main
+
+import "embed"
+
+var embeddedFS embed.FS
+
+const hasEmbedded = true