build: Add make nuke; speed-up make start (#5583)

Signed-off-by: Alex Collins <alex_collins@intuit.com>

.gitignore

@@ -28,6 +28,7 @@ git-ask-pass.sh
 /pkg/apiclient/event/event.swagger.json
 /pkg/apiclient/eventsource/eventsource.swagger.json
 /pkg/apiclient/info/info.swagger.json
+/pkg/apiclient/pipeline/pipeline.swagger.json
 /pkg/apiclient/sensor/sensor.swagger.json
 /pkg/apiclient/workflow/workflow.swagger.json
 /pkg/apiclient/workflowarchive/workflow-archive.swagger.json
@@ -37,5 +38,3 @@ git-ask-pass.sh
 /go-diagrams/
 /.run/
 pprof
-pkg/apiclient/sensor/sensor.swagger.json
-pkg/apiclient/eventsource/eventsource.swagger.json

Makefile

@@ -39,8 +39,6 @@ else
 STATIC_FILES          ?= $(shell [ $(DEV_BRANCH) = true ] && echo false || echo true)
 endif
 
-$(info GIT_COMMIT=$(GIT_COMMIT) GIT_BRANCH=$(GIT_BRANCH) GIT_TAG=$(GIT_TAG) GIT_TREE_STATE=$(GIT_TREE_STATE) RELEASE_TAG=$(RELEASE_TAG) DEV_BRANCH=$(DEV_BRANCH) VERSION=$(VERSION) STATIC_FILES=$(STATIC_FILES))
-
 START_UI              ?= $(shell [ "$(CI)" != "" ] && echo true || echo false)
 GOTEST                ?= go test -v
 PROFILE               ?= minimal
@@ -57,7 +55,9 @@ endif
 # * `local` run the workflow–controller and argo-server as single replicas on the local machine (default)
 # * `kubernetes` run the workflow-controller and argo-server on the Kubernetes cluster
 RUN_MODE              := local
-K3D                   := $(shell if [[ "`which kubectl`" != '' ]] && [[ "`kubectl config current-context`" == "k3d-"* ]]; then echo true; else echo false; fi)
+KUBECTX               := $(shell [[ "`which kubectl`" != '' ]] && kubectl config current-context || echo none)
+DOCKER_DESKTOP        := $(shell [[ "$(KUBECTX)" == "docker-desktop" ]] && echo true || echo false)
+K3D                   := $(shell [[ "$(KUBECTX)" == "k3d-"* ]] && echo true || echo false)
 LOG_LEVEL             := debug
 UPPERIO_DB_DEBUG      := 0
 NAMESPACED            := true
@@ -70,6 +70,10 @@ endif
 
 ALWAYS_OFFLOAD_NODE_STATUS := false
 
+$(info GIT_COMMIT=$(GIT_COMMIT) GIT_BRANCH=$(GIT_BRANCH) GIT_TAG=$(GIT_TAG) GIT_TREE_STATE=$(GIT_TREE_STATE) RELEASE_TAG=$(RELEASE_TAG) DEV_BRANCH=$(DEV_BRANCH) VERSION=$(VERSION))
+$(info KUBECTX=$(KUBECTX) DOCKER_DESKTOP=$(DOCKER_DESKTOP) K3D=$(K3D) DOCKER_PUSH=$(DOCKER_PUSH))
+$(info RUN_MODE=$(RUN_MODE) PROFILE=$(PROFILE) AUTH_MODE=$(AUTH_MODE) SECURE=$(SECURE) STATIC_FILES=$(STATIC_FILES) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) LOG_LEVEL=$(LOG_LEVEL) NAMESPACED=$(NAMESPACED))
+
 override LDFLAGS += \
   -X github.com/argoproj/argo-workflows/v3.version=$(VERSION) \
   -X github.com/argoproj/argo-workflows/v3.buildDate=${BUILD_DATE} \
@@ -103,7 +107,7 @@ PROTO_BINARIES := $(GOPATH)/bin/protoc-gen-gogo $(GOPATH)/bin/protoc-gen-gogofas
 
 # go_install,path
 define go_install
-	[ -e ./vendor ] || go mod vendor
+	go mod vendor
 	go install -mod=vendor ./vendor/$(1)
 endef
 
@@ -178,10 +182,10 @@ dist/argo-linux-s390x: GOARGS = GOOS=linux GOARCH=s390x
 dist/argo-%.gz: dist/argo-%
 	gzip --force --keep dist/argo-$*
 
-dist/argo-%: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS)
+dist/argo-%: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS) go.sum
 	CGO_ENABLED=0 $(GOARGS) go build -v -i -ldflags '${LDFLAGS} -extldflags -static' -o $@ ./cmd/argo
 
-dist/argo: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS)
+dist/argo: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS) go.sum
 ifeq ($(shell uname -s),Darwin)
 	# if local, then build fast: use CGO and dynamic-linking
 	go build -v -i -ldflags '${LDFLAGS}' -o $@ ./cmd/argo
@@ -197,7 +201,7 @@ argo-server.key:
 .PHONY: cli-image
 cli-image: dist/argocli.image
 
-dist/argocli.image: $(CLI_PKGS) argo-server.crt argo-server.key
+dist/argocli.image: $(CLI_PKGS) go.sum argo-server.crt argo-server.key
 	$(call docker_build,argocli)
 	touch dist/argocli.image
 
@@ -209,7 +213,7 @@ clis: dist/argo-linux-amd64.gz dist/argo-linux-arm64.gz dist/argo-linux-ppc64le.
 .PHONY: controller
 controller: dist/workflow-controller
 
-dist/workflow-controller: $(CONTROLLER_PKGS)
+dist/workflow-controller: $(CONTROLLER_PKGS) go.sum
 ifeq ($(shell uname -s),Darwin)
 	# if local, then build fast: use CGO and dynamic-linking
 	go build -v -i -ldflags '${LDFLAGS}' -o $@ ./cmd/workflow-controller
@@ -220,13 +224,13 @@ endif
 .PHONY: controller-image
 controller-image: dist/controller.image
 
-dist/controller.image: $(CONTROLLER_PKGS) Dockerfile
+dist/controller.image: $(CONTROLLER_PKGS) go.sum Dockerfile
 	$(call docker_build,workflow-controller)
 	touch dist/controller.image
 
 # argoexec
 
-dist/argoexec: $(ARGOEXEC_PKGS)
+dist/argoexec: $(ARGOEXEC_PKGS) go.sum
 ifeq ($(shell uname -s),Darwin)
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS} -extldflags -static' -o $@ ./cmd/argoexec
 else
@@ -243,7 +247,7 @@ dist/argoexec.image: dist/argoexec
 	mv argoexec dist/
 	docker tag argoproj/argoexec-dev:latest argoproj/argoexec:latest
 else
-dist/argoexec.image: $(ARGOEXEC_PKGS)
+dist/argoexec.image: $(ARGOEXEC_PKGS) go.sum
 	$(call docker_build,argoexec)
 endif
 	touch dist/argoexec.image
@@ -416,6 +420,18 @@ ifeq ($(RUN_MODE),kubernetes)
 	kubectl -n $(KUBE_NAMESPACE) scale deploy/argo-server --replicas 1
 endif
 
+# nuke is like "clean" but attempts to return you to a state as if it had never been installed at all
+# this only available in "safe" local development
+ifeq (true,$(filter true, $(DOCKER_DESKTOP) $(K3D)))
+nuke:
+	kubectl delete --ignore-not-found workflow,cronworkflow,clusterworkflowtemplate,workflowtemplate,workfloweventbinding --all || true
+	kubectl delete --ignore-not-found ns argo
+	kubectl delete --ignore-not-found -k manifests/base/crds/minimal
+	git clean -fxd
+	docker image rm argoproj/argoexec  argoproj/argoexec-dev argoproj/argocli argoproj/workflow-controller || true
+	docker system prune -f
+endif
+
 .PHONY: argosay
 argosay:
 	cd test/e2e/images/argosay/v2 && docker build . -t argoproj/argosay:v2
@@ -444,7 +460,7 @@ $(GOPATH)/bin/goreman:
 
 .PHONY: start
 ifeq ($(RUN_MODE),local)
-start: install executor-image controller cli $(GOPATH)/bin/goreman
+start: install controller cli $(GOPATH)/bin/goreman
 else
 start: install executor-image controller-image cli-image
 endif
@@ -458,8 +474,7 @@ endif
 	grep '127.0.0.1[[:blank:]]*mysql' /etc/hosts
 	./hack/port-forward.sh
 ifeq ($(RUN_MODE),local)
-	killall goreman node || true
-	env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start controller argo-server logs watch-pods $(shell [ $(START_UI) = false ]&& echo ui || echo) $(shell if [ -z $GREP_LOGS ]; then echo; else echo "| grep \"$(GREP_LOGS)\""; fi)
+	env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start $(shell if [ -z $GREP_LOGS ]; then echo; else echo "| grep \"$(GREP_LOGS)\""; fi)
 endif
 
 $(GOPATH)/bin/stern:
@@ -478,7 +493,7 @@ watch-pods:
 	  -w
 
 .PHONY: wait
-wait:
+wait: executor-image
 	# Wait for workflow controller
 	until lsof -i :9090 > /dev/null ; do sleep 10s ; done
 	# Wait for Argo Server
@@ -575,7 +590,7 @@ docs/fields.md: api/openapi-spec/swagger.json $(shell find examples -type f) hac
 	env ARGO_SECURE=false ARGO_INSECURE_SKIP_VERIFY=false ARGO_SERVER= ARGO_INSTANCEID= go run ./hack docgen
 
 # generates several other files
-docs/cli/argo.md: $(CLI_PKGS) server/static/files.go hack/cli/main.go
+docs/cli/argo.md: $(CLI_PKGS) go.sum server/static/files.go hack/cli/main.go
 	go run ./hack/cli
 
 .PHONY: validate-examples

Procfile

@@ -1,5 +1,6 @@
-controller: PNS_PRIVILEGED=true DEFAULT_REQUEUE_TIME=${DEFAULT_REQUEUE_TIME} LEADER_ELECTION_IDENTITY=local ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} OFFLOAD_NODE_STATUS_TTL=30s WORKFLOW_GC_PERIOD=30s UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ARCHIVED_WORKFLOW_GC_PERIOD=30s ./dist/workflow-controller --executor-image ${IMAGE_NAMESPACE}/argoexec:${VERSION} --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --loglevel ${LOG_LEVEL}
-argo-server: UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ./dist/argo --loglevel ${LOG_LEVEL} server --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --auth-mode ${AUTH_MODE} --secure=$SECURE --x-frame-options=SAMEORIGIN
-ui: yarn --cwd ui install && yarn --cwd ui start
+executor-image : make  executor-image
+controller: ./hack/free-port.sh 9090 && PNS_PRIVILEGED=true DEFAULT_REQUEUE_TIME=${DEFAULT_REQUEUE_TIME} LEADER_ELECTION_IDENTITY=local ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} OFFLOAD_NODE_STATUS_TTL=30s WORKFLOW_GC_PERIOD=30s UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ARCHIVED_WORKFLOW_GC_PERIOD=30s ./dist/workflow-controller --executor-image ${IMAGE_NAMESPACE}/argoexec:${VERSION} --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --loglevel ${LOG_LEVEL}
+argo-server: ./hack/free-port.sh 2746 && UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ./dist/argo --loglevel ${LOG_LEVEL} server --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --auth-mode ${AUTH_MODE} --secure=$SECURE --x-frame-options=SAMEORIGIN
+ui: ./hack/free-port.sh 8080 && yarn --cwd ui install && yarn --cwd ui start
 logs: make logs
 watch-pods: make watch-pods

hack/free-port.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -eux
+
+port=$1
+
+lsof -i ":$port" | grep -v PID | awk '{print $2}' | xargs kill || true
+
+

hack/port-forward.sh

@@ -7,10 +7,7 @@ pf() {
   resource=$2
   port=$3
   dest_port=${4:-"$port"}
-  pid=$(lsof -i ":$port" | grep -v PID | awk '{print $2}' || true)
-  if [ "$pid" != "" ]; then
-    kill $pid
-  fi
+  ./hack/free-port.sh $port
   kubectl -n argo port-forward "$resource" "$port:$dest_port" > /dev/null &
   # wait until port forward is established
 	until lsof -i ":$port" > /dev/null ; do sleep 1s ; done