Fix docker image builds with an actually-reliable dependency skip (ca…

…dence-workflow#5071)

* Fix docker image builds with an actually-reliable dependency skip

Honestly this is so obvious in retrospect that I don't know why I tried to do the fake-codegen thing.
The easiest way to skip dependencies is... to not declare that dependency in the first place.

Dockerfile

@@ -20,13 +20,13 @@ COPY go.* ./
 RUN go mod download
 
 COPY . .
-RUN rm -fr .bin .build
+RUN rm -fr .bin .build idls
 
 ENV CADENCE_RELEASE_VERSION=$RELEASE_VERSION
 
-# bypass codegen, use committed files.  must be run separately, before building things.
-RUN make .fake-codegen
-RUN CGO_ENABLED=0 make copyright cadence-cassandra-tool cadence-sql-tool cadence cadence-server cadence-bench cadence-canary
+# don't do anything fancy, just build.  must be run separately, before building things.
+RUN make .just-build
+RUN CGO_ENABLED=0 make cadence-cassandra-tool cadence-sql-tool cadence cadence-server cadence-bench cadence-canary
 
 
 # Download dockerize

Makefile

@@ -320,28 +320,11 @@ $(BUILD)/protoc: $(PROTO_FILES) $(STABLE_BIN)/$(PROTOC_VERSION_BIN) $(BIN)/proto
 # Rule-breaking targets intended ONLY for special cases with no good alternatives.
 # ====================================
 
-.PHONY: .fake-codegen .fake-protoc .fake-thriftrw
-
-# buildkite / release-only target to avoid building / running codegen tools (protoc is unable to be run on alpine).
-# this will ensure that committed code will be used rather than re-generating.
-# must be manually run before (nearly) any other targets.
-.fake-codegen: .fake-protoc .fake-thrift
-	$(warning build-tool binaries have been faked, you will need to delete the $(BIN) folder if you wish to build real ones)
-	$Q # touch a marker-file for a `make clean` warning.  this does not impact behavior.
-	touch $(STABLE_BIN)/fake-codegen
-
-# "build" fake binaries, and touch the book-keeping files, so Make thinks codegen has been run.
-# order matters, as e.g. a $(BIN) newer than a $(BUILD) implies Make should run the $(BIN).
-.fake-protoc: | $(STABLE_BIN) $(BUILD) $(BIN)
-	touch $(STABLE_BIN)/$(PROTOC_VERSION_BIN) $(BIN)/protoc-gen-gogofast $(BIN)/protoc-gen-yarpc-go
-	touch $(BUILD)/protoc
-
-.fake-thrift: | $(BUILD) $(BIN)
-	touch $(BIN)/thriftrw $(BIN)/thriftrw-plugin-yarpc
-	$Q # if the submodule exists, touch thrift_gen markers to fake their generation.
-	$Q # if it does not, do nothing - there are none.
-	$(if $(THRIFT_GEN),touch $(THRIFT_GEN),)
-	touch $(BUILD)/thrift
+# used to bypass checks when building binaries.
+# this is primarily intended for docker image builds, but can be used to skip things locally.
+.PHONY: .just-build
+.just-build: | $(BUILD)
+	touch $(BUILD)/just-build
 
 # ====================================
 # other intermediates
@@ -419,39 +402,48 @@ copyright: $(BIN)/copyright | $(BUILD) ## update copyright headers
 GOOS ?= $(shell go env GOOS)
 GOARCH ?= $(shell go env GOARCH)
 
+# normally, depend on lint, so a full build and check and codegen runs.
+# docker builds though must *not* do this, and need to rely entirely on committed code.
+ifeq (,$(wildcard $(BUILD)/just-build))
+BINS_DEPEND_ON := $(BUILD)/lint
+else
+BINS_DEPEND_ON :=
+$(warning !!!!! lint and codegen disabled, validations skipped !!!!!)
+endif
+
 BINS =
 TOOLS =
 
 BINS  += cadence-cassandra-tool
 TOOLS += cadence-cassandra-tool
-cadence-cassandra-tool: $(BUILD)/lint
+cadence-cassandra-tool: $(BINS_DEPEND_ON)
 	$Q echo "compiling cadence-cassandra-tool with OS: $(GOOS), ARCH: $(GOARCH)"
 	$Q go build -o $@ cmd/tools/cassandra/main.go
 
 BINS  += cadence-sql-tool
 TOOLS += cadence-sql-tool
-cadence-sql-tool: $(BUILD)/lint
+cadence-sql-tool: $(BINS_DEPEND_ON)
 	$Q echo "compiling cadence-sql-tool with OS: $(GOOS), ARCH: $(GOARCH)"
 	$Q go build -o $@ cmd/tools/sql/main.go
 
 BINS  += cadence
 TOOLS += cadence
-cadence: $(BUILD)/lint
+cadence: $(BINS_DEPEND_ON)
 	$Q echo "compiling cadence with OS: $(GOOS), ARCH: $(GOARCH)"
 	$Q go build -ldflags '$(GO_BUILD_LDFLAGS)' -o $@ cmd/tools/cli/main.go
 
 BINS += cadence-server
-cadence-server: $(BUILD)/lint
+cadence-server: $(BINS_DEPEND_ON)
 	$Q echo "compiling cadence-server with OS: $(GOOS), ARCH: $(GOARCH)"
 	$Q go build -ldflags '$(GO_BUILD_LDFLAGS)' -o $@ cmd/server/main.go
 
 BINS += cadence-canary
-cadence-canary: $(BUILD)/lint
+cadence-canary: $(BINS_DEPEND_ON)
 	$Q echo "compiling cadence-canary with OS: $(GOOS), ARCH: $(GOARCH)"
 	$Q go build -o $@ cmd/canary/main.go
 
 BINS += cadence-bench
-cadence-bench: $(BUILD)/lint
+cadence-bench: $(BINS_DEPEND_ON)
 	$Q echo "compiling cadence-bench with OS: $(GOOS), ARCH: $(GOARCH)"
 	$Q go build -o $@ cmd/bench/main.go
 
@@ -476,8 +468,8 @@ clean: ## Clean build products
 	rm -f $(BINS)
 	rm -Rf $(BUILD)
 	$(if \
-		$(filter $(BIN)/fake-codegen, $(wildcard $(BIN)/*)), \
-		$(warning fake build tools may exist, delete the $(BIN) folder to get real ones if desired),)
+		$(wildcard $(STABLE_BIN)/*), \
+		$(warning usually-stable build tools still exist, delete the $(STABLE_BIN) folder to rebuild them),)
 
 # v----- not yet cleaned up -----v
 

docker/README.md

@@ -109,6 +109,31 @@ docker-compose down
 docker-compose up
 ```
 
+DIY: Troubleshooting docker builds
+----------------------------------
+
+Note that Docker has been making changes to its build system, and the new system is currently missing some capabilities
+that the old one had, and makes major changes to how you control it.
+When searching for workarounds, make sure you are looking at modern answers, and consider specifically searching for
+"buildkit" solutions.  
+You can also disable buildkit explicitly with `DOCKER_BUILDKIT=0 docker build ...`.
+
+For output limiting (e.g. `[output clipped ...]` messages), or for anything that requires changing buildkit environment
+variables or other options, start a new builder and use it to build with:
+```
+# create a new builder with your options
+docker buildx create ...
+# which will print out a name, use it in the build step.
+
+# now use the exact same command as normal, but it prepends `buildx` and adds a builder flag.
+docker buildx build . -t ubercadence/<imageName>:YOUR_TAG --builder <that_builder_name>
+```
+
+For output limiting (e.g. `[output clipped ...]` messages), you can fix this with some buildkit env variables:
+```
+docker buildx create --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=-1 --driver-opt env.BUILDKIT_STEP_LOG_MAX_SPEED=-1
+```
+
 DIY: Running a custom cadence server locally alongside cadence requirements
 ---------------------------------------------------------------------------
 If you want to test out a custom-built cadence server, while running all the normal cadence dependencies, there's a simple workflow to do that: