build: migrate op-node and op-proposer to just (#13042)

* build(just): factor out flags code

This will enable making use of JUSTFLAGS in the main Makefile for
migrated targets.

* fix(just): parallel support

For some unclear reason runs in CI environment are invalid (they
generate some "empty" calls to the private target).
Sidestep the issue for now.

* build(just): add more helpers

- VERSION_META variable
- go_generate target

* build(op-proposer): migrate build to just

* build(op-node): migrate build to just

* fix(build): rewire just-migrated targets

---------

Co-authored-by: Matthew Slipper <me@matthewslipper.com>

Makefile

@@ -1,3 +1,6 @@
+# provide JUSTFLAGS for just-backed targets
+include ./just/flags.mk
+
 COMPOSEFLAGS=-d
 ITESTS_L2_HOST=http://localhost:9545
 BEDROCK_TAGS_REMOTE?=origin
@@ -94,7 +97,7 @@ submodules: ## Updates git submodules
 
 
 op-node: ## Builds op-node binary
-	make -C ./op-node op-node
+	just $(JUSTFLAGS) ./op-node/op-node
 .PHONY: op-node
 
 generate-mocks-op-node: ## Generates mocks for op-node
@@ -106,11 +109,11 @@ generate-mocks-op-service: ## Generates mocks for op-service
 .PHONY: generate-mocks-op-service
 
 op-batcher: ## Builds op-batcher binary
-	make -C ./op-batcher op-batcher
+	just $(JUSTFLAGS) ./op-batcher/op-batcher
 .PHONY: op-batcher
 
 op-proposer: ## Builds op-proposer binary
-	make -C ./op-proposer op-proposer
+	just $(JUSTFLAGS) ./op-proposer/op-proposer
 .PHONY: op-proposer
 
 op-challenger: ## Builds op-challenger binary

just/default.just

@@ -1,5 +1,6 @@
 set shell := ["bash", "-c"]
 
-PARALLEL := num_cpus()
+PARALLEL_JOBS := num_cpus()
 
-MAP_JUST := "/usr/bin/env -S parallel --shebang --jobs " + PARALLEL + " --colsep ' ' -r " + just_executable()
+# TODO: this fails in CI for some reason
+MAP_JUST := "/usr/bin/env -S parallel --shebang --jobs " + PARALLEL_JOBS + " --colsep ' ' -r " + just_executable()

just/deprecated.mk

@@ -1,42 +1,24 @@
 ifeq (, $(shell which tput))
   # CI environment typically does not support tput.
   banner-style = $1
+else ifeq (, $(TERM))
+  # Terminal type not set, so tput would fail.
+  banner-style = $1
 else
   # print in bold red to bring attention.
   banner-style = $(shell tput bold)$(shell tput setaf 1)$1$(shell tput sgr0)
 endif
 
-# Variable assignments can affect the semantic of the make targets.
-# Typical use-case: setting VERSION in a release build, since CI
-# doesn't preserve the git environment.
-#
-# We need to translate:
-# "make target VAR=val" to "just VAR=val target"
-#
-# MAKEFLAGS is a string of the form:
-# "abc --foo --bar=baz -- VAR1=val1 VAR2=val2", namely:
-# - abc is the concatnation of all short flags
-# - --foo and --bar=baz are long options,
-# - -- is the separator between flags and variable assignments,
-# - VAR1=val1 and VAR2=val2 are variable assignments
-#
-# Goal: ignore all CLI flags, keep only variable assignments.
-#
-# First remove the short flags at the beginning, or the first long-flag,
-# or if there is no flag at all, the -- separator (which then makes the
-# next step a noop). If there's no flag and no variable assignment, the
-# result is empty anyway, so the wordlist call is safe (everything is a noop).
-tmp-flags = $(wordlist 2,$(words $(MAKEFLAGS)),$(MAKEFLAGS))
-# Then remove all long options, including the -- separator, if needed. That
-# leaves only variable assignments.
-just-flags = $(patsubst --%,,$(tmp-flags))
+SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+include $(SELF_DIR)/flags.mk
 
 define make-deprecated-target
 $1:
 	@echo
-	@printf %s\\n '$(call banner-style,"make $1 $(just-flags)" is deprecated. Please use "just $(just-flags) $1" instead.)'
+	@printf %s\\n '$(call banner-style,Deprecated make call: make $1 $(JUSTFLAGS))'
+	@printf %s\\n '$(call banner-style,Consider using just instead: just $(JUSTFLAGS) $1)'
 	@echo
-	just $(just-flags) $1
+	just $(JUSTFLAGS) $1
 endef
 
 $(foreach element,$(DEPRECATED_TARGETS),$(eval $(call make-deprecated-target,$(element))))

just/flags.mk

@@ -0,0 +1,24 @@
+# Variable assignments can affect the semantic of the make targets.
+# Typical use-case: setting VERSION in a release build, since CI
+# doesn't preserve the git environment.
+#
+# We need to translate:
+# "make target VAR=val" to "just VAR=val target"
+#
+# MAKEFLAGS is a string of the form:
+# "abc --foo --bar=baz -- VAR1=val1 VAR2=val2", namely:
+# - abc is the concatnation of all short flags
+# - --foo and --bar=baz are long options,
+# - -- is the separator between flags and variable assignments,
+# - VAR1=val1 and VAR2=val2 are variable assignments
+#
+# Goal: ignore all CLI flags, keep only variable assignments.
+#
+# First remove the short flags at the beginning, or the first long-flag,
+# or if there is no flag at all, the -- separator (which then makes the
+# next step a noop). If there's no flag and no variable assignment, the
+# result is empty anyway, so the wordlist call is safe (everything is a noop).
+tmp-flags := $(wordlist 2,$(words $(MAKEFLAGS)),$(MAKEFLAGS))
+# Then remove all long options, including the -- separator, if needed. That
+# leaves only variable assignments.
+JUSTFLAGS := $(patsubst --%,,$(tmp-flags))

just/git.just

@@ -24,3 +24,5 @@ VERSION := shell('if [ -z "$1" ]; then
 else
     echo $1
 fi', _PREFERRED_TAG, _LAST_TAG)
+
+VERSION_META := ""

just/go.just

@@ -25,3 +25,7 @@ go_test SELECTOR *FLAGS:
 
 [private]
 go_fuzz FUZZ TIME='10s' PKG='': (go_test PKG _EXTRALDFLAGS "-fuzztime" TIME "-fuzz" FUZZ "-run" "NOTAREALTEST")
+
+[private]
+go_generate SELECTOR *FLAGS:
+    go generate -v {{FLAGS}} {{SELECTOR}}

op-batcher/justfile

@@ -24,12 +24,13 @@ batcher_fuzz_task FUZZ TIME='10s': (go_fuzz FUZZ TIME "./batcher")
 
 # Run fuzzing tests
 fuzz:
-    #!{{MAP_JUST}} batcher_fuzz_task
-    FuzzChannelConfig_CheckTimeout
-    FuzzDurationZero
-    FuzzDurationTimeoutMaxChannelDuration
-    FuzzDurationTimeoutZeroMaxChannelDuration
-    FuzzChannelCloseTimeout
-    FuzzChannelZeroCloseTimeout
-    FuzzSeqWindowClose
-    FuzzSeqWindowZeroTimeoutClose
+    printf "%s\n" \
+        "FuzzChannelConfig_CheckTimeout" \
+        "FuzzDurationZero" \
+        "FuzzDurationTimeoutMaxChannelDuration" \
+        "FuzzDurationTimeoutZeroMaxChannelDuration" \
+        "FuzzChannelCloseTimeout" \
+        "FuzzChannelZeroCloseTimeout" \
+        "FuzzSeqWindowClose" \
+        "FuzzSeqWindowZeroTimeoutClose" \
+    | parallel -j {{PARALLEL_JOBS}} {{just_executable()}} batcher_fuzz_task {}

op-node/Makefile

@@ -1,65 +1,3 @@
-GITCOMMIT ?= $(shell git rev-parse HEAD)
-GITDATE ?= $(shell git show -s --format='%ct')
-# Find the github tag that points to this commit. If none are found, set the version string to "untagged"
-# Prioritizes release tag, if one exists, over tags suffixed with "-rc"
-VERSION ?= $(shell tags=$$(git tag --points-at $(GITCOMMIT) | grep '^op-node/' | sed 's/op-node\///' | sort -V); \
-             preferred_tag=$$(echo "$$tags" | grep -v -- '-rc' | tail -n 1); \
-             if [ -z "$$preferred_tag" ]; then \
-                 if [ -z "$$tags" ]; then \
-                     echo "untagged"; \
-                 else \
-                     echo "$$tags" | tail -n 1; \
-                 fi \
-             else \
-                 echo $$preferred_tag; \
-             fi)
+DEPRECATED_TARGETS := op-node clean test fuzz generate-mocks readme
 
-LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
-LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
-LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-node/version.Version=$(VERSION)
-LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-node/version.Meta=$(VERSION_META)
-LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
-
-# Use the old Apple linker to workaround broken xcode - https://github.com/golang/go/issues/65169
-ifeq ($(shell uname),Darwin)
-	FUZZLDFLAGS := -ldflags=-extldflags=-Wl,-ld_classic
-endif
-
-op-node:
-	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) CGO_ENABLED=0 go build -v $(LDFLAGS) -o ./bin/op-node ./cmd/main.go
-
-clean:
-	rm bin/op-node
-
-test:
-	go test -v ./...
-
-fuzz:
-	printf "%s\n" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzL1InfoBedrockRoundTrip ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzL1InfoEcotoneRoundTrip ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzL1InfoAgainstContract ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzUnmarshallLogEvent ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzParseFrames ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzFrameUnmarshalBinary ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzBatchRoundTrip ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzDeriveDepositsRoundTrip ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzDeriveDepositsBadVersion ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzParseL1InfoDepositTxDataValid ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzParseL1InfoDepositTxDataBadLength ./rollup/derive" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzRejectCreateBlockBadTimestamp ./rollup/driver" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzDecodeDepositTxDataToL1Info ./rollup/driver" \
-	| parallel -j 8 {}
-
-generate-mocks:
-	go generate ./...
-
-readme:
-	doctoc README.md
-
-.PHONY: \
-	op-node \
-	clean \
-	test \
-	fuzz \
-	readme
+include ../just/deprecated.mk

op-node/justfile

@@ -0,0 +1,49 @@
+import '../just/go.just'
+
+# Build ldflags string
+_LDFLAGSSTRING := "'" + trim(
+    "-X main.GitCommit=" + GITCOMMIT + " " + \
+    "-X main.GitDate=" + GITDATE + " " + \
+    "-X github.com/ethereum-optimism/optimism/op-node/version.Version=" + VERSION + " " + \
+    "-X github.com/ethereum-optimism/optimism/op-node/version.Meta=" + VERSION_META + " " + \
+    "") + "'"
+
+BINARY := "./bin/op-node"
+
+# Build op-node binary
+op-node: (go_build BINARY "./cmd" "-ldflags" _LDFLAGSSTRING)
+
+# Clean build artifacts
+clean:
+    rm -f {{BINARY}}
+
+# Run tests
+test: (go_test "./...")
+
+# Generate mocks
+generate-mocks: (go_generate "./...")
+
+# Update readme
+readme:
+    doctoc README.md
+
+[private]
+node_fuzz_task FUZZ TIME='10s': (go_fuzz FUZZ TIME "./rollup/derive")
+
+# Run fuzz tests
+fuzz:
+	printf "%s\n" \
+		"FuzzL1InfoBedrockRoundTrip" \
+		"FuzzL1InfoEcotoneRoundTrip" \
+		"FuzzL1InfoAgainstContract" \
+		"FuzzUnmarshallLogEvent" \
+		"FuzzParseFrames" \
+		"FuzzFrameUnmarshalBinary" \
+		"FuzzBatchRoundTrip" \
+		"FuzzDeriveDepositsRoundTrip" \
+		"FuzzDeriveDepositsBadVersion" \
+		"FuzzParseL1InfoDepositTxDataValid" \
+		"FuzzParseL1InfoDepositTxDataBadLength" \
+		"FuzzRejectCreateBlockBadTimestamp" \
+		"FuzzDecodeDepositTxDataToL1Info" \
+	| parallel -j {{PARALLEL_JOBS}} {{just_executable()}} node_fuzz_task {}

op-proposer/Makefile

@@ -1,34 +1,3 @@
-GITCOMMIT ?= $(shell git rev-parse HEAD)
-GITDATE ?= $(shell git show -s --format='%ct')
-# Find the github tag that points to this commit. If none are found, set the version string to "untagged"
-# Prioritizes release tag, if one exists, over tags suffixed with "-rc"
-VERSION ?= $(shell tags=$$(git tag --points-at $(GITCOMMIT) | grep '^op-proposer/' | sed 's/op-proposer\///' | sort -V); \
-             preferred_tag=$$(echo "$$tags" | grep -v -- '-rc' | tail -n 1); \
-             if [ -z "$$preferred_tag" ]; then \
-                 if [ -z "$$tags" ]; then \
-                     echo "untagged"; \
-                 else \
-                     echo "$$tags" | tail -n 1; \
-                 fi \
-             else \
-                 echo $$preferred_tag; \
-             fi)
+DEPRECATED_TARGETS := op-proposer clean test
 
-LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
-LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
-LDFLAGSSTRING +=-X main.Version=$(VERSION)
-LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
-
-op-proposer:
-	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) CGO_ENABLED=0 go build -v $(LDFLAGS) -o ./bin/op-proposer ./cmd
-
-clean:
-	rm bin/op-proposer
-
-test:
-	go test -v ./...
-
-.PHONY: \
-	clean \
-	op-proposer \
-	test
+include ../just/deprecated.mk

op-proposer/justfile

@@ -0,0 +1,20 @@
+import '../just/go.just'
+
+# Build ldflags string
+_LDFLAGSSTRING := "'" + trim(
+    "-X main.GitCommit=" + GITCOMMIT + " " + \
+    "-X main.GitDate=" + GITDATE + " " + \
+    "-X main.Version=" + VERSION + " " + \
+    "") + "'"
+
+BINARY := "./bin/op-proposer"
+
+# Build op-proposer binary
+op-proposer: (go_build BINARY "./cmd" "-ldflags" _LDFLAGSSTRING)
+
+# Clean build artifacts
+clean:
+    rm -f {{BINARY}}
+
+# Run tests
+test: (go_test "./...")