Add generated go/proto files to git

Add the generated *.capnp.go and structs.gen.go files in go/proto to
version control.
This intends to simplify the build process when using the go toolchain,
especially for go projects that consume scion as a library.

Add make target 'gogen' to update the generated files in go/proto.

Use new bazel pkg_tar rule 'build-tools' to compile canpnpc-go for
generating the *.capnpc.go files.

Add CI check for generated go/proto files; detect uncommitted changes to
generated go/proto files by cleaning and regenerating the files.

.buildkite/steps/test.yml

@@ -1,3 +1,6 @@
+- label: "Check generated go/proto files in git"
+  command: $BASE/run_step checkgogen
+  timeout_in_minutes: 10
 - label: "Unit tests :bazel:"
   command: $BASE/run_step test
   artifact_paths:

BUILD.bazel

@@ -66,6 +66,17 @@ pkg_tar(
     mode = "0755",
 )
 
+# This is a package of tools used for generating code that is checked in
+# to git (i.e. outside of bazels reach).
+pkg_tar(
+    name = "build-tools",
+    package_dir = "",
+    srcs = [
+        "@com_zombiezen_go_capnproto2//capnpc-go:capnpc-go",
+    ],
+    mode = "0755",
+)
+
 # Nogo - Go code analysis tool
 nogo(
     name = "nogo",

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all clean goenv gogenlinks gogenlinks_clean vendor mocks bazel bazel_bin_clean gazelle setcap clibs libscion libfilter dispatcher uninstall tags
+.PHONY: all clean goenv gogen vendor mocks bazel bazel_bin_clean gazelle setcap clibs libscion libfilter dispatcher uninstall tags
 
 BRACCEPT = bin/braccept
 
@@ -8,26 +8,26 @@ SRC_DIRS = c/lib/scion c/lib/filter c/dispatcher
 
 all: tags clibs dispatcher bazel
 
-clean: gogenlinks_clean
+clean:
 	$(foreach var,$(SRC_DIRS),$(MAKE) -C $(var) clean || exit 1;)
 	bazel clean
 	rm -f bin/* tags
 	if [ -e go/vendor ]; then rm -r go/vendor; fi
 
-goenv: vendor gogenlinks
+gogen:
+ifndef GOGEN_SKIP
+	$(MAKE) -C go/proto
+else
+	@echo "gogen: skipped"
+endif
 
-gogenlinks: gogenlinks_clean
-	bazel build //go/proto:go_default_library
-	find bazel-genfiles/go/proto -maxdepth 1 -type f -exec ln -snf ../../{} go/proto \;
-
-gogenlinks_clean:
-	find ./go/proto -maxdepth 1 -type l -exec rm {} +
+goenv: vendor
 
 vendor:
 	if [ -e go/vendor ]; then rm -r go/vendor; fi
 	bzlcompat -vendorBase=go
 
-bazel: vendor bazel_bin_clean
+bazel: vendor gogen bazel_bin_clean
 	bazel build //:scion //:scion-ci --workspace_status_command=./tools/bazel-build-env
 	tar -kxf bazel-bin/scion.tar -C bin
 	tar -kxf bazel-bin/scion-ci.tar -C bin

docker/perapp/base/Dockerfile.builder

@@ -1,3 +1,3 @@
 FROM scion:latest
 COPY bazelrc.quiet ~/.bazelrc
-RUN make -s all setcap && bazel clean
+RUN make -s GOGEN_SKIP=1 all setcap && bazel clean

go/proto/BUILD.bazel

@@ -8,43 +8,7 @@ go_library(
         "cereal.go",
         "doc.go",
         "sign.go",
-        ":structs",
-        ":capnp",
-    ],
-    importpath = "github.com/scionproto/scion/go/proto",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//go/lib/common:go_default_library",
-        "//go/lib/scrypto:go_default_library",
-        "//go/lib/util:go_default_library",
-        "@com_zombiezen_go_capnproto2//:go_default_library",
-        "@com_zombiezen_go_capnproto2//encoding/text:go_default_library",
-        "@com_zombiezen_go_capnproto2//pogs:go_default_library",
-        "@com_zombiezen_go_capnproto2//schemas:go_default_library",
-    ],
-)
-
-genrule(
-    name = "structs",
-    srcs = [],
-    outs = ["structs.gen.go"],
-    cmd = "./$(location structs_gen_go.sh) > \"$@\"",
-    tools = ["structs_gen_go.sh"],
-    visibility = ["//visibility:public"],
-)
-
-# TODO(sustrik): At the moment we have a copied version of go.capnp in proto directory.
-# Once python is Bazelified we can copy it as part of the build process.
-# Specifically we would need to add "@com_zombiezen_go_capnproto2//std:go.capnp" to
-# srcs and -I$$(dirname \"$(location @com_zombiezen_go_capnproto2//std:go.capnp)\")
-# to the command line. Also "go.capnp" inside individual capnp files should be changed
-# to "/go.capnp".
-genrule(
-    name = "capnp",
-    srcs = [
-        "//proto:sources",
-    ],
-    outs = [
+        "structs.gen.go",
         "ack.capnp.go",
         "asm_exts.capnp.go",
         "cert_mgmt.capnp.go",
@@ -64,6 +28,15 @@ genrule(
         "svc_resolution.capnp.go",
         "zkid.capnp.go",
     ],
-    cmd = "capnp compile -o$(location @com_zombiezen_go_capnproto2//capnpc-go:capnpc-go):$(@D) --src-prefix=proto $(locations //proto:sources)",
-    tools = ["@com_zombiezen_go_capnproto2//capnpc-go:capnpc-go"],
+    importpath = "github.com/scionproto/scion/go/proto",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//go/lib/common:go_default_library",
+        "//go/lib/scrypto:go_default_library",
+        "//go/lib/util:go_default_library",
+        "@com_zombiezen_go_capnproto2//:go_default_library",
+        "@com_zombiezen_go_capnproto2//encoding/text:go_default_library",
+        "@com_zombiezen_go_capnproto2//pogs:go_default_library",
+        "@com_zombiezen_go_capnproto2//schemas:go_default_library",
+    ],
 )

go/proto/Makefile

@@ -0,0 +1,19 @@
+.PHONY: all capnp
+
+PROTO_DIR := ../../proto
+SRCS := $(wildcard $(PROTO_DIR)/*.capnp)
+SRCS := $(filter-out $(PROTO_DIR)/go.capnp, $(SRCS))
+
+all: capnp structs.gen.go
+
+capnp:
+	set -eu; \
+	TMPDIR=$$(mktemp -d -t scion-build-tools.XXXXXXXXXX); \
+	bazel build //:build-tools; \
+	tar -xf ../../bazel-bin/build-tools.tar -C $$TMPDIR; \
+	rm -f *.capnp.go; \
+	capnp compile --verbose -o$$TMPDIR/capnpc-go --src-prefix $(PROTO_DIR) $(SRCS); \
+	rm -rf $$TMPDIR
+
+structs.gen.go: structs_gen_go.sh
+	./structs_gen_go.sh > structs.gen.go