From de026a00ac4f4aed8ffefefbf1c199ad799c95e3 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 24 Oct 2019 15:50:11 +0200 Subject: [PATCH] Fix compiled logproto protobuf (#1183) * Enforce recompile of protobuf sources on 'make check-generated-files' * Pinned version of protobuf in loki-build-image * Pinned version of goyacc in loki-build-image * Added 'make build-image-push' to Makefile * Documented how to release loki-build-image --- .circleci/config.yml | 2 +- .drone/drone.yml | 10 +++++----- Makefile | 19 +++++++++++++++---- cmd/docker-driver/Dockerfile | 2 +- cmd/loki-canary/Dockerfile.cross | 2 +- cmd/loki/Dockerfile.cross | 2 +- cmd/promtail/Dockerfile.cross | 2 +- docs/maintaining/README.md | 1 + docs/maintaining/release-loki-build-image.md | 8 ++++++++ go.mod | 2 +- loki-build-image/Dockerfile | 15 ++++++++++----- 11 files changed, 45 insertions(+), 20 deletions(-) create mode 100644 docs/maintaining/release-loki-build-image.md diff --git a/.circleci/config.yml b/.circleci/config.yml index eceaf0baaf498..138b9aec12e69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ workflows: # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ .defaults: &defaults docker: - - image: grafana/loki-build-image:0.7.3 + - image: grafana/loki-build-image:0.7.4 working_directory: /src/loki jobs: diff --git a/.drone/drone.yml b/.drone/drone.yml index 66906d1a85b8b..96c42f734d3da 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -5,19 +5,19 @@ steps: - make BUILD_IN_CONTAINER=false test depends_on: - clone - image: grafana/loki-build-image:0.7.3 + image: grafana/loki-build-image:0.7.4 name: test - commands: - make BUILD_IN_CONTAINER=false lint depends_on: - clone - image: grafana/loki-build-image:0.7.3 + image: grafana/loki-build-image:0.7.4 name: lint - commands: - make BUILD_IN_CONTAINER=false check-generated-files depends_on: - clone - image: grafana/loki-build-image:0.7.3 + image: grafana/loki-build-image:0.7.4 name: check-generated-files - commands: - make BUILD_IN_CONTAINER=false check-mod @@ -25,7 +25,7 @@ steps: - clone - test - lint - image: grafana/loki-build-image:0.7.3 + image: grafana/loki-build-image:0.7.4 name: check-mod workspace: base: /src @@ -492,7 +492,7 @@ steps: environment: CIRCLE_TOKEN: from_secret: circle_token - image: grafana/loki-build-image:0.7.3 + image: grafana/loki-build-image:0.7.4 name: trigger trigger: ref: diff --git a/Makefile b/Makefile index ab290c3065a45..c838045d9d5e5 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := all -.PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos +.PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos touch-protobuf-sources .PHONY: helm helm-install helm-upgrade helm-publish helm-debug helm-clean .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test @@ -20,7 +20,7 @@ IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%, # make BUILD_IN_CONTAINER=false target # or you can override this with an environment variable BUILD_IN_CONTAINER ?= true -BUILD_IMAGE_VERSION := 0.7.3 +BUILD_IMAGE_VERSION := 0.7.4 # Docker image info IMAGE_PREFIX ?= grafana @@ -106,15 +106,22 @@ binfmt: all: promtail logcli loki loki-canary check-generated-files # This is really a check for the CI to make sure generated files are built and checked in manually -check-generated-files: yacc protos pkg/promtail/server/ui/assets_vfsdata.go +check-generated-files: touch-protobuf-sources yacc protos pkg/promtail/server/ui/assets_vfsdata.go @if ! (git diff --exit-code $(YACC_GOS) $(PROTO_GOS) $(PROMTAIL_GENERATED_FILE)); then \ echo "\nChanges found in generated files"; \ - echo "Run 'make all' and commit the changes to fix this error."; \ + echo "Run 'make check-generated-files' and commit the changes to fix this error."; \ echo "If you are actively developing these files you can ignore this error"; \ echo "(Don't forget to check in the generated files when finished)\n"; \ exit 1; \ fi +# Trick used to ensure that protobuf files are always compiled even if not changed, because the +# tooling may have been upgraded and the compiled output may be different. We're not using a +# PHONY target so that we can control where we want to touch it. +touch-protobuf-sources: + for def in $(PROTO_DEFS); do \ + touch $$def; \ + done ########## # Logcli # @@ -463,6 +470,10 @@ loki-canary-push: loki-canary-image-cross build-image: OCI_PLATFORMS= build-image: $(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG) ./loki-build-image +build-image-push: build-image + $(call push,loki-build-image,$(BUILD_IMAGE_VERSION)) + $(call push,loki-build-image,latest) + ######## # Misc # diff --git a/cmd/docker-driver/Dockerfile b/cmd/docker-driver/Dockerfile index 1b7baeedcc6bc..1abf9454da3d9 100644 --- a/cmd/docker-driver/Dockerfile +++ b/cmd/docker-driver/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3 +ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/loki -f cmd/loki/Dockerfile . diff --git a/cmd/loki-canary/Dockerfile.cross b/cmd/loki-canary/Dockerfile.cross index 5e42d8a921c0e..9cd830a7e69ab 100644 --- a/cmd/loki-canary/Dockerfile.cross +++ b/cmd/loki-canary/Dockerfile.cross @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3 +ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/promtail -f cmd/promtail/Dockerfile . diff --git a/cmd/loki/Dockerfile.cross b/cmd/loki/Dockerfile.cross index ad9fb0d16c653..f061f7c65a2b1 100644 --- a/cmd/loki/Dockerfile.cross +++ b/cmd/loki/Dockerfile.cross @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3 +ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/loki -f cmd/loki/Dockerfile . diff --git a/cmd/promtail/Dockerfile.cross b/cmd/promtail/Dockerfile.cross index 334fcde962955..1962e40a2af06 100644 --- a/cmd/promtail/Dockerfile.cross +++ b/cmd/promtail/Dockerfile.cross @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3 +ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/promtail -f cmd/promtail/Dockerfile . diff --git a/docs/maintaining/README.md b/docs/maintaining/README.md index f5ba6aaf5cb07..fca08fe4d2260 100644 --- a/docs/maintaining/README.md +++ b/docs/maintaining/README.md @@ -3,3 +3,4 @@ This section details information for maintainers of Loki. 1. [Releasing Loki](./release.md) +2. [Releasing `loki-build-image`](./release-loki-build-image.md) diff --git a/docs/maintaining/release-loki-build-image.md b/docs/maintaining/release-loki-build-image.md new file mode 100644 index 0000000000000..f96af559e7dd7 --- /dev/null +++ b/docs/maintaining/release-loki-build-image.md @@ -0,0 +1,8 @@ +# Releasing `loki-build-image` + +The [`loki-build-image`](../../loki-build-image/) is the Docker image used to run tests and build Loki binaries in CI. + +## How To Perform a Release + +1. Update `BUILD_IMAGE_VERSION` in the `Makefile` +2. Run `make build-image-push` diff --git a/go.mod b/go.mod index a4681d9cfabbb..b775eb1871195 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/fluent/fluent-bit-go v0.0.0-20190925192703-ea13c021720c github.com/go-kit/kit v0.9.0 github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b // indirect - github.com/gogo/protobuf v1.3.0 + github.com/gogo/protobuf v1.3.0 // remember to update loki-build-image/Dockerfile too github.com/golang/snappy v0.0.1 github.com/gorilla/mux v1.6.2 github.com/gorilla/websocket v1.4.0 diff --git a/loki-build-image/Dockerfile b/loki-build-image/Dockerfile index 6a9e86b5bbe2a..d29afa516f59b 100644 --- a/loki-build-image/Dockerfile +++ b/loki-build-image/Dockerfile @@ -26,12 +26,17 @@ RUN apt-get update && \ COPY --from=docker /usr/bin/docker /usr/bin/docker COPY --from=helm /usr/bin/helm /usr/bin/helm COPY --from=golangci /bin/golangci-lint /usr/local/bin -RUN go get \ - github.com/golang/protobuf/protoc-gen-go \ - github.com/gogo/protobuf/protoc-gen-gogoslick \ - github.com/gogo/protobuf/gogoproto \ + +# Enable go 1.11 modules to be able to install pinned version of dependencies +# even if we're installing them in the GOPATH +RUN GO111MODULE=on go get \ + github.com/golang/protobuf/protoc-gen-go@v1.3.0 \ + github.com/gogo/protobuf/protoc-gen-gogoslick@v1.3.0 \ + github.com/gogo/protobuf/gogoproto@v1.3.0 \ github.com/go-delve/delve/cmd/dlv \ - golang.org/x/tools/cmd/goyacc \ + # Due to the lack of a proper release tag, we use the commit hash of + # https://github.com/golang/tools/releases v0.1.7 + golang.org/x/tools/cmd/goyacc@58d531046acdc757f177387bc1725bfa79895d69 \ github.com/mitchellh/gox \ github.com/tcnksm/ghr && \ rm -rf /go/pkg /go/src