e2e: Moved to Cortex e2e test and GithubActions. (thanos-io#2138)

* e2e: Moved to Cortex e2e test and GithubActions. It's docker based now, and should be less flaky. Fixes: thanos-io#2046 Fixes: thanos-io#2045 Partially: thanos-io#1704 Learnings: * Receiver is like extremely with default backoff retry of Prometheus remote write (100ms-300ms). This is way to extreme. Either we need to increase client backoff or return status code that will stop retry for certain errors (e.g misconfiguration). * When block is removed we still fill cache with old entries from it. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com> * e2e: Moved to Cortex e2e test and GithubActions. It's docker based now, and should be less flaky. Fixes: thanos-io#2046 Fixes: thanos-io#2045 Partially: thanos-io#1704 Learnings: * Receiver is like extremely with default backoff retry of Prometheus remote write (100ms-300ms). This is way to extreme. Either we need to increase client backoff or return status code that will stop retry for certain errors (e.g misconfiguration). * When block is removed we still fill cache with old entries from it. Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
alex-laties · Feb 22, 2020 · febddd1 · febddd1
1 parent e68765c
commit febddd1
Show file tree

Hide file tree

Showing 20 changed files with 1,371 additions and 1,802 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -33,7 +33,7 @@ jobs:
       - run: make check-docs
       - run: make format
       - run:
-          name: "Run all tests"
+          name: "Run unit tests."
           # taskset sets CPU affinity to 2 (current CPU limit).
           command: |
             if [ -z ${GCP_PROJECT} ]; then

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -6,6 +6,11 @@ jobs:
     name: Cross build check
     runs-on: ubuntu-latest
     steps:
+      - name: Install Go.
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13.6
+
       - name: Check out code into the Go module directory
         uses: actions/checkout@v1
 

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -0,0 +1,22 @@
+name: e2e
+on: [pull_request]
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go.
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13.6
+
+      - name: Check out code into the Go module directory.
+        uses: actions/checkout@v2
+
+      - uses: actions/cache@v1
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+
+      - name: Run e2e docker-based tests.
+        run: make test-e2e
diff --git a/Makefile b/Makefile
@@ -124,11 +124,14 @@ define require_clean_work_tree
 
 endef
 
+help: ## Displays help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+
 .PHONY: all
 all: format build
 
-# assets repacks all static assets into go file for easier deploy.
 .PHONY: assets
+assets: # Repacks all static assets into go file for easier deploy.
 assets: $(GOBINDATA)
 	@echo ">> deleting asset file"
 	@rm pkg/ui/bindata.go || true
@@ -137,126 +140,122 @@ assets: $(GOBINDATA)
 	@go fmt ./pkg/ui
 
 
-# build builds Thanos binary using `promu`.
 .PHONY: build
+build: ## Builds Thanos binary using `promu`.
 build: check-git deps $(PROMU)
 	@echo ">> building binaries $(GOBIN)"
 	@$(PROMU) build --prefix $(PREFIX)
 
-# crossbuild builds all binaries for all platforms.
 .PHONY: crossbuild
+crossbuild: ## Builds all binaries for all platforms.
 crossbuild: $(PROMU)
 	@echo ">> crossbuilding all binaries"
 	$(PROMU) crossbuild -v
 
-# deps ensures fresh go.mod and go.sum.
 .PHONY: deps
-deps:
+deps: ## Ensures fresh go.mod and go.sum.
 	@go mod tidy
 	@go mod verify
 
-# docker builds docker with no tag.
 .PHONY: docker
+docker: ## Builds 'thanos' docker with no tag.
 docker: build
 	@echo ">> building docker image 'thanos'"
 	@docker build -t "thanos" .
 
-#docker-multi-stage builds docker image using multi-stage.
 .PHONY: docker-multi-stage
+docker-multi-stage: ## Builds 'thanos' docker image using multi-stage.
 docker-multi-stage:
 	@echo ">> building docker image 'thanos' with Dockerfile.multi-stage"
 	@docker build -f Dockerfile.multi-stage -t "thanos" .
 
-# docker-push pushes docker image build under `thanos` to "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)"
 .PHONY: docker-push
+docker-push: ## Pushes 'thanos' docker image build to "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)".
 docker-push:
 	@echo ">> pushing image"
 	@docker tag "thanos" "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)"
 	@docker push "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)"
 
-# docs regenerates flags in docs for all thanos commands.
 .PHONY: docs
+docs: ## Regenerates flags in docs for all thanos commands.
 docs: $(EMBEDMD) build
 	@EMBEDMD_BIN="$(EMBEDMD)" SED_BIN="$(SED)" scripts/genflagdocs.sh
 	@find . -type f -name "*.md" | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh
 
-# check-docs checks:
-# - discrepancy with flags is valid
-# - links are valid
-# - white noise
 .PHONY: check-docs
+check-docs: ## checks docs against discrepancy with flags, links, white noise.
 check-docs: $(EMBEDMD) $(LICHE) build
 	@EMBEDMD_BIN="$(EMBEDMD)" SED_BIN="$(SED)" scripts/genflagdocs.sh check
 	@$(LICHE) --recursive docs --exclude "(couchdb.apache.org/bylaws.html|cloud.tencent.com|alibabacloud.com)" --document-root .
 	@$(LICHE) --exclude "goreportcard.com" --document-root . *.md
 	@find . -type f -name "*.md" | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh
 	$(call require_clean_work_tree,"check documentation")
 
-# checks Go code comments if they have trailing period (excludes protobuffers and vendor files).
-# Comments with more than 3 spaces at beginning are omitted from the check, example: '//    - foo'.
 .PHONY: check-comments
-check-comments:
+check-comments: ## Checks Go code comments if they have trailing period (excludes protobuffers and vendor files). Comments with more than 3 spaces at beginning are omitted from the check, example: '//    - foo'.
 	@printf ">> checking Go comments trailing periods\n\n\n"
 	@./scripts/build-check-comments.sh
 
-# format the code:
-# - format code (including imports format)
-# - clean up all white noise
 .PHONY: format
+format: ## Formats Go code including imports and cleans up white noise.
 format: $(GOIMPORTS) check-comments
 	@echo ">> formatting code"
 	@$(GOIMPORTS) -w $(FILES_TO_FMT)
 	@SED_BIN="$(SED)" scripts/cleanup-white-noise.sh $(FILES_TO_FMT)
 
-# proto generates golang files from Thanos proto files.
 .PHONY: proto
+proto: ## Generates Go files from Thanos proto files.
 proto: check-git  $(GOIMPORTS) $(PROTOC)
 	@GOIMPORTS_BIN="$(GOIMPORTS)" PROTOC_BIN="$(PROTOC)" scripts/genproto.sh
 
 .PHONY: promu
 promu: $(PROMU)
 
 .PHONY: tarballs-release
+tarballs-release: ## Build tarballs.
 tarballs-release: $(PROMU)
 	@echo ">> Publishing tarballs"
 	$(PROMU) crossbuild -v tarballs
 	$(PROMU) checksum -v .tarballs
 	$(PROMU) release -v .tarballs
 
-# test runs all Thanos golang tests against each supported version of Prometheus.
 .PHONY: test
+test: ## Runs all Thanos Go unit tests against each supported version of Prometheus. This excludes tests in ./test/e2e.
 test: export GOCACHE= $(TMP_GOPATH)/gocache
 test: export THANOS_TEST_MINIO_PATH= $(MINIO_SERVER)
 test: export THANOS_TEST_PROMETHEUS_VERSIONS= $(PROM_VERSIONS)
 test: export THANOS_TEST_ALERTMANAGER_PATH= $(ALERTMANAGER)
 test: check-git install-deps
 	@echo ">> install thanos GOOPTS=${GOOPTS}"
-	# Thanos binary is required by e2e tests.
-	@go install github.com/thanos-io/thanos/cmd/thanos
-	# Be careful on GOCACHE. Those tests are sometimes using built Thanos/Prometheus binaries directly. Don't cache those.
-	@rm -rf ${GOCACHE}
-	@echo ">> running all tests. Do export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS if you want to skip e2e tests against all real store buckets. Current value: ${THANOS_TEST_OBJSTORE_SKIP}"
-	@go test $(shell go list ./... | grep -v /vendor/);
+	@echo ">> running unit tests (without /test/e2e). Do export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS if you want to skip e2e tests against all real store buckets. Current value: ${THANOS_TEST_OBJSTORE_SKIP}"
+	@go test $(shell go list ./... | grep -v /vendor/ | grep -v /test/e2e);
 
 .PHONY: test-ci
+test-ci: ## Runs test for CI, so excluding object storage integrations that we don't have configured yet.
 test-ci: export THANOS_TEST_OBJSTORE_SKIP=AZURE,SWIFT,COS,ALIYUNOSS
 test-ci:
 	@echo ">> Skipping ${THANOS_TEST_OBJSTORE_SKIP} tests"
 	$(MAKE) test
 
 .PHONY: test-local
+test-local: ## Runs test excluding tests for ALL  object storage integrations.
 test-local: export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS
 test-local:
 	$(MAKE) test
 
-# install-deps installs dependencies for e2e tetss.
-# It installs supported versions of Prometheus and alertmanager to test against in e2e.
+.PHONY: test-e2e
+test-e2e: ## Runs all Thanos e2e docker-based e2e tests from test/e2e. Required access to docker daemon.
+test-e2e: docker
+	@echo ">> running /test/e2e tests."
+	@go test -v ./test/e2e/...
+
 .PHONY: install-deps
+install-deps: ## Installs dependencies for integration tests. It installs supported versions of Prometheus and alertmanager to test against in integration tests.
 install-deps: $(ALERTMANAGER) $(MINIO_SERVER) $(PROMS)
 	@echo ">>GOBIN=$(GOBIN)"
 
 .PHONY: docker-ci
-# To be run by Thanos maintainer.
+docker-ci: ## Builds and pushes docker image used by our CI. This is done to cache our tools and dependencies. To be run by Thanos maintainer.
 docker-ci: install-deps
 	# Copy all to tmp local dir as this is required by docker.
 	@rm -rf ./tmp/bin
@@ -267,7 +266,6 @@ docker-ci: install-deps
 	@docker tag "thanos-ci" "quay.io/thanos/thanos-ci:$(DOCKER_CI_TAG)"
 	@docker push "quay.io/thanos/thanos-ci:$(DOCKER_CI_TAG)"
 
-# tooling deps. TODO(bwplotka): Pin them all to certain version!
 .PHONY: check-git
 check-git:
 ifneq ($(GIT),)
@@ -282,19 +280,20 @@ web-pre-process:
 	@bash scripts/websitepreprocess.sh
 
 .PHONY: web
+web: ## Builds our website.
 web: web-pre-process $(HUGO)
 	@echo ">> building documentation website"
 	# TODO(bwplotka): Make it --gc
 	@cd $(WEB_DIR) && HUGO_ENV=production $(HUGO) --config hugo.yaml --minify -v -b $(WEBSITE_BASE_URL)
 
-.PHONY: lint
 # PROTIP:
 # Add
 #      --cpu-profile-path string   Path to CPU profile output file
 #      --mem-profile-path string   Path to memory profile output file
-#
 # to debug big allocations during linting.
-lint: check-git $(GOLANGCILINT) $(MISSPELL) $(FAILLINT)
+lint: ## Runs various static analysis against our code.
+lint: check-git deps $(GOLANGCILINT) $(MISSPELL) $(FAILLINT)
+	$(call require_clean_work_tree,"detected not clean master before running lint")
 	@echo ">> verifying modules being imported"
 	@$(FAILLINT) -paths $(MODULES_TO_AVOID) ./...
 	@echo ">> examining all of the Go files"
@@ -311,6 +310,7 @@ lint: check-git $(GOLANGCILINT) $(MISSPELL) $(FAILLINT)
 	$(call require_clean_work_tree,"detected files without copyright")
 
 .PHONY: web-serve
+web-serve: ## Builds and serves Thanos website on localhost.
 web-serve: web-pre-process $(HUGO)
 	@echo ">> serving documentation website"
 	@cd $(WEB_DIR) && $(HUGO) --config hugo.yaml -v server

diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go
@@ -284,6 +284,9 @@ func runReceive(
 					if !ok {
 						return nil
 					}
+
+					level.Info(logger).Log("msg", "updating DB")
+
 					if err := db.Flush(); err != nil {
 						return errors.Wrap(err, "flushing storage")
 					}
@@ -298,7 +301,7 @@ func runReceive(
 					localStorage.Set(db.Get(), startTimeMargin)
 					webHandler.SetWriter(receive.NewWriter(log.With(logger, "component", "receive-writer"), localStorage))
 					statusProber.Ready()
-					level.Info(logger).Log("msg", "server is ready to receive web requests.")
+					level.Info(logger).Log("msg", "server is ready to receive web requests")
 					dbReady <- struct{}{}
 				}
 			}
@@ -318,6 +321,7 @@ func runReceive(
 		if cw != nil {
 			ctx, cancel := context.WithCancel(context.Background())
 			g.Add(func() error {
+				// TODO: If config is empty we never start receiver. We might want to fail in this case.
 				receive.HashringFromConfig(ctx, updates, cw)
 				return nil
 			}, func(error) {

diff --git a/docs/contributing/how-to-change-go-version.md b/docs/contributing/how-to-change-go-version.md
@@ -15,3 +15,4 @@ To update Thanos build system to newer Golang:
 1. Edit [.circleci/config.yaml](/.circleci/config.yml) and update ` - image: circleci/golang:<go version>` to desired
   Golang version. This will ensure that all docker images and go tests are using desired Golang version. How to verify? Invoke `docker pull quay.io/thanos/thanos:<version> --version`
 1. Edit [.Dockerfile.thanos-ci](/Dockerfile.thanos-ci) and update Go version. Run `make docker-ci DOCKER_CI_TAG=<new tag>`. Update [.circleci/config.yaml](/.circleci/config.yml) thanos-ci image to `<new tag>`.
+1. Edit [.github/workflows/e2e.yaml](/.github/workflows/e2e.yaml), [.github/workflows/ci.yaml](/.github/workflows/ci.yaml) and update Go version.