Merge remote-tracking branch 'upstream/master'

.gitignore

@@ -27,3 +27,6 @@ website/docs-pre-processed/
 
 tmp/bin
 examples/tmp/
+
+# Ignore the MacOS Trash (DS-Store)
+.DS_Store

CHANGELOG.md

@@ -13,16 +13,44 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ### Fixed
 
-- [#2033](https://github.com/thanos-io/thanos/pull/2033) minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS
-- [#1985](https://github.com/thanos-io/thanos/pull/1985) store gateway: Fixed case where series entry is larger than 64KB in index.
-- [#2051](https://github.com/thanos-io/thanos/pull/2051) ruler: Fixed issue where ruler does not expose shipper metrics.
-- [#2101](https://github.com/thanos-io/thanos/pull/2101) ruler: Fixed bug where thanos_alert_sender_errors_total was not registered.
+- [#2238](https://github.com/thanos-io/thanos/pull/2238) Ruler: Fixed Issue #2204 bug in alert queue signalling filled up queue and alerts were dropped
+- [#2231](https://github.com/thanos-io/thanos/pull/2231) Bucket Web - Sort chunks by thanos.downsample.resolution for better grouping
+- [#2254](https://github.com/thanos-io/thanos/pull/2254) Bucket: Fix metrics registered multiple times in bucket replicate
 
 ### Added
 
-- [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags
-- [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction
-- [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`.
+- [#2265](https://github.com/thanos-io/thanos/pull/2265) Compactor: Add `--wait-interval` to specify compaction wait interval between consecutive compact runs when `--wait` enabled.
+
+### Changed
+
+- [#2136](https://github.com/thanos-io/thanos/pull/2136) store, compact, bucket: schedule block deletion by adding deletion-mark.json. This adds a consistent way for multiple readers and writers to access object storage.
+Since there are no consistency guarantees provided by some Object Storage providers, this PR adds a consistent lock-free way of dealing with Object Storage irrespective of the choice of object storage. In order to achieve this co-ordination, blocks are not deleted directly. Instead, blocks are marked for deletion by uploading `deletion-mark.json` file for the block that was chosen to be deleted. This file contains unix time of when the block was marked for deletion.
+
+- [#2090](https://github.com/thanos-io/thanos/issues/2090) *breaking* Downsample command: the `downsample` command has moved as the `thanos bucket` sub-command, and cannot be called via `thanos downsample` any more.
+
+## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0) - 2020.03.02
+
+### Fixed
+
+- [#2033](https://github.com/thanos-io/thanos/pull/2033) Minio-go: Fixed Issue #1494 support Web Identity providers for IAM credentials for AWS EKS.
+- [#1985](https://github.com/thanos-io/thanos/pull/1985) Store Gateway: Fixed case where series entry is larger than 64KB in index.
+- [#2051](https://github.com/thanos-io/thanos/pull/2051) Ruler: Fixed issue where ruler does not expose shipper metrics.
+- [#2101](https://github.com/thanos-io/thanos/pull/2101) Ruler: Fixed bug where thanos_alert_sender_errors_total was not registered.
+- [#1789](https://github.com/thanos-io/thanos/pull/1789) Store Gateway: Improve timeouts.
+- [#2139](https://github.com/thanos-io/thanos/pull/2139) Properly handle SIGHUP for reloading.
+- [#2040](https://github.com/thanos-io/thanos/pull/2040) UI: Fix URL of alerts in Ruler
+- [#2033](https://github.com/thanos-io/thanos/pull/1978) Ruler: Fix tracing in Thanos Ruler
+
+### Added
+
+- [#2003](https://github.com/thanos-io/thanos/pull/2003) Query: Support downsampling for /series.
+- [#1952](https://github.com/thanos-io/thanos/pull/1952) Store Gateway: Implemented [binary index header](https://thanos.io/proposals/201912_thanos_binary_index_header.md/). This significantly reduces resource consumption (memory, CPU, net bandwidth) for startup and data loading processes as well as baseline memory. This means that adding more blocks into object storage, without querying them will use almost no resources. This, however, **still means that querying large amounts of data** will result in high spikes of memory and CPU use as before, due to simply fetching large amounts of metrics data. Since we fixed baseline, we are now focusing on query performance optimizations in separate initiatives. To enable experimental `index-header` mode run store with hidden `experimental.enable-index-header` flag.
+- [#2009](https://github.com/thanos-io/thanos/pull/2009) Store Gateway: Minimum age of all blocks before they are being read. Set it to a safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent.
+- [#1963](https://github.com/thanos-io/thanos/pull/1963) Mixin: Add Thanos Ruler alerts.
+- [#1984](https://github.com/thanos-io/thanos/pull/1984) Query: Add cache-control header to not cache on error.
+- [#1870](https://github.com/thanos-io/thanos/pull/1870) UI: Persist settings in query.
+- [#1969](https://github.com/thanos-io/thanos/pull/1969) Sidecar: allow setting http connection pool size via flags.
+- [#1967](https://github.com/thanos-io/thanos/issues/1967) Receive: Allow local TSDB compaction.
 - [#1939](https://github.com/thanos-io/thanos/pull/1939) Ruler: Add TLS and authentication support for query endpoints with the `--query.config` and `--query.config-file` CLI flags. See [documentation](docs/components/rule.md/#configuration) for further information.
 - [#1982](https://github.com/thanos-io/thanos/pull/1982) Ruler: Add support for Alertmanager v2 API endpoints.
 - [#2030](https://github.com/thanos-io/thanos/pull/2030) Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores.
@@ -34,6 +62,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ### Changed
 
+- [#1970](https://github.com/thanos-io/thanos/issues/1970) *breaking* Receive: Use gRPC for forwarding requests between peers. Note that existing values for the `--receive.local-endpoint` flag and the endpoints in the hashring configuration file must now specify the receive gRPC port and must be updated to be a simple `host:port` combination, e.g. `127.0.0.1:10901`, rather than a full HTTP URL, e.g. `http://127.0.0.1:10902/api/v1/receive`.
 - [#1933](https://github.com/thanos-io/thanos/pull/1933) Add a flag `--tsdb.wal-compression` to configure whether to enable tsdb wal compression in ruler and receiver.
 - [#2021](https://github.com/thanos-io/thanos/pull/2021) Rename metric `thanos_query_duplicated_store_address` to `thanos_query_duplicated_store_addresses_total` and `thanos_rule_duplicated_query_address` to `thanos_rule_duplicated_query_addresses_total`.
 - [#2166](https://github.com/thanos-io/thanos/pull/2166) Improve tooltip for bucket web UI.

Makefile

@@ -44,8 +44,9 @@ MISSPELL ?= $(GOBIN)/misspell-$(MISSPELL_VERSION)
 GOJSONTOYAML_VERSION ?= e8bd32d46b3d764bef60f12b3bada1c132c4be55
 GOJSONTOYAML ?= $(GOBIN)/gojsontoyaml-$(GOJSONTOYAML_VERSION)
 # v0.14.0
-JSONNET_VERSION ?= fbde25be2182caa4345b03f1532450911ac7d1f3
+JSONNET_VERSION ?= 724650d358b67909a7bea00ea443e23afc3d2a17
 JSONNET ?= $(GOBIN)/jsonnet-$(JSONNET_VERSION)
+JSONNETFMT ?= $(GOBIN)/jsonnetfmt-$(JSONNET_VERSION)
 JSONNET_BUNDLER_VERSION ?= efe0c9e864431e93d5c3376bd5931d0fb9b2a296
 JSONNET_BUNDLER ?= $(GOBIN)/jb-$(JSONNET_BUNDLER_VERSION)
 # Prometheus v2.14.0
@@ -78,9 +79,8 @@ ALERTMANAGER ?= $(GOBIN)/alertmanager-$(ALERTMANAGER_VERSION)
 MINIO_SERVER_VERSION ?= RELEASE.2018-10-06T00-15-16Z
 MINIO_SERVER ?=$(GOBIN)/minio-$(MINIO_SERVER_VERSION)
 
-FAILLINT_VERSION ?= v1.0.1
+FAILLINT_VERSION ?= v1.2.0
 FAILLINT ?=$(GOBIN)/faillint-$(FAILLINT_VERSION)
-MODULES_TO_AVOID ?=errors,github.com/prometheus/tsdb,github.com/prometheus/prometheus/pkg/testutils
 
 # fetch_go_bin_version downloads (go gets) the binary from specific version and installs it in $(GOBIN)/<bin>-<version>
 # arguments:
@@ -139,7 +139,6 @@ assets: $(GOBINDATA)
  @$(GOBINDATA) $(bindata_flags) -pkg ui -o pkg/ui/bindata.go -ignore '(.*\.map|bootstrap\.js|bootstrap-theme\.css|bootstrap\.css)' pkg/ui/templates/... pkg/ui/static/...
  @go fmt ./pkg/ui
 
-
 .PHONY: build
 build: ## Builds Thanos binary using `promu`.
 build: check-git deps $(PROMU)
@@ -295,7 +294,15 @@ lint: ## Runs various static analysis against our code.
 lint: check-git deps $(GOLANGCILINT) $(MISSPELL) $(FAILLINT)
  $(call require_clean_work_tree,"detected not clean master before running lint")
  @echo ">> verifying modules being imported"
- @$(FAILLINT) -paths $(MODULES_TO_AVOID) ./...
+ @# TODO(bwplotka): Add, Printf, DefaultRegisterer, NewGaugeFunc and MustRegister once exception are accepted. Add fmt.{Errorf}=github.com/pkg/errors.{Errorf} once https://github.com/fatih/faillint/issues/10 is addressed.
+ @$(FAILLINT) -paths "errors=github.com/pkg/errors,\
+github.com/prometheus/tsdb=github.com/prometheus/prometheus/tsdb,\
+github.com/prometheus/prometheus/pkg/testutils=github.com/thanos-io/thanos/pkg/testutil,\
+github.com/prometheus/client_golang/prometheus.{DefaultGatherer,DefBuckets,NewUntypedFunc,UntypedFunc},\
+github.com/prometheus/client_golang/prometheus.{NewCounter,NewCounterVec,NewCounterVec,NewGauge,NewGaugeVec,NewGaugeFunc,\
+NewHistorgram,NewHistogramVec,NewSummary,NewSummaryVec}=github.com/prometheus/client_golang/prometheus/promauto.{NewCounter,\
+NewCounterVec,NewCounterVec,NewGauge,NewGaugeVec,NewGaugeFunc,NewHistorgram,NewHistogramVec,NewSummary,NewSummaryVec}" ./...
+ @$(FAILLINT) -paths "fmt.{Print,Println,Sprint}" -ignore-tests ./...
  @echo ">> examining all of the Go files"
  @go vet -stdmethods=false ./pkg/... ./cmd/... && go vet doc.go
  @echo ">> linting all of the Go files GOGC=${GOGC}"
@@ -365,19 +372,17 @@ jsonnet-vendor: $(JSONNET_BUNDLER) $(MIXIN_ROOT)/jsonnetfile.json $(MIXIN_ROOT)/
  rm -rf ${JSONNET_VENDOR_DIR}
  cd ${MIXIN_ROOT} && $(JSONNET_BUNDLER) install
 
-JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s
+JSONNETFMT_CMD := $(JSONNETFMT) -n 2 --max-blank-lines 2 --string-style s --comment-style s
 
 .PHONY: jsonnet-format
-jsonnet-format:
- @which jsonnetfmt 2>&1 >/dev/null || ( \
- echo "Cannot find jsonnetfmt command, please install from https://github.com/google/jsonnet/releases.\nIf your C++ does not support GLIBCXX_3.4.20, please use xxx-in-container target like jsonnet-format-in-container." \
- && exit 1)
+jsonnet-format: $(JSONNETFMT)
  find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
- xargs -n 1 -- $(JSONNET_FMT) -i
+ xargs -n 1 -- $(JSONNETFMT_CMD) -i
 
 .PHONY: jsonnet-format-in-container
 jsonnet-format-in-container:
- $(JSONNET_CONTAINER_CMD) make $(MFLAGS) jsonnet-format
+ $(JSONNET_CONTAINER_CMD) find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+ xargs -n 1 -- jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s -i
 
 .PHONY: example-rules-lint
 example-rules-lint: $(PROMTOOL) examples/alerts/alerts.yaml examples/alerts/rules.yaml
@@ -441,6 +446,9 @@ $(PROTOC):
 $(JSONNET):
  $(call fetch_go_bin_version,github.com/google/go-jsonnet/cmd/jsonnet,$(JSONNET_VERSION))
 
+$(JSONNETFMT):
+ $(call fetch_go_bin_version,github.com/google/go-jsonnet/cmd/jsonnetfmt,$(JSONNET_VERSION))
+
 $(GOJSONTOYAML):
  $(call fetch_go_bin_version,github.com/brancz/gojsontoyaml,$(GOJSONTOYAML_VERSION))
 

VERSION

@@ -1 +1 @@
-0.10.1-dev
+0.12.0-dev

cmd/thanos/bucket.go

@@ -73,6 +73,7 @@ func registerBucket(m map[string]setupFunc, app *kingpin.Application, name strin
  registerBucketInspect(m, cmd, name, objStoreConfig)
  registerBucketWeb(m, cmd, name, objStoreConfig)
  registerBucketReplicate(m, cmd, name, objStoreConfig)
+ registerBucketDownsample(m, cmd, name, objStoreConfig)
 }
 
 func registerBucketVerify(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) {
@@ -84,6 +85,12 @@ func registerBucketVerify(m map[string]setupFunc, root *kingpin.CmdClause, name
  Short('i').Default(verifier.IndexIssueID, verifier.OverlappedBlocksIssueID).Strings()
  idWhitelist := cmd.Flag("id-whitelist", "Block IDs to verify (and optionally repair) only. "+
  "If none is specified, all blocks will be verified. Repeated field").Strings()
+ deleteDelay := modelDuration(cmd.Flag("delete-delay", "Duration after which blocks marked for deletion would be deleted permanently from source bucket by compactor component. "+
+ "If delete-delay is non zero, blocks will be marked for deletion and compactor component is required to delete blocks from source bucket. "+
+ "If delete-delay is 0, blocks will be deleted straight away. Use this if you want to get rid of or move the block immediately. "+
+ "Note that deleting blocks immediately can cause query failures, if store gateway still has the block loaded, "+
+ "or compactor is ignoring the deletion because it's compacting the block at the same time.").
+ Default("0s"))
  m[name+" verify"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error {
  confContentYaml, err := objStoreConfig.Content()
  if err != nil {
@@ -139,9 +146,9 @@ func registerBucketVerify(m map[string]setupFunc, root *kingpin.CmdClause, name
  }
 
  if *repair {
- v = verifier.NewWithRepair(logger, bkt, backupBkt, fetcher, issues)
+ v = verifier.NewWithRepair(logger, reg, bkt, backupBkt, fetcher, time.Duration(*deleteDelay), issues)
  } else {
- v = verifier.New(logger, bkt, fetcher, issues)
+ v = verifier.New(logger, reg, bkt, fetcher, time.Duration(*deleteDelay), issues)
  }
 
  var idMatcher func(ulid.ULID) bool = nil
@@ -269,7 +276,7 @@ func registerBucketInspect(m map[string]setupFunc, root *kingpin.CmdClause, name
  // Parse selector.
  selectorLabels, err := parseFlagLabels(*selector)
  if err != nil {
- return fmt.Errorf("error parsing selector flag: %v", err)
+ return errors.Wrap(err, "error parsing selector flag")
  }
 
  confContentYaml, err := objStoreConfig.Content()
@@ -424,6 +431,21 @@ func registerBucketReplicate(m map[string]setupFunc, root *kingpin.CmdClause, na
 
 }
 
+func registerBucketDownsample(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) {
+
+ comp := component.Downsample
+ cmd := root.Command(comp.String(), "continuously downsamples blocks in an object store bucket")
+
+ httpAddr, httpGracePeriod := regHTTPFlags(cmd)
+
+ dataDir := cmd.Flag("data-dir", "Data directory in which to cache blocks and process downsamplings.").
+ Default("./data").String()
+
+ m[name+" "+comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
+ return RunDownsample(g, logger, reg, *httpAddr, time.Duration(*httpGracePeriod), *dataDir, objStoreConfig, comp)
+ }
+}
+
 // refresh metadata from remote storage periodically and update UI.
 func refresh(ctx context.Context, logger log.Logger, bucketUI *ui.Bucket, duration time.Duration, timeout time.Duration, name string, reg *prometheus.Registry, objStoreConfig *extflag.PathOrContent) error {
  confContentYaml, err := objStoreConfig.Content()
@@ -523,7 +545,7 @@ func printTable(blockMetas []*metadata.Meta, selectorLabels labels.Labels, sortB
  for _, col := range sortBy {
  index := getIndex(header, col)
  if index == -1 {
- return fmt.Errorf("column %s not found", col)
+ return errors.Errorf("column %s not found", col)
  }
  sortByColNum = append(sortByColNum, index)
  }