Update Prometheus to cd73b3d33e064bbd846fc7a26dc8c313d46af382 (#2596)

* Fix all the breaking things Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Fix tests Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Fixes Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Add CHANGELOG entry Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in>
thanos-io · May 18, 2020 · 748740d · 748740d
1 parent 04f6cd3
commit 748740d
Show file tree

Hide file tree

Showing 35 changed files with 563 additions and 274 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,6 +35,9 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 moved `thanos check rules` to `thanos tools rules-check`. `thanos tools rules-check` also takes rules by `--rules` repeated flag not argument
 anymore.
 - [#2548](https://github.com/thanos-io/thanos/pull/2548/commits/53e69bd89b2b08c18df298eed7d90cb7179cc0ec) Store, Querier: remove duplicated chunks on StoreAPI.
+- [#2596](https://github.com/thanos-io/thanos/pull/2596) Update to Prometheus [@cd73b3d33e064bbd846fc7a26dc8c313d46af382](https://github.com/prometheus/prometheus/commit/cd73b3d33e064bbd846fc7a26dc8c313d46af382) which falls in between v2.17.0 and v2.18.0.
+    - TSDB now supports isolation of append and queries.
+    - TSDB now holds less WAL files after Head Truncation.
 
 ## [v0.12.2](https://github.com/thanos-io/thanos/releases/tag/v0.12.2) - 2020.04.30
 

diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go
@@ -257,9 +257,8 @@ func runQuery(
 		queryableCreator = query.NewQueryableCreator(logger, proxy)
 		engine           = promql.NewEngine(
 			promql.EngineOpts{
-				Logger:        logger,
-				Reg:           reg,
-				MaxConcurrent: maxConcurrentQueries,
+				Logger: logger,
+				Reg:    reg,
 				// TODO(bwplotka): Expose this as a flag: https://github.com/thanos-io/thanos/issues/703.
 				MaxSamples: math.MaxInt32,
 				Timeout:    queryTimeout,

diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go
@@ -19,7 +19,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/pkg/labels"
-	"github.com/prometheus/prometheus/storage/tsdb"
+	"github.com/prometheus/prometheus/tsdb"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/thanos-io/thanos/pkg/component"
@@ -102,9 +102,9 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) {
 		}
 
 		tsdbOpts := &tsdb.Options{
-			MinBlockDuration:  *tsdbMinBlockDuration,
-			MaxBlockDuration:  *tsdbMaxBlockDuration,
-			RetentionDuration: *retention,
+			MinBlockDuration:  int64(time.Duration(*tsdbMinBlockDuration) / time.Millisecond),
+			MaxBlockDuration:  int64(time.Duration(*tsdbMaxBlockDuration) / time.Millisecond),
+			RetentionDuration: int64(time.Duration(*retention) / time.Millisecond),
 			NoLockfile:        true,
 			WALCompression:    *walCompression,
 		}
@@ -213,7 +213,7 @@ func runReceive(
 	if upload {
 		if tsdbOpts.MinBlockDuration != tsdbOpts.MaxBlockDuration {
 			if !ignoreBlockSize {
-				return errors.Errorf("found that TSDB Max time is %s and Min time is %s. "+
+				return errors.Errorf("found that TSDB Max time is %d and Min time is %d. "+
 					"Compaction needs to be disabled (tsdb.min-block-duration = tsdb.max-block-duration)", tsdbOpts.MaxBlockDuration, tsdbOpts.MinBlockDuration)
 			}
 			level.Warn(logger).Log("msg", "flag to ignore min/max block duration flags differing is being used. If the upload of a 2h block fails and a tsdb compaction happens that block may be missing from your Thanos bucket storage.")

diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go
@@ -25,7 +25,7 @@ import (
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/rules"
-	"github.com/prometheus/prometheus/storage/tsdb"
+	tsdb "github.com/prometheus/prometheus/tsdb"
 	tsdberrors "github.com/prometheus/prometheus/tsdb/errors"
 	"github.com/prometheus/prometheus/util/strutil"
 	"github.com/thanos-io/thanos/pkg/alert"
@@ -125,9 +125,9 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) {
 		}
 
 		tsdbOpts := &tsdb.Options{
-			MinBlockDuration:  *tsdbBlockDuration,
-			MaxBlockDuration:  *tsdbBlockDuration,
-			RetentionDuration: *tsdbRetention,
+			MinBlockDuration:  int64(time.Duration(*tsdbBlockDuration) / time.Millisecond),
+			MaxBlockDuration:  int64(time.Duration(*tsdbBlockDuration) / time.Millisecond),
+			RetentionDuration: int64(time.Duration(*tsdbRetention) / time.Millisecond),
 			NoLockfile:        true,
 			WALCompression:    *walCompression,
 		}
@@ -425,7 +425,7 @@ func runRule(
 			}
 			alertQ.Push(res)
 		}
-		st := tsdb.Adapter(db, 0)
+		st := db
 
 		opts := rules.ManagerOptions{
 			NotifyFunc:  notify,

diff --git a/cmd/thanos/tools.go b/cmd/thanos/tools.go
@@ -4,17 +4,22 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/oklog/run"
 	"github.com/opentracing/opentracing-go"
+	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/pkg/rulefmt"
-	"github.com/prometheus/prometheus/tsdb/errors"
+	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
 	"gopkg.in/alecthomas/kingpin.v2"
 	"gopkg.in/yaml.v2"
+	yamlv3 "gopkg.in/yaml.v3"
+
+	thanosrule "github.com/thanos-io/thanos/pkg/rule"
 )
 
 func registerTools(m map[string]setupFunc, app *kingpin.Application) {
@@ -36,7 +41,7 @@ func registerCheckRules(m map[string]setupFunc, app *kingpin.CmdClause, pre stri
 }
 
 func checkRulesFiles(logger log.Logger, files *[]string) error {
-	failed := errors.MultiError{}
+	failed := tsdb_errors.MultiError{}
 
 	for _, f := range *files {
 		n, errs := checkRules(logger, f)
@@ -58,17 +63,66 @@ func checkRulesFiles(logger log.Logger, files *[]string) error {
 }
 
 type ThanosRuleGroup struct {
-	PartialResponseStrategy string `yaml:"partial_response_strategy"`
-	rulefmt.RuleGroup       `yaml:",inline"`
+	PartialResponseStrategy  string `yaml:"partial_response_strategy"`
+	thanosrule.PromRuleGroup `yaml:",inline"`
 }
 
 type ThanosRuleGroups struct {
 	Groups []ThanosRuleGroup `yaml:"groups"`
 }
 
-func checkRules(logger log.Logger, filename string) (int, errors.MultiError) {
+// Validate validates all rules in the rule groups.
+// This is copied from Prometheus.
+// TODO: Replace this with upstream implementation after https://github.com/prometheus/prometheus/issues/7128 is fixed.
+func (g *ThanosRuleGroups) Validate() (errs []error) {
+	set := map[string]struct{}{}
+
+	for _, g := range g.Groups {
+		if g.Name == "" {
+			errs = append(errs, errors.New("Groupname should not be empty"))
+		}
+
+		if _, ok := set[g.Name]; ok {
+			errs = append(
+				errs,
+				fmt.Errorf("groupname: %q is repeated in the same file", g.Name),
+			)
+		}
+
+		set[g.Name] = struct{}{}
+
+		for i, r := range g.Rules {
+			ruleNode := rulefmt.RuleNode{
+				Record:      yamlv3.Node{Value: r.Record},
+				Alert:       yamlv3.Node{Value: r.Alert},
+				Expr:        yamlv3.Node{Value: r.Expr},
+				For:         r.For,
+				Labels:      r.Labels,
+				Annotations: r.Annotations,
+			}
+			for _, node := range ruleNode.Validate() {
+				var ruleName string
+				if r.Alert != "" {
+					ruleName = r.Alert
+				} else {
+					ruleName = r.Record
+				}
+				errs = append(errs, &rulefmt.Error{
+					Group:    g.Name,
+					Rule:     i,
+					RuleName: ruleName,
+					Err:      node,
+				})
+			}
+		}
+	}
+
+	return errs
+}
+
+func checkRules(logger log.Logger, filename string) (int, tsdb_errors.MultiError) {
 	level.Info(logger).Log("msg", "checking", "filename", filename)
-	checkErrors := errors.MultiError{}
+	checkErrors := tsdb_errors.MultiError{}
 
 	b, err := ioutil.ReadFile(filename)
 	if err != nil {
@@ -82,9 +136,7 @@ func checkRules(logger log.Logger, filename string) (int, errors.MultiError) {
 		return 0, checkErrors
 	}
 
-	// We need to convert Thanos rules to Prometheus rules so we can use their validation.
-	promRgs := thanosRuleGroupsToPromRuleGroups(rgs)
-	if errs := promRgs.Validate(); errs != nil {
+	if errs := rgs.Validate(); errs != nil {
 		for _, e := range errs {
 			checkErrors.Add(e)
 		}
@@ -98,32 +150,3 @@ func checkRules(logger log.Logger, filename string) (int, errors.MultiError) {
 
 	return numRules, checkErrors
 }
-
-func thanosRuleGroupsToPromRuleGroups(ruleGroups ThanosRuleGroups) rulefmt.RuleGroups {
-	promRuleGroups := rulefmt.RuleGroups{Groups: []rulefmt.RuleGroup{}}
-	for _, g := range ruleGroups.Groups {
-		group := rulefmt.RuleGroup{
-			Name:     g.Name,
-			Interval: g.Interval,
-			Rules:    []rulefmt.Rule{},
-		}
-		for _, r := range g.Rules {
-			group.Rules = append(
-				group.Rules,
-				rulefmt.Rule{
-					Record:      r.Record,
-					Alert:       r.Alert,
-					Expr:        r.Expr,
-					For:         r.For,
-					Labels:      r.Labels,
-					Annotations: r.Annotations,
-				},
-			)
-		}
-		promRuleGroups.Groups = append(
-			promRuleGroups.Groups,
-			group,
-		)
-	}
-	return promRuleGroups
-}
diff --git a/go.mod b/go.mod
@@ -16,51 +16,53 @@ require (
 	github.com/fatih/structtag v1.1.0
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/fsnotify/fsnotify v1.4.7
-	github.com/go-kit/kit v0.9.0
-	github.com/go-openapi/strfmt v0.19.2
+	github.com/go-kit/kit v0.10.0
+	github.com/go-openapi/strfmt v0.19.4
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9
 	github.com/golang/snappy v0.0.1
 	github.com/googleapis/gax-go v2.0.2+incompatible
-	github.com/gophercloud/gophercloud v0.6.0
+	github.com/gophercloud/gophercloud v0.8.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/hashicorp/golang-lru v0.5.3
 	github.com/leanovate/gopter v0.2.4
-	github.com/lightstep/lightstep-tracer-go v0.18.0
+	github.com/lightstep/lightstep-tracer-go v0.18.1
 	github.com/lovoo/gcloud-opentracing v0.3.0
-	github.com/miekg/dns v1.1.22
+	github.com/miekg/dns v1.1.27
 	github.com/minio/minio-go/v6 v6.0.53
 	github.com/mozillazg/go-cos v0.13.0
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
-	github.com/oklog/run v1.0.0
+	github.com/oklog/run v1.1.0
 	github.com/oklog/ulid v1.3.1
 	github.com/olekukonko/tablewriter v0.0.2
 	github.com/opentracing/basictracer-go v1.0.0
 	github.com/opentracing/opentracing-go v1.1.1-0.20200124165624-2876d2018785
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/alertmanager v0.20.0
-	github.com/prometheus/client_golang v1.5.0
+	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.9.1
-	github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33
+	github.com/prometheus/prometheus v1.8.2-0.20200407102557-cd73b3d33e06
+	github.com/sercand/kuberesolver v2.4.0+incompatible // indirect
 	github.com/uber/jaeger-client-go v2.20.1+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible
 	go.elastic.co/apm v1.5.0
 	go.elastic.co/apm/module/apmot v1.5.0
 	go.uber.org/automaxprocs v1.2.0
-	golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708
-	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/text v0.3.2
 	golang.org/x/tools v0.0.0-20200306191617-51e69f71924f // indirect
-	google.golang.org/api v0.14.0
-	google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9
-	google.golang.org/grpc v1.25.1
+	google.golang.org/api v0.20.0
+	google.golang.org/genproto v0.0.0-20200305110556-506484158171
+	google.golang.org/grpc v1.27.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/fsnotify.v1 v1.4.7
-	gopkg.in/yaml.v2 v2.2.7
+	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71
 )
 
 // We want to replace the client-go version with a specific commit hash,
@@ -70,7 +72,7 @@ replace (
 	// Mitigation for: https://github.com/Azure/go-autorest/issues/414
 	github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.3.0+incompatible
 	// Make sure Cortex is not forcing us to some other Prometheus version.
-	github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 // master ~ v2.15.2
+	github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20200407102557-cd73b3d33e06 // @cd73b3d33e064bbd846fc7a26dc8c313d46af382 (after v2.17.0 and before v2.18.0).
 	k8s.io/api => k8s.io/api v0.0.0-20190620084959-7cf5895f2711
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190620085554-14e95df34f1f
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719