Revendor/cortex (#3446)

* Upgrade Cortex in Loki Signed-off-by: Marco Pracucci <marco@pracucci.com> * proto compat * gofmt * changelog for deprecated querier.split-queries-by-day Co-authored-by: Marco Pracucci <marco@pracucci.com>
grafana · Mar 10, 2021 · 306cc72 · 306cc72
1 parent 8012362
commit 306cc72
Show file tree

Hide file tree

Showing 964 changed files with 63,425 additions and 28,512 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## Master
+
+* [3446](https://github.com/grafana/loki/pull/3446) **pracucci, owen-d**: Remove deprecated config `querier.split-queries-by-day` in favor of `querier.split-queries-by-interval`
+
 ## 2.1.0 (2020/12/23)
 
 Happy Holidays from the Loki team! Please enjoy a new Loki release to welcome in the New Year!

diff --git a/cmd/loki/main.go b/cmd/loki/main.go
@@ -82,7 +82,7 @@ func main() {
 
 	// Validate the config once both the config file has been loaded
 	// and CLI flags parsed.
-	err := config.Validate(util_log.Logger)
+	err := config.Validate()
 	if err != nil {
 		level.Error(util_log.Logger).Log("msg", "validating config", "err", err.Error())
 		os.Exit(1)

diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go
@@ -18,7 +18,6 @@ import (
 	"github.com/prometheus/common/model"
 
 	"github.com/cortexproject/cortex/pkg/chunk"
-	gokit "github.com/go-kit/kit/log"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/weaveworks/common/user"
 
@@ -80,13 +79,12 @@ func main() {
 		log.Println("Failed to create limit overrides:", err)
 		os.Exit(1)
 	}
-	lg := gokit.NewLogfmtLogger(gokit.NewSyncWriter(os.Stdout))
-	err = sourceConfig.Validate(lg)
+	err = sourceConfig.Validate()
 	if err != nil {
 		log.Println("Failed to validate source store config:", err)
 		os.Exit(1)
 	}
-	err = destConfig.Validate(lg)
+	err = destConfig.Validate()
 	if err != nil {
 		log.Println("Failed to validate dest store config:", err)
 		os.Exit(1)

diff --git a/go.mod b/go.mod
@@ -11,9 +11,9 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
-	github.com/cortexproject/cortex v1.6.1-0.20210204145131-7dac81171c66
+	github.com/cortexproject/cortex v1.7.1-0.20210308081334-2dae12a14c48
 	github.com/davecgh/go-spew v1.1.1
-	github.com/docker/docker v20.10.1+incompatible
+	github.com/docker/docker v20.10.3+incompatible
 	github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect
 	github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8
 	github.com/drone/envsubst v1.0.2
@@ -24,11 +24,11 @@ require (
 	github.com/go-kit/kit v0.10.0
 	github.com/go-logfmt/logfmt v0.5.0
 	github.com/gofrs/flock v0.7.1 // indirect
-	github.com/gogo/protobuf v1.3.1 // remember to update loki-build-image/Dockerfile too
+	github.com/gogo/protobuf v1.3.2 // remember to update loki-build-image/Dockerfile too
 	github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.2
-	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
 	github.com/hashicorp/golang-lru v0.5.4
@@ -49,28 +49,28 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.9.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.15.0
-	github.com/prometheus/prometheus v1.8.2-0.20210124145330-b5dfa2414b9e
+	github.com/prometheus/common v0.18.0
+	github.com/prometheus/prometheus v1.8.2-0.20210215121130-6f488061dfb4
 	github.com/segmentio/fasthash v1.0.2
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
 	github.com/spf13/afero v1.2.2
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/tonistiigi/fifo v0.0.0-20190226154929-a9fb20d87448
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	github.com/ugorji/go v1.1.7 // indirect
 	github.com/weaveworks/common v0.0.0-20210112142934-23c8d7fa6120
-	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50
+	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.7.0
 	golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9
-	golang.org/x/net v0.0.0-20201224014010-6772e930b67b
-	golang.org/x/sys v0.0.0-20201223074533-0d417f636930
-	google.golang.org/api v0.36.0
-	google.golang.org/grpc v1.33.2
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
+	google.golang.org/api v0.39.0
+	google.golang.org/grpc v1.34.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/klog v1.0.0
 )
 

diff --git a/go.sum b/go.sum
diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go
@@ -235,7 +235,7 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log
 	}
 
 	now := time.Now()
-	if !d.ingestionRateLimiter.AllowN(now, userID, validatedSamplesSize) {
+	if ok, _ := d.ingestionRateLimiter.AllowN(now, userID, validatedSamplesSize); !ok {
 		// Return a 429 to indicate to the client they are being rate limited
 		validation.DiscardedSamples.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedSamplesCount))
 		validation.DiscardedBytes.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedSamplesSize))

diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go
@@ -421,3 +421,5 @@ func (r mockRing) ShuffleShard(identifier string, size int) ring.ReadRing {
 func (r mockRing) ShuffleShardWithLookback(identifier string, size int, lookbackPeriod time.Duration, now time.Time) ring.ReadRing {
 	return r
 }
+
+func (r mockRing) CleanupShuffleShardCache(identifier string) {}
diff --git a/pkg/ingester/checkpoint.pb.go b/pkg/ingester/checkpoint.pb.go
diff --git a/pkg/ingester/checkpoint.proto b/pkg/ingester/checkpoint.proto
@@ -6,7 +6,7 @@ option go_package = "ingester";
 
 import "google/protobuf/timestamp.proto";
 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
-import "github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto";
+import "github.com/cortexproject/cortex/pkg/cortexpb/cortex.proto";
 
 // Chunk is a {de,}serializable intermediate type for chunkDesc which allows
 // efficient loading/unloading to disk during WAL checkpoint recovery.
@@ -28,7 +28,7 @@ message Series {
   string userID = 1;
   // post mapped fingerprint is necessary because subsequent wal writes will reference it.
   uint64 fingerprint = 2;
-  repeated cortex.LabelPair labels = 3 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];
+  repeated cortexpb.LabelPair labels = 3 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter"];
   repeated Chunk chunks = 4 [(gogoproto.nullable) = false];
   // Last timestamp of the last chunk.
   google.protobuf.Timestamp to = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];

diff --git a/pkg/logcli/query/query.go b/pkg/logcli/query/query.go
@@ -182,7 +182,7 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string
 		return err
 	}
 
-	if err := conf.Validate(util_log.Logger); err != nil {
+	if err := conf.Validate(); err != nil {
 		return err
 	}
 

diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go
@@ -9,6 +9,7 @@ import (
 
 	frontend "github.com/cortexproject/cortex/pkg/frontend/v1"
 	"github.com/cortexproject/cortex/pkg/querier/worker"
+	"github.com/cortexproject/cortex/pkg/ruler/rulestore"
 	"github.com/felixge/fgprof"
 
 	"github.com/grafana/loki/pkg/storage/stores/shipper/compactor"
@@ -23,13 +24,11 @@ import (
 	"github.com/cortexproject/cortex/pkg/ring"
 	"github.com/cortexproject/cortex/pkg/ring/kv/memberlist"
 	cortex_ruler "github.com/cortexproject/cortex/pkg/ruler"
-	"github.com/cortexproject/cortex/pkg/ruler/rules"
 	"github.com/cortexproject/cortex/pkg/util"
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
 	"github.com/cortexproject/cortex/pkg/util/runtimeconfig"
 	"github.com/cortexproject/cortex/pkg/util/services"
 
-	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/weaveworks/common/middleware"
@@ -113,14 +112,14 @@ func (c *Config) Clone() flagext.Registerer {
 
 // Validate the config and returns an error if the validation
 // doesn't pass
-func (c *Config) Validate(log log.Logger) error {
+func (c *Config) Validate() error {
 	if err := c.SchemaConfig.Validate(); err != nil {
 		return errors.Wrap(err, "invalid schema config")
 	}
 	if err := c.StorageConfig.Validate(); err != nil {
 		return errors.Wrap(err, "invalid storage config")
 	}
-	if err := c.QueryRange.Validate(log); err != nil {
+	if err := c.QueryRange.Validate(); err != nil {
 		return errors.Wrap(err, "invalid queryrange config")
 	}
 	if err := c.TableManager.Validate(); err != nil {
@@ -155,7 +154,7 @@ type Loki struct {
 	tableManager    *chunk.TableManager
 	frontend        *frontend.Frontend
 	ruler           *cortex_ruler.Ruler
-	RulerStorage    rules.RuleStore
+	RulerStorage    rulestore.RuleStore
 	rulerAPI        *cortex_ruler.API
 	stopper         queryrange.Stopper
 	runtimeConfig   *runtimeconfig.Manager

diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
@@ -1,6 +1,7 @@
 package loki
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -459,8 +460,15 @@ func (t *Loki) initQueryFrontend() (_ services.Service, err error) {
 	t.Server.HTTP.Handle("/loki/api/v1/tail", defaultHandler)
 	t.Server.HTTP.Handle("/api/prom/tail", defaultHandler)
 
-	return services.NewIdleService(nil, func(_ error) error {
-		t.frontend.Close()
+	return services.NewIdleService(func(ctx context.Context) error {
+		return services.StartAndAwaitRunning(ctx, t.frontend)
+	}, func(_ error) error {
+		// Log but not return in case of error, so that other following dependencies
+		// are stopped too.
+		if err := services.StopAndAwaitTerminated(context.Background(), t.frontend); err != nil {
+			level.Warn(util_log.Logger).Log("msg", "failed to stop frontend service", "err", err)
+		}
+
 		if t.stopper != nil {
 			t.stopper.Stop()
 		}
@@ -492,7 +500,7 @@ func (t *Loki) initRulerStorage() (_ services.Service, err error) {
 		}
 	}
 
-	t.RulerStorage, err = cortex_ruler.NewRuleStorage(t.cfg.Ruler.StoreConfig, manager.GroupLoader{})
+	t.RulerStorage, err = cortex_ruler.NewLegacyRuleStore(t.cfg.Ruler.StoreConfig, manager.GroupLoader{}, util_log.Logger)
 
 	return
 }
@@ -525,7 +533,7 @@ func (t *Loki) initRuler() (_ services.Service, err error) {
 		return
 	}
 
-	t.rulerAPI = cortex_ruler.NewAPI(t.ruler, t.RulerStorage)
+	t.rulerAPI = cortex_ruler.NewAPI(t.ruler, t.RulerStorage, util_log.Logger)
 
 	// Expose HTTP endpoints.
 	if t.cfg.Ruler.EnableAPI {

diff --git a/pkg/querier/querier_mock_test.go b/pkg/querier/querier_mock_test.go
@@ -351,6 +351,8 @@ func (r *readRingMock) ShuffleShardWithLookback(identifier string, size int, loo
 	return r
 }
 
+func (r *readRingMock) CleanupShuffleShardCache(identifier string) {}
+
 func mockReadRingWithOneActiveIngester() *readRingMock {
 	return newReadRingMock([]ring.InstanceDesc{
 		{Addr: "test", Timestamp: time.Now().UnixNano(), State: ring.ACTIVE, Tokens: []uint32{1, 2, 3}},

diff --git a/pkg/querier/queryrange/limits.go b/pkg/querier/queryrange/limits.go
@@ -50,10 +50,6 @@ func WithDefaultLimits(l Limits, conf queryrange.Config) Limits {
 		overrides: true,
 	}
 
-	if conf.SplitQueriesByDay {
-		res.splitDuration = 24 * time.Hour
-	}
-
 	if conf.SplitQueriesByInterval != 0 {
 		res.splitDuration = conf.SplitQueriesByInterval
 	}

diff --git a/pkg/querier/queryrange/limits_test.go b/pkg/querier/queryrange/limits_test.go
@@ -31,14 +31,6 @@ func TestLimits(t *testing.T) {
 	require.Equal(t, l.QuerySplitDuration("b"), time.Duration(0))
 
 	wrapped := WithDefaultLimits(l, queryrange.Config{
-		SplitQueriesByDay: true,
-	})
-
-	require.Equal(t, wrapped.QuerySplitDuration("a"), time.Minute)
-	require.Equal(t, wrapped.QuerySplitDuration("b"), 24*time.Hour)
-
-	wrapped = WithDefaultLimits(l, queryrange.Config{
-		SplitQueriesByDay:      true, // should be overridden by SplitQueriesByInterval
 		SplitQueriesByInterval: time.Hour,
 	})