diff --git a/Makefile b/Makefile index 3b9394a8ee..70ef87aacd 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,9 @@ ME ?= $(shell whoami) # E2e test deps. # Referenced by github.com/improbable-eng/thanos/blob/master/docs/getting_started.md#prometheus -# Limitied prom version, because testing was not possible. This should fix it: https://github.com/improbable-eng/thanos/issues/758 -PROM_VERSIONS ?=v2.4.3 v2.5.0 +# Limited prom version, because testing was not possible. This should fix it: https://github.com/improbable-eng/thanos/issues/758 +PROM_VERSIONS ?=v2.4.3 v2.5.0 v2.8.1 + ALERTMANAGER_VERSION ?=v0.15.2 MINIO_SERVER_VERSION ?=RELEASE.2018-10-06T00-15-16Z diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 89cf8cb07a..bec9746c55 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -197,9 +197,11 @@ func runCompact( level.Warn(logger).Log("msg", "Max compaction level is lower than should be", "current", maxCompactionLevel, "default", compactions.maxLevel()) } + ctx, cancel := context.WithCancel(context.Background()) + // Instantiate the compactor with different time slices. Timestamps in TSDB // are in milliseconds. - comp, err := tsdb.NewLeveledCompactor(reg, logger, levels, downsample.NewPool()) + comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, levels, downsample.NewPool()) if err != nil { return errors.Wrap(err, "create compactor") } @@ -229,7 +231,6 @@ func runCompact( level.Info(logger).Log("msg", "retention policy of 1 hour aggregated samples is enabled", "duration", retentionByResolution[compact.ResolutionLevel1h]) } - ctx, cancel := context.WithCancel(context.Background()) f := func() error { if err := compactor.Compact(ctx); err != nil { return errors.Wrap(err, "compaction failed") diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 63a3ba9c36..59790bf75a 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -15,13 +15,13 @@ import ( "github.com/improbable-eng/thanos/pkg/store" "github.com/improbable-eng/thanos/pkg/store/storepb" "github.com/oklog/run" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/storage/tsdb" "google.golang.org/grpc" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/alecthomas/kingpin.v2" ) func registerReceive(m map[string]setupFunc, app *kingpin.Application, name string) { @@ -70,10 +70,10 @@ func runReceive( level.Warn(logger).Log("msg", "setting up receive; the Thanos receive component is EXPERIMENTAL, it may break significantly without notice") tsdbCfg := &tsdb.Options{ - Retention: model.Duration(time.Hour * 24 * 15), - NoLockfile: true, - MinBlockDuration: model.Duration(time.Hour * 2), - MaxBlockDuration: model.Duration(time.Hour * 2), + RetentionDuration: model.Duration(time.Hour * 24 * 15), + NoLockfile: true, + MinBlockDuration: model.Duration(time.Hour * 2), + MaxBlockDuration: model.Duration(time.Hour * 2), } localStorage := &tsdb.ReadyStorage{} diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 17bb3273b5..9f0a97df06 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -39,7 +39,7 @@ import ( "github.com/improbable-eng/thanos/pkg/tracing" "github.com/improbable-eng/thanos/pkg/ui" "github.com/oklog/run" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -53,7 +53,7 @@ import ( "github.com/prometheus/prometheus/util/strutil" "github.com/prometheus/tsdb/labels" "google.golang.org/grpc" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/alecthomas/kingpin.v2" ) // registerRule registers a rule command. @@ -122,10 +122,10 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) } tsdbOpts := &tsdb.Options{ - MinBlockDuration: *tsdbBlockDuration, - MaxBlockDuration: *tsdbBlockDuration, - Retention: *tsdbRetention, - NoLockfile: true, + MinBlockDuration: *tsdbBlockDuration, + MaxBlockDuration: *tsdbBlockDuration, + RetentionDuration: *tsdbRetention, + NoLockfile: true, } lookupQueries := map[string]struct{}{} diff --git a/go.mod b/go.mod index 8f4fe3ad9a..d104168e16 100644 --- a/go.mod +++ b/go.mod @@ -36,8 +36,8 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.2 github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 - github.com/prometheus/prometheus v0.0.0-20190118110214-3bd41cc92c78 - github.com/prometheus/tsdb v0.4.0 + github.com/prometheus/prometheus v0.0.0-20190328180107-4d60eb36dcbe + github.com/prometheus/tsdb v0.6.1 go.opencensus.io v0.19.0 // indirect golang.org/x/net v0.0.0-20190213061140-3a22650c66bd golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 diff --git a/go.sum b/go.sum index 26b7619663..e8269186b4 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,16 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +contrib.go.opencensus.io/exporter/ocagent v0.4.1/go.mod h1:b6YwD5Q3Yvj4yk0CDK5vGXexygNzI09aXUdDEakQBgA= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.0.0-20181218151757-9b75e4fe745a/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-sdk-for-go v0.0.0-20161028183111-bd73d950fa44/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v23.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c h1:Y5ueznoCekgCWBytF1Q9lTpZ3tJeX37dQtCcGjMCLYI= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69/go.mod h1:L1AbZdiDllfyYH5l5OkAaZtk7VkWe89bPJFmnDBNHxg= -github.com/BurntSushi/toml v0.0.0-20170626110600-a368813c5e64/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/NYTimes/gziphandler v1.0.1 h1:iLrQrdwjDd52kHDA5op2UBJFjmOb9g+7scBan4RN8F0= +github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -31,6 +33,7 @@ github.com/bep/gitmap v1.0.0/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kY github.com/bep/go-tocss v0.6.0/go.mod h1:d9d3crzlTl+PUZLFzBUjfFCpp68K+ku10mzTlnqU/+A= github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8/go.mod h1:Iev9Q3MErcn+w3UOJD/DkEzllvugfdx7bGcMOFhvr/4= github.com/cenk/backoff v2.0.0+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= +github.com/census-instrumentation/opencensus-proto v0.1.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chaseadamsio/goorgeous v1.1.0/go.mod h1:6QaC0vFoKWYDth94dHFNgRT2YkT5FHdQp/Yx15aAAi0= @@ -66,6 +69,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-ini/ini v1.21.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -145,9 +150,9 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw= +github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= +github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 h1:jw16EimP5oAEM/2wt+SiEUov/YDyTCTDuPtIKgQIvk0= github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJuf5HmJAJrKV64MmozrSsk+or0PB5dzdfspg= github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -198,8 +203,10 @@ github.com/prometheus/common v0.0.0-20181218105931-67670fe90761/go.mod h1:daVV7q github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/prometheus v0.0.0-20190118110214-3bd41cc92c78/go.mod h1:aNuqjQW47j/yc/kN0cNZ8cOTBlBU8VbspEvNDfxe1To= -github.com/prometheus/tsdb v0.4.0/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/prometheus v0.0.0-20190328180107-4d60eb36dcbe h1:Wpwxfdtx0+/3dgqBvUAB2CTRtHRimQrH4RTEJ5yxCUo= +github.com/prometheus/prometheus v0.0.0-20190328180107-4d60eb36dcbe/go.mod h1:mVHnNN4pf1p9fM7dRvZQAKj0Ex93s+mIhtsml8SJy5E= +github.com/prometheus/tsdb v0.6.1 h1:wUp5iY/rmwv6VNzZg4yaUVpkSs4nX5O5+aoviWTNeUo= +github.com/prometheus/tsdb v0.6.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -241,8 +248,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/wellington/go-libsass v0.9.3-0.20181113175235-c63644206701/go.mod h1:mxgxgam0N0E+NAUMHLcu20Ccfc3mVpDkyrLDayqfiTs= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= -go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= -go.opencensus.io v0.19.0/go.mod h1:AYeH0+ZxYyghG8diqaaIq/9P3VgCCt5GF2ldCY4dkFg= +go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.19.0 h1:+jrnNy8MR4GZXvwF9PEuSyHxA4NaTf6601oNRwCSXq0= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -279,7 +286,6 @@ golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181023010539-40a48ad93fbe/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -google.golang.org/api v0.0.0-20180506000402-20530fd5d65a/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -291,6 +297,7 @@ google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -305,7 +312,6 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= -gopkg.in/yaml.v2 v2.2.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go index 80c10e8e6e..36a21f3ea3 100644 --- a/pkg/block/index_test.go +++ b/pkg/block/index_test.go @@ -1,6 +1,7 @@ package block import ( + "context" "io/ioutil" "os" "path/filepath" @@ -12,11 +13,13 @@ import ( ) func TestWriteReadIndexCache(t *testing.T) { + ctx := context.Background() + tmpDir, err := ioutil.TempDir("", "test-compact-prepare") testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() - b, err := testutil.CreateBlock(tmpDir, []labels.Labels{ + b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 738048c5c2..f58862c0ac 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -178,7 +178,7 @@ func TestGroup_Compact_e2e(t *testing.T) { var metas []*metadata.Meta extLset := labels.Labels{{Name: "e1", Value: "1"}} - b1, err := testutil.CreateBlock(prepareDir, []labels.Labels{ + b1, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, @@ -190,7 +190,7 @@ func TestGroup_Compact_e2e(t *testing.T) { testutil.Ok(t, err) metas = append(metas, meta) - b3, err := testutil.CreateBlock(prepareDir, []labels.Labels{ + b3, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, {{Name: "a", Value: "5"}}, @@ -215,7 +215,7 @@ func TestGroup_Compact_e2e(t *testing.T) { metas = append(metas, meta) // Due to TSDB compaction delay (not compacting fresh block), we need one more block to be pushed to trigger compaction. - freshB, err := testutil.CreateBlock(prepareDir, []labels.Labels{ + freshB, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, @@ -251,7 +251,7 @@ func TestGroup_Compact_e2e(t *testing.T) { ) testutil.Ok(t, err) - comp, err := tsdb.NewLeveledCompactor(nil, log.NewLogfmtLogger(os.Stderr), []int64{1000, 3000}, nil) + comp, err := tsdb.NewLeveledCompactor(ctx, nil, log.NewLogfmtLogger(os.Stderr), []int64{1000, 3000}, nil) testutil.Ok(t, err) shouldRerun, id, err := g.Compact(ctx, dir, comp) diff --git a/pkg/compact/downsample/downsample_test.go b/pkg/compact/downsample/downsample_test.go index d54d31f0e9..731faa5b53 100644 --- a/pkg/compact/downsample/downsample_test.go +++ b/pkg/compact/downsample/downsample_test.go @@ -390,6 +390,8 @@ type memBlock struct { chunks []chunkenc.Chunk numberOfChunks uint64 + + minTime, maxTime int64 } type series struct { @@ -398,7 +400,7 @@ type series struct { } func newMemBlock() *memBlock { - return &memBlock{symbols: map[string]struct{}{}} + return &memBlock{symbols: map[string]struct{}{}, minTime: -1, maxTime: -1} } func (b *memBlock) addSeries(s *series) { @@ -412,12 +414,34 @@ func (b *memBlock) addSeries(s *series) { } for i, cm := range s.chunks { + if b.minTime == -1 || cm.MinTime < b.minTime { + b.minTime = cm.MinTime + } + if b.maxTime == -1 || cm.MaxTime < b.maxTime { + b.maxTime = cm.MaxTime + } s.chunks[i].Ref = b.numberOfChunks b.chunks = append(b.chunks, cm.Chunk) b.numberOfChunks++ } } +func (b *memBlock) MinTime() int64 { + if b.minTime == -1 { + return 0 + } + + return b.minTime +} + +func (b *memBlock) MaxTime() int64 { + if b.maxTime == -1 { + return 0 + } + + return b.maxTime +} + func (b *memBlock) Postings(name, val string) (index.Postings, error) { allName, allVal := index.AllPostingsKey() diff --git a/pkg/promclient/promclient_e2e_test.go b/pkg/promclient/promclient_e2e_test.go index 1cb7c968af..9f30996dde 100644 --- a/pkg/promclient/promclient_e2e_test.go +++ b/pkg/promclient/promclient_e2e_test.go @@ -72,8 +72,8 @@ func TestConfiguredFlags_e2e(t *testing.T) { testutil.Assert(t, !flags.WebEnableLifecycle, "") testutil.Equals(t, p.Dir(), flags.TSDBPath) testutil.Equals(t, int64(2*time.Hour), int64(flags.TSDBMinTime)) - testutil.Equals(t, int64(36*time.Hour), int64(flags.TSDBMaxTime)) - testutil.Equals(t, int64(15*24*time.Hour), int64(flags.TSDBRetention)) + testutil.Equals(t, int64(4.8*float64(time.Hour)), int64(flags.TSDBMaxTime)) + testutil.Equals(t, int64(2*24*time.Hour), int64(flags.TSDBRetention)) }) } @@ -81,8 +81,10 @@ func TestSnapshot_e2e(t *testing.T) { testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { now := time.Now() + ctx := context.Background() // Create artificial block. id, err := testutil.CreateBlockWithTombstone( + ctx, p.Dir(), []labels.Labels{labels.FromStrings("a", "b")}, 10, @@ -99,7 +101,10 @@ func TestSnapshot_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - dir, err := Snapshot(context.Background(), log.NewNopLogger(), u, false) + // Prometheus since 2.7.0 don't write empty blocks even if it's head block. So it's no matter passing skip_head true or false here + // Pass skipHead = true to support all prometheus versions and assert that snapshot creates only one file + // https://github.com/prometheus/tsdb/pull/374 + dir, err := Snapshot(ctx, log.NewNopLogger(), u, true) testutil.Ok(t, err) _, err = os.Stat(path.Join(p.Dir(), dir, id.String())) @@ -113,7 +118,7 @@ func TestSnapshot_e2e(t *testing.T) { testutil.Ok(t, err) } - testutil.Equals(t, 2, len(files)) + testutil.Equals(t, 1, len(files)) }) } diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 997767d055..bc8e55e359 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -66,9 +66,9 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m // Create two blocks per time slot. Only add 10 samples each so only one chunk // gets created each. This way we can easily verify we got 10 chunks per series below. - id1, err := testutil.CreateBlock(dir, series[:4], 10, mint, maxt, extLset, 0) + id1, err := testutil.CreateBlock(ctx, dir, series[:4], 10, mint, maxt, extLset, 0) testutil.Ok(t, err) - id2, err := testutil.CreateBlock(dir, series[4:], 10, mint, maxt, extLset, 0) + id2, err := testutil.CreateBlock(ctx, dir, series[4:], 10, mint, maxt, extLset, 0) testutil.Ok(t, err) dir1, dir2 := filepath.Join(dir, id1.String()), filepath.Join(dir, id2.String()) diff --git a/pkg/testutil/prometheus.go b/pkg/testutil/prometheus.go index 3843068469..ee65642470 100644 --- a/pkg/testutil/prometheus.go +++ b/pkg/testutil/prometheus.go @@ -163,16 +163,16 @@ func (p *Prometheus) Start() error { ) } p.addr = fmt.Sprintf("localhost:%d", port) - p.cmd = exec.Command( - prometheusBin(p.version), - append([]string{ - "--storage.tsdb.path=" + p.db.Dir(), - "--web.listen-address=" + p.addr, - "--web.route-prefix=" + p.prefix, - "--web.enable-admin-api", - "--config.file=" + filepath.Join(p.db.Dir(), "prometheus.yml"), - }, extra...)..., - ) + args := append([]string{ + "--storage.tsdb.retention=2d", // Pass retention cause prometheus since 2.8.0 don't show default value for that flags in web/api: https://github.com/prometheus/prometheus/pull/5433 + "--storage.tsdb.path=" + p.db.Dir(), + "--web.listen-address=" + p.addr, + "--web.route-prefix=" + p.prefix, + "--web.enable-admin-api", + "--config.file=" + filepath.Join(p.db.Dir(), "prometheus.yml"), + }, extra...) + + p.cmd = exec.Command(prometheusBin(p.version), args...) go func() { if b, err := p.cmd.CombinedOutput(); err != nil { fmt.Fprintln(os.Stderr, "running Prometheus failed", err) @@ -264,6 +264,7 @@ func (p *Prometheus) Appender() tsdb.Appender { // CreateBlock writes a block with the given series and numSamples samples each. // Samples will be in the time range [mint, maxt). func CreateBlock( + ctx context.Context, dir string, series []labels.Labels, numSamples int, @@ -271,11 +272,12 @@ func CreateBlock( extLset labels.Labels, resolution int64, ) (id ulid.ULID, err error) { - return createBlock(dir, series, numSamples, mint, maxt, extLset, resolution, false) + return createBlock(ctx, dir, series, numSamples, mint, maxt, extLset, resolution, false) } // CreateBlockWithTombstone is same as CreateBlock but leaves tombstones which mimics the Prometheus local block. func CreateBlockWithTombstone( + ctx context.Context, dir string, series []labels.Labels, numSamples int, @@ -283,10 +285,11 @@ func CreateBlockWithTombstone( extLset labels.Labels, resolution int64, ) (id ulid.ULID, err error) { - return createBlock(dir, series, numSamples, mint, maxt, extLset, resolution, true) + return createBlock(ctx, dir, series, numSamples, mint, maxt, extLset, resolution, true) } func createBlock( + ctx context.Context, dir string, series []labels.Labels, numSamples int, @@ -340,7 +343,7 @@ func createBlock( if err := g.Wait(); err != nil { return id, err } - c, err := tsdb.NewLeveledCompactor(nil, log.NewNopLogger(), []int64{maxt - mint}, nil) + c, err := tsdb.NewLeveledCompactor(ctx, nil, log.NewNopLogger(), []int64{maxt - mint}, nil) if err != nil { return id, errors.Wrap(err, "create compactor") } diff --git a/pkg/verifier/overlapped_blocks.go b/pkg/verifier/overlapped_blocks.go index 072fe54aec..af28fccb0c 100644 --- a/pkg/verifier/overlapped_blocks.go +++ b/pkg/verifier/overlapped_blocks.go @@ -2,6 +2,8 @@ package verifier import ( "context" + "sort" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/block" @@ -10,7 +12,6 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/tsdb" - "sort" ) const OverlappedBlocksIssueID = "overlapped_blocks" diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 8648dc5b3d..094db34f9a 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -68,10 +68,10 @@ func TestStoreGatewayQuery(t *testing.T) { extLset2 := labels.FromStrings("ext1", "value1", "replica", "2") now := time.Now() - id1, err := testutil.CreateBlock(dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0) + id1, err := testutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0) testutil.Ok(t, err) - id2, err := testutil.CreateBlock(dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset2, 0) + id2, err := testutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset2, 0) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout)