From 13c0591c585d822d59d4ffba23d0c175f021344f Mon Sep 17 00:00:00 2001 From: vvitkovskiy Date: Wed, 3 Apr 2019 12:09:34 +0300 Subject: [PATCH] Prometheus library updated to v2.8.1 --- Makefile | 2 +- cmd/thanos/compact.go | 5 ++-- go.mod | 5 ++-- go.sum | 29 ++++++++++++----------- pkg/block/index_test.go | 5 +++- pkg/compact/compact_e2e_test.go | 8 +++---- pkg/compact/downsample/downsample_test.go | 26 +++++++++++++++++++- pkg/promclient/promclient_e2e_test.go | 9 +++++-- pkg/store/bucket_e2e_test.go | 4 ++-- pkg/testutil/prometheus.go | 29 +++++++++++++---------- pkg/verifier/overlapped_blocks.go | 3 ++- test/e2e/store_gateway_test.go | 4 ++-- 12 files changed, 83 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 88b9ebb2ffe..e0804243c61 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ BZR ?= $(shell which bzr) # Referenced by github.com/improbable-eng/thanos/blob/master/docs/getting_started.md#prometheus # Limitied prom version, because testing was not possibe. This should fix it: https://github.com/improbable-eng/thanos/issues/758 -PROM_VERSIONS ?=v2.4.3 v2.5.0 +PROM_VERSIONS ?=v2.4.3 v2.5.0 v2.8.1 ALERTMANAGER_VERSION ?=v0.15.2 MINIO_SERVER_VERSION ?=RELEASE.2018-10-06T00-15-16Z diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index c95cf113d68..6de2203a0d7 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -182,9 +182,11 @@ func runCompact( level.Warn(logger).Log("msg", "Max compaction level is lower than should be", "current", maxCompactionLevel, "default", compactions.maxLevel()) } + ctx, cancel := context.WithCancel(context.Background()) + // Instantiate the compactor with different time slices. Timestamps in TSDB // are in milliseconds. - comp, err := tsdb.NewLeveledCompactor(reg, logger, levels, downsample.NewPool()) + comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, levels, downsample.NewPool()) if err != nil { return errors.Wrap(err, "create compactor") } @@ -210,7 +212,6 @@ func runCompact( level.Info(logger).Log("msg", "retention policy of 1 hour aggregated samples is enabled", "duration", retentionByResolution[compact.ResolutionLevel1h]) } - ctx, cancel := context.WithCancel(context.Background()) f := func() error { if err := compactor.Compact(ctx); err != nil { return errors.Wrap(err, "compaction failed") diff --git a/go.mod b/go.mod index 01b4034b4b4..cb84921caf6 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/hashicorp/memberlist v0.1.0 github.com/julienschmidt/httprouter v1.1.0 // indirect github.com/lovoo/gcloud-opentracing v0.3.0 - github.com/miekg/dns v1.0.8 // indirect github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 github.com/mozillazg/go-cos v0.11.0 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 @@ -34,8 +33,8 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.2 github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 - github.com/prometheus/prometheus v0.0.0-20190118110214-3bd41cc92c78 - github.com/prometheus/tsdb v0.4.0 + github.com/prometheus/prometheus v0.0.0-20190328180107-4d60eb36dcbe + github.com/prometheus/tsdb v0.6.1 go.opencensus.io v0.19.0 // indirect golang.org/x/net v0.0.0-20190213061140-3a22650c66bd golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 diff --git a/go.sum b/go.sum index 645506c5e04..d4af7a0725e 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,15 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +contrib.go.opencensus.io/exporter/ocagent v0.4.1/go.mod h1:b6YwD5Q3Yvj4yk0CDK5vGXexygNzI09aXUdDEakQBgA= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.0.0-20181218151757-9b75e4fe745a/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-pipeline-go v0.1.8 h1:KmVRa8oFMaargVesEuuEoiLCQ4zCCwQ8QX/xg++KS20= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-sdk-for-go v0.0.0-20161028183111-bd73d950fa44 h1:L4fLiifszjLnCRGi6Xhp0MgUwjIMbVXKbayoRiVxkU8= -github.com/Azure/azure-sdk-for-go v0.0.0-20161028183111-bd73d950fa44/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v23.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c h1:Y5ueznoCekgCWBytF1Q9lTpZ3tJeX37dQtCcGjMCLYI= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= -github.com/Azure/go-autorest v10.8.1+incompatible h1:u0jVQf+a6k6x8A+sT60l6EY9XZu+kHdnZVPAYqpVRo0= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/NYTimes/gziphandler v1.0.1 h1:iLrQrdwjDd52kHDA5op2UBJFjmOb9g+7scBan4RN8F0= github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -29,6 +28,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8/go.mod h1:Iev9Q3MErcn+w3UOJD/DkEzllvugfdx7bGcMOFhvr/4= github.com/cenk/backoff v2.0.0+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= +github.com/census-instrumentation/opencensus-proto v0.1.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -64,6 +64,8 @@ github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= @@ -105,7 +107,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:BWIsLfhgKhV5g/oF34aRjniBHLTZe5DNekSjbAjIS6c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20181025070259-68e3a13e4117 h1:v9uUYPE4RHQHA0C9XfpAX9uzWQvgIDYjPh6m/mQgrzs= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20181025070259-68e3a13e4117/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -170,9 +172,8 @@ github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8Bz github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI= -github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= +github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 h1:jw16EimP5oAEM/2wt+SiEUov/YDyTCTDuPtIKgQIvk0= github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJuf5HmJAJrKV64MmozrSsk+or0PB5dzdfspg= github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -240,10 +241,10 @@ github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/prometheus v0.0.0-20190118110214-3bd41cc92c78 h1:XGymJlAz2gJI0W/v9hZP7tBYTDdaK1Yq2lMNcXGb2Lo= -github.com/prometheus/prometheus v0.0.0-20190118110214-3bd41cc92c78/go.mod h1:aNuqjQW47j/yc/kN0cNZ8cOTBlBU8VbspEvNDfxe1To= -github.com/prometheus/tsdb v0.4.0 h1:pXJyEi/5p6UBmOrnzsZmYxLrZjxnRlEB78/qj3+a8Gk= -github.com/prometheus/tsdb v0.4.0/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/prometheus v0.0.0-20190328180107-4d60eb36dcbe h1:Wpwxfdtx0+/3dgqBvUAB2CTRtHRimQrH4RTEJ5yxCUo= +github.com/prometheus/prometheus v0.0.0-20190328180107-4d60eb36dcbe/go.mod h1:mVHnNN4pf1p9fM7dRvZQAKj0Ex93s+mIhtsml8SJy5E= +github.com/prometheus/tsdb v0.6.1 h1:wUp5iY/rmwv6VNzZg4yaUVpkSs4nX5O5+aoviWTNeUo= +github.com/prometheus/tsdb v0.6.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= @@ -269,6 +270,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.19.0 h1:+jrnNy8MR4GZXvwF9PEuSyHxA4NaTf6601oNRwCSXq0= go.opencensus.io v0.19.0/go.mod h1:AYeH0+ZxYyghG8diqaaIq/9P3VgCCt5GF2ldCY4dkFg= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -311,7 +313,6 @@ golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181023010539-40a48ad93fbe/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -google.golang.org/api v0.0.0-20180506000402-20530fd5d65a/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI= @@ -327,6 +328,7 @@ google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb h1:dQshZyyJ5W/Xk8myF4GKBak1pZW6EywJuQ8+44EQhGA= google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -349,7 +351,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/vmihailenco/msgpack.v2 v2.9.1 h1:kb0VV7NuIojvRfzwslQeP3yArBqJHW9tOl4t38VS1jM= gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= -gopkg.in/yaml.v2 v2.2.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go index 80c10e8e6ed..36a21f3ea30 100644 --- a/pkg/block/index_test.go +++ b/pkg/block/index_test.go @@ -1,6 +1,7 @@ package block import ( + "context" "io/ioutil" "os" "path/filepath" @@ -12,11 +13,13 @@ import ( ) func TestWriteReadIndexCache(t *testing.T) { + ctx := context.Background() + tmpDir, err := ioutil.TempDir("", "test-compact-prepare") testutil.Ok(t, err) defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() - b, err := testutil.CreateBlock(tmpDir, []labels.Labels{ + b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 738048c5c20..f58862c0ac0 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -178,7 +178,7 @@ func TestGroup_Compact_e2e(t *testing.T) { var metas []*metadata.Meta extLset := labels.Labels{{Name: "e1", Value: "1"}} - b1, err := testutil.CreateBlock(prepareDir, []labels.Labels{ + b1, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ {{Name: "a", Value: "1"}}, {{Name: "a", Value: "2"}, {Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, @@ -190,7 +190,7 @@ func TestGroup_Compact_e2e(t *testing.T) { testutil.Ok(t, err) metas = append(metas, meta) - b3, err := testutil.CreateBlock(prepareDir, []labels.Labels{ + b3, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, {{Name: "a", Value: "5"}}, @@ -215,7 +215,7 @@ func TestGroup_Compact_e2e(t *testing.T) { metas = append(metas, meta) // Due to TSDB compaction delay (not compacting fresh block), we need one more block to be pushed to trigger compaction. - freshB, err := testutil.CreateBlock(prepareDir, []labels.Labels{ + freshB, err := testutil.CreateBlock(ctx, prepareDir, []labels.Labels{ {{Name: "a", Value: "2"}}, {{Name: "a", Value: "3"}}, {{Name: "a", Value: "4"}}, @@ -251,7 +251,7 @@ func TestGroup_Compact_e2e(t *testing.T) { ) testutil.Ok(t, err) - comp, err := tsdb.NewLeveledCompactor(nil, log.NewLogfmtLogger(os.Stderr), []int64{1000, 3000}, nil) + comp, err := tsdb.NewLeveledCompactor(ctx, nil, log.NewLogfmtLogger(os.Stderr), []int64{1000, 3000}, nil) testutil.Ok(t, err) shouldRerun, id, err := g.Compact(ctx, dir, comp) diff --git a/pkg/compact/downsample/downsample_test.go b/pkg/compact/downsample/downsample_test.go index d54d31f0e99..731faa5b535 100644 --- a/pkg/compact/downsample/downsample_test.go +++ b/pkg/compact/downsample/downsample_test.go @@ -390,6 +390,8 @@ type memBlock struct { chunks []chunkenc.Chunk numberOfChunks uint64 + + minTime, maxTime int64 } type series struct { @@ -398,7 +400,7 @@ type series struct { } func newMemBlock() *memBlock { - return &memBlock{symbols: map[string]struct{}{}} + return &memBlock{symbols: map[string]struct{}{}, minTime: -1, maxTime: -1} } func (b *memBlock) addSeries(s *series) { @@ -412,12 +414,34 @@ func (b *memBlock) addSeries(s *series) { } for i, cm := range s.chunks { + if b.minTime == -1 || cm.MinTime < b.minTime { + b.minTime = cm.MinTime + } + if b.maxTime == -1 || cm.MaxTime < b.maxTime { + b.maxTime = cm.MaxTime + } s.chunks[i].Ref = b.numberOfChunks b.chunks = append(b.chunks, cm.Chunk) b.numberOfChunks++ } } +func (b *memBlock) MinTime() int64 { + if b.minTime == -1 { + return 0 + } + + return b.minTime +} + +func (b *memBlock) MaxTime() int64 { + if b.maxTime == -1 { + return 0 + } + + return b.maxTime +} + func (b *memBlock) Postings(name, val string) (index.Postings, error) { allName, allVal := index.AllPostingsKey() diff --git a/pkg/promclient/promclient_e2e_test.go b/pkg/promclient/promclient_e2e_test.go index 1cb7c968afe..b50e072c1d1 100644 --- a/pkg/promclient/promclient_e2e_test.go +++ b/pkg/promclient/promclient_e2e_test.go @@ -81,8 +81,10 @@ func TestSnapshot_e2e(t *testing.T) { testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) { now := time.Now() + ctx := context.Background() // Create artificial block. id, err := testutil.CreateBlockWithTombstone( + ctx, p.Dir(), []labels.Labels{labels.FromStrings("a", "b")}, 10, @@ -99,7 +101,10 @@ func TestSnapshot_e2e(t *testing.T) { u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr())) testutil.Ok(t, err) - dir, err := Snapshot(context.Background(), log.NewNopLogger(), u, false) + // Prometheus since 2.7.0 don't write empty blocks even if it's head block. So it's no matter passing skip_head true or false here + // Pass skipHead = true to support all prometheus versions and assert that snapshot creates only one file + // https://github.com/prometheus/tsdb/pull/374 + dir, err := Snapshot(ctx, log.NewNopLogger(), u, true) testutil.Ok(t, err) _, err = os.Stat(path.Join(p.Dir(), dir, id.String())) @@ -113,7 +118,7 @@ func TestSnapshot_e2e(t *testing.T) { testutil.Ok(t, err) } - testutil.Equals(t, 2, len(files)) + testutil.Equals(t, 1, len(files)) }) } diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 997767d0556..bc8e55e359e 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -66,9 +66,9 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m // Create two blocks per time slot. Only add 10 samples each so only one chunk // gets created each. This way we can easily verify we got 10 chunks per series below. - id1, err := testutil.CreateBlock(dir, series[:4], 10, mint, maxt, extLset, 0) + id1, err := testutil.CreateBlock(ctx, dir, series[:4], 10, mint, maxt, extLset, 0) testutil.Ok(t, err) - id2, err := testutil.CreateBlock(dir, series[4:], 10, mint, maxt, extLset, 0) + id2, err := testutil.CreateBlock(ctx, dir, series[4:], 10, mint, maxt, extLset, 0) testutil.Ok(t, err) dir1, dir2 := filepath.Join(dir, id1.String()), filepath.Join(dir, id2.String()) diff --git a/pkg/testutil/prometheus.go b/pkg/testutil/prometheus.go index 38430684692..f23cb2803bd 100644 --- a/pkg/testutil/prometheus.go +++ b/pkg/testutil/prometheus.go @@ -163,16 +163,16 @@ func (p *Prometheus) Start() error { ) } p.addr = fmt.Sprintf("localhost:%d", port) - p.cmd = exec.Command( - prometheusBin(p.version), - append([]string{ - "--storage.tsdb.path=" + p.db.Dir(), - "--web.listen-address=" + p.addr, - "--web.route-prefix=" + p.prefix, - "--web.enable-admin-api", - "--config.file=" + filepath.Join(p.db.Dir(), "prometheus.yml"), - }, extra...)..., - ) + args := append([]string{ + "--storage.tsdb.retention=15d", // Pass retention cause prometheus since 2.8.0 don't show default value for that flags in web/api: https://github.com/prometheus/prometheus/pull/5433 + "--storage.tsdb.path=" + p.db.Dir(), + "--web.listen-address=" + p.addr, + "--web.route-prefix=" + p.prefix, + "--web.enable-admin-api", + "--config.file=" + filepath.Join(p.db.Dir(), "prometheus.yml"), + }, extra...) + + p.cmd = exec.Command(prometheusBin(p.version), args...) go func() { if b, err := p.cmd.CombinedOutput(); err != nil { fmt.Fprintln(os.Stderr, "running Prometheus failed", err) @@ -264,6 +264,7 @@ func (p *Prometheus) Appender() tsdb.Appender { // CreateBlock writes a block with the given series and numSamples samples each. // Samples will be in the time range [mint, maxt). func CreateBlock( + ctx context.Context, dir string, series []labels.Labels, numSamples int, @@ -271,11 +272,12 @@ func CreateBlock( extLset labels.Labels, resolution int64, ) (id ulid.ULID, err error) { - return createBlock(dir, series, numSamples, mint, maxt, extLset, resolution, false) + return createBlock(ctx, dir, series, numSamples, mint, maxt, extLset, resolution, false) } // CreateBlockWithTombstone is same as CreateBlock but leaves tombstones which mimics the Prometheus local block. func CreateBlockWithTombstone( + ctx context.Context, dir string, series []labels.Labels, numSamples int, @@ -283,10 +285,11 @@ func CreateBlockWithTombstone( extLset labels.Labels, resolution int64, ) (id ulid.ULID, err error) { - return createBlock(dir, series, numSamples, mint, maxt, extLset, resolution, true) + return createBlock(ctx, dir, series, numSamples, mint, maxt, extLset, resolution, true) } func createBlock( + ctx context.Context, dir string, series []labels.Labels, numSamples int, @@ -340,7 +343,7 @@ func createBlock( if err := g.Wait(); err != nil { return id, err } - c, err := tsdb.NewLeveledCompactor(nil, log.NewNopLogger(), []int64{maxt - mint}, nil) + c, err := tsdb.NewLeveledCompactor(ctx, nil, log.NewNopLogger(), []int64{maxt - mint}, nil) if err != nil { return id, errors.Wrap(err, "create compactor") } diff --git a/pkg/verifier/overlapped_blocks.go b/pkg/verifier/overlapped_blocks.go index 072fe54aecb..af28fccb0ca 100644 --- a/pkg/verifier/overlapped_blocks.go +++ b/pkg/verifier/overlapped_blocks.go @@ -2,6 +2,8 @@ package verifier import ( "context" + "sort" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/improbable-eng/thanos/pkg/block" @@ -10,7 +12,6 @@ import ( "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/tsdb" - "sort" ) const OverlappedBlocksIssueID = "overlapped_blocks" diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 8648dc5b3d3..094db34f9a8 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -68,10 +68,10 @@ func TestStoreGatewayQuery(t *testing.T) { extLset2 := labels.FromStrings("ext1", "value1", "replica", "2") now := time.Now() - id1, err := testutil.CreateBlock(dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0) + id1, err := testutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0) testutil.Ok(t, err) - id2, err := testutil.CreateBlock(dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset2, 0) + id2, err := testutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset2, 0) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout)