Potential fix for e2e flake from 6114 (thanos-io#6128)

This commit is a potential fix for the flake e2e test intruduced
in thanos-io#6114. The commit uses strict endpoints in queriers to make sure
all stores are ready in order for the query to be considered successful.

I verified the fix by running the test locally using
go test ./test/e2e/... -run TestDistributedQueryExecution -count 20.

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>

test/e2e/distributed_query_test.go

@@ -28,13 +28,13 @@ func TestDistributedQueryExecution(t *testing.T) {
 	prom2, sidecar2 := e2ethanos.NewPrometheusWithSidecar(e, "prom2", e2ethanos.DefaultPromConfig("prom2", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "", "remote-write-receiver")
 	testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, sidecar1, sidecar2))
 
-	qry1 := e2ethanos.NewQuerierBuilder(e, "1", sidecar1.InternalEndpoint("grpc")).Init()
-	qry2 := e2ethanos.NewQuerierBuilder(e, "2", sidecar2.InternalEndpoint("grpc")).Init()
+	qry1 := e2ethanos.NewQuerierBuilder(e, "1").WithStrictEndpoints(sidecar1.InternalEndpoint("grpc")).Init()
+	qry2 := e2ethanos.NewQuerierBuilder(e, "2").WithStrictEndpoints(sidecar2.InternalEndpoint("grpc")).Init()
 	testutil.Ok(t, e2e.StartAndWaitReady(qry1, qry2))
 
 	qryEndpoints := []string{qry1.InternalEndpoint("grpc"), qry2.InternalEndpoint("grpc")}
-	fanoutQry := e2ethanos.NewQuerierBuilder(e, "3", qryEndpoints...).Init()
-	distQry := e2ethanos.NewQuerierBuilder(e, "4", qryEndpoints...).
+	fanoutQry := e2ethanos.NewQuerierBuilder(e, "3").WithStrictEndpoints(qryEndpoints...).Init()
+	distQry := e2ethanos.NewQuerierBuilder(e, "4").WithStrictEndpoints(qryEndpoints...).
 		WithEngine("thanos").
 		WithQueryMode("distributed").
 		Init()

test/e2e/e2ethanos/services.go

@@ -252,6 +252,7 @@ type QuerierBuilder struct {
 	exemplarAddresses       []string
 	enableFeatures          []string
 	endpoints               []string
+	strictEndpoints         []string
 
 	engine    string
 	queryMode string
@@ -327,6 +328,11 @@ func (q *QuerierBuilder) WithEndpoints(endpoints ...string) *QuerierBuilder {
 	return q
 }
 
+func (q *QuerierBuilder) WithStrictEndpoints(strictEndpoints ...string) *QuerierBuilder {
+	q.strictEndpoints = strictEndpoints
+	return q
+}
+
 func (q *QuerierBuilder) WithRoutePrefix(routePrefix string) *QuerierBuilder {
 	q.routePrefix = routePrefix
 	return q
@@ -420,6 +426,9 @@ func (q *QuerierBuilder) collectArgs() ([]string, error) {
 	for _, addr := range q.endpoints {
 		args = append(args, "--endpoint="+addr)
 	}
+	for _, addr := range q.strictEndpoints {
+		args = append(args, "--endpoint-strict="+addr)
+	}
 	if len(q.fileSDStoreAddresses) > 0 {
 		if err := os.MkdirAll(q.Dir(), 0750); err != nil {
 			return nil, errors.Wrap(err, "create query dir failed")