From 7975d40c0fe1b533d7d88da122aa6f9e87e47d73 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Fri, 27 Aug 2021 15:25:09 +0200 Subject: [PATCH 1/5] Adjust e2ethanos pkg - Adjusts services to use e2e - Simplifies some func signatures, remove redundant funcs etc. - Adds service for reverse proxy (instead of using one on host) - Adds service for Minio (a workaround, see issue in the func comment) Signed-off-by: Matej Gera --- test/e2e/e2ethanos/helpers.go | 4 +- test/e2e/e2ethanos/service.go | 49 +++-- test/e2e/e2ethanos/services.go | 356 ++++++++++++++++++++------------- 3 files changed, 250 insertions(+), 159 deletions(-) diff --git a/test/e2e/e2ethanos/helpers.go b/test/e2e/e2ethanos/helpers.go index 0bcd21c912..d5c8519e9e 100644 --- a/test/e2e/e2ethanos/helpers.go +++ b/test/e2e/e2ethanos/helpers.go @@ -11,12 +11,12 @@ import ( "strings" "testing" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/thanos-io/thanos/pkg/testutil" ) -func CleanScenario(t testing.TB, s *e2e.Scenario) func() { +func CleanScenario(t testing.TB, s *e2e.DockerEnvironment) func() { return func() { // Make sure Clean can properly delete everything. testutil.Ok(t, exec.Command("chmod", "-R", "777", s.SharedDir()).Run()) diff --git a/test/e2e/e2ethanos/service.go b/test/e2e/e2ethanos/service.go index 57f5dea453..35d8d4300f 100644 --- a/test/e2e/e2ethanos/service.go +++ b/test/e2e/e2ethanos/service.go @@ -4,35 +4,48 @@ package e2ethanos import ( - "github.com/cortexproject/cortex/integration/e2e" -) + "os" + "strconv" -type Service struct { - *e2e.HTTPService + "github.com/efficientgo/e2e" +) - grpc int +type Port struct { + name string + portNum int + isMetrics bool } func NewService( + e e2e.Environment, name string, image string, - command *e2e.Command, + command e2e.Command, readiness *e2e.HTTPReadinessProbe, http, grpc int, - otherPorts ...int, -) *Service { - return &Service{ - HTTPService: e2e.NewHTTPService(name, image, command, readiness, http, append(otherPorts, grpc)...), - grpc: grpc, + otherPorts ...Port, +) *e2e.InstrumentedRunnable { + metricsPorts := "http" + ports := map[string]int{ + "http": http, + "grpc": grpc, } -} -func (s *Service) GRPCEndpoint() string { return s.Endpoint(s.grpc) } + for _, op := range otherPorts { + ports[op.name] = op.portNum -func (s *Service) GRPCNetworkEndpoint() string { - return s.NetworkEndpoint(s.grpc) -} + if op.isMetrics { + metricsPorts = op.name + } + } -func (s *Service) GRPCNetworkEndpointFor(networkName string) string { - return s.NetworkEndpointFor(networkName, s.grpc) + return e2e.NewInstrumentedRunnable(e, name, ports, metricsPorts).Init( + e2e.StartOptions{ + Image: image, + Command: command, + Readiness: readiness, + User: strconv.Itoa(os.Getuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, + ) } diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 3d83325774..17f79aae92 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -13,8 +13,9 @@ import ( "strings" "time" - "github.com/cortexproject/cortex/integration/e2e" - "github.com/cortexproject/cortex/pkg/util" + "github.com/efficientgo/e2e" + e2edb "github.com/efficientgo/e2e/db" + "github.com/efficientgo/tools/core/pkg/backoff" "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -28,12 +29,15 @@ import ( "github.com/thanos-io/thanos/pkg/receive" ) -const infoLogLevel = "info" +const ( + infoLogLevel = "info" + ContainerSharedDir = "/shared" +) // Same as default for now. -var defaultBackoffConfig = util.BackoffConfig{ - MinBackoff: 300 * time.Millisecond, - MaxBackoff: 600 * time.Millisecond, +var defaultBackoffConfig = backoff.Config{ + Min: 300 * time.Millisecond, + Max: 600 * time.Millisecond, MaxRetries: 50, } @@ -60,9 +64,9 @@ func DefaultImage() string { return "thanos" } -func NewPrometheus(sharedDir, name, config, promImage string, enableFeatures ...string) (*e2e.HTTPService, string, error) { - dir := filepath.Join(sharedDir, "data", "prometheus", name) - container := filepath.Join(e2e.ContainerSharedDir, "data", "prometheus", name) +func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, string, error) { + dir := filepath.Join(e.SharedDir(), "data", "prometheus", name) + container := filepath.Join(ContainerSharedDir, "data", "prometheus", name) if err := os.MkdirAll(dir, 0750); err != nil { return nil, "", errors.Wrap(err, "create prometheus dir") } @@ -82,27 +86,31 @@ func NewPrometheus(sharedDir, name, config, promImage string, enableFeatures ... if len(enableFeatures) > 0 { args = append(args, fmt.Sprintf("--enable-feature=%s", strings.Join(enableFeatures, ","))) } - prom := e2e.NewHTTPService( + prom := e2e.NewInstrumentedRunnable( + e, fmt.Sprintf("prometheus-%s", name), - promImage, - e2e.NewCommandWithoutEntrypoint("prometheus", args...), - e2e.NewHTTPReadinessProbe(9090, "/-/ready", 200, 200), - 9090, + map[string]int{"http": 9090}, + "http").Init( + e2e.StartOptions{ + Image: promImage, + Command: e2e.NewCommandWithoutEntrypoint("prometheus", args...), + Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + User: strconv.Itoa(os.Getuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, ) - prom.SetUser(strconv.Itoa(os.Getuid())) - prom.SetBackoff(defaultBackoffConfig) return prom, container, nil } -func NewPrometheusWithSidecar(sharedDir, netName, name, config, promImage string, enableFeatures ...string) (*e2e.HTTPService, *Service, error) { - prom, dataDir, err := NewPrometheus(sharedDir, name, config, promImage, enableFeatures...) +func NewPrometheusWithSidecar(e e2e.Environment, netName, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { + prom, dataDir, err := NewPrometheus(e, name, config, promImage, enableFeatures...) if err != nil { return nil, nil, err } - prom.SetBackoff(defaultBackoffConfig) sidecar := NewService( + e, fmt.Sprintf("sidecar-%s", name), DefaultImage(), e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ @@ -110,21 +118,20 @@ func NewPrometheusWithSidecar(sharedDir, netName, name, config, promImage string "--grpc-address": ":9091", "--grpc-grace-period": "0s", "--http-address": ":8080", - "--prometheus.url": "http://" + prom.NetworkEndpointFor(netName, 9090), + "--prometheus.url": "http://" + prom.InternalEndpoint("http"), "--tsdb.path": dataDir, "--log.level": infoLogLevel, })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, ) - sidecar.SetUser(strconv.Itoa(os.Getuid())) - sidecar.SetBackoff(defaultBackoffConfig) return prom, sidecar, nil } type QuerierBuilder struct { + environment e2e.Environment sharedDir string name string routePrefix string @@ -140,9 +147,10 @@ type QuerierBuilder struct { tracingConfig string } -func NewQuerierBuilder(sharedDir, name string, storeAddresses []string) *QuerierBuilder { +func NewQuerierBuilder(e e2e.Environment, name string, storeAddresses []string) *QuerierBuilder { return &QuerierBuilder{ - sharedDir: sharedDir, + environment: e, + sharedDir: e.SharedDir(), name: name, storeAddresses: storeAddresses, } @@ -188,7 +196,7 @@ func (q *QuerierBuilder) WithTracingConfig(tracingConfig string) *QuerierBuilder return q } -func (q *QuerierBuilder) Build() (*Service, error) { +func (q *QuerierBuilder) Build() (*e2e.InstrumentedRunnable, error) { const replicaLabel = "replica" args := e2e.BuildArgs(map[string]string{ @@ -224,7 +232,7 @@ func (q *QuerierBuilder) Build() (*Service, error) { if len(q.fileSDStoreAddresses) > 0 { queryFileSDDir := filepath.Join(q.sharedDir, "data", "querier", q.name) - container := filepath.Join(e2e.ContainerSharedDir, "data", "querier", q.name) + container := filepath.Join(ContainerSharedDir, "data", "querier", q.name) if err := os.MkdirAll(queryFileSDDir, 0750); err != nil { return nil, errors.Wrap(err, "create query dir failed") } @@ -259,15 +267,14 @@ func (q *QuerierBuilder) Build() (*Service, error) { } querier := NewService( + q.environment, fmt.Sprintf("querier-%v", q.name), DefaultImage(), e2e.NewCommand("query", args...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, ) - querier.SetUser(strconv.Itoa(os.Getuid())) - querier.SetBackoff(defaultBackoffConfig) return querier, nil } @@ -275,16 +282,18 @@ func (q *QuerierBuilder) Build() (*Service, error) { func RemoteWriteEndpoint(addr string) string { return fmt.Sprintf("http://%s/api/v1/receive", addr) } // NewRoutingAndIngestingReceiver creates a Thanos Receive instances that is configured both for ingesting samples and routing samples to other receivers. -func NewRoutingAndIngestingReceiver(sharedDir, networkName, name string, replicationFactor int, hashring ...receive.HashringConfig) (*Service, error) { - - localEndpoint := NewService(fmt.Sprintf("receive-%v", name), "", e2e.NewCommand("", ""), nil, 8080, 9091, 8081).GRPCNetworkEndpointFor(networkName) +func NewRoutingAndIngestingReceiver(e e2e.Environment, networkName, name string, replicationFactor int, hashring ...receive.HashringConfig) (*e2e.InstrumentedRunnable, error) { + var localEndpoint string if len(hashring) == 0 { + localEndpoint = "0.0.0.0:9091" hashring = []receive.HashringConfig{{Endpoints: []string{localEndpoint}}} + } else { + localEndpoint = networkName + fmt.Sprintf("-receive-%v", name) + ":9091" } - dir := filepath.Join(sharedDir, "data", "receive", name) + dir := filepath.Join(e.SharedDir(), "data", "receive", name) dataDir := filepath.Join(dir, "data") - container := filepath.Join(e2e.ContainerSharedDir, "data", "receive", name) + container := filepath.Join(ContainerSharedDir, "data", "receive", name) if err := os.MkdirAll(dataDir, 0750); err != nil { return nil, errors.Wrap(err, "create receive dir") } @@ -293,7 +302,7 @@ func NewRoutingAndIngestingReceiver(sharedDir, networkName, name string, replica return nil, errors.Wrapf(err, "generate hashring file: %v", hashring) } - receiver := NewService( + receiver := NewService(e, fmt.Sprintf("receive-%v", name), DefaultImage(), // TODO(bwplotka): BuildArgs should be interface. @@ -310,27 +319,25 @@ func NewRoutingAndIngestingReceiver(sharedDir, networkName, name string, replica "--receive.local-endpoint": localEndpoint, "--receive.hashrings": string(b), })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, - 8081, + Port{name: "remote-write", portNum: 8081}, ) - receiver.SetUser(strconv.Itoa(os.Getuid())) - receiver.SetBackoff(defaultBackoffConfig) return receiver, nil } // NewRoutingReceiver creates a Thanos Receive instance that is only configured to route to other receive instances. It has no local storage. -func NewRoutingReceiver(sharedDir, name string, replicationFactor int, hashring ...receive.HashringConfig) (*Service, error) { +func NewRoutingReceiver(e e2e.Environment, name string, replicationFactor int, hashring ...receive.HashringConfig) (*e2e.InstrumentedRunnable, error) { if len(hashring) == 0 { return nil, errors.New("hashring should not be empty for receive-distributor mode") } - dir := filepath.Join(sharedDir, "data", "receive", name) + dir := filepath.Join(e.SharedDir(), "data", "receive", name) dataDir := filepath.Join(dir, "data") - container := filepath.Join(e2e.ContainerSharedDir, "data", "receive", name) + container := filepath.Join(ContainerSharedDir, "data", "receive", name) if err := os.MkdirAll(dataDir, 0750); err != nil { return nil, errors.Wrap(err, "create receive dir") } @@ -340,6 +347,7 @@ func NewRoutingReceiver(sharedDir, name string, replicationFactor int, hashring } receiver := NewService( + e, fmt.Sprintf("receive-%v", name), DefaultImage(), // TODO(bwplotka): BuildArgs should be interface. @@ -355,26 +363,24 @@ func NewRoutingReceiver(sharedDir, name string, replicationFactor int, hashring "--receive.replication-factor": strconv.Itoa(replicationFactor), "--receive.hashrings": string(b), })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, - 8081, + Port{name: "remote-write", portNum: 8081}, ) - receiver.SetUser(strconv.Itoa(os.Getuid())) - receiver.SetBackoff(defaultBackoffConfig) return receiver, nil } // NewIngestingReceiver creates a Thanos Receive instance that is only configured to ingest, not route to other receivers. -func NewIngestingReceiver(sharedDir, name string) (*Service, error) { - dir := filepath.Join(sharedDir, "data", "receive", name) +func NewIngestingReceiver(e e2e.Environment, name string) (*e2e.InstrumentedRunnable, error) { + dir := filepath.Join(e.SharedDir(), "data", "receive", name) dataDir := filepath.Join(dir, "data") - container := filepath.Join(e2e.ContainerSharedDir, "data", "receive", name) + container := filepath.Join(ContainerSharedDir, "data", "receive", name) if err := os.MkdirAll(dataDir, 0750); err != nil { return nil, errors.Wrap(err, "create receive dir") } - receiver := NewService( + receiver := NewService(e, fmt.Sprintf("receive-%v", name), DefaultImage(), // TODO(bwplotka): BuildArgs should be interface. @@ -388,26 +394,27 @@ func NewIngestingReceiver(sharedDir, name string) (*Service, error) { "--tsdb.path": filepath.Join(container, "data"), "--log.level": infoLogLevel, })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, - 8081, + Port{name: "remote-write", portNum: 8081}, ) - receiver.SetUser(strconv.Itoa(os.Getuid())) - receiver.SetBackoff(defaultBackoffConfig) return receiver, nil } -func NewRoutingAndIngestingReceiverWithConfigWatcher(sharedDir, networkName, name string, replicationFactor int, hashring ...receive.HashringConfig) (*Service, error) { - localEndpoint := NewService(fmt.Sprintf("receive-%v", name), "", e2e.NewCommand("", ""), nil, 8080, 9091, 8081).GRPCNetworkEndpointFor(networkName) +func NewRoutingAndIngestingReceiverWithConfigWatcher(e e2e.Environment, networkName, name string, replicationFactor int, hashring ...receive.HashringConfig) (*e2e.InstrumentedRunnable, error) { + var localEndpoint string if len(hashring) == 0 { + localEndpoint = "0.0.0.0:9091" hashring = []receive.HashringConfig{{Endpoints: []string{localEndpoint}}} + } else { + localEndpoint = networkName + fmt.Sprintf("-receive-%v", name) + ":9091" } - dir := filepath.Join(sharedDir, "data", "receive", name) + dir := filepath.Join(e.SharedDir(), "data", "receive", name) dataDir := filepath.Join(dir, "data") - container := filepath.Join(e2e.ContainerSharedDir, "data", "receive", name) + container := filepath.Join(ContainerSharedDir, "data", "receive", name) if err := os.MkdirAll(dataDir, 0750); err != nil { return nil, errors.Wrap(err, "create receive dir") } @@ -420,7 +427,7 @@ func NewRoutingAndIngestingReceiverWithConfigWatcher(sharedDir, networkName, nam return nil, errors.Wrap(err, "creating receive config") } - receiver := NewService( + receiver := NewService(e, fmt.Sprintf("receive-%v", name), DefaultImage(), // TODO(bwplotka): BuildArgs should be interface. @@ -438,20 +445,18 @@ func NewRoutingAndIngestingReceiverWithConfigWatcher(sharedDir, networkName, nam "--receive.hashrings-file": filepath.Join(container, "hashrings.json"), "--receive.hashrings-file-refresh-interval": "5s", })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, - 8081, + Port{name: "remote-write", portNum: 8081}, ) - receiver.SetUser(strconv.Itoa(os.Getuid())) - receiver.SetBackoff(defaultBackoffConfig) return receiver, nil } -func NewRuler(sharedDir, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []query.Config) (*Service, error) { - dir := filepath.Join(sharedDir, "data", "rule", name) - container := filepath.Join(e2e.ContainerSharedDir, "data", "rule", name) +func NewRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []query.Config) (*e2e.InstrumentedRunnable, error) { + dir := filepath.Join(e.SharedDir(), "data", "rule", name) + container := filepath.Join(ContainerSharedDir, "data", "rule", name) if err := os.MkdirAll(dir, 0750); err != nil { return nil, errors.Wrap(err, "create rule dir") } @@ -468,7 +473,7 @@ func NewRuler(sharedDir, name, ruleSubDir string, amCfg []alert.AlertmanagerConf return nil, errors.Wrapf(err, "generate query file: %v", queryCfg) } - ruler := NewService( + ruler := NewService(e, fmt.Sprintf("rule-%v", name), DefaultImage(), e2e.NewCommand("rule", e2e.BuildArgs(map[string]string{ @@ -478,7 +483,7 @@ func NewRuler(sharedDir, name, ruleSubDir string, amCfg []alert.AlertmanagerConf "--http-address": ":8080", "--label": fmt.Sprintf(`replica="%s"`, name), "--data-dir": container, - "--rule-file": filepath.Join(e2e.ContainerSharedDir, ruleSubDir, "*.yaml"), + "--rule-file": filepath.Join(ContainerSharedDir, ruleSubDir, "*.yaml"), "--eval-interval": "3s", "--alertmanagers.config": string(amCfgBytes), "--alertmanagers.sd-dns-interval": "1s", @@ -487,19 +492,17 @@ func NewRuler(sharedDir, name, ruleSubDir string, amCfg []alert.AlertmanagerConf "--query.sd-dns-interval": "1s", "--resend-delay": "5s", })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, ) - ruler.SetUser(strconv.Itoa(os.Getuid())) - ruler.SetBackoff(defaultBackoffConfig) return ruler, nil } -func NewAlertmanager(sharedDir, name string) (*e2e.HTTPService, error) { - dir := filepath.Join(sharedDir, "data", "am", name) - container := filepath.Join(e2e.ContainerSharedDir, "data", "am", name) +func NewAlertmanager(e e2e.Environment, name string) (*e2e.InstrumentedRunnable, error) { + dir := filepath.Join(e.SharedDir(), "data", "am", name) + container := filepath.Join(ContainerSharedDir, "data", "am", name) if err := os.MkdirAll(dir, 0750); err != nil { return nil, errors.Wrap(err, "create am dir") } @@ -516,29 +519,30 @@ receivers: return nil, errors.Wrap(err, "creating alertmanager config file failed") } - s := e2e.NewHTTPService( - fmt.Sprintf("alertmanager-%v", name), - DefaultAlertmanagerImage(), - e2e.NewCommandWithoutEntrypoint("/bin/alertmanager", e2e.BuildArgs(map[string]string{ - "--config.file": filepath.Join(container, "config.yaml"), - "--web.listen-address": "0.0.0.0:8080", - "--log.level": infoLogLevel, - "--storage.path": container, - "--web.get-concurrency": "1", - "--web.timeout": "2m", - })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), - 8080, + s := e2e.NewInstrumentedRunnable( + e, fmt.Sprintf("alertmanager-%v", name), map[string]int{"http": 8080}, "http").Init( + e2e.StartOptions{ + Image: DefaultAlertmanagerImage(), + Command: e2e.NewCommandWithoutEntrypoint("/bin/alertmanager", e2e.BuildArgs(map[string]string{ + "--config.file": filepath.Join(container, "config.yaml"), + "--web.listen-address": "0.0.0.0:8080", + "--log.level": infoLogLevel, + "--storage.path": container, + "--web.get-concurrency": "1", + "--web.timeout": "2m", + })...), + Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + User: strconv.Itoa(os.Geteuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, ) - s.SetUser(strconv.Itoa(os.Getuid())) - s.SetBackoff(defaultBackoffConfig) return s, nil } -func NewStoreGW(sharedDir, name string, bucketConfig client.BucketConfig, relabelConfig ...relabel.Config) (*Service, error) { - dir := filepath.Join(sharedDir, "data", "store", name) - container := filepath.Join(e2e.ContainerSharedDir, "data", "store", name) +func NewStoreGW(e e2e.Environment, name string, bucketConfig client.BucketConfig, relabelConfig ...relabel.Config) (*e2e.InstrumentedRunnable, error) { + dir := filepath.Join(e.SharedDir(), "data", "store", name) + container := filepath.Join(ContainerSharedDir, "data", "store", name) if err := os.MkdirAll(dir, 0750); err != nil { return nil, errors.Wrap(err, "create store dir") } @@ -554,6 +558,7 @@ func NewStoreGW(sharedDir, name string, bucketConfig client.BucketConfig, relabe } store := NewService( + e, fmt.Sprintf("store-gw-%v", name), DefaultImage(), e2e.NewCommand("store", e2e.BuildArgs(map[string]string{ @@ -571,19 +576,17 @@ func NewStoreGW(sharedDir, name string, bucketConfig client.BucketConfig, relabe "--selector.relabel-config": string(relabelConfigBytes), "--consistency-delay": "30m", })...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, ) - store.SetUser(strconv.Itoa(os.Getuid())) - store.SetBackoff(defaultBackoffConfig) return store, nil } -func NewCompactor(sharedDir, name string, bucketConfig client.BucketConfig, relabelConfig []relabel.Config, extArgs ...string) (*e2e.HTTPService, error) { - dir := filepath.Join(sharedDir, "data", "compact", name) - container := filepath.Join(e2e.ContainerSharedDir, "data", "compact", name) +func NewCompactor(e e2e.Environment, name string, bucketConfig client.BucketConfig, relabelConfig []relabel.Config, extArgs ...string) (*e2e.InstrumentedRunnable, error) { + dir := filepath.Join(e.SharedDir(), "data", "compact", name) + container := filepath.Join(ContainerSharedDir, "data", "compact", name) if err := os.MkdirAll(dir, 0750); err != nil { return nil, errors.Wrap(err, "create compact dir") @@ -599,29 +602,30 @@ func NewCompactor(sharedDir, name string, bucketConfig client.BucketConfig, rela return nil, errors.Wrapf(err, "generate compact relabel file: %v", relabelConfig) } - compactor := e2e.NewHTTPService( - fmt.Sprintf("compact-%s", name), - DefaultImage(), - e2e.NewCommand("compact", append(e2e.BuildArgs(map[string]string{ - "--debug.name": fmt.Sprintf("compact-%s", name), - "--log.level": infoLogLevel, - "--data-dir": container, - "--objstore.config": string(bktConfigBytes), - "--http-address": ":8080", - "--block-sync-concurrency": "20", - "--selector.relabel-config": string(relabelConfigBytes), - "--wait": "", - }), extArgs...)...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), - 8080, + compactor := e2e.NewInstrumentedRunnable( + e, fmt.Sprintf("compact-%s", name), map[string]int{"http": 8080}, "http").Init( + e2e.StartOptions{ + Image: DefaultImage(), + Command: e2e.NewCommand("compact", append(e2e.BuildArgs(map[string]string{ + "--debug.name": fmt.Sprintf("compact-%s", name), + "--log.level": infoLogLevel, + "--data-dir": container, + "--objstore.config": string(bktConfigBytes), + "--http-address": ":8080", + "--block-sync-concurrency": "20", + "--selector.relabel-config": string(relabelConfigBytes), + "--wait": "", + }), extArgs...)...), + Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + User: strconv.Itoa(os.Getuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, ) - compactor.SetUser(strconv.Itoa(os.Getuid())) - compactor.SetBackoff(defaultBackoffConfig) return compactor, nil } -func NewQueryFrontend(name, downstreamURL string, cacheConfig queryfrontend.CacheProviderConfig) (*e2e.HTTPService, error) { +func NewQueryFrontend(e e2e.Environment, name, downstreamURL string, cacheConfig queryfrontend.CacheProviderConfig) (*e2e.InstrumentedRunnable, error) { cacheConfigBytes, err := yaml.Marshal(cacheConfig) if err != nil { return nil, errors.Wrapf(err, "marshal response cache config file: %v", cacheConfig) @@ -635,41 +639,117 @@ func NewQueryFrontend(name, downstreamURL string, cacheConfig queryfrontend.Cach "--query-range.response-cache-config": string(cacheConfigBytes), }) - queryFrontend := e2e.NewHTTPService( - fmt.Sprintf("query-frontend-%s", name), - DefaultImage(), - e2e.NewCommand("query-frontend", args...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), - 8080, + queryFrontend := e2e.NewInstrumentedRunnable( + e, fmt.Sprintf("query-frontend-%s", name), map[string]int{"http": 8080}, "http").Init( + e2e.StartOptions{ + Image: DefaultImage(), + Command: e2e.NewCommand("query-frontend", args...), + Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + User: strconv.Itoa(os.Getuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, ) - queryFrontend.SetUser(strconv.Itoa(os.Getuid())) - queryFrontend.SetBackoff(defaultBackoffConfig) return queryFrontend, nil } -func NewMemcached(name string) *e2e.ConcreteService { - memcached := e2e.NewConcreteService( - fmt.Sprintf("memcached-%s", name), - "docker.io/memcached:1.6.3-alpine", - e2e.NewCommand("memcached", []string{"-m 1024", "-I 1m", "-c 1024", "-v"}...), - nil, - 11211, +func NewReverseProxy(e e2e.Environment, name, tenantID, target string) (*e2e.InstrumentedRunnable, error) { + conf := fmt.Sprintf(` +events { + worker_connections 1024; +} + +http { + server { + listen 80; + server_name _; + + location / { + proxy_set_header THANOS-TENANT %s; + proxy_pass %s; + } + } +} +`, tenantID, target) + + dir := filepath.Join(e.SharedDir(), "data", "nginx", name) + if err := os.MkdirAll(dir, 0750); err != nil { + return nil, errors.Wrap(err, "create store dir") + } + + if err := ioutil.WriteFile(filepath.Join(dir, "nginx.conf"), []byte(conf), 0600); err != nil { + return nil, errors.Wrap(err, "creating nginx config file failed") + } + + nginx := e2e.NewInstrumentedRunnable(e, fmt.Sprintf("nginx-%s", name), map[string]int{"http": 80}, "http").Init( + e2e.StartOptions{ + Image: "docker.io/nginx:1.21.1-alpine", + Volumes: []string{filepath.Join(dir, "/nginx.conf") + ":/etc/nginx/nginx.conf:ro"}, + WaitReadyBackoff: &defaultBackoffConfig, + }, + ) + + return nginx, nil +} + +// NewMinio returns minio server, used as a local replacement for S3. +// TODO(@matej-g): This is a temporary workaround for https://github.com/efficientgo/e2e/issues/11 +func NewMinio(env e2e.Environment, name, bktName string) *e2e.InstrumentedRunnable { + image := "minio/minio:RELEASE.2019-12-30T05-45-39Z" + minioKESGithubContent := "https://raw.githubusercontent.com/minio/kes/master" + commands := []string{ + "curl -sSL --tlsv1.2 -O '%s/root.key' -O '%s/root.cert'", + "mkdir -p /data/%s && minio server --address :%v --quiet /data", + } + + return e2e.NewInstrumentedRunnable( + env, + name, + map[string]int{"http": 8090}, + "http").Init( + e2e.StartOptions{ + Image: image, + // Create the required bucket before starting minio. + Command: e2e.NewCommandWithoutEntrypoint("sh", "-c", fmt.Sprintf(strings.Join(commands, " && "), minioKESGithubContent, minioKESGithubContent, bktName, 8090)), + Readiness: e2e.NewHTTPReadinessProbe("http", "/minio/health/ready", 200, 200), + EnvVars: map[string]string{ + "MINIO_ACCESS_KEY": e2edb.MinioAccessKey, + "MINIO_SECRET_KEY": e2edb.MinioSecretKey, + "MINIO_BROWSER": "off", + "ENABLE_HTTPS": "0", + // https://docs.min.io/docs/minio-kms-quickstart-guide.html + "MINIO_KMS_KES_ENDPOINT": "https://play.min.io:7373", + "MINIO_KMS_KES_KEY_FILE": "root.key", + "MINIO_KMS_KES_CERT_FILE": "root.cert", + "MINIO_KMS_KES_KEY_NAME": "my-minio-key", + }, + }, + ) +} + +func NewMemcached(e e2e.Environment, name string) *e2e.InstrumentedRunnable { + memcached := e2e.NewInstrumentedRunnable(e, fmt.Sprintf("memcached-%s", name), map[string]int{"memcached": 11211}, "memcached").Init( + e2e.StartOptions{ + Image: "docker.io/memcached:1.6.3-alpine", + Command: e2e.NewCommand("memcached", []string{"-m 1024", "-I 1m", "-c 1024", "-v"}...), + User: strconv.Itoa(os.Getuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, ) - memcached.SetUser(strconv.Itoa(os.Getuid())) - memcached.SetBackoff(defaultBackoffConfig) return memcached } func NewToolsBucketWeb( + e e2e.Environment, name string, bucketConfig client.BucketConfig, routePrefix, externalPrefix string, minTime string, maxTime string, - relabelConfig string) (*Service, error) { + relabelConfig string, +) (*e2e.InstrumentedRunnable, error) { bktConfigBytes, err := yaml.Marshal(bucketConfig) if err != nil { return nil, errors.Wrapf(err, "generate tools bucket web config file: %v", bucketConfig) @@ -703,16 +783,14 @@ func NewToolsBucketWeb( args = append([]string{"bucket", "web"}, args...) - toolsBucketWeb := NewService( + toolsBucketWeb := NewService(e, fmt.Sprintf("toolsBucketWeb-%s", name), DefaultImage(), e2e.NewCommand("tools", args...), - e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, ) - toolsBucketWeb.SetUser(strconv.Itoa(os.Getuid())) - toolsBucketWeb.SetBackoff(defaultBackoffConfig) return toolsBucketWeb, nil } From e6c572ebbffa23dc33e03dba023f2d9be4f74d7e Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Fri, 27 Aug 2021 16:22:17 +0200 Subject: [PATCH 2/5] Adjust e2e tests - Mechanical replacement / adjustments to new services, package imports etc. Signed-off-by: Matej Gera --- pkg/objstore/s3/s3_e2e_test.go | 16 +- test/e2e/compact_test.go | 73 ++++--- test/e2e/e2ethanos/helpers.go | 6 +- test/e2e/e2ethanos/services.go | 3 +- test/e2e/exemplars_api_test.go | 67 +++--- test/e2e/metadata_api_test.go | 37 ++-- test/e2e/query_frontend_test.go | 153 +++++++------- test/e2e/query_test.go | 122 +++++------ test/e2e/receive_test.go | 336 +++++++++++++----------------- test/e2e/rule_test.go | 56 ++--- test/e2e/rules_api_test.go | 48 ++--- test/e2e/store_gateway_test.go | 47 +++-- test/e2e/targets_api_test.go | 22 +- test/e2e/tools_bucket_web_test.go | 82 ++++---- 14 files changed, 512 insertions(+), 556 deletions(-) diff --git a/pkg/objstore/s3/s3_e2e_test.go b/pkg/objstore/s3/s3_e2e_test.go index e837b9baa5..8991acdbfe 100644 --- a/pkg/objstore/s3/s3_e2e_test.go +++ b/pkg/objstore/s3/s3_e2e_test.go @@ -9,8 +9,8 @@ import ( "strings" "testing" - "github.com/cortexproject/cortex/integration/e2e" - e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/efficientgo/e2e" + e2edb "github.com/efficientgo/e2e/db" "github.com/go-kit/kit/log" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/test/e2e/e2ethanos" @@ -23,19 +23,19 @@ func BenchmarkUpload(b *testing.B) { b.ReportAllocs() ctx := context.Background() - s, err := e2e.NewScenario("e2e_bench_mino_client") + e, err := e2e.NewDockerEnvironment("e2e_bench_mino_client") testutil.Ok(b, err) - b.Cleanup(e2ethanos.CleanScenario(b, s)) + b.Cleanup(e2ethanos.CleanScenario(b, e)) - const bucket = "test" - m := e2edb.NewMinio(8080, bucket) - testutil.Ok(b, s.StartAndWaitReady(m)) + const bucket = "benchmark" + m := e2ethanos.NewMinio(e, "benchmark", bucket) + testutil.Ok(b, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{ Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.HTTPEndpoint(), + Endpoint: m.Endpoint("http"), Insecure: true, }, "test-feed") testutil.Ok(b, err) diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index d19bc778ec..037d987b38 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -16,8 +16,9 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" - e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/efficientgo/e2e" + e2edb "github.com/efficientgo/e2e/db" + "github.com/efficientgo/e2e/matchers" "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" @@ -33,6 +34,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/promclient" + "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" "github.com/thanos-io/thanos/pkg/testutil/e2eutil" "github.com/thanos-io/thanos/test/e2e/e2ethanos" @@ -336,22 +338,22 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { if penaltyDedup { name = "e2e_test_compact_penalty_dedup" } - s, err := e2e.NewScenario(name) + e, err := e2e.NewDockerEnvironment(name) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) - dir := filepath.Join(s.SharedDir(), "tmp") + dir := filepath.Join(e.SharedDir(), "tmp") testutil.Ok(t, os.MkdirAll(dir, os.ModePerm)) const bucket = "compact_test" - m := e2edb.NewMinio(8080, bucket) - testutil.Ok(t, s.StartAndWaitReady(m)) + m := e2ethanos.NewMinio(e, "minio", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, s3.Config{ Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.HTTPEndpoint(), // We need separate client config, when connecting to minio from outside. + Endpoint: m.Endpoint("http"), // We need separate client config, when connecting to minio from outside. Insecure: true, }, "test-feed") testutil.Ok(t, err) @@ -363,7 +365,10 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { for _, b := range blocks { id, err := b.Create(ctx, dir, justAfterConsistencyDelay, b.hashFunc) testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadDir(ctx, logger, bkt, path.Join(dir, id.String()), id.String())) + testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { + return objstore.UploadDir(ctx, logger, bkt, path.Join(dir, id.String()), id.String()) + })) + rawBlockIDs[id] = struct{}{} if b.markedForNoCompact { testutil.Ok(t, block.MarkForNoCompact(ctx, logger, bkt, id, metadata.ManualNoCompactReason, "why not", promauto.With(nil).NewCounter(prometheus.CounterOpts{}))) @@ -442,26 +447,26 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.NetworkHTTPEndpoint(), + Endpoint: m.InternalEndpoint("http"), Insecure: true, }, } - str, err := e2ethanos.NewStoreGW(s.SharedDir(), "1", svcConfig) + str, err := e2ethanos.NewStoreGW(e, "1", svcConfig) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(str)) + testutil.Ok(t, e2e.StartAndWaitReady(str)) testutil.Ok(t, str.WaitSumMetrics(e2e.Equals(float64(len(rawBlockIDs)+7)), "thanos_blocks_meta_synced")) testutil.Ok(t, str.WaitSumMetrics(e2e.Equals(0), "thanos_blocks_meta_sync_failures_total")) testutil.Ok(t, str.WaitSumMetrics(e2e.Equals(0), "thanos_blocks_meta_modified")) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{str.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{str.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) // Check if query detects current series, even if overlapped. - queryAndAssert(t, ctx, q.HTTPEndpoint(), + queryAndAssert(t, ctx, q.Endpoint("http"), fmt.Sprintf(`count_over_time({a="1"}[13h] offset %ds)`, int64(time.Since(now.Add(12*time.Hour)).Seconds())), promclient.QueryOptions{ Deduplicate: false, // This should be false, so that we can be sure deduplication was offline. @@ -599,7 +604,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { // Precreate a directory. It should be deleted. // In a hypothetical scenario, the directory could be a left-over from // a compaction that had crashed. - p := filepath.Join(s.SharedDir(), "data", "compact", "expect-to-halt", "compact") + p := filepath.Join(e.SharedDir(), "data", "compact", "expect-to-halt", "compact") testutil.Assert(t, len(blocksWithHashes) > 0) @@ -613,9 +618,9 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, err) testutil.Ok(t, f.Close()) - c, err := e2ethanos.NewCompactor(s.SharedDir(), "expect-to-halt", svcConfig, nil) + c, err := e2ethanos.NewCompactor(e, "expect-to-halt", svcConfig, nil) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(c)) + testutil.Ok(t, e2e.StartAndWaitReady(c)) // Expect compactor halted and for one cleanup iteration to happen. testutil.Ok(t, c.WaitSumMetrics(e2e.Equals(1), "thanos_compact_halted")) @@ -626,10 +631,10 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, c.WaitSumMetrics(e2e.Equals(0), "thanos_blocks_meta_modified")) // The compact directory is still there. - dataDir := filepath.Join(s.SharedDir(), "data", "compact", "expect-to-halt") + dataDir := filepath.Join(e.SharedDir(), "data", "compact", "expect-to-halt") empty, err := isEmptyDir(dataDir) testutil.Ok(t, err) - testutil.Equals(t, false, empty, "directory %s should not be empty", dataDir) + testutil.Equals(t, false, empty, "directory %e should not be empty", dataDir) // We expect no ops. testutil.Ok(t, c.WaitSumMetrics(e2e.Equals(0), "thanos_compact_iterations_total")) @@ -646,10 +651,10 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, c.WaitSumMetrics(e2e.Equals(2), "thanos_compact_blocks_cleaned_total")) // Ensure bucket UI. - ensureGETStatusCode(t, http.StatusOK, "http://"+path.Join(c.HTTPEndpoint(), "global")) - ensureGETStatusCode(t, http.StatusOK, "http://"+path.Join(c.HTTPEndpoint(), "loaded")) + ensureGETStatusCode(t, http.StatusOK, "http://"+path.Join(c.Endpoint("http"), "global")) + ensureGETStatusCode(t, http.StatusOK, "http://"+path.Join(c.Endpoint("http"), "loaded")) - testutil.Ok(t, s.Stop(c)) + testutil.Ok(t, c.Stop()) _, err = os.Stat(randBlockDir) testutil.NotOk(t, err) @@ -661,7 +666,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { // Dedup enabled; compactor should work as expected. { // Predownload block dirs with hashes. We should not try downloading them again. - p := filepath.Join(s.SharedDir(), "data", "compact", "working") + p := filepath.Join(e.SharedDir(), "data", "compact", "working") for _, id := range blocksWithHashes { m, err := block.DownloadMeta(ctx, logger, bkt, id) @@ -677,9 +682,9 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { } // We expect 2x 4-block compaction, 2-block vertical compaction, 2x 3-block compaction. - c, err := e2ethanos.NewCompactor(s.SharedDir(), "working", svcConfig, nil, extArgs...) + c, err := e2ethanos.NewCompactor(e, "working", svcConfig, nil, extArgs...) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(c)) + testutil.Ok(t, e2e.StartAndWaitReady(c)) // NOTE: We cannot assert on intermediate `thanos_blocks_meta_` metrics as those are gauge and change dynamically due to many // compaction groups. Wait for at least first compaction iteration (next is in 5m). @@ -706,9 +711,9 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, c.WaitSumMetrics(e2e.Equals(0), "thanos_compact_halted")) - bucketMatcher, err := labels.NewMatcher(labels.MatchEqual, "bucket", bucket) + bucketMatcher, err := matchers.NewMatcher(matchers.MatchEqual, "bucket", bucket) testutil.Ok(t, err) - operationMatcher, err := labels.NewMatcher(labels.MatchEqual, "operation", "get") + operationMatcher, err := matchers.NewMatcher(matchers.MatchEqual, "operation", "get") testutil.Ok(t, err) testutil.Ok(t, c.WaitSumMetricsWithOptions(e2e.Equals(478), []string{"thanos_objstore_bucket_operations_total"}, e2e.WithLabelMatchers( @@ -718,13 +723,13 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { ) // Make sure compactor does not modify anything else over time. - testutil.Ok(t, s.Stop(c)) + testutil.Ok(t, c.Stop()) ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) // Check if query detects new blocks. - queryAndAssert(t, ctx, q.HTTPEndpoint(), + queryAndAssert(t, ctx, q.Endpoint("http"), fmt.Sprintf(`count_over_time({a="1"}[13h] offset %ds)`, int64(time.Since(now.Add(12*time.Hour)).Seconds())), promclient.QueryOptions{ Deduplicate: false, // This should be false, so that we can be sure deduplication was offline. @@ -742,9 +747,9 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { if penaltyDedup { extArgs = append(extArgs, "--deduplication.func=penalty") } - c, err := e2ethanos.NewCompactor(s.SharedDir(), "working", svcConfig, nil, extArgs...) + c, err := e2ethanos.NewCompactor(e, "working-dedup", svcConfig, nil, extArgs...) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(c)) + testutil.Ok(t, e2e.StartAndWaitReady(c)) // NOTE: We cannot assert on intermediate `thanos_blocks_meta_` metrics as those are gauge and change dynamically due to many // compaction groups. Wait for at least first compaction iteration (next is in 5m). @@ -767,13 +772,13 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, c.WaitSumMetrics(e2e.Equals(0), "thanos_compact_halted")) // Make sure compactor does not modify anything else over time. - testutil.Ok(t, s.Stop(c)) + testutil.Ok(t, c.Stop()) ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) // Check if query detects new blocks. - queryAndAssert(t, ctx, q.HTTPEndpoint(), + queryAndAssert(t, ctx, q.Endpoint("http"), fmt.Sprintf(`count_over_time({a="1"}[13h] offset %ds)`, int64(time.Since(now.Add(12*time.Hour)).Seconds())), promclient.QueryOptions{ Deduplicate: false, // This should be false, so that we can be sure deduplication was offline. diff --git a/test/e2e/e2ethanos/helpers.go b/test/e2e/e2ethanos/helpers.go index d5c8519e9e..22ef3d8141 100644 --- a/test/e2e/e2ethanos/helpers.go +++ b/test/e2e/e2ethanos/helpers.go @@ -16,11 +16,11 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) -func CleanScenario(t testing.TB, s *e2e.DockerEnvironment) func() { +func CleanScenario(t testing.TB, e *e2e.DockerEnvironment) func() { return func() { // Make sure Clean can properly delete everything. - testutil.Ok(t, exec.Command("chmod", "-R", "777", s.SharedDir()).Run()) - s.Close() + testutil.Ok(t, exec.Command("chmod", "-R", "777", e.SharedDir()).Run()) + e.Close() } } diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 17f79aae92..af7cf2d445 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -693,7 +693,8 @@ http { } // NewMinio returns minio server, used as a local replacement for S3. -// TODO(@matej-g): This is a temporary workaround for https://github.com/efficientgo/e2e/issues/11 +// TODO(@matej-g): This is a temporary workaround for https://github.com/efficientgo/e2e/issues/11; +// after this is addresses fixed all calls should be replaced with e2edb.NewMinio. func NewMinio(env e2e.Environment, name, bktName string) *e2e.InstrumentedRunnable { image := "minio/minio:RELEASE.2019-12-30T05-45-39Z" minioKESGithubContent := "https://raw.githubusercontent.com/minio/kes/master" diff --git a/test/e2e/exemplars_api_test.go b/test/e2e/exemplars_api_test.go index 0379258c7c..0e9bea9597 100644 --- a/test/e2e/exemplars_api_test.go +++ b/test/e2e/exemplars_api_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" "github.com/thanos-io/thanos/pkg/store/labelpb" @@ -27,83 +27,64 @@ func TestExemplarsAPI_Fanout(t *testing.T) { netName := "e2e_test_exemplars_fanout" var ( - prom1, prom2 *e2e.HTTPService - sidecar1, sidecar2 *e2ethanos.Service + prom1, prom2 *e2e.InstrumentedRunnable + sidecar1, sidecar2 *e2e.InstrumentedRunnable err error - s *e2e.Scenario + e *e2e.DockerEnvironment ) - s, err = e2e.NewScenario(netName) - testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) - - // 2x Prometheus. - _, sidecar1, err = e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), - netName, - "prom1", - defaultPromConfig("ha", 0, "", ""), - e2ethanos.DefaultPrometheusImage(), - e2ethanos.FeatureExemplarStorage, - ) - testutil.Ok(t, err) - _, sidecar2, err = e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), - netName, - "prom2", - defaultPromConfig("ha", 1, "", ""), - e2ethanos.DefaultPrometheusImage(), - e2ethanos.FeatureExemplarStorage, - ) + e, err = e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) tracingCfg := fmt.Sprintf(`type: JAEGER config: sampler_type: const sampler_param: 1 - service_name: %s`, s.NetworkName()+"-query") + service_name: %s`, netName+"-query") - stores := []string{sidecar1.NetworkEndpointFor(s.NetworkName(), 9091), sidecar2.NetworkEndpointFor(s.NetworkName(), 9091)} - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "query", stores). + stores := []string{netName + "-" + "sidecar-prom1:9091", netName + "-" + "sidecar-prom2:9091"} + q, err := e2ethanos.NewQuerierBuilder( + e, "query", stores). WithExemplarAddresses(stores). WithTracingConfig(tracingCfg). Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) // Recreate Prometheus and sidecar with Thanos query scrape target. prom1, sidecar1, err = e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom1", - defaultPromConfig("ha", 0, "", "", "localhost:9090", q.NetworkHTTPEndpoint()), + defaultPromConfig("ha", 0, "", "", "localhost:9090", q.InternalEndpoint("http")), e2ethanos.DefaultPrometheusImage(), - "exemplar-storage", + e2ethanos.FeatureExemplarStorage, ) testutil.Ok(t, err) prom2, sidecar2, err = e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom2", - defaultPromConfig("ha", 1, "", "", "localhost:9090", q.NetworkHTTPEndpoint()), + defaultPromConfig("ha", 1, "", "", "localhost:9090", q.InternalEndpoint("http")), e2ethanos.DefaultPrometheusImage(), - "exemplar-storage", + e2ethanos.FeatureExemplarStorage, ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_query_exemplar_apis_dns_provider_results"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_query_exemplar_apis_dns_provider_results"}, e2e.WaitMissingMetrics())) now := time.Now() start := timestamp.FromTime(now.Add(-time.Hour)) end := timestamp.FromTime(now.Add(time.Hour)) // Send HTTP requests to thanos query to trigger exemplars. - labelNames(t, ctx, q.HTTPEndpoint(), nil, start, end, func(res []string) bool { + labelNames(t, ctx, q.Endpoint("http"), nil, start, end, func(res []string) bool { return true }) @@ -115,7 +96,7 @@ config: "method": "get", "prometheus": "ha", } - queryExemplars(t, ctx, q.HTTPEndpoint(), `http_request_duration_seconds_bucket{handler="label_names"}`, + queryExemplars(t, ctx, q.Endpoint("http"), `http_request_duration_seconds_bucket{handler="label_names"}`, start, end, func(data []*exemplarspb.ExemplarData) bool { if len(data) != 1 { return false @@ -152,7 +133,7 @@ config: "prometheus": "ha", } // Here replica is an external label. - queryExemplars(t, ctx, q.HTTPEndpoint(), `http_request_duration_seconds_bucket{handler="label_names", replica="0"}`, + queryExemplars(t, ctx, q.Endpoint("http"), `http_request_duration_seconds_bucket{handler="label_names", replica="0"}`, start, end, func(data []*exemplarspb.ExemplarData) bool { if len(data) != 1 { return false @@ -182,7 +163,7 @@ config: t.Run("Exemplars query doesn't match external label", func(t *testing.T) { // Here replica is an external label, but it doesn't match. - queryExemplars(t, ctx, q.HTTPEndpoint(), `http_request_duration_seconds_bucket{handler="label_names", replica="foo"}`, + queryExemplars(t, ctx, q.Endpoint("http"), `http_request_duration_seconds_bucket{handler="label_names", replica="foo"}`, start, end, func(data []*exemplarspb.ExemplarData) bool { return len(data) == 0 }) diff --git a/test/e2e/metadata_api_test.go b/test/e2e/metadata_api_test.go index 83ca8dea08..a607575acc 100644 --- a/test/e2e/metadata_api_test.go +++ b/test/e2e/metadata_api_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/thanos-io/thanos/pkg/metadata/metadatapb" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/runutil" @@ -23,14 +23,14 @@ func TestMetadataAPI_Fanout(t *testing.T) { netName := "e2e_test_metadata_fanout" - s, err := e2e.NewScenario(netName) + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // 2x Prometheus. // Each Prometheus scrapes its own metrics and Sidecar's metrics. prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "sidecar-prom1:8080"), @@ -39,32 +39,33 @@ func TestMetadataAPI_Fanout(t *testing.T) { testutil.Ok(t, err) prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "sidecar-prom2:8080"), e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) - stores := []string{sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint()} - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "query", stores). + stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc")} + q, err := e2ethanos.NewQuerierBuilder( + e, "query", stores). WithMetadataAddresses(stores). Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_query_metadata_apis_dns_provider_results"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_query_metadata_apis_dns_provider_results"}, e2e.WaitMissingMetrics())) var promMeta map[string][]metadatapb.Meta // Wait metadata response to be ready as Prometheus gets metadata after scrape. - testutil.Ok(t, runutil.Retry(3*time.Second, ctx.Done(), func() error { - promMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+prom1.HTTPEndpoint()), "", -1) + testutil.Ok(t, runutil.Retry(5*time.Second, ctx.Done(), func() error { + promMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+prom1.Endpoint("http")), "", -1) testutil.Ok(t, err) if len(promMeta) > 0 { return nil @@ -72,7 +73,7 @@ func TestMetadataAPI_Fanout(t *testing.T) { return fmt.Errorf("empty metadata response from Prometheus") })) - thanosMeta, err := promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.HTTPEndpoint()), "", -1) + thanosMeta, err := promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.Endpoint("http")), "", -1) testutil.Ok(t, err) testutil.Assert(t, len(thanosMeta) > 0, "got empty metadata response from Thanos") @@ -80,22 +81,22 @@ func TestMetadataAPI_Fanout(t *testing.T) { metadataEqual(t, thanosMeta, promMeta) // We only expect to see one metadata returned. - thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.HTTPEndpoint()), "", 1) + thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.Endpoint("http")), "", 1) testutil.Ok(t, err) testutil.Equals(t, len(thanosMeta), 1) // We only expect to see ten metadata returned. - thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.HTTPEndpoint()), "", 10) + thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.Endpoint("http")), "", 10) testutil.Ok(t, err) testutil.Equals(t, len(thanosMeta), 10) // No metadata returned. - thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.HTTPEndpoint()), "", 0) + thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.Endpoint("http")), "", 0) testutil.Ok(t, err) testutil.Equals(t, len(thanosMeta), 0) // Only prometheus_build_info metric will be returned. - thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.HTTPEndpoint()), "prometheus_build_info", -1) + thanosMeta, err = promclient.NewDefaultClient().MetricMetadataInGRPC(ctx, mustURLParse(t, "http://"+q.Endpoint("http")), "prometheus_build_info", -1) testutil.Ok(t, err) testutil.Assert(t, len(thanosMeta) == 1 && len(thanosMeta["prometheus_build_info"]) > 0, "expected one prometheus_build_info metadata from Thanos, got %v", thanosMeta) } diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index dafb39de36..1e218bc2ba 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" - + "github.com/efficientgo/e2e" + "github.com/efficientgo/e2e/matchers" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" @@ -24,19 +24,21 @@ import ( func TestQueryFrontend(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_frontend") + netName := "e2e_test_query_frontend" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), s.NetworkName(), "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom, sidecar)) + testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{sidecar.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) inMemoryCacheConfig := queryfrontend.CacheProviderConfig{ Type: queryfrontend.INMEMORY, @@ -46,18 +48,18 @@ func TestQueryFrontend(t *testing.T) { }, } - queryFrontend, err := e2ethanos.NewQueryFrontend("1", "http://"+q.NetworkHTTPEndpoint(), inMemoryCacheConfig) + queryFrontend, err := e2ethanos.NewQueryFrontend(e, "1", "http://"+q.InternalEndpoint("http"), inMemoryCacheConfig) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(queryFrontend)) + testutil.Ok(t, e2e.StartAndWaitReady(queryFrontend)) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) // Ensure we can get the result from Querier first so that it // doesn't need to retry when we send queries to the frontend later. - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -67,14 +69,15 @@ func TestQueryFrontend(t *testing.T) { }, }) - vals, err := q.SumMetrics([]string{"http_requests_total"}, e2e.WithLabelMatchers( - labels.MustNewMatcher(labels.MatchEqual, "handler", "query"))) + vals, err := q.SumMetrics([]string{"http_requests_total"}) + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query")) + testutil.Ok(t, err) testutil.Equals(t, 1, len(vals)) queryTimes := vals[0] t.Run("query frontend works for instant query", func(t *testing.T) { - queryAndAssertSeries(t, ctx, queryFrontend.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, queryFrontend.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -87,21 +90,21 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "query"))), - ) + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query")), + )) testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(queryTimes+1), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "query"))), - ) + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query")), + )) }) t.Run("query frontend works for range query and it can cache results", func(t *testing.T) { rangeQuery( t, ctx, - queryFrontend.HTTPEndpoint(), + queryFrontend.Endpoint("http"), queryUpWithoutInstance, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), @@ -115,8 +118,8 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "query_range"))), - ) + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query_range")), + )) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(1), "cortex_cache_fetched_keys")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(0), "cortex_cache_hits")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(1), "querier_cache_added_new_total")) @@ -131,8 +134,8 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "query_range"))), - ) + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range")), + )) }) t.Run("same range query, cache hit.", func(t *testing.T) { @@ -140,7 +143,7 @@ func TestQueryFrontend(t *testing.T) { rangeQuery( t, ctx, - queryFrontend.HTTPEndpoint(), + queryFrontend.Endpoint("http"), queryUpWithoutInstance, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), @@ -154,7 +157,7 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(2), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query_range"))), ) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(2), "cortex_cache_fetched_keys")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(1), "cortex_cache_hits")) @@ -167,14 +170,14 @@ func TestQueryFrontend(t *testing.T) { // Query is only 2h so it won't be split. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(2), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "query_range"))), ) // One more request is needed in order to satisfy the req range. testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(2), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range"))), ) }) @@ -182,7 +185,7 @@ func TestQueryFrontend(t *testing.T) { rangeQuery( t, ctx, - queryFrontend.HTTPEndpoint(), + queryFrontend.Endpoint("http"), queryUpWithoutInstance, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(24*time.Hour)), @@ -196,7 +199,7 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(3), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query_range"))), ) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(3), "cortex_cache_fetched_keys")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(2), "cortex_cache_hits")) @@ -209,94 +212,94 @@ func TestQueryFrontend(t *testing.T) { // Query is 25h so it will be split to 2 requests. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(4), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "query_range"))), ) testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(4), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range"))), ) }) t.Run("query frontend splitting works for labels names API", func(t *testing.T) { // LabelNames and LabelValues API should still work via query frontend. - labelNames(t, ctx, queryFrontend.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "label_names"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "label_names"))), ) testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "label_names"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "label_names"))), ) // Query is only 2h so it won't be split. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) - labelNames(t, ctx, queryFrontend.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(3), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "label_names"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "label_names"))), ) testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(2), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "label_names"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "label_names"))), ) // Query is 25h so split to 2 requests. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(3), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) }) t.Run("query frontend splitting works for labels values API", func(t *testing.T) { - labelValues(t, ctx, queryFrontend.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "label_values"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "label_values"))), ) testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "label_values"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "label_values"))), ) // Query is only 2h so it won't be split. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(4), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) - labelValues(t, ctx, queryFrontend.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(3), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "label_values"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "label_values"))), ) testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(2), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "label_values"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "label_values"))), ) // Query is 25h so split to 2 requests. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(6), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) }) @@ -304,7 +307,7 @@ func TestQueryFrontend(t *testing.T) { series( t, ctx, - queryFrontend.HTTPEndpoint(), + queryFrontend.Endpoint("http"), []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "up")}, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), @@ -324,23 +327,23 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "series"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "series"))), ) testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "series"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "series"))), ) // Query is only 2h so it won't be split. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(7), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) series( t, ctx, - queryFrontend.HTTPEndpoint(), + queryFrontend.Endpoint("http"), []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "up")}, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), @@ -360,17 +363,17 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, q.WaitSumMetricsWithOptions( e2e.Equals(3), []string{"http_requests_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "handler", "series"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "series"))), ) testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(2), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "series"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "series"))), ) // Query is only 2h so it won't be split. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(9), []string{"thanos_frontend_split_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) }) } @@ -378,28 +381,30 @@ func TestQueryFrontend(t *testing.T) { func TestQueryFrontendMemcachedCache(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_frontend_memcached") + netName := "e2e_test_query_frontend_memcached" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), s.NetworkName(), "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom, sidecar)) + testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{sidecar.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) - memcached := e2ethanos.NewMemcached("1") - testutil.Ok(t, s.StartAndWaitReady(memcached)) + memcached := e2ethanos.NewMemcached(e, "1") + testutil.Ok(t, e2e.StartAndWaitReady(memcached)) memCachedConfig := queryfrontend.CacheProviderConfig{ Type: queryfrontend.MEMCACHED, Config: queryfrontend.MemcachedResponseCacheConfig{ Memcached: cacheutil.MemcachedClientConfig{ - Addresses: []string{memcached.NetworkEndpoint(11211)}, + Addresses: []string{memcached.InternalEndpoint("memcached")}, MaxIdleConnections: 100, MaxAsyncConcurrency: 20, MaxGetMultiConcurrency: 100, @@ -411,20 +416,20 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { }, } - queryFrontend, err := e2ethanos.NewQueryFrontend("1", "http://"+q.NetworkHTTPEndpoint(), memCachedConfig) + queryFrontend, err := e2ethanos.NewQueryFrontend(e, "1", "http://"+q.InternalEndpoint("http"), memCachedConfig) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(queryFrontend)) + testutil.Ok(t, e2e.StartAndWaitReady(queryFrontend)) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(1), "cortex_memcache_client_servers")) // Ensure we can get the result from Querier first so that it // doesn't need to retry when we send queries to the frontend later. - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -435,14 +440,14 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { }) vals, err := q.SumMetrics([]string{"http_requests_total"}, e2e.WithLabelMatchers( - labels.MustNewMatcher(labels.MatchEqual, "handler", "query"))) + matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query"))) testutil.Ok(t, err) testutil.Equals(t, 1, len(vals)) rangeQuery( t, ctx, - queryFrontend.HTTPEndpoint(), + queryFrontend.Endpoint("http"), queryUpWithoutInstance, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), @@ -456,7 +461,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(1), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query_range"))), ) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2e.Equals(1), "cortex_cache_fetched_keys")) @@ -469,7 +474,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { rangeQuery( t, ctx, - queryFrontend.HTTPEndpoint(), + queryFrontend.Endpoint("http"), queryUpWithoutInstance, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), @@ -483,7 +488,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( e2e.Equals(2), []string{"thanos_query_frontend_queries_total"}, - e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "op", "query_range"))), + e2e.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query_range"))), ) // Query is only 2h so it won't be split. diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index cbae749af5..0a98669e1b 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -17,7 +17,7 @@ import ( "github.com/chromedp/cdproto/network" "github.com/chromedp/chromedp" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/go-kit/kit/log" "github.com/pkg/errors" "github.com/prometheus/common/model" @@ -94,36 +94,38 @@ func sortResults(res model.Vector) { func TestQuery(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query") + netName := "e2e_test_query" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) - receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1) + receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(receiver)) + testutil.Ok(t, e2e.StartAndWaitReady(receiver)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_query", "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_query", "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_query", "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2e.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_query", "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2e.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3, prom4, sidecar4)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3, prom4, sidecar4)) // Querier. Both fileSD and directly by flags. - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint(), receiver.GRPCNetworkEndpoint()}). - WithFileSDStoreAddresses([]string{sidecar3.GRPCNetworkEndpoint(), sidecar4.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), receiver.InternalEndpoint("grpc")}). + WithFileSDStoreAddresses([]string{sidecar3.InternalEndpoint("grpc"), sidecar4.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(5), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(5), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -156,7 +158,7 @@ func TestQuery(t *testing.T) { }) // With deduplication. - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: true, }, []model.Metric{ { @@ -183,35 +185,35 @@ func TestQuery(t *testing.T) { func TestQueryExternalPrefixWithoutReverseProxy(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_route_prefix") + e, err := e2e.NewDockerEnvironment("e2e_test_query_route_prefix") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) externalPrefix := "test" - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", nil). + q, err := e2ethanos.NewQuerierBuilder(e, "1", nil). WithExternalPrefix(externalPrefix).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) - checkNetworkRequests(t, "http://"+q.HTTPEndpoint()+"/"+externalPrefix+"/graph") + checkNetworkRequests(t, "http://"+q.Endpoint("http")+"/"+externalPrefix+"/graph") } func TestQueryExternalPrefix(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_external_prefix") + e, err := e2e.NewDockerEnvironment("e2e_test_query_external_prefix") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) externalPrefix := "thanos" - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", nil). + q, err := e2ethanos.NewQuerierBuilder(e, "1", nil). WithExternalPrefix(externalPrefix).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) - querierURL := mustURLParse(t, "http://"+q.HTTPEndpoint()+"/"+externalPrefix) + querierURL := mustURLParse(t, "http://"+q.Endpoint("http")+"/"+externalPrefix) querierProxy := httptest.NewServer(e2ethanos.NewSingleHostReverseProxy(querierURL, externalPrefix)) t.Cleanup(querierProxy.Close) @@ -222,21 +224,21 @@ func TestQueryExternalPrefix(t *testing.T) { func TestQueryExternalPrefixAndRoutePrefix(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_external_prefix_and_route_prefix") + e, err := e2e.NewDockerEnvironment("e2e_test_query_external_prefix_and_route_prefix") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) externalPrefix := "thanos" routePrefix := "test" - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", nil). + q, err := e2ethanos.NewQuerierBuilder(e, "1", nil). WithRoutePrefix(routePrefix). WithExternalPrefix(externalPrefix). Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) - querierURL := mustURLParse(t, "http://"+q.HTTPEndpoint()+"/"+routePrefix) + querierURL := mustURLParse(t, "http://"+q.Endpoint("http")+"/"+routePrefix) querierProxy := httptest.NewServer(e2ethanos.NewSingleHostReverseProxy(querierURL, externalPrefix)) t.Cleanup(querierProxy.Close) @@ -247,38 +249,40 @@ func TestQueryExternalPrefixAndRoutePrefix(t *testing.T) { func TestQueryLabelNames(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_label_names") + netName := "e2e_test_query_label_names" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) - receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1) + receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(receiver)) + testutil.Ok(t, e2e.StartAndWaitReady(receiver)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), s.NetworkName(), "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), s.NetworkName(), "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint(), receiver.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), receiver.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) t.Cleanup(cancel) now := time.Now() - labelNames(t, ctx, q.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, q.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) // Outside time range. - labelNames(t, ctx, q.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { + labelNames(t, ctx, q.Endpoint("http"), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { return len(res) == 0 }) - labelNames(t, ctx, q.HTTPEndpoint(), []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "up"}}, + labelNames(t, ctx, q.Endpoint("http"), []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "up"}}, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { // Expected result: [__name__, instance, job, prometheus, replica] return len(res) == 7 @@ -286,7 +290,7 @@ func TestQueryLabelNames(t *testing.T) { ) // There is no matched series. - labelNames(t, ctx, q.HTTPEndpoint(), []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "foobar"}}, + labelNames(t, ctx, q.Endpoint("http"), []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "foobar"}}, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 0 }, @@ -296,44 +300,46 @@ func TestQueryLabelNames(t *testing.T) { func TestQueryLabelValues(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_label_values") + netName := "e2e_test_query_label_values" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) - receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1) + receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(receiver)) + testutil.Ok(t, e2e.StartAndWaitReady(receiver)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), s.NetworkName(), "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), s.NetworkName(), "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint(), receiver.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), receiver.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) t.Cleanup(cancel) now := time.Now() - labelValues(t, ctx, q.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, q.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) // Outside time range. - labelValues(t, ctx, q.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { + labelValues(t, ctx, q.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { return len(res) == 0 }) - labelValues(t, ctx, q.HTTPEndpoint(), "__name__", []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "up"}}, + labelValues(t, ctx, q.Endpoint("http"), "__name__", []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "up"}}, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "up" }, ) - labelValues(t, ctx, q.HTTPEndpoint(), "__name__", []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "foobar"}}, + labelValues(t, ctx, q.Endpoint("http"), "__name__", []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "foobar"}}, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 0 }, @@ -387,7 +393,7 @@ func instantQuery(t *testing.T, ctx context.Context, addr, q string, opts promcl logger := log.NewLogfmtLogger(os.Stdout) logger = log.With(logger, "ts", log.DefaultTimestampUTC) - testutil.Ok(t, runutil.RetryWithLog(logger, time.Second, ctx.Done(), func() error { + testutil.Ok(t, runutil.RetryWithLog(logger, 3*time.Second, ctx.Done(), func() error { res, warnings, err := promclient.NewDefaultClient().QueryInstant(ctx, mustURLParse(t, "http://"+addr), q, time.Now(), opts) if err != nil { return err diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index e63656f54b..2c0f3d6675 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -8,11 +8,10 @@ import ( "log" "net/http" "net/http/httputil" - "net/url" "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/prometheus/common/model" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/receive" @@ -20,12 +19,6 @@ import ( "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) -type ReverseProxyConfig struct { - tenantId string - port string - target string -} - type DebugTransport struct{} func (DebugTransport) RoundTrip(r *http.Request) (*http.Response, error) { @@ -36,19 +29,6 @@ func (DebugTransport) RoundTrip(r *http.Request) (*http.Response, error) { return http.DefaultTransport.RoundTrip(r) } -func generateProxy(conf ReverseProxyConfig) { - targetURL, _ := url.Parse(conf.target) - proxy := httputil.NewSingleHostReverseProxy(targetURL) - d := proxy.Director - proxy.Director = func(r *http.Request) { - d(r) // call default director - r.Header.Add("THANOS-TENANT", conf.tenantId) - } - proxy.ErrorHandler = ErrorHandler - proxy.Transport = DebugTransport{} - log.Fatal(http.ListenAndServe(conf.port, proxy)) -} - func ErrorHandler(_ http.ResponseWriter, _ *http.Request, err error) { log.Print("Response from receiver") log.Print(err) @@ -74,32 +54,34 @@ func TestReceive(t *testing.T) { NB: Made with asciiflow.com - you can copy & paste the above there to modify. */ + netName := "e2e_receive_single_ingestor" + t.Parallel() - s, err := e2e.NewScenario("e2e_receive_single_ingestor") + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // Setup Router Ingestor. - i, err := e2ethanos.NewIngestingReceiver(s.SharedDir(), "ingestor") + i, err := e2ethanos.NewIngestingReceiver(e, "ingestor") testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(i)) + testutil.Ok(t, e2e.StartAndWaitReady(i)) // Setup Prometheus - prom, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(i.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(i.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom)) + testutil.Ok(t, e2e.StartAndWaitReady(prom)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{i.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{i.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) // We expect the data from each Prometheus instance to be replicated twice across our ingesting instances - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -143,51 +125,51 @@ func TestReceive(t *testing.T) { */ t.Parallel() - s, err := e2e.NewScenario("e2e_receive_router_replication") + e, err := e2e.NewDockerEnvironment("e2e_receive_router_replication") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // Setup 3 ingestors. - i1, err := e2ethanos.NewIngestingReceiver(s.SharedDir(), "i1") + i1, err := e2ethanos.NewIngestingReceiver(e, "i1") testutil.Ok(t, err) - i2, err := e2ethanos.NewIngestingReceiver(s.SharedDir(), "i2") + i2, err := e2ethanos.NewIngestingReceiver(e, "i2") testutil.Ok(t, err) - i3, err := e2ethanos.NewIngestingReceiver(s.SharedDir(), "i3") + i3, err := e2ethanos.NewIngestingReceiver(e, "i3") testutil.Ok(t, err) h := receive.HashringConfig{ Endpoints: []string{ - i1.GRPCNetworkEndpointFor(s.NetworkName()), - i2.GRPCNetworkEndpointFor(s.NetworkName()), - i3.GRPCNetworkEndpointFor(s.NetworkName()), + i1.InternalEndpoint("grpc"), + i2.InternalEndpoint("grpc"), + i3.InternalEndpoint("grpc"), }, } // Setup 1 distributor - r1, err := e2ethanos.NewRoutingReceiver(s.SharedDir(), "r1", 2, h) + r1, err := e2ethanos.NewRoutingReceiver(e, "r1", 2, h) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(i1, i2, i3, r1)) + testutil.Ok(t, e2e.StartAndWaitReady(i1, i2, i3, r1)) - prom1, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, prom2, prom3)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{i1.GRPCNetworkEndpoint(), i2.GRPCNetworkEndpoint(), i3.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{i1.InternalEndpoint("grpc"), i2.InternalEndpoint("grpc"), i3.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) expectedReplicationFactor := 2.0 - queryAndAssert(t, ctx, q.HTTPEndpoint(), "count(up) by (prometheus)", promclient.QueryOptions{ + queryAndAssert(t, ctx, q.Endpoint("http"), "count(up) by (prometheus)", promclient.QueryOptions{ Deduplicate: false, }, model.Vector{ &model.Sample{ @@ -250,57 +232,57 @@ func TestReceive(t *testing.T) { */ t.Parallel() - s, err := e2e.NewScenario("e2e_receive_routing_tree") + e, err := e2e.NewDockerEnvironment("e2e_receive_routing_tree") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // Setup ingestors. - i1, err := e2ethanos.NewIngestingReceiver(s.SharedDir(), "i1") + i1, err := e2ethanos.NewIngestingReceiver(e, "i1") testutil.Ok(t, err) - i2, err := e2ethanos.NewIngestingReceiver(s.SharedDir(), "i2") + i2, err := e2ethanos.NewIngestingReceiver(e, "i2") testutil.Ok(t, err) - i3, err := e2ethanos.NewIngestingReceiver(s.SharedDir(), "i3") + i3, err := e2ethanos.NewIngestingReceiver(e, "i3") testutil.Ok(t, err) // Setup distributors - r2, err := e2ethanos.NewRoutingReceiver(s.SharedDir(), "r2", 2, receive.HashringConfig{ + r2, err := e2ethanos.NewRoutingReceiver(e, "r2", 2, receive.HashringConfig{ Endpoints: []string{ - i2.GRPCNetworkEndpointFor(s.NetworkName()), - i3.GRPCNetworkEndpointFor(s.NetworkName()), + i2.InternalEndpoint("grpc"), + i3.InternalEndpoint("grpc"), }, }) testutil.Ok(t, err) - r1, err := e2ethanos.NewRoutingReceiver(s.SharedDir(), "r1", 2, receive.HashringConfig{ + r1, err := e2ethanos.NewRoutingReceiver(e, "r1", 2, receive.HashringConfig{ Endpoints: []string{ - r2.GRPCNetworkEndpointFor(s.NetworkName()), - i1.GRPCNetworkEndpointFor(s.NetworkName()), + i1.InternalEndpoint("grpc"), + r2.InternalEndpoint("grpc"), }, }) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(i1, i2, i3, r1, r2)) + testutil.Ok(t, e2e.StartAndWaitReady(i1, i2, i3, r1, r2)) //Setup Prometheuses - prom1, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, prom2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2)) //Setup Querier - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{i1.GRPCNetworkEndpoint(), i2.GRPCNetworkEndpoint(), i3.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{i1.InternalEndpoint("grpc"), i2.InternalEndpoint("grpc"), i3.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) expectedReplicationFactor := 3.0 - queryAndAssert(t, ctx, q.HTTPEndpoint(), "count(up) by (prometheus)", promclient.QueryOptions{ + queryAndAssert(t, ctx, q.Endpoint("http"), "count(up) by (prometheus)", promclient.QueryOptions{ Deduplicate: false, }, model.Vector{ &model.Sample{ @@ -355,52 +337,48 @@ func TestReceive(t *testing.T) { └───────┘ */ t.Parallel() - s, err := e2e.NewScenario("e2e_test_receive_hashring") - testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1) - testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "2", 1) - testutil.Ok(t, err) - r3, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "3", 1) + netName := "e2e_test_receive_hashring" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) h := receive.HashringConfig{ Endpoints: []string{ - r1.GRPCNetworkEndpointFor(s.NetworkName()), - r2.GRPCNetworkEndpointFor(s.NetworkName()), - r3.GRPCNetworkEndpointFor(s.NetworkName()), + netName + "-receive-1:9091", + netName + "-receive-2:9091", + netName + "-receive-3:9091", }, } - // Recreate again, but with hashring config. - r1, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1, h) + // Create with hashring config. + r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1, h) testutil.Ok(t, err) - r2, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "2", 1, h) + r2, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "2", 1, h) testutil.Ok(t, err) - r3, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "3", 1, h) + r3, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "3", 1, h) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(r1, r2, r3)) + testutil.Ok(t, e2e.StartAndWaitReady(r1, r2, r3)) - prom1, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, prom2, prom3)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{r1.GRPCNetworkEndpoint(), r2.GRPCNetworkEndpoint(), r3.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{r1.InternalEndpoint("grpc"), r2.InternalEndpoint("grpc"), r3.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -430,53 +408,48 @@ func TestReceive(t *testing.T) { t.Run("hashring with config watcher", func(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_receive_hashring_config_watcher") - testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + netName := "e2e_test_receive_hashring_config_watcher" - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1) - testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "2", 1) - testutil.Ok(t, err) - r3, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "3", 1) + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) h := receive.HashringConfig{ Endpoints: []string{ - r1.GRPCNetworkEndpointFor(s.NetworkName()), - r2.GRPCNetworkEndpointFor(s.NetworkName()), - r3.GRPCNetworkEndpointFor(s.NetworkName()), + netName + "-receive-1:9091", + netName + "-receive-2:9091", + netName + "-receive-3:9091", }, } - // Recreate again, but with hashring config. + // Create with hashring config. // TODO(kakkoyun): Update config file and wait config watcher to reconcile hashring. - r1, err = e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(s.SharedDir(), s.NetworkName(), "1", 1, h) + r1, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(e, netName, "1", 1, h) testutil.Ok(t, err) - r2, err = e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(s.SharedDir(), s.NetworkName(), "2", 1, h) + r2, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(e, netName, "2", 1, h) testutil.Ok(t, err) - r3, err = e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(s.SharedDir(), s.NetworkName(), "3", 1, h) + r3, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(e, netName, "3", 1, h) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(r1, r2, r3)) + testutil.Ok(t, e2e.StartAndWaitReady(r1, r2, r3)) - prom1, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, prom2, prom3)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{r1.GRPCNetworkEndpoint(), r2.GRPCNetworkEndpoint(), r3.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{r1.InternalEndpoint("grpc"), r2.InternalEndpoint("grpc"), r3.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -506,52 +479,48 @@ func TestReceive(t *testing.T) { t.Run("replication", func(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_receive_replication") + netName := "e2e_test_receive_replication" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // The replication suite creates three receivers but only one // receives Prometheus remote-written data. The querier queries all // receivers and the test verifies that the time series are // replicated to all of the nodes. - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 3) - testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "2", 3) - testutil.Ok(t, err) - r3, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "3", 3) - testutil.Ok(t, err) h := receive.HashringConfig{ Endpoints: []string{ - r1.GRPCNetworkEndpointFor(s.NetworkName()), - r2.GRPCNetworkEndpointFor(s.NetworkName()), - r3.GRPCNetworkEndpointFor(s.NetworkName()), + netName + "-receive-1:9091", + netName + "-receive-2:9091", + netName + "-receive-3:9091", }, } - // Recreate again, but with hashring config. - r1, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 3, h) + // Create with hashring config. + r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 3, h) testutil.Ok(t, err) - r2, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "2", 3, h) + r2, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "2", 3, h) testutil.Ok(t, err) - r3, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "3", 3, h) + r3, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "3", 3, h) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(r1, r2, r3)) + testutil.Ok(t, e2e.StartAndWaitReady(r1, r2, r3)) - prom1, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{r1.GRPCNetworkEndpoint(), r2.GRPCNetworkEndpoint(), r3.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{r1.InternalEndpoint("grpc"), r2.InternalEndpoint("grpc"), r3.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(3), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -581,49 +550,45 @@ func TestReceive(t *testing.T) { t.Run("replication_with_outage", func(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_receive_replication_with_outage") + netName := "e2e_test_receive_replication_with_outage" + + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // The replication suite creates a three-node hashring but one of the // receivers is dead. In this case, replication should still // succeed and the time series should be replicated to the other nodes. - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 3) - testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "2", 3) - testutil.Ok(t, err) - notRunningR3, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "3", 3) - testutil.Ok(t, err) h := receive.HashringConfig{ Endpoints: []string{ - r1.GRPCNetworkEndpointFor(s.NetworkName()), - r2.GRPCNetworkEndpointFor(s.NetworkName()), - notRunningR3.GRPCNetworkEndpointFor(s.NetworkName()), + netName + "-receive-1:9091", + netName + "-receive-2:9091", + netName + "-receive-3:9091", }, } - // Recreate again, but with hashring config. - r1, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 3, h) + // Create with hashring config. + r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 3, h) testutil.Ok(t, err) - r2, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "2", 3, h) + r2, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "2", 3, h) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(r1, r2)) + testutil.Ok(t, e2e.StartAndWaitReady(r1, r2)) - prom1, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.NetworkEndpoint(8081)), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{r1.GRPCNetworkEndpoint(), r2.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{r1.InternalEndpoint("grpc"), r2.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -646,57 +611,42 @@ func TestReceive(t *testing.T) { t.Run("multitenancy", func(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_for_multitenancy") - testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + netName := "e2e_test_for_multitenancy" - // The replication suite creates a three-node hashring but one of the - // receivers is dead. In this case, replication should still - // succeed and the time series should be replicated to the other nodes. - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1) + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) h := receive.HashringConfig{ Endpoints: []string{ - r1.GRPCNetworkEndpointFor(s.NetworkName()), + netName + "-receive-1:9091", }, } - // Recreate again, but with hashring config. - r1, err = e2ethanos.NewRoutingAndIngestingReceiver(s.SharedDir(), s.NetworkName(), "1", 1, h) + r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1, h) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(r1)) - testutil.Ok(t, err) - - conf1 := ReverseProxyConfig{ - tenantId: "tenant-1", - port: ":9097", - target: "http://" + r1.Endpoint(8081), - } - conf2 := ReverseProxyConfig{ - tenantId: "tenant-2", - port: ":9098", - target: "http://" + r1.Endpoint(8081), - } + testutil.Ok(t, e2e.StartAndWaitReady(r1)) - go generateProxy(conf1) - go generateProxy(conf2) + rp1, err := e2ethanos.NewReverseProxy(e, "1", "tenant-1", "http://"+r1.InternalEndpoint("remote-write")) + testutil.Ok(t, err) + rp2, err := e2ethanos.NewReverseProxy(e, "2", "tenant-2", "http://"+r1.InternalEndpoint("remote-write")) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(rp1, rp2)) - prom1, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "1", defaultPromConfig("prom1", 0, "http://172.17.0.1:9097/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, "http://"+rp1.InternalEndpoint("http")+"/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(s.SharedDir(), "2", defaultPromConfig("prom1", 0, "http://172.17.0.1:9098/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, "http://"+rp2.InternalEndpoint("http")+"/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1)) - testutil.Ok(t, s.StartAndWaitReady(prom2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{r1.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{r1.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -708,7 +658,7 @@ func TestReceive(t *testing.T) { }, { "job": "myself", - "prometheus": "prom1", + "prometheus": "prom2", "receive": "1", "replica": "0", "tenant_id": "tenant-2", diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 3e7f6a2069..17c23d1974 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -15,7 +15,7 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" "gopkg.in/yaml.v2" @@ -127,7 +127,7 @@ func reloadRulesHTTP(t *testing.T, ctx context.Context, endpoint string) { testutil.Equals(t, 200, resp.StatusCode) } -func reloadRulesSignal(t *testing.T, r *e2ethanos.Service) { +func reloadRulesSignal(t *testing.T, r *e2e.InstrumentedRunnable) { c := e2e.NewCommand("kill", "-1", "1") _, _, err := r.Exec(c) testutil.Ok(t, err) @@ -197,41 +197,41 @@ func writeTargets(t *testing.T, path string, addrs ...string) { func TestRule(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_rule") + e, err := e2e.NewDockerEnvironment("e2e_test_rule") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) t.Cleanup(cancel) // Prepare work dirs. rulesSubDir := filepath.Join("rules") - rulesPath := filepath.Join(s.SharedDir(), rulesSubDir) + rulesPath := filepath.Join(e.SharedDir(), rulesSubDir) testutil.Ok(t, os.MkdirAll(rulesPath, os.ModePerm)) createRuleFiles(t, rulesPath) amTargetsSubDir := filepath.Join("rules_am_targets") - testutil.Ok(t, os.MkdirAll(filepath.Join(s.SharedDir(), amTargetsSubDir), os.ModePerm)) + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), amTargetsSubDir), os.ModePerm)) queryTargetsSubDir := filepath.Join("rules_query_targets") - testutil.Ok(t, os.MkdirAll(filepath.Join(s.SharedDir(), queryTargetsSubDir), os.ModePerm)) + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), queryTargetsSubDir), os.ModePerm)) - am1, err := e2ethanos.NewAlertmanager(s.SharedDir(), "1") + am1, err := e2ethanos.NewAlertmanager(e, "1") testutil.Ok(t, err) - am2, err := e2ethanos.NewAlertmanager(s.SharedDir(), "2") + am2, err := e2ethanos.NewAlertmanager(e, "2") testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(am1, am2)) + testutil.Ok(t, e2e.StartAndWaitReady(am1, am2)) - r, err := e2ethanos.NewRuler(s.SharedDir(), "1", rulesSubDir, []alert.AlertmanagerConfig{ + r, err := e2ethanos.NewRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{ { EndpointsConfig: http_util.EndpointsConfig{ FileSDConfigs: []http_util.FileSDConfig{ { // FileSD which will be used to register discover dynamically am1. - Files: []string{filepath.Join(e2e.ContainerSharedDir, amTargetsSubDir, "*.yaml")}, + Files: []string{filepath.Join(e2ethanos.ContainerSharedDir, amTargetsSubDir, "*.yaml")}, RefreshInterval: model.Duration(time.Second), }, }, StaticAddresses: []string{ - am2.NetworkHTTPEndpoint(), + am2.InternalEndpoint("http"), }, Scheme: "http", }, @@ -245,7 +245,7 @@ func TestRule(t *testing.T) { FileSDConfigs: []http_util.FileSDConfig{ { // FileSD which will be used to register discover dynamically q. - Files: []string{filepath.Join(e2e.ContainerSharedDir, queryTargetsSubDir, "*.yaml")}, + Files: []string{filepath.Join(e2ethanos.ContainerSharedDir, queryTargetsSubDir, "*.yaml")}, RefreshInterval: model.Duration(time.Second), }, }, @@ -254,11 +254,11 @@ func TestRule(t *testing.T) { }, }) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(r)) + testutil.Ok(t, e2e.StartAndWaitReady(r)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{r.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{r.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) t.Run("no query configured", func(t *testing.T) { // Check for a few evaluations, check all of them failed. @@ -272,9 +272,9 @@ func TestRule(t *testing.T) { var currentFailures float64 t.Run("attach query", func(t *testing.T) { // Attach querier to target files. - writeTargets(t, filepath.Join(s.SharedDir(), queryTargetsSubDir, "targets.yaml"), q.NetworkHTTPEndpoint()) + writeTargets(t, filepath.Join(e.SharedDir(), queryTargetsSubDir, "targets.yaml"), q.InternalEndpoint("http")) - testutil.Ok(t, r.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_rule_query_apis_dns_provider_results"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, r.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_rule_query_apis_dns_provider_results"}, e2e.WaitMissingMetrics())) testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(1), "thanos_rule_alertmanagers_dns_provider_results")) var currentVal float64 @@ -305,7 +305,7 @@ func TestRule(t *testing.T) { }) t.Run("attach am1", func(t *testing.T) { // Attach am1 to target files. - writeTargets(t, filepath.Join(s.SharedDir(), amTargetsSubDir, "targets.yaml"), am1.NetworkHTTPEndpoint()) + writeTargets(t, filepath.Join(e.SharedDir(), amTargetsSubDir, "targets.yaml"), am1.InternalEndpoint("http")) testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(1), "thanos_rule_query_apis_dns_provider_results")) testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(2), "thanos_rule_alertmanagers_dns_provider_results")) @@ -329,7 +329,7 @@ func TestRule(t *testing.T) { }) t.Run("am1 drops again", func(t *testing.T) { - testutil.Ok(t, os.RemoveAll(filepath.Join(s.SharedDir(), amTargetsSubDir, "targets.yaml"))) + testutil.Ok(t, os.RemoveAll(filepath.Join(e.SharedDir(), amTargetsSubDir, "targets.yaml"))) testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(1), "thanos_rule_query_apis_dns_provider_results")) testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(1), "thanos_rule_alertmanagers_dns_provider_results")) @@ -358,31 +358,31 @@ func TestRule(t *testing.T) { t.Run("duplicate am ", func(t *testing.T) { // am2 is already registered in static addresses. - writeTargets(t, filepath.Join(s.SharedDir(), amTargetsSubDir, "targets.yaml"), am2.NetworkHTTPEndpoint()) + writeTargets(t, filepath.Join(e.SharedDir(), amTargetsSubDir, "targets.yaml"), am2.InternalEndpoint("http")) testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(1), "thanos_rule_query_apis_dns_provider_results")) testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(1), "thanos_rule_alertmanagers_dns_provider_results")) }) t.Run("rule groups have last evaluation and evaluation duration set", func(t *testing.T) { - rulegroupCorrectData(t, ctx, r.HTTPEndpoint()) + rulegroupCorrectData(t, ctx, r.Endpoint("http")) }) t.Run("signal reload works", func(t *testing.T) { // Add a new rule via sending sighup createRuleFile(t, fmt.Sprintf("%s/newrule.yaml", rulesPath), testAlertRuleAddedLaterSignal) reloadRulesSignal(t, r) - checkReloadSuccessful(t, ctx, r.HTTPEndpoint(), 4) + checkReloadSuccessful(t, ctx, r.Endpoint("http"), 4) }) t.Run("http reload works", func(t *testing.T) { // Add a new rule via /-/reload. createRuleFile(t, fmt.Sprintf("%s/newrule.yaml", rulesPath), testAlertRuleAddedLaterWebHandler) - reloadRulesHTTP(t, ctx, r.HTTPEndpoint()) - checkReloadSuccessful(t, ctx, r.HTTPEndpoint(), 3) + reloadRulesHTTP(t, ctx, r.Endpoint("http")) + checkReloadSuccessful(t, ctx, r.Endpoint("http"), 3) }) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), "ALERTS", promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "ALERTS", promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -426,7 +426,7 @@ func TestRule(t *testing.T) { }, } - alrts, err := promclient.NewDefaultClient().AlertmanagerAlerts(ctx, mustURLParse(t, "http://"+am2.HTTPEndpoint())) + alrts, err := promclient.NewDefaultClient().AlertmanagerAlerts(ctx, mustURLParse(t, "http://"+am2.Endpoint("http"))) testutil.Ok(t, err) testutil.Equals(t, len(expAlertLabels), len(alrts)) diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index 5b993cd812..77adc6efa7 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/pkg/errors" http_util "github.com/thanos-io/thanos/pkg/http" @@ -31,74 +31,68 @@ func TestRulesAPI_Fanout(t *testing.T) { netName := "e2e_test_rules_fanout" - s, err := e2e.NewScenario(netName) + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) promRulesSubDir := filepath.Join("rules") - testutil.Ok(t, os.MkdirAll(filepath.Join(s.SharedDir(), promRulesSubDir), os.ModePerm)) + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), promRulesSubDir), os.ModePerm)) // Create the abort_on_partial_response alert for Prometheus. // We don't create the warn_on_partial_response alert as Prometheus has strict yaml unmarshalling. - createRuleFile(t, filepath.Join(s.SharedDir(), promRulesSubDir, "rules.yaml"), testAlertRuleAbortOnPartialResponse) + createRuleFile(t, filepath.Join(e.SharedDir(), promRulesSubDir, "rules.yaml"), testAlertRuleAbortOnPartialResponse) thanosRulesSubDir := filepath.Join("thanos-rules") - testutil.Ok(t, os.MkdirAll(filepath.Join(s.SharedDir(), thanosRulesSubDir), os.ModePerm)) - createRuleFiles(t, filepath.Join(s.SharedDir(), thanosRulesSubDir)) + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), thanosRulesSubDir), os.ModePerm)) + createRuleFiles(t, filepath.Join(e.SharedDir(), thanosRulesSubDir)) // 2x Prometheus. prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom1", - defaultPromConfig("ha", 0, "", filepath.Join(e2e.ContainerSharedDir, promRulesSubDir, "*.yaml")), + defaultPromConfig("ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom2", - defaultPromConfig("ha", 1, "", filepath.Join(e2e.ContainerSharedDir, promRulesSubDir, "*.yaml")), + defaultPromConfig("ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) - // 2x Rulers. - r1, err := e2ethanos.NewRuler(s.SharedDir(), "rule1", thanosRulesSubDir, nil, nil) - testutil.Ok(t, err) - r2, err := e2ethanos.NewRuler(s.SharedDir(), "rule2", thanosRulesSubDir, nil, nil) - testutil.Ok(t, err) - - stores := []string{sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint(), r1.NetworkEndpointFor(s.NetworkName(), 9091), r2.NetworkEndpointFor(s.NetworkName(), 9091)} - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "query", stores). + stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), netName + "-rule-rule1:9091", netName + "-rule-rule2:9091"} + q, err := e2ethanos.NewQuerierBuilder(e, "query", stores). WithRuleAddresses(stores). Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) queryCfg := []query.Config{ { EndpointsConfig: http_util.EndpointsConfig{ - StaticAddresses: []string{q.NetworkHTTPEndpoint()}, + StaticAddresses: []string{q.InternalEndpoint("http")}, Scheme: "http", }, }, } // Recreate rulers with the corresponding query config. - r1, err = e2ethanos.NewRuler(s.SharedDir(), "rule1", thanosRulesSubDir, nil, queryCfg) + r1, err := e2ethanos.NewRuler(e, "rule1", thanosRulesSubDir, nil, queryCfg) testutil.Ok(t, err) - r2, err = e2ethanos.NewRuler(s.SharedDir(), "rule2", thanosRulesSubDir, nil, queryCfg) + r2, err := e2ethanos.NewRuler(e, "rule2", thanosRulesSubDir, nil, queryCfg) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(r1, r2)) + testutil.Ok(t, e2e.StartAndWaitReady(r1, r2)) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(4), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(4), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - ruleAndAssert(t, ctx, q.HTTPEndpoint(), "", []*rulespb.RuleGroup{ + ruleAndAssert(t, ctx, q.Endpoint("http"), "", []*rulespb.RuleGroup{ { Name: "example_abort", File: "/shared/rules/rules.yaml", diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index f34919d901..1366ec2a0d 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" - e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/efficientgo/e2e" + e2edb "github.com/efficientgo/e2e/db" "github.com/go-kit/kit/log" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" @@ -36,20 +36,21 @@ import ( func TestStoreGateway(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_store_gateway") + e, err := e2e.NewDockerEnvironment("e2e_test_store_gateway") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) - m := e2edb.NewMinio(8080, "thanos") - testutil.Ok(t, s.StartAndWaitReady(m)) + const bucket = "store_gateway_test" + m := e2ethanos.NewMinio(e, "thanos-minio", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) - s1, err := e2ethanos.NewStoreGW(s.SharedDir(), "1", client.BucketConfig{ + s1, err := e2ethanos.NewStoreGW(e, "1", client.BucketConfig{ Type: client.S3, Config: s3.Config{ - Bucket: "thanos", + Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.NetworkHTTPEndpoint(), + Endpoint: m.InternalEndpoint("http"), Insecure: true, }, }, relabel.Config{ @@ -58,16 +59,16 @@ func TestStoreGateway(t *testing.T) { SourceLabels: model.LabelNames{"ext1"}, }) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(s1)) + testutil.Ok(t, e2e.StartAndWaitReady(s1)) // Ensure bucket UI. - ensureGETStatusCode(t, http.StatusOK, "http://"+path.Join(s1.HTTPEndpoint(), "loaded")) + ensureGETStatusCode(t, http.StatusOK, "http://"+path.Join(s1.Endpoint("http"), "loaded")) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{s1.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{s1.InternalEndpoint("grpc")}).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) - dir := filepath.Join(s.SharedDir(), "tmp") - testutil.Ok(t, os.MkdirAll(filepath.Join(s.SharedDir(), dir), os.ModePerm)) + dir := filepath.Join(e.SharedDir(), "tmp") + testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), dir), os.ModePerm)) series := []labels.Labels{labels.FromStrings("a", "1", "b", "2")} extLset := labels.FromStrings("ext1", "value1", "replica", "1") @@ -75,7 +76,7 @@ func TestStoreGateway(t *testing.T) { extLset3 := labels.FromStrings("ext1", "value2", "replica", "3") extLset4 := labels.FromStrings("ext1", "value1", "replica", "3") - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) t.Cleanup(cancel) now := time.Now() @@ -89,10 +90,10 @@ func TestStoreGateway(t *testing.T) { testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, s3.Config{ - Bucket: "thanos", + Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.HTTPEndpoint(), // We need separate client config, when connecting to minio from outside. + Endpoint: m.Endpoint("http"), // We need separate client config, when connecting to minio from outside. Insecure: true, }, "test-feed") testutil.Ok(t, err) @@ -112,7 +113,7 @@ func TestStoreGateway(t *testing.T) { testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_load_failures_total")) t.Run("query works", func(t *testing.T) { - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), "{a=\"1\"}", + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", promclient.QueryOptions{ Deduplicate: false, }, @@ -137,7 +138,7 @@ func TestStoreGateway(t *testing.T) { testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(6), "thanos_bucket_store_series_data_fetched")) testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(2), "thanos_bucket_store_series_blocks_queried")) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), "{a=\"1\"}", + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", promclient.QueryOptions{ Deduplicate: true, }, @@ -167,7 +168,7 @@ func TestStoreGateway(t *testing.T) { testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_load_failures_total")) // TODO(bwplotka): Entries are still in LRU cache. - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), "{a=\"1\"}", + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", promclient.QueryOptions{ Deduplicate: false, }, @@ -196,7 +197,7 @@ func TestStoreGateway(t *testing.T) { testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(1), "thanos_bucket_store_block_drops_total")) testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_load_failures_total")) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), "{a=\"1\"}", + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", promclient.QueryOptions{ Deduplicate: false, }, @@ -229,7 +230,7 @@ func TestStoreGateway(t *testing.T) { testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(1+1), "thanos_bucket_store_block_drops_total")) testutil.Ok(t, s1.WaitSumMetrics(e2e.Equals(0), "thanos_bucket_store_block_load_failures_total")) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), "{a=\"1\"}", + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "{a=\"1\"}", promclient.QueryOptions{ Deduplicate: false, }, diff --git a/test/e2e/targets_api_test.go b/test/e2e/targets_api_test.go index 15fee88d25..84f4a0f00c 100644 --- a/test/e2e/targets_api_test.go +++ b/test/e2e/targets_api_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" + "github.com/efficientgo/e2e" "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/promclient" @@ -29,13 +29,13 @@ func TestTargetsAPI_Fanout(t *testing.T) { netName := "e2e_test_targets_fanout" - s, err := e2e.NewScenario(netName) + e, err := e2e.NewDockerEnvironment(netName) testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // 2x Prometheus. prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "localhost:80"), @@ -43,28 +43,28 @@ func TestTargetsAPI_Fanout(t *testing.T) { ) testutil.Ok(t, err) prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), + e, netName, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "localhost:80"), e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) - stores := []string{sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint()} - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "query", stores). + stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc")} + q, err := e2ethanos.NewQuerierBuilder(e, "query", stores). WithTargetAddresses(stores). Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - targetAndAssert(t, ctx, q.HTTPEndpoint(), "", &targetspb.TargetDiscovery{ + targetAndAssert(t, ctx, q.Endpoint("http"), "", &targetspb.TargetDiscovery{ ActiveTargets: []*targetspb.ActiveTarget{ { DiscoveredLabels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ diff --git a/test/e2e/tools_bucket_web_test.go b/test/e2e/tools_bucket_web_test.go index 97d6c0db00..0c9f03bac6 100644 --- a/test/e2e/tools_bucket_web_test.go +++ b/test/e2e/tools_bucket_web_test.go @@ -15,8 +15,8 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" - e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/efficientgo/e2e" + e2edb "github.com/efficientgo/e2e/db" "github.com/go-kit/kit/log" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" @@ -25,6 +25,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/s3" + "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) @@ -32,26 +33,29 @@ import ( func TestToolsBucketWebExternalPrefixWithoutReverseProxy(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_tools_bucket_web_route_prefix") + e, err := e2e.NewDockerEnvironment("e2e_test_tools_bucket_web_route_prefix") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) externalPrefix := "testThanos" - m := e2edb.NewMinio(8080, "thanos") - testutil.Ok(t, s.StartAndWaitReady(m)) + + const bucket = "compact_test" + m := e2ethanos.NewMinio(e, "thanos", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) svcConfig := client.BucketConfig{ Type: client.S3, Config: s3.Config{ - Bucket: "thanos", + Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.NetworkHTTPEndpoint(), + Endpoint: m.Endpoint("http"), Insecure: true, }, } b, err := e2ethanos.NewToolsBucketWeb( + e, "1", svcConfig, "", @@ -61,22 +65,22 @@ func TestToolsBucketWebExternalPrefixWithoutReverseProxy(t *testing.T) { "", ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(b)) + testutil.Ok(t, e2e.StartAndWaitReady(b)) - checkNetworkRequests(t, "http://"+b.HTTPEndpoint()+"/"+externalPrefix+"/blocks") + checkNetworkRequests(t, "http://"+b.Endpoint("http")+"/"+externalPrefix+"/blocks") } func TestToolsBucketWebExternalPrefix(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_tools_bucket_web_external_prefix") + e, err := e2e.NewDockerEnvironment("e2e_test_tools_bucket_web_external_prefix") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) externalPrefix := "testThanos" const bucket = "toolsBucketWeb_test" - m := e2edb.NewMinio(8080, bucket) - testutil.Ok(t, s.StartAndWaitReady(m)) + m := e2ethanos.NewMinio(e, "thanos", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) svcConfig := client.BucketConfig{ Type: client.S3, @@ -84,12 +88,13 @@ func TestToolsBucketWebExternalPrefix(t *testing.T) { Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.NetworkHTTPEndpoint(), + Endpoint: m.Endpoint("http"), Insecure: true, }, } b, err := e2ethanos.NewToolsBucketWeb( + e, "1", svcConfig, "", @@ -99,9 +104,9 @@ func TestToolsBucketWebExternalPrefix(t *testing.T) { "", ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(b)) + testutil.Ok(t, e2e.StartAndWaitReady(b)) - toolsBucketWebURL := mustURLParse(t, "http://"+b.HTTPEndpoint()+"/"+externalPrefix) + toolsBucketWebURL := mustURLParse(t, "http://"+b.Endpoint("http")+"/"+externalPrefix) toolsBucketWebProxy := httptest.NewServer(e2ethanos.NewSingleHostReverseProxy(toolsBucketWebURL, externalPrefix)) t.Cleanup(toolsBucketWebProxy.Close) @@ -112,15 +117,15 @@ func TestToolsBucketWebExternalPrefix(t *testing.T) { func TestToolsBucketWebExternalPrefixAndRoutePrefix(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_tools_bucket_web_and_route_prefix") + e, err := e2e.NewDockerEnvironment("e2e_test_tools_bucket_web_and_route_prefix") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) externalPrefix := "testThanos" routePrefix := "test" const bucket = "toolsBucketWeb_test" - m := e2edb.NewMinio(8080, bucket) - testutil.Ok(t, s.StartAndWaitReady(m)) + m := e2ethanos.NewMinio(e, "thanos", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) svcConfig := client.BucketConfig{ Type: client.S3, @@ -128,12 +133,13 @@ func TestToolsBucketWebExternalPrefixAndRoutePrefix(t *testing.T) { Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.NetworkHTTPEndpoint(), + Endpoint: m.Endpoint("http"), Insecure: true, }, } b, err := e2ethanos.NewToolsBucketWeb( + e, "1", svcConfig, routePrefix, @@ -143,9 +149,9 @@ func TestToolsBucketWebExternalPrefixAndRoutePrefix(t *testing.T) { "", ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(b)) + testutil.Ok(t, e2e.StartAndWaitReady(b)) - toolsBucketWebURL := mustURLParse(t, "http://"+b.HTTPEndpoint()+"/"+routePrefix) + toolsBucketWebURL := mustURLParse(t, "http://"+b.Endpoint("http")+"/"+routePrefix) toolsBucketWebProxy := httptest.NewServer(e2ethanos.NewSingleHostReverseProxy(toolsBucketWebURL, externalPrefix)) t.Cleanup(toolsBucketWebProxy.Close) @@ -156,25 +162,25 @@ func TestToolsBucketWebExternalPrefixAndRoutePrefix(t *testing.T) { func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { t.Parallel() // Create network. - s, err := e2e.NewScenario("e2e_test_tools_bucket_web_time_and_relabel_filter") + e, err := e2e.NewDockerEnvironment("e2e_test_tools_bucket_web_time_and_relabel_filter") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) // Create Minio. const bucket = "toolsBucketWeb_test" - m := e2edb.NewMinio(8080, bucket) - testutil.Ok(t, s.StartAndWaitReady(m)) + m := e2ethanos.NewMinio(e, "thanos", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) // Create bucket. logger := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(logger, s3.Config{ Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.HTTPEndpoint(), + Endpoint: m.Endpoint("http"), Insecure: true, }, "tools") testutil.Ok(t, err) // Create share dir for upload. - dir := filepath.Join(s.SharedDir(), "tmp") + dir := filepath.Join(e.SharedDir(), "tmp") testutil.Ok(t, os.MkdirAll(dir, os.ModePerm)) // Upload blocks. now, err := time.Parse(time.RFC3339, "2021-07-24T08:00:00Z") @@ -200,9 +206,14 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { }, } for _, b := range blocks { - id, err := b.Create(context.Background(), dir, 0, b.hashFunc) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + + id, err := b.Create(ctx, dir, 0, b.hashFunc) testutil.Ok(t, err) - testutil.Ok(t, objstore.UploadDir(context.Background(), logger, bkt, path.Join(dir, id.String()), id.String())) + testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { + return objstore.UploadDir(ctx, logger, bkt, path.Join(dir, id.String()), id.String()) + })) } // Start thanos tool bucket web. svcConfig := client.BucketConfig{ @@ -211,11 +222,12 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { Bucket: bucket, AccessKey: e2edb.MinioAccessKey, SecretKey: e2edb.MinioSecretKey, - Endpoint: m.NetworkHTTPEndpoint(), + Endpoint: m.InternalEndpoint("http"), Insecure: true, }, } b, err := e2ethanos.NewToolsBucketWeb( + e, "1", svcConfig, "", @@ -228,9 +240,9 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { source_labels: ["tenant_id"]`, ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(b)) + testutil.Ok(t, e2e.StartAndWaitReady(b)) // Request blocks api. - resp, err := http.DefaultClient.Get("http://" + b.HTTPEndpoint() + "/api/v1/blocks") + resp, err := http.DefaultClient.Get("http://" + b.Endpoint("http") + "/api/v1/blocks") testutil.Ok(t, err) testutil.Equals(t, http.StatusOK, resp.StatusCode) defer resp.Body.Close() From ffcf82381e2e1237781f22d972e050890bb45f2c Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Fri, 27 Aug 2021 16:26:48 +0200 Subject: [PATCH 3/5] go mod update; adjust gitignore Signed-off-by: Matej Gera --- .gitignore | 2 +- go.mod | 3 ++- go.sum | 13 +++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 4e1783a9b8..e5b6dfce00 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ kube/.minikube # Ignore e2e working dirs. data/ -test/e2e/e2e_integration_test* +test/e2e/e2e_* # Ignore promu artifacts. /.build diff --git a/go.mod b/go.mod index 0586501dd0..6da7149008 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/cortexproject/cortex v1.8.1-0.20210422151339-cf1c444e0905 github.com/davecgh/go-spew v1.1.1 github.com/efficientgo/e2e v0.9.0 + github.com/efficientgo/tools/core v0.0.0-20210129205121-421d0828c9a6 github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb github.com/fatih/structtag v1.1.0 @@ -86,7 +87,7 @@ replace ( github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab // TODO(bwplotka): Remove when dev finishes. - github.com/efficientgo/e2e => github.com/efficientgo/e2e v0.10.0 + github.com/efficientgo/e2e => github.com/efficientgo/e2e v0.11.1-0.20210825193253-b28b6c291075 github.com/efficientgo/tools/core => github.com/efficientgo/tools/core v0.0.0-20210731122119-5d4a0645ce9a // Update to v1.1.1 to make sure windows CI pass. github.com/elastic/go-sysinfo => github.com/elastic/go-sysinfo v1.1.1 diff --git a/go.sum b/go.sum index 89e4cc817d..e2bb140b21 100644 --- a/go.sum +++ b/go.sum @@ -229,6 +229,7 @@ github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aa github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -244,6 +245,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -335,8 +338,8 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/efficientgo/e2e v0.10.0 h1:QFgE7W06nYLmASJ4HKUXanHCkKhmz0sIu5ym4zv0Ibs= -github.com/efficientgo/e2e v0.10.0/go.mod h1:5Z90zeIm2FTFD0xmhNn+vXGlOnFiizmnOMZjtLAFIUw= +github.com/efficientgo/e2e v0.11.1-0.20210825193253-b28b6c291075 h1:V29Xn64yy+6C2l0c7x618EhmsWzVJ1g3WC+BwQ7b9Z0= +github.com/efficientgo/e2e v0.11.1-0.20210825193253-b28b6c291075/go.mod h1:vDnF4AAEZmO0mvyFIATeDJPFaSRM7ywaOnKd61zaSoE= github.com/efficientgo/tools/core v0.0.0-20210731122119-5d4a0645ce9a h1:Az9zRvQubUIHE+tHAm0gG7Dwge08V8Q/9uNSIFjFm+A= github.com/efficientgo/tools/core v0.0.0-20210731122119-5d4a0645ce9a/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 h1:kM/ALyvAnTrwSB+nlKqoKaDnZbInp1YImZvW+gtHwc8= @@ -374,6 +377,7 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -541,7 +545,9 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= github.com/gocql/gocql v0.0.0-20200121121104-95d072f1b5bb/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= github.com/gocql/gocql v0.0.0-20200526081602-cd04bd7f22a7/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968 h1:s+PDl6lozQ+dEUtUtQnO7+A2iPG3sK1pI4liU+jxn90= github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -1033,6 +1039,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e h1:4cPxUYdgaGzZIT5/j0IfqOrrXmq6bG8AwvwisMXpdrg= github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= @@ -1284,6 +1292,7 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/weaveworks/common v0.0.0-20200206153930-760e36ae819a/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw= github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY= From 49c3e43feb94a7a61d459a1973174928ef17d2c9 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Sat, 11 Sep 2021 00:22:13 +0200 Subject: [PATCH 4/5] Fix condition in Query FE Signed-off-by: Matej Gera --- test/e2e/query_frontend_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index 9a5f25dc80..99a2818838 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -113,7 +113,7 @@ func TestQueryFrontend(t *testing.T) { 14, promclient.QueryOptions{}, func(res model.Matrix) error { - if len(res) > 0 { + if len(res) == 0 { return errors.Errorf("expected some results, got nothing") } return nil @@ -155,7 +155,7 @@ func TestQueryFrontend(t *testing.T) { 14, promclient.QueryOptions{}, func(res model.Matrix) error { - if len(res) > 0 { + if len(res) == 0 { return errors.Errorf("expected some results, got nothing") } return nil @@ -200,7 +200,7 @@ func TestQueryFrontend(t *testing.T) { 14, promclient.QueryOptions{}, func(res model.Matrix) error { - if len(res) > 0 { + if len(res) == 0 { return errors.Errorf("expected some results, got nothing") } return nil @@ -465,7 +465,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { 14, promclient.QueryOptions{}, func(res model.Matrix) error { - if len(res) > 0 { + if len(res) == 0 { return errors.Errorf("expected some results, got nothing") } return nil @@ -495,7 +495,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { 14, promclient.QueryOptions{}, func(res model.Matrix) error { - if len(res) > 0 { + if len(res) == 0 { return errors.Errorf("expected some results, got nothing") } return nil From 36f921a595041b1f424792a51639980d6605d28d Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Tue, 21 Sep 2021 21:50:28 +0200 Subject: [PATCH 5/5] Simplify propagation of container hostnames - Remove manually building hostname with netName - Use InternalEndpoint instead - Take advantage of future runnable to get hostname addresses for configs Signed-off-by: Matej Gera --- test/e2e/e2ethanos/service.go | 40 +++++-- test/e2e/e2ethanos/services.go | 196 +++++++++++++++++++------------- test/e2e/exemplars_api_test.go | 45 ++++---- test/e2e/metadata_api_test.go | 6 +- test/e2e/query_frontend_test.go | 12 +- test/e2e/query_test.go | 65 +++++------ test/e2e/receive_test.go | 129 +++++++++++---------- test/e2e/rule_test.go | 102 +++++++++-------- test/e2e/rules_api_test.go | 21 ++-- test/e2e/targets_api_test.go | 6 +- 10 files changed, 337 insertions(+), 285 deletions(-) diff --git a/test/e2e/e2ethanos/service.go b/test/e2e/e2ethanos/service.go index 35d8d4300f..2a8d0bd31f 100644 --- a/test/e2e/e2ethanos/service.go +++ b/test/e2e/e2ethanos/service.go @@ -11,9 +11,9 @@ import ( ) type Port struct { - name string - portNum int - isMetrics bool + Name string + PortNum int + IsMetrics bool } func NewService( @@ -25,6 +25,23 @@ func NewService( http, grpc int, otherPorts ...Port, ) *e2e.InstrumentedRunnable { + return newUninitiatedService(e, name, http, grpc, otherPorts...).Init( + e2e.StartOptions{ + Image: image, + Command: command, + Readiness: readiness, + User: strconv.Itoa(os.Getuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, + ) +} + +func newUninitiatedService( + e e2e.Environment, + name string, + http, grpc int, + otherPorts ...Port, +) *e2e.FutureInstrumentedRunnable { metricsPorts := "http" ports := map[string]int{ "http": http, @@ -32,14 +49,23 @@ func NewService( } for _, op := range otherPorts { - ports[op.name] = op.portNum + ports[op.Name] = op.PortNum - if op.isMetrics { - metricsPorts = op.name + if op.IsMetrics { + metricsPorts = op.Name } } - return e2e.NewInstrumentedRunnable(e, name, ports, metricsPorts).Init( + return e2e.NewInstrumentedRunnable(e, name, ports, metricsPorts) +} + +func initiateService( + service *e2e.FutureInstrumentedRunnable, + image string, + command e2e.Command, + readiness *e2e.HTTPReadinessProbe, +) *e2e.InstrumentedRunnable { + return service.Init( e2e.StartOptions{ Image: image, Command: command, diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index e471baff8b..e129d1dd6c 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -103,11 +103,11 @@ func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeat return prom, container, nil } -func NewPrometheusWithSidecar(e e2e.Environment, netName, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { - return NewPrometheusWithSidecarCustomImage(e, netName, name, config, promImage, DefaultImage(), enableFeatures...) +func NewPrometheusWithSidecar(e e2e.Environment, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { + return NewPrometheusWithSidecarCustomImage(e, name, config, promImage, DefaultImage(), enableFeatures...) } -func NewPrometheusWithSidecarCustomImage(e e2e.Environment, netName, name, config, promImage string, sidecarImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { +func NewPrometheusWithSidecarCustomImage(e e2e.Environment, name, config, promImage string, sidecarImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { prom, dataDir, err := NewPrometheus(e, name, config, promImage, enableFeatures...) if err != nil { return nil, nil, err @@ -207,7 +207,52 @@ func (q *QuerierBuilder) WithTracingConfig(tracingConfig string) *QuerierBuilder return q } +func (q *QuerierBuilder) BuildUninitiated() *e2e.FutureInstrumentedRunnable { + return newUninitiatedService( + q.environment, + fmt.Sprintf("querier-%v", q.name), + 8080, + 9091, + ) +} + +func (q *QuerierBuilder) Initiate(service *e2e.FutureInstrumentedRunnable, storeAddresses ...string) (*e2e.InstrumentedRunnable, error) { + q.storeAddresses = storeAddresses + args, err := q.collectArgs() + if err != nil { + return nil, err + } + + querier := initiateService( + service, + q.image, + e2e.NewCommand("query", args...), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + ) + + return querier, nil +} + func (q *QuerierBuilder) Build() (*e2e.InstrumentedRunnable, error) { + args, err := q.collectArgs() + if err != nil { + return nil, err + } + + querier := NewService( + q.environment, + fmt.Sprintf("querier-%v", q.name), + q.image, + e2e.NewCommand("query", args...), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + 8080, + 9091, + ) + + return querier, nil +} + +func (q *QuerierBuilder) collectArgs() ([]string, error) { const replicaLabel = "replica" args := e2e.BuildArgs(map[string]string{ @@ -277,34 +322,31 @@ func (q *QuerierBuilder) Build() (*e2e.InstrumentedRunnable, error) { args = append(args, "--tracing.config="+q.tracingConfig) } - querier := NewService( - q.environment, - fmt.Sprintf("querier-%v", q.name), - q.image, - e2e.NewCommand("query", args...), - e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), - 8080, - 9091, - ) - - return querier, nil + return args, nil } func RemoteWriteEndpoint(addr string) string { return fmt.Sprintf("http://%s/api/v1/receive", addr) } -// NewRoutingAndIngestingReceiver creates a Thanos Receive instances that is configured both for ingesting samples and routing samples to other receivers. -func NewRoutingAndIngestingReceiver(e e2e.Environment, networkName, name string, replicationFactor int, hashring ...receive.HashringConfig) (*e2e.InstrumentedRunnable, error) { +// NewUninitiatedReceiver returns a future receiver that can be initiated. It is useful +// for obtaining a receiver address for hashring before the receiver is started. +func NewUninitiatedReceiver(e e2e.Environment, name string) *e2e.FutureInstrumentedRunnable { + return newUninitiatedService(e, fmt.Sprintf("receive-%v", name), 8080, 9091, Port{Name: "remote-write", PortNum: 8081}) +} + +// NewRoutingAndIngestingReceiverFromService creates a Thanos Receive instances from an unitiated service. +// It is configured both for ingesting samples and routing samples to other receivers. +func NewRoutingAndIngestingReceiverFromService(service *e2e.FutureInstrumentedRunnable, sharedDir string, replicationFactor int, hashring ...receive.HashringConfig) (*e2e.InstrumentedRunnable, error) { var localEndpoint string if len(hashring) == 0 { localEndpoint = "0.0.0.0:9091" hashring = []receive.HashringConfig{{Endpoints: []string{localEndpoint}}} } else { - localEndpoint = networkName + fmt.Sprintf("-receive-%v", name) + ":9091" + localEndpoint = service.InternalEndpoint("grpc") } - dir := filepath.Join(e.SharedDir(), "data", "receive", name) + dir := filepath.Join(sharedDir, "data", "receive", service.Name()) dataDir := filepath.Join(dir, "data") - container := filepath.Join(ContainerSharedDir, "data", "receive", name) + container := filepath.Join(ContainerSharedDir, "data", "receive", service.Name()) if err := os.MkdirAll(dataDir, 0750); err != nil { return nil, errors.Wrap(err, "create receive dir") } @@ -313,17 +355,17 @@ func NewRoutingAndIngestingReceiver(e e2e.Environment, networkName, name string, return nil, errors.Wrapf(err, "generate hashring file: %v", hashring) } - receiver := NewService(e, - fmt.Sprintf("receive-%v", name), + receiver := initiateService( + service, DefaultImage(), // TODO(bwplotka): BuildArgs should be interface. e2e.NewCommand("receive", e2e.BuildArgs(map[string]string{ - "--debug.name": fmt.Sprintf("receive-%v", name), + "--debug.name": service.Name(), "--grpc-address": ":9091", "--grpc-grace-period": "0s", "--http-address": ":8080", "--remote-write.address": ":8081", - "--label": fmt.Sprintf(`receive="%s"`, name), + "--label": fmt.Sprintf(`receive="%s"`, service.Name()), "--tsdb.path": filepath.Join(container, "data"), "--log.level": infoLogLevel, "--receive.replication-factor": strconv.Itoa(replicationFactor), @@ -331,9 +373,54 @@ func NewRoutingAndIngestingReceiver(e e2e.Environment, networkName, name string, "--receive.hashrings": string(b), })...), e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), - 8080, - 9091, - Port{name: "remote-write", portNum: 8081}, + ) + + return receiver, nil +} + +func NewRoutingAndIngestingReceiverWithConfigWatcher(service *e2e.FutureInstrumentedRunnable, sharedDir string, replicationFactor int, hashring ...receive.HashringConfig) (*e2e.InstrumentedRunnable, error) { + var localEndpoint string + if len(hashring) == 0 { + localEndpoint = "0.0.0.0:9091" + hashring = []receive.HashringConfig{{Endpoints: []string{localEndpoint}}} + } else { + localEndpoint = service.InternalEndpoint("grpc") + } + + dir := filepath.Join(sharedDir, "data", "receive", service.Name()) + dataDir := filepath.Join(dir, "data") + container := filepath.Join(ContainerSharedDir, "data", "receive", service.Name()) + if err := os.MkdirAll(dataDir, 0750); err != nil { + return nil, errors.Wrap(err, "create receive dir") + } + b, err := json.Marshal(hashring) + if err != nil { + return nil, errors.Wrapf(err, "generate hashring file: %v", hashring) + } + + if err := ioutil.WriteFile(filepath.Join(dir, "hashrings.json"), b, 0600); err != nil { + return nil, errors.Wrap(err, "creating receive config") + } + + receiver := initiateService( + service, + DefaultImage(), + // TODO(bwplotka): BuildArgs should be interface. + e2e.NewCommand("receive", e2e.BuildArgs(map[string]string{ + "--debug.name": service.Name(), + "--grpc-address": ":9091", + "--grpc-grace-period": "0s", + "--http-address": ":8080", + "--remote-write.address": ":8081", + "--label": fmt.Sprintf(`receive="%s"`, service.Name()), + "--tsdb.path": filepath.Join(container, "data"), + "--log.level": infoLogLevel, + "--receive.replication-factor": strconv.Itoa(replicationFactor), + "--receive.local-endpoint": localEndpoint, + "--receive.hashrings-file": filepath.Join(container, "hashrings.json"), + "--receive.hashrings-file-refresh-interval": "5s", + })...), + e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), ) return receiver, nil @@ -377,7 +464,7 @@ func NewRoutingReceiver(e e2e.Environment, name string, replicationFactor int, h e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, - Port{name: "remote-write", portNum: 8081}, + Port{Name: "remote-write", PortNum: 8081}, ) return receiver, nil @@ -408,58 +495,7 @@ func NewIngestingReceiver(e e2e.Environment, name string) (*e2e.InstrumentedRunn e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, - Port{name: "remote-write", portNum: 8081}, - ) - - return receiver, nil -} - -func NewRoutingAndIngestingReceiverWithConfigWatcher(e e2e.Environment, networkName, name string, replicationFactor int, hashring ...receive.HashringConfig) (*e2e.InstrumentedRunnable, error) { - var localEndpoint string - if len(hashring) == 0 { - localEndpoint = "0.0.0.0:9091" - hashring = []receive.HashringConfig{{Endpoints: []string{localEndpoint}}} - } else { - localEndpoint = networkName + fmt.Sprintf("-receive-%v", name) + ":9091" - } - - dir := filepath.Join(e.SharedDir(), "data", "receive", name) - dataDir := filepath.Join(dir, "data") - container := filepath.Join(ContainerSharedDir, "data", "receive", name) - if err := os.MkdirAll(dataDir, 0750); err != nil { - return nil, errors.Wrap(err, "create receive dir") - } - b, err := json.Marshal(hashring) - if err != nil { - return nil, errors.Wrapf(err, "generate hashring file: %v", hashring) - } - - if err := ioutil.WriteFile(filepath.Join(dir, "hashrings.json"), b, 0600); err != nil { - return nil, errors.Wrap(err, "creating receive config") - } - - receiver := NewService(e, - fmt.Sprintf("receive-%v", name), - DefaultImage(), - // TODO(bwplotka): BuildArgs should be interface. - e2e.NewCommand("receive", e2e.BuildArgs(map[string]string{ - "--debug.name": fmt.Sprintf("receive-%v", name), - "--grpc-address": ":9091", - "--grpc-grace-period": "0s", - "--http-address": ":8080", - "--remote-write.address": ":8081", - "--label": fmt.Sprintf(`receive="%s"`, name), - "--tsdb.path": filepath.Join(container, "data"), - "--log.level": infoLogLevel, - "--receive.replication-factor": strconv.Itoa(replicationFactor), - "--receive.local-endpoint": localEndpoint, - "--receive.hashrings-file": filepath.Join(container, "hashrings.json"), - "--receive.hashrings-file-refresh-interval": "5s", - })...), - e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), - 8080, - 9091, - Port{name: "remote-write", portNum: 8081}, + Port{Name: "remote-write", PortNum: 8081}, ) return receiver, nil @@ -495,7 +531,7 @@ func NewRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Alertman "--label": fmt.Sprintf(`replica="%s"`, name), "--data-dir": container, "--rule-file": filepath.Join(ContainerSharedDir, ruleSubDir, "*.yaml"), - "--eval-interval": "3s", + "--eval-interval": "1s", "--alertmanagers.config": string(amCfgBytes), "--alertmanagers.sd-dns-interval": "1s", "--log.level": infoLogLevel, diff --git a/test/e2e/exemplars_api_test.go b/test/e2e/exemplars_api_test.go index 1fda00afee..13aaffca85 100644 --- a/test/e2e/exemplars_api_test.go +++ b/test/e2e/exemplars_api_test.go @@ -25,9 +25,6 @@ const ( func TestExemplarsAPI_Fanout(t *testing.T) { t.Parallel() - - netName := "e2e_test_exemplars_fanout" - var ( prom1, prom2 *e2e.InstrumentedRunnable sidecar1, sidecar2 *e2e.InstrumentedRunnable @@ -35,43 +32,45 @@ func TestExemplarsAPI_Fanout(t *testing.T) { e *e2e.DockerEnvironment ) - e, err = e2e.NewDockerEnvironment(netName) + e, err = e2e.NewDockerEnvironment("e2e_test_exemplars_fanout") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) - tracingCfg := fmt.Sprintf(`type: JAEGER -config: - sampler_type: const - sampler_param: 1 - service_name: %s`, netName+"-query") - - stores := []string{netName + "-" + "sidecar-prom1:9091", netName + "-" + "sidecar-prom2:9091"} - q, err := e2ethanos.NewQuerierBuilder( - e, "query", stores...). - WithExemplarAddresses(stores...). - WithTracingConfig(tracingCfg). - Build() - testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(q)) + qBuilder := e2ethanos.NewQuerierBuilder(e, "query") + qUnitiated := qBuilder.BuildUninitiated() - // Recreate Prometheus and sidecar with Thanos query scrape target. prom1, sidecar1, err = e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom1", - defaultPromConfig("ha", 0, "", "", "localhost:9090", q.InternalEndpoint("http")), + defaultPromConfig("ha", 0, "", "", "localhost:9090", qUnitiated.InternalEndpoint("http")), e2ethanos.DefaultPrometheusImage(), e2ethanos.FeatureExemplarStorage, ) testutil.Ok(t, err) prom2, sidecar2, err = e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom2", - defaultPromConfig("ha", 1, "", "", "localhost:9090", q.InternalEndpoint("http")), + defaultPromConfig("ha", 1, "", "", "localhost:9090", qUnitiated.InternalEndpoint("http")), e2ethanos.DefaultPrometheusImage(), e2ethanos.FeatureExemplarStorage, ) + testutil.Ok(t, err) + + tracingCfg := fmt.Sprintf(`type: JAEGER +config: + sampler_type: const + sampler_param: 1 + service_name: %s`, qUnitiated.Name()) + + stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc")} + + qBuilder = qBuilder.WithExemplarAddresses(stores...). + WithTracingConfig(tracingCfg) + + q, err := qBuilder.Initiate(qUnitiated, stores...) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(q)) + testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) diff --git a/test/e2e/metadata_api_test.go b/test/e2e/metadata_api_test.go index 7cc4c6d1a6..096560e64e 100644 --- a/test/e2e/metadata_api_test.go +++ b/test/e2e/metadata_api_test.go @@ -21,9 +21,7 @@ import ( func TestMetadataAPI_Fanout(t *testing.T) { t.Parallel() - netName := "e2e_test_metadata_fanout" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_metadata_fanout") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) @@ -31,7 +29,6 @@ func TestMetadataAPI_Fanout(t *testing.T) { // Each Prometheus scrapes its own metrics and Sidecar's metrics. prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "sidecar-prom1:8080"), e2ethanos.DefaultPrometheusImage(), @@ -40,7 +37,6 @@ func TestMetadataAPI_Fanout(t *testing.T) { prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "sidecar-prom2:8080"), e2ethanos.DefaultPrometheusImage(), diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index 99a2818838..6635555ed0 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -26,15 +26,13 @@ import ( func TestQueryFrontend(t *testing.T) { t.Parallel() - netName := "e2e_test_query_frontend" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_query_frontend") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) @@ -392,15 +390,13 @@ func TestQueryFrontend(t *testing.T) { func TestQueryFrontendMemcachedCache(t *testing.T) { t.Parallel() - netName := "e2e_test_query_frontend_memcached" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_query_frontend_memcached") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index a2bbc514f4..514e91270f 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -98,23 +98,22 @@ func sortResults(res model.Vector) { func TestQuery(t *testing.T) { t.Parallel() - netName := "e2e_test_query" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_query") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) - receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1) + receiver := e2ethanos.NewUninitiatedReceiver(e, "1") + receiverRunnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(receiver, e.SharedDir(), 1) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(receiver)) + testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(e, "e2e_test_query", "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(e, "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3, prom4, sidecar4)) @@ -140,7 +139,7 @@ func TestQuery(t *testing.T) { { "job": "myself", "prometheus": "prom-both-remote-write-and-sidecar", - "receive": "1", + "receive": "receive-1", "replica": "1234", "tenant_id": "default-tenant", }, @@ -172,7 +171,7 @@ func TestQuery(t *testing.T) { { "job": "myself", "prometheus": "prom-both-remote-write-and-sidecar", - "receive": "1", + "receive": "receive-1", "tenant_id": "default-tenant", }, { @@ -253,19 +252,18 @@ func TestQueryExternalPrefixAndRoutePrefix(t *testing.T) { func TestQueryLabelNames(t *testing.T) { t.Parallel() - netName := "e2e_test_query_label_names" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_query_label_names") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) - receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1) + receiver := e2ethanos.NewUninitiatedReceiver(e, "1") + receiverRunnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(receiver, e.SharedDir(), 1) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(receiver)) + testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) @@ -306,19 +304,18 @@ func TestQueryLabelNames(t *testing.T) { func TestQueryLabelValues(t *testing.T) { t.Parallel() - netName := "e2e_test_query_label_values" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_query_label_values") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) - receiver, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1) + receiver := e2ethanos.NewUninitiatedReceiver(e, "1") + receiverRunnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(receiver, e.SharedDir(), 1) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(receiver)) + testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, netName, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) @@ -370,9 +367,7 @@ func TestQueryCompatibilityWithPreInfoAPI(t *testing.T) { } { i := i t.Run(fmt.Sprintf("%+v", tcase), func(t *testing.T) { - netName := fmt.Sprintf("e2e_test_query_comp_query_%d", i) - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment(fmt.Sprintf("e2e_test_query_comp_query_%d", i)) testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) @@ -382,11 +377,13 @@ func TestQueryCompatibilityWithPreInfoAPI(t *testing.T) { // We don't create the warn_on_partial_response alert as Prometheus has strict yaml unmarshalling. createRuleFile(t, filepath.Join(e.SharedDir(), promRulesSubDir, "rules.yaml"), testAlertRuleAbortOnPartialResponse) + qBuilder := e2ethanos.NewQuerierBuilder(e, "1") + qUninit := qBuilder.BuildUninitiated() + p1, s1, err := e2ethanos.NewPrometheusWithSidecarCustomImage( e, - netName, "p1", - defaultPromConfig("p1", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml"), "localhost:9090", netName+"-querier-1:8080"), // TODO(bwplotka): Use newer e2e lib to handle this in type safe manner. + defaultPromConfig("p1", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml"), "localhost:9090", qUninit.InternalEndpoint("http")), e2ethanos.DefaultPrometheusImage(), tcase.sidecarImage, e2ethanos.FeatureExemplarStorage, @@ -395,7 +392,7 @@ func TestQueryCompatibilityWithPreInfoAPI(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(p1, s1)) // Newest querier with old --rules --meta etc flags. - q, err := e2ethanos.NewQuerierBuilder(e, "1", s1.InternalEndpoint("grpc")). + q, err := qBuilder. WithMetadataAddresses(s1.InternalEndpoint("grpc")). WithExemplarAddresses(s1.InternalEndpoint("grpc")). WithTargetAddresses(s1.InternalEndpoint("grpc")). @@ -404,9 +401,9 @@ func TestQueryCompatibilityWithPreInfoAPI(t *testing.T) { config: sampler_type: const sampler_param: 1 - service_name: %s`, netName+"-query")). // Use fake tracing config to trigger exemplar. + service_name: %s`, qUninit.Name())). // Use fake tracing config to trigger exemplar. WithImage(tcase.queryImage). - Build() + Initiate(qUninit, s1.InternalEndpoint("grpc")) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(q)) @@ -583,7 +580,7 @@ func instantQuery(t *testing.T, ctx context.Context, addr, q string, opts promcl logger := log.NewLogfmtLogger(os.Stdout) logger = log.With(logger, "ts", log.DefaultTimestampUTC) - testutil.Ok(t, runutil.RetryWithLog(logger, 3*time.Second, ctx.Done(), func() error { + testutil.Ok(t, runutil.RetryWithLog(logger, 5*time.Second, ctx.Done(), func() error { res, warnings, err := promclient.NewDefaultClient().QueryInstant(ctx, mustURLParse(t, "http://"+addr), q, time.Now(), opts) if err != nil { return err diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index 1150d49142..c41c9b7ddb 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -54,10 +54,8 @@ func TestReceive(t *testing.T) { NB: Made with asciiflow.com - you can copy & paste the above there to modify. */ - netName := "e2e_receive_single_ingestor" - t.Parallel() - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_receive_single_ingestor") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) @@ -338,28 +336,30 @@ func TestReceive(t *testing.T) { */ t.Parallel() - netName := "e2e_test_receive_hashring" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_receive_hashring") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) + r1 := e2ethanos.NewUninitiatedReceiver(e, "1") + r2 := e2ethanos.NewUninitiatedReceiver(e, "2") + r3 := e2ethanos.NewUninitiatedReceiver(e, "3") + h := receive.HashringConfig{ Endpoints: []string{ - netName + "-receive-1:9091", - netName + "-receive-2:9091", - netName + "-receive-3:9091", + r1.InternalEndpoint("grpc"), + r2.InternalEndpoint("grpc"), + r3.InternalEndpoint("grpc"), }, } // Create with hashring config. - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1, h) + r1Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r1, e.SharedDir(), 1, h) testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "2", 1, h) + r2Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r2, e.SharedDir(), 1, h) testutil.Ok(t, err) - r3, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "3", 1, h) + r3Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r3, e.SharedDir(), 1, h) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(r1, r2, r3)) + testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) @@ -384,21 +384,21 @@ func TestReceive(t *testing.T) { { "job": "myself", "prometheus": "prom1", - "receive": "2", + "receive": "receive-2", "replica": "0", "tenant_id": "default-tenant", }, { "job": "myself", "prometheus": "prom2", - "receive": "1", + "receive": "receive-1", "replica": "0", "tenant_id": "default-tenant", }, { "job": "myself", "prometheus": "prom3", - "receive": "2", + "receive": "receive-2", "replica": "0", "tenant_id": "default-tenant", }, @@ -408,29 +408,31 @@ func TestReceive(t *testing.T) { t.Run("hashring with config watcher", func(t *testing.T) { t.Parallel() - netName := "e2e_test_receive_hashring_config_watcher" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_receive_hashring_config_watcher") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) + r1 := e2ethanos.NewUninitiatedReceiver(e, "1") + r2 := e2ethanos.NewUninitiatedReceiver(e, "2") + r3 := e2ethanos.NewUninitiatedReceiver(e, "3") + h := receive.HashringConfig{ Endpoints: []string{ - netName + "-receive-1:9091", - netName + "-receive-2:9091", - netName + "-receive-3:9091", + r1.InternalEndpoint("grpc"), + r2.InternalEndpoint("grpc"), + r3.InternalEndpoint("grpc"), }, } // Create with hashring config. // TODO(kakkoyun): Update config file and wait config watcher to reconcile hashring. - r1, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(e, netName, "1", 1, h) + r1Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(r1, e.SharedDir(), 1, h) testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(e, netName, "2", 1, h) + r2Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(r2, e.SharedDir(), 1, h) testutil.Ok(t, err) - r3, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(e, netName, "3", 1, h) + r3Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverWithConfigWatcher(r3, e.SharedDir(), 1, h) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(r1, r2, r3)) + testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) @@ -455,21 +457,21 @@ func TestReceive(t *testing.T) { { "job": "myself", "prometheus": "prom1", - "receive": "2", + "receive": "receive-2", "replica": "0", "tenant_id": "default-tenant", }, { "job": "myself", "prometheus": "prom2", - "receive": "1", + "receive": "receive-1", "replica": "0", "tenant_id": "default-tenant", }, { "job": "myself", "prometheus": "prom3", - "receive": "2", + "receive": "receive-2", "replica": "0", "tenant_id": "default-tenant", }, @@ -479,9 +481,7 @@ func TestReceive(t *testing.T) { t.Run("replication", func(t *testing.T) { t.Parallel() - netName := "e2e_test_receive_replication" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_receive_replication") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) @@ -490,22 +490,26 @@ func TestReceive(t *testing.T) { // receivers and the test verifies that the time series are // replicated to all of the nodes. + r1 := e2ethanos.NewUninitiatedReceiver(e, "1") + r2 := e2ethanos.NewUninitiatedReceiver(e, "2") + r3 := e2ethanos.NewUninitiatedReceiver(e, "3") + h := receive.HashringConfig{ Endpoints: []string{ - netName + "-receive-1:9091", - netName + "-receive-2:9091", - netName + "-receive-3:9091", + r1.InternalEndpoint("grpc"), + r2.InternalEndpoint("grpc"), + r3.InternalEndpoint("grpc"), }, } // Create with hashring config. - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 3, h) + r1Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r1, e.SharedDir(), 3, h) testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "2", 3, h) + r2Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r2, e.SharedDir(), 3, h) testutil.Ok(t, err) - r3, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "3", 3, h) + r3Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r3, e.SharedDir(), 3, h) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(r1, r2, r3)) + testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) @@ -526,21 +530,21 @@ func TestReceive(t *testing.T) { { "job": "myself", "prometheus": "prom1", - "receive": "1", + "receive": "receive-1", "replica": "0", "tenant_id": "default-tenant", }, { "job": "myself", "prometheus": "prom1", - "receive": "2", + "receive": "receive-2", "replica": "0", "tenant_id": "default-tenant", }, { "job": "myself", "prometheus": "prom1", - "receive": "3", + "receive": "receive-3", "replica": "0", "tenant_id": "default-tenant", }, @@ -550,9 +554,7 @@ func TestReceive(t *testing.T) { t.Run("replication_with_outage", func(t *testing.T) { t.Parallel() - netName := "e2e_test_receive_replication_with_outage" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_receive_replication_with_outage") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) @@ -560,20 +562,24 @@ func TestReceive(t *testing.T) { // receivers is dead. In this case, replication should still // succeed and the time series should be replicated to the other nodes. + r1 := e2ethanos.NewUninitiatedReceiver(e, "1") + r2 := e2ethanos.NewUninitiatedReceiver(e, "2") + r3 := e2ethanos.NewUninitiatedReceiver(e, "3") + h := receive.HashringConfig{ Endpoints: []string{ - netName + "-receive-1:9091", - netName + "-receive-2:9091", - netName + "-receive-3:9091", + r1.InternalEndpoint("grpc"), + r2.InternalEndpoint("grpc"), + r3.InternalEndpoint("grpc"), }, } // Create with hashring config. - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 3, h) + r1Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r1, e.SharedDir(), 3, h) testutil.Ok(t, err) - r2, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "2", 3, h) + r2Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r2, e.SharedDir(), 3, h) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(r1, r2)) + testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable)) prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) @@ -594,14 +600,14 @@ func TestReceive(t *testing.T) { { "job": "myself", "prometheus": "prom1", - "receive": "1", + "receive": "receive-1", "replica": "0", "tenant_id": "default-tenant", }, { "job": "myself", "prometheus": "prom1", - "receive": "2", + "receive": "receive-2", "replica": "0", "tenant_id": "default-tenant", }, @@ -611,21 +617,22 @@ func TestReceive(t *testing.T) { t.Run("multitenancy", func(t *testing.T) { t.Parallel() - netName := "e2e_test_for_multitenancy" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_for_multitenancy") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) + r1 := e2ethanos.NewUninitiatedReceiver(e, "1") + h := receive.HashringConfig{ Endpoints: []string{ - netName + "-receive-1:9091", + r1.InternalEndpoint("grpc"), }, } - r1, err := e2ethanos.NewRoutingAndIngestingReceiver(e, netName, "1", 1, h) + // Create with hashring config. + r1Runnable, err := e2ethanos.NewRoutingAndIngestingReceiverFromService(r1, e.SharedDir(), 1, h) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(r1)) + testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable)) rp1, err := e2ethanos.NewReverseProxy(e, "1", "tenant-1", "http://"+r1.InternalEndpoint("remote-write")) testutil.Ok(t, err) @@ -652,14 +659,14 @@ func TestReceive(t *testing.T) { { "job": "myself", "prometheus": "prom1", - "receive": "1", + "receive": "receive-1", "replica": "0", "tenant_id": "tenant-1", }, { "job": "myself", "prometheus": "prom2", - "receive": "1", + "receive": "receive-1", "replica": "0", "tenant_id": "tenant-2", }, diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 8543a1e0a6..633d70b1ed 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -201,7 +201,7 @@ func TestRule(t *testing.T) { testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) t.Cleanup(cancel) // Prepare work dirs. @@ -356,7 +356,7 @@ func TestRule(t *testing.T) { testutil.Ok(t, am1.WaitSumMetrics(e2e.Equals(currentValAm1), "alertmanager_alerts_received_total")) }) - t.Run("duplicate am ", func(t *testing.T) { + t.Run("duplicate am", func(t *testing.T) { // am2 is already registered in static addresses. writeTargets(t, filepath.Join(e.SharedDir(), amTargetsSubDir, "targets.yaml"), am2.InternalEndpoint("http")) @@ -382,57 +382,59 @@ func TestRule(t *testing.T) { checkReloadSuccessful(t, ctx, r.Endpoint("http"), 3) }) - queryAndAssertSeries(t, ctx, q.Endpoint("http"), "ALERTS", promclient.QueryOptions{ - Deduplicate: false, - }, []model.Metric{ - { - "__name__": "ALERTS", - "severity": "page", - "alertname": "TestAlert_AbortOnPartialResponse", - "alertstate": "firing", - "replica": "1", - }, - { - "__name__": "ALERTS", - "severity": "page", - "alertname": "TestAlert_HasBeenLoadedViaWebHandler", - "alertstate": "firing", - "replica": "1", - }, - { - "__name__": "ALERTS", - "severity": "page", - "alertname": "TestAlert_WarnOnPartialResponse", - "alertstate": "firing", - "replica": "1", - }, - }) + t.Run("query alerts", func(t *testing.T) { + queryAndAssertSeries(t, ctx, q.Endpoint("http"), "ALERTS", promclient.QueryOptions{ + Deduplicate: false, + }, []model.Metric{ + { + "__name__": "ALERTS", + "severity": "page", + "alertname": "TestAlert_AbortOnPartialResponse", + "alertstate": "firing", + "replica": "1", + }, + { + "__name__": "ALERTS", + "severity": "page", + "alertname": "TestAlert_HasBeenLoadedViaWebHandler", + "alertstate": "firing", + "replica": "1", + }, + { + "__name__": "ALERTS", + "severity": "page", + "alertname": "TestAlert_WarnOnPartialResponse", + "alertstate": "firing", + "replica": "1", + }, + }) - expAlertLabels := []model.LabelSet{ - { - "severity": "page", - "alertname": "TestAlert_AbortOnPartialResponse", - "replica": "1", - }, - { - "severity": "page", - "alertname": "TestAlert_HasBeenLoadedViaWebHandler", - "replica": "1", - }, - { - "severity": "page", - "alertname": "TestAlert_WarnOnPartialResponse", - "replica": "1", - }, - } + expAlertLabels := []model.LabelSet{ + { + "severity": "page", + "alertname": "TestAlert_AbortOnPartialResponse", + "replica": "1", + }, + { + "severity": "page", + "alertname": "TestAlert_HasBeenLoadedViaWebHandler", + "replica": "1", + }, + { + "severity": "page", + "alertname": "TestAlert_WarnOnPartialResponse", + "replica": "1", + }, + } - alrts, err := promclient.NewDefaultClient().AlertmanagerAlerts(ctx, mustURLParse(t, "http://"+am2.Endpoint("http"))) - testutil.Ok(t, err) + alrts, err := promclient.NewDefaultClient().AlertmanagerAlerts(ctx, mustURLParse(t, "http://"+am2.Endpoint("http"))) + testutil.Ok(t, err) - testutil.Equals(t, len(expAlertLabels), len(alrts)) - for i, a := range alrts { - testutil.Assert(t, a.Labels.Equal(expAlertLabels[i]), "unexpected labels %s", a.Labels) - } + testutil.Equals(t, len(expAlertLabels), len(alrts)) + for i, a := range alrts { + testutil.Assert(t, a.Labels.Equal(expAlertLabels[i]), "unexpected labels %s", a.Labels) + } + }) } // Test Ruler behavior on different storepb.PartialResponseStrategy when having partial response from single `failingStoreAPI`. diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index ae7d2caad2..f55a1175ae 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -30,9 +30,7 @@ import ( func TestRulesAPI_Fanout(t *testing.T) { t.Parallel() - netName := "e2e_test_rules_fanout" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_rules_fanout") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) @@ -49,7 +47,6 @@ func TestRulesAPI_Fanout(t *testing.T) { // 2x Prometheus. prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom1", defaultPromConfig("ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), e2ethanos.DefaultPrometheusImage(), @@ -57,7 +54,6 @@ func TestRulesAPI_Fanout(t *testing.T) { testutil.Ok(t, err) prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom2", defaultPromConfig("ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), e2ethanos.DefaultPrometheusImage(), @@ -65,17 +61,13 @@ func TestRulesAPI_Fanout(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) - stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), netName + "-rule-rule1:9091", netName + "-rule-rule2:9091"} - q, err := e2ethanos.NewQuerierBuilder(e, "query", stores...). - WithRuleAddresses(stores...). - Build() - testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(q)) + qBuilder := e2ethanos.NewQuerierBuilder(e, "query") + qUninit := qBuilder.BuildUninitiated() queryCfg := []query.Config{ { EndpointsConfig: http_util.EndpointsConfig{ - StaticAddresses: []string{q.InternalEndpoint("http")}, + StaticAddresses: []string{qUninit.InternalEndpoint("http")}, Scheme: "http", }, }, @@ -88,6 +80,11 @@ func TestRulesAPI_Fanout(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1, r2)) + stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), r1.InternalEndpoint("grpc"), r2.InternalEndpoint("grpc")} + q, err := qBuilder.WithRuleAddresses(stores...).Initiate(qUninit, stores...) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(q)) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) diff --git a/test/e2e/targets_api_test.go b/test/e2e/targets_api_test.go index 1ab6cd4bd9..a3b2d4a615 100644 --- a/test/e2e/targets_api_test.go +++ b/test/e2e/targets_api_test.go @@ -29,16 +29,13 @@ func TestTargetsAPI_Fanout(t *testing.T) { t.Parallel() - netName := "e2e_test_targets_fanout" - - e, err := e2e.NewDockerEnvironment(netName) + e, err := e2e.NewDockerEnvironment("e2e_test_targets_fanout") testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) // 2x Prometheus. prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "localhost:80"), e2ethanos.DefaultPrometheusImage(), @@ -46,7 +43,6 @@ func TestTargetsAPI_Fanout(t *testing.T) { testutil.Ok(t, err) prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar( e, - netName, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "localhost:80"), e2ethanos.DefaultPrometheusImage(),