Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

E2E Tests: Improve Prometheus configurability and make query tests more resilient #5181

Merged
merged 5 commits into from
Mar 16, 2022
Merged

E2E Tests: Improve Prometheus configurability and make query tests more resilient #5181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
92a7e3e
Move prom config to services; optional self scraping
matej-g Feb 24, 2022
1caf880
Bump wait time for minio ready
matej-g Feb 24, 2022
0e8ee80
Address feedback
matej-g Feb 25, 2022
a8aa8a1
Adjust DefaultPromConfig signature; use constant instead
matej-g Mar 3, 2022
1f3c383
Merge remote-tracking branch 'origin/main' into query-tests-improve-p…
matej-g Mar 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions test/e2e/e2ethanos/services.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1037,3 +1037,67 @@ func NewS3Config(bucket, endpoint, basePath string) s3.Config {
},
}
}

// NOTE: by using aggregation all results are now unsorted.
var QueryUpWithoutInstance = func() string { return "sum(up) without (instance)" }

// LocalPrometheusTarget is a constant to be used in the Prometheus config if you
// wish to enable Prometheus to scrape itself in a test.
const LocalPrometheusTarget = "localhost:9090"

// DefaultPromConfig returns Prometheus config that sets Prometheus to:
// * expose 2 external labels, source and replica.
// * optionallly scrape self. This will produce up == 0 metric which we can assert on.
// * optionally remote write endpoint to write into.
func DefaultPromConfig(name string, replica int, remoteWriteEndpoint, ruleFile string, scrapeTargets ...string) string {
var targets string
if len(scrapeTargets) > 0 {
targets = strings.Join(scrapeTargets, ",")
}

config := fmt.Sprintf(`
global:
external_labels:
prometheus: %v
replica: %v
`, name, replica)

if targets != "" {
config = fmt.Sprintf(`
%s
scrape_configs:
- job_name: 'myself'
# Quick scrapes for test purposes.
scrape_interval: 1s
scrape_timeout: 1s
static_configs:
- targets: [%s]
relabel_configs:
- source_labels: ['__address__']
regex: '^.+:80$'
action: drop
`, config, targets)
}

if remoteWriteEndpoint != "" {
config = fmt.Sprintf(`
%s
remote_write:
- url: "%s"
# Don't spam receiver on mistake.
queue_config:
min_backoff: 2s
max_backoff: 10s
`, config, remoteWriteEndpoint)
}

if ruleFile != "" {
config = fmt.Sprintf(`
%s
rule_files:
- "%s"
`, config, ruleFile)
}

return config
}
4 changes: 2 additions & 2 deletions test/e2e/exemplars_api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestExemplarsAPI_Fanout(t *testing.T) {
prom1, sidecar1, err = e2ethanos.NewPrometheusWithSidecar(
e,
"prom1",
defaultPromConfig("ha", 0, "", "", "localhost:9090", qUnitiated.Future().InternalEndpoint("http")),
e2ethanos.DefaultPromConfig("ha", 0, "", "", "localhost:9090", qUnitiated.Future().InternalEndpoint("http"), e2ethanos.LocalPrometheusTarget),
"",
e2ethanos.DefaultPrometheusImage(),
"",
Expand All @@ -52,7 +52,7 @@ func TestExemplarsAPI_Fanout(t *testing.T) {
prom2, sidecar2, err = e2ethanos.NewPrometheusWithSidecar(
e,
"prom2",
defaultPromConfig("ha", 1, "", "", "localhost:9090", qUnitiated.Future().InternalEndpoint("http")),
e2ethanos.DefaultPromConfig("ha", 1, "", "", "localhost:9090", qUnitiated.Future().InternalEndpoint("http"), e2ethanos.LocalPrometheusTarget),
"",
e2ethanos.DefaultPrometheusImage(),
"",
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/info_api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ func TestInfo(t *testing.T) {
testutil.Ok(t, err)
t.Cleanup(e2ethanos.CleanScenario(t, e))

prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone1", defaultPromConfig("prom-alone1", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "")
prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone1", e2ethanos.DefaultPromConfig("prom-alone1", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "")
testutil.Ok(t, err)
prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "alone2", defaultPromConfig("prom-alone2", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "")
prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "alone2", e2ethanos.DefaultPromConfig("prom-alone2", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "")
testutil.Ok(t, err)
prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "alone3", defaultPromConfig("prom-alone3", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "")
prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "alone3", e2ethanos.DefaultPromConfig("prom-alone3", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "")
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3))

Expand Down
4 changes: 2 additions & 2 deletions test/e2e/metadata_api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func TestMetadataAPI_Fanout(t *testing.T) {
prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(
e,
"prom1",
defaultPromConfig("ha", 0, "", "", "localhost:9090", "sidecar-prom1:8080"),
e2ethanos.DefaultPromConfig("ha", 0, "", "", e2ethanos.LocalPrometheusTarget, "sidecar-prom1:8080"),
"",
e2ethanos.DefaultPrometheusImage(), "",
)
Expand All @@ -39,7 +39,7 @@ func TestMetadataAPI_Fanout(t *testing.T) {
prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(
e,
"prom2",
defaultPromConfig("ha", 1, "", "", "localhost:9090", "sidecar-prom2:8080"),
e2ethanos.DefaultPromConfig("ha", 1, "", "", e2ethanos.LocalPrometheusTarget, "sidecar-prom2:8080"),
"",
e2ethanos.DefaultPrometheusImage(), "",
)
Expand Down
20 changes: 10 additions & 10 deletions test/e2e/query_frontend_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestQueryFrontend(t *testing.T) {

now := time.Now()

prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "")
prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", e2ethanos.DefaultPromConfig("test", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "")
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar))

Expand All @@ -59,7 +59,7 @@ func TestQueryFrontend(t *testing.T) {

// Ensure we can get the result from Querier first so that it
// doesn't need to retry when we send queries to the frontend later.
queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, time.Now, promclient.QueryOptions{
queryAndAssertSeries(t, ctx, q.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, time.Now, promclient.QueryOptions{
Deduplicate: false,
}, []model.Metric{
{
Expand All @@ -77,7 +77,7 @@ func TestQueryFrontend(t *testing.T) {
queryTimes := vals[0]

t.Run("query frontend works for instant query", func(t *testing.T) {
queryAndAssertSeries(t, ctx, queryFrontend.Endpoint("http"), queryUpWithoutInstance, time.Now, promclient.QueryOptions{
queryAndAssertSeries(t, ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, time.Now, promclient.QueryOptions{
Deduplicate: false,
}, []model.Metric{
{
Expand Down Expand Up @@ -105,7 +105,7 @@ func TestQueryFrontend(t *testing.T) {
t,
ctx,
queryFrontend.Endpoint("http"),
queryUpWithoutInstance,
e2ethanos.QueryUpWithoutInstance,
timestamp.FromTime(now.Add(-time.Hour)),
timestamp.FromTime(now.Add(time.Hour)),
14,
Expand Down Expand Up @@ -147,7 +147,7 @@ func TestQueryFrontend(t *testing.T) {
t,
ctx,
queryFrontend.Endpoint("http"),
queryUpWithoutInstance,
e2ethanos.QueryUpWithoutInstance,
timestamp.FromTime(now.Add(-time.Hour)),
timestamp.FromTime(now.Add(time.Hour)),
14,
Expand Down Expand Up @@ -192,7 +192,7 @@ func TestQueryFrontend(t *testing.T) {
t,
ctx,
queryFrontend.Endpoint("http"),
queryUpWithoutInstance,
e2ethanos.QueryUpWithoutInstance,
timestamp.FromTime(now.Add(-time.Hour)),
timestamp.FromTime(now.Add(24*time.Hour)),
14,
Expand Down Expand Up @@ -396,7 +396,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) {

now := time.Now()

prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "")
prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", e2ethanos.DefaultPromConfig("test", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "")
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar))

Expand Down Expand Up @@ -436,7 +436,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) {

// Ensure we can get the result from Querier first so that it
// doesn't need to retry when we send queries to the frontend later.
queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, time.Now, promclient.QueryOptions{
queryAndAssertSeries(t, ctx, q.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, time.Now, promclient.QueryOptions{
Deduplicate: false,
}, []model.Metric{
{
Expand All @@ -455,7 +455,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) {
t,
ctx,
queryFrontend.Endpoint("http"),
queryUpWithoutInstance,
e2ethanos.QueryUpWithoutInstance,
timestamp.FromTime(now.Add(-time.Hour)),
timestamp.FromTime(now.Add(time.Hour)),
14,
Expand Down Expand Up @@ -485,7 +485,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) {
t,
ctx,
queryFrontend.Endpoint("http"),
queryUpWithoutInstance,
e2ethanos.QueryUpWithoutInstance,
timestamp.FromTime(now.Add(-time.Hour)),
timestamp.FromTime(now.Add(time.Hour)),
14,
Expand Down
Loading