e2e: Fixed WaitSumMetrics to fail on non existing metric

.circleci/config.yml

@@ -155,7 +155,7 @@ jobs:
           export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:${CIRCLE_TAG:-$(./tools/image-tag)}"
           export CORTEX_CHECKOUT_DIR="/home/circleci/.go_workspace/src/github.com/cortexproject/cortex"
           echo "Running integration tests with image: $CORTEX_IMAGE"
-          go test -tags=integration -timeout 300s -v -count=1 ./integration/...
+          go test -tags=requires_docker -timeout 300s -v -count=1 ./integration/...
 
   build:
     <<: *defaults

integration/README.md

@@ -7,9 +7,10 @@
 
 ## Running
 
-Integration tests have `integration` tag (`// +build integration` line followed by empty line on top of Go files), to avoid running them unintentionally, e.g. by `go test ./...` in main Cortex package.
+Integration tests have `requires_docker` tag (`// +build requires_docker` line followed by empty line on top of Go files),
+to avoid running them unintentionally as they require docker, e.g. by `go test ./...` in main Cortex package.
 
-To run integration tests, one needs to run `go test -tags=integration ./integration/...` (or just `go test -tags=integration ./...` to run unit tests as well).
+To run integration tests, one needs to run `go test -tags=requires_docker ./integration/...` (or just `go test -tags=requires_docker ./...` to run unit tests as well).
 
 ## Owners
 

integration/alertmanager_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 

integration/api_config_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 

integration/backward_compatibility_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 
@@ -36,20 +36,23 @@ func TestBackwardCompatibilityWithChunksStorage(t *testing.T) {
 		"-config-yaml":        ChunksStorageFlags["-schema-config-file"],
 	})
 
-	// Start Cortex components (ingester running on previous version).
 	require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml)))
-	tableManager := e2ecortex.NewTableManager("table-manager", flagsForOldImage, previousVersionImage)
-	// Old table-manager doesn't expose a readiness probe, so we just check if the / returns 404
-	tableManager.SetReadinessProbe(e2e.NewReadinessProbe(tableManager.HTTPPort(), "/", 404))
+
+	// Start Cortex table-manager (running on current version since the backward compatibility
+	// test is about testing a rolling update of other services).
+	tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags, "")
+	require.NoError(t, s.StartAndWaitReady(tableManager))
+
+	// Wait until the first table-manager sync has completed, so that we're
+	// sure the tables have been created.
+	require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds"))
+
+	// Start other Cortex components (ingester running on previous version).
 	ingester1 := e2ecortex.NewIngester("ingester-1", consul.NetworkHTTPEndpoint(), flagsForOldImage, "")
 	distributor := e2ecortex.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), flagsForOldImage, "")
 	// Old ring didn't have /ready probe, use /ring instead.
 	distributor.SetReadinessProbe(e2e.NewReadinessProbe(distributor.HTTPPort(), "/ring", 200))
-	require.NoError(t, s.StartAndWaitReady(distributor, ingester1, tableManager))
-
-	// Wait until the first table-manager sync has completed, so that we're
-	// sure the tables have been created.
-	require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_dynamo_sync_tables_seconds"))
+	require.NoError(t, s.StartAndWaitReady(distributor, ingester1))
 
 	// Wait until the distributor has updated the ring.
 	require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total"))

integration/chunks_storage_backends_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 
@@ -55,7 +55,7 @@ func TestChunksStorageAllIndexBackends(t *testing.T) {
 
 		// Wait until the first table-manager sync has completed, so that we're
 		// sure the tables have been created.
-		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_dynamo_sync_tables_seconds"))
+		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds"))
 		require.NoError(t, s.Stop(tableManager))
 	}
 

integration/configs.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 

integration/e2e/scenario_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package e2e_test
 

integration/e2e/service.go

@@ -461,6 +461,7 @@ func sumValues(family *io_prometheus_client.MetricFamily) float64 {
 	return sum
 }
 
+// Equals is an isExpected function for WaitSumMetrics that returns true if given single sum is equals to given value.
 func Equals(value float64) func(sums ...float64) bool {
 	return func(sums ...float64) bool {
 		if len(sums) != 1 {
@@ -470,6 +471,7 @@ func Equals(value float64) func(sums ...float64) bool {
 	}
 }
 
+// Greater is an isExpected function for WaitSumMetrics that returns true if given single sum is greater than given value.
 func Greater(value float64) func(sums ...float64) bool {
 	return func(sums ...float64) bool {
 		if len(sums) != 1 {
@@ -479,6 +481,7 @@ func Greater(value float64) func(sums ...float64) bool {
 	}
 }
 
+// Less is an isExpected function for WaitSumMetrics that returns true if given single sum is less than given value.
 func Less(value float64) func(sums ...float64) bool {
 	return func(sums ...float64) bool {
 		if len(sums) != 1 {
@@ -488,7 +491,7 @@ func Less(value float64) func(sums ...float64) bool {
 	}
 }
 
-// EqualsAmongTwo returns true if first sum is equal to the second.
+// EqualsAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is equal to the second.
 // NOTE: Be careful on scrapes in between of process that changes two metrics. Those are
 // usually not atomic.
 func EqualsAmongTwo(sums ...float64) bool {
@@ -498,7 +501,7 @@ func EqualsAmongTwo(sums ...float64) bool {
 	return sums[0] == sums[1]
 }
 
-// GreaterAmongTwo returns true if first sum is greater than second.
+// GreaterAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is greater than second.
 // NOTE: Be careful on scrapes in between of process that changes two metrics. Those are
 // usually not atomic.
 func GreaterAmongTwo(sums ...float64) bool {
@@ -508,7 +511,7 @@ func GreaterAmongTwo(sums ...float64) bool {
 	return sums[0] > sums[1]
 }
 
-// LessAmongTwo returns true if first sum is smaller than second.
+// LessAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is smaller than second.
 // NOTE: Be careful on scrapes in between of process that changes two metrics. Those are
 // usually not atomic.
 func LessAmongTwo(sums ...float64) bool {
@@ -518,6 +521,8 @@ func LessAmongTwo(sums ...float64) bool {
 	return sums[0] < sums[1]
 }
 
+// WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true
+// when passed to given isExpected(...).
 func (s *HTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metricNames ...string) error {
 	sums := make([]float64, len(metricNames))
 
@@ -537,10 +542,11 @@ func (s *HTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metr
 			sums[i] = 0.0
 
 			// Check if the metric is exported.
-			mf, ok := families[m]
-			if ok {
+			if mf, ok := families[m]; ok {
 				sums[i] = sumValues(mf)
+				continue
 			}
+			return errors.Errorf("metric %s not found in %s metric page", m, s.name)
 		}
 
 		if isExpected(sums...) {

integration/e2e/service_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package e2e
 
@@ -107,6 +107,8 @@ metric_b_summary_count 1
 
 	require.NoError(t, s.WaitSumMetrics(LessAmongTwo, "metric_a", "metric_b"))
 	require.Error(t, s.WaitSumMetrics(LessAmongTwo, "metric_b", "metric_a"))
+
+	require.Error(t, s.WaitSumMetrics(Equals(0), "non_existing_metric"))
 }
 
 func TestWaitSumMetric_Nan(t *testing.T) {

integration/getting_started_single_process_config_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 

integration/ingester_flush_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 
@@ -42,7 +42,7 @@ func TestIngesterFlushWithChunksStorage(t *testing.T) {
 
 	// Wait until the first table-manager sync has completed, so that we're
 	// sure the tables have been created.
-	require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_dynamo_sync_tables_seconds"))
+	require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds"))
 
 	// Wait until both the distributor and querier have updated the ring.
 	require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total"))

integration/ingester_hand_over_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 
@@ -34,7 +34,7 @@ func TestIngesterHandOverWithChunksStorage(t *testing.T) {
 
 		// Wait until the first table-manager sync has completed, so that we're
 		// sure the tables have been created.
-		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_dynamo_sync_tables_seconds"))
+		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds"))
 	})
 }
 

integration/integration_memberlist_single_binary_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 

integration/metrics_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 
@@ -44,7 +44,7 @@ func TestExportedMetrics(t *testing.T) {
 
 	// Wait until the first table-manager sync has completed, so that we're
 	// sure the tables have been created.
-	require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_dynamo_sync_tables_seconds"))
+	require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds"))
 
 	// Wait until both the distributor and querier have updated the ring.
 	require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total"))

integration/querier_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 
@@ -83,9 +83,9 @@ func TestQuerierWithBlocksStorage(t *testing.T) {
 			// Wait until the TSDB head is compacted and shipped to the storage.
 			// The shipped block contains the 1st series, while the 2ns series in in the head.
 			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_shipper_uploads_total"))
+			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_memory_series"))
 			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_created_total"))
 			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_memory_series_removed_total"))
-			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_memory_series"))
 
 			// Push another series to further compact another block and delete the first block
 			// due to expired retention.
@@ -97,9 +97,9 @@ func TestQuerierWithBlocksStorage(t *testing.T) {
 			require.Equal(t, 200, res.StatusCode)
 
 			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_shipper_uploads_total"))
+			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_memory_series"))
 			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(3), "cortex_ingester_memory_series_created_total"))
 			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_removed_total"))
-			require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_memory_series"))
 
 			// Wait until the querier has synched the new uploaded blocks.
 			require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "cortex_querier_bucket_store_blocks_loaded"))

integration/query_frontend_test.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 
@@ -52,7 +52,7 @@ func TestQueryFrontendWithChunksStorageViaFlags(t *testing.T) {
 
 		// Wait until the first table-manager sync has completed, so that we're
 		// sure the tables have been created.
-		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_dynamo_sync_tables_seconds"))
+		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds"))
 
 		return "", ChunksStorageFlags
 	})
@@ -71,7 +71,7 @@ func TestQueryFrontendWithChunksStorageViaConfigFile(t *testing.T) {
 
 		// Wait until the first table-manager sync has completed, so that we're
 		// sure the tables have been created.
-		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_dynamo_sync_tables_seconds"))
+		require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds"))
 
 		return cortexConfigFile, e2e.EmptyFlags()
 	})

integration/util.go

@@ -1,4 +1,4 @@
-// +build integration
+// +build requires_docker
 
 package main
 

pkg/alertmanager/multitenant.go

@@ -38,6 +38,9 @@ const (
 	// a URL derived from Config.AutoWebhookRoot
 	autoWebhookURL = "http://internal.monitor"
 
+	configStatusValid   = "valid"
+	configStatusInvalid = "invalid"
+
 	statusPage = `
 <!doctype html>
 <html>
@@ -76,6 +79,10 @@ var (
 )
 
 func init() {
+	// Ensure the metric values are initialized.
+	totalConfigs.WithLabelValues(configStatusInvalid).Set(0)
+	totalConfigs.WithLabelValues(configStatusValid).Set(0)
+
 	prometheus.MustRegister(totalConfigs)
 	statusTemplate = template.Must(template.New("statusPage").Funcs(map[string]interface{}{
 		"state": func(enabled bool) string {
@@ -313,8 +320,8 @@ func (am *MultitenantAlertmanager) syncConfigs(cfgs map[string]alerts.AlertConfi
 			level.Info(am.logger).Log("msg", "deactivated per-tenant alertmanager", "user", user)
 		}
 	}
-	totalConfigs.WithLabelValues("invalid").Set(float64(invalid))
-	totalConfigs.WithLabelValues("valid").Set(float64(len(am.cfgs) - invalid))
+	totalConfigs.WithLabelValues(configStatusInvalid).Set(float64(invalid))
+	totalConfigs.WithLabelValues(configStatusValid).Set(float64(len(am.cfgs) - invalid))
 }
 
 func (am *MultitenantAlertmanager) transformConfig(userID string, amConfig *amconfig.Config) (*amconfig.Config, error) {

pkg/chunk/table_manager.go

@@ -29,10 +29,11 @@ const (
 )
 
 type tableManagerMetrics struct {
-	syncTableDuration *prometheus.HistogramVec
-	tableCapacity     *prometheus.GaugeVec
-	createFailures    prometheus.Gauge
-	deleteFailures    prometheus.Gauge
+	syncTableDuration  *prometheus.HistogramVec
+	tableCapacity      *prometheus.GaugeVec
+	createFailures     prometheus.Gauge
+	deleteFailures     prometheus.Gauge
+	lastSuccessfulSync prometheus.Gauge
 }
 
 func newTableManagerMetrics(r prometheus.Registerer) *tableManagerMetrics {
@@ -61,12 +62,19 @@ func newTableManagerMetrics(r prometheus.Registerer) *tableManagerMetrics {
 		Help:      "Number of table deletion failures during the last table-manager reconciliation",
 	})
 
+	m.lastSuccessfulSync = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "cortex",
+		Name:      "table_manager_sync_success_timestamp_seconds",
+		Help:      "Timestamp of the last successful table manager sync.",
+	})
+
 	if r != nil {
 		r.MustRegister(
 			m.syncTableDuration,
 			m.tableCapacity,
 			m.createFailures,
 			m.deleteFailures,
+			m.lastSuccessfulSync,
 		)
 	}
 
@@ -259,7 +267,12 @@ func (m *TableManager) SyncTables(ctx context.Context) error {
 		return err
 	}
 
-	return m.updateTables(ctx, toCheckThroughput)
+	if err := m.updateTables(ctx, toCheckThroughput); err != nil {
+		return err
+	}
+
+	m.metrics.lastSuccessfulSync.SetToCurrentTime()
+	return nil
 }
 
 func (m *TableManager) calculateExpectedTables() []TableDesc {