fix(metrics): reset backup gauges after schedule deactivation (#176)

Author: ulya-sidorina

internal/metrics/metrics.go

@@ -359,6 +359,18 @@ func (s *MetricsRegistryImpl) ResetScheduleCounters(schedule *types.BackupSchedu
 		schedule.ID,
 		scheduleNameLabel,
 	)
+
+	s.backupsFailedCount.DeleteLabelValues(
+		schedule.ContainerID,
+		schedule.DatabaseName,
+		scheduleNameLabel,
+	)
+
+	s.backupsSucceededCount.DeleteLabelValues(
+		schedule.ContainerID,
+		schedule.DatabaseName,
+		scheduleNameLabel,
+	)
 }
 
 func InitializeMetricsRegistry(

internal/metrics/metrics_test.go

@@ -10,9 +10,11 @@ import (
 	"sync"
 	"testing"
 	"time"
+	"ydbcp/internal/types"
 
 	"ydbcp/internal/config"
 
+	"github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -70,3 +72,99 @@ func TestMetricsCount(t *testing.T) {
 	cancel()
 	wg.Wait()
 }
+
+func TestResetScheduleMetrics(t *testing.T) {
+	var wg sync.WaitGroup
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	cfg := &config.MetricsServerConfig{
+		BindPort:    8080,
+		BindAddress: "127.0.0.1",
+	}
+	InitializeMetricsRegistry(ctx, &wg, cfg, clockwork.NewFakeClock())
+	s := GlobalMetricsRegistry.(*MetricsRegistryImpl)
+
+	scheduleName := "test_schedule"
+	schedule := types.BackupSchedule{
+		ID:           "123",
+		ContainerID:  "test_container",
+		DatabaseName: "test_db",
+		Name:         &scheduleName,
+	}
+
+	// set schedule metrics
+	s.scheduleLastBackupTimestamp.WithLabelValues(
+		schedule.ContainerID,
+		schedule.DatabaseName,
+		schedule.ID,
+		scheduleName,
+	).Set(float64(time.Now().Unix()))
+
+	s.backupsSucceededCount.WithLabelValues(schedule.ContainerID, schedule.DatabaseName, scheduleName).Set(1)
+	s.backupsFailedCount.WithLabelValues(schedule.ContainerID, schedule.DatabaseName, scheduleName).Set(0)
+
+	getMetricValue := func(metricFamilies []*io_prometheus_client.MetricFamily, familyName string) (float64, bool) {
+		for _, mf := range metricFamilies {
+			if mf.GetName() == familyName {
+				var val float64
+				for _, m := range mf.Metric {
+					if m.Counter != nil {
+						val += m.Counter.GetValue()
+					}
+
+					if m.Gauge != nil {
+						val += m.Gauge.GetValue()
+					}
+				}
+				return val, true
+			}
+		}
+		return 0, false
+	}
+
+	// check metrics before ResetScheduleCounters
+	metricFamilies, err := GlobalMetricsRegistry.GetReg().Gather()
+	assert.Equal(t, nil, err)
+
+	{ // check schedules_last_backup_timestamp
+		_, found := getMetricValue(metricFamilies, "schedules_last_backup_timestamp")
+		assert.Equal(t, true, found)
+	}
+
+	{ // check backups_succeeded_count (should be 1)
+		val, found := getMetricValue(metricFamilies, "backups_succeeded_count")
+		assert.Equal(t, true, found)
+		assert.Equal(t, 1, int(val))
+	}
+
+	{ // check backups_succeeded_count (should be 0)
+		val, found := getMetricValue(metricFamilies, "backups_failed_count")
+		assert.Equal(t, true, found)
+		assert.Equal(t, 0, int(val))
+	}
+
+	GlobalMetricsRegistry.ResetScheduleCounters(&schedule)
+
+	// check metrics after ResetScheduleCounters
+	metricFamilies, err = GlobalMetricsRegistry.GetReg().Gather()
+	assert.Equal(t, nil, err)
+
+	{ // check schedules_last_backup_timestamp (should be deleted)
+		_, found := getMetricValue(metricFamilies, "schedules_last_backup_timestamp")
+		assert.Equal(t, false, found)
+	}
+
+	{ // check backups_succeeded_count (should be deleted)
+		_, found := getMetricValue(metricFamilies, "backups_succeeded_count")
+		assert.Equal(t, false, found)
+	}
+
+	{ // check backups_succeeded_count (should be deleted)
+		_, found := getMetricValue(metricFamilies, "backups_failed_count")
+		assert.Equal(t, false, found)
+	}
+
+	cancel()
+	wg.Wait()
+}