Add check for running backups

pkg/controller/mysqlbackupcron/job_backup.go

@@ -54,43 +54,63 @@ func (j *job) Run() {
 	}
 
 	// create the backup
-	if err := j.createBackup(); err != nil {
+	if _, err := j.createBackup(); err != nil {
 		log.Error(err, "failed to create backup")
 	}
 }
 
 func (j *job) anyScheduledBackupRunning() bool {
-	return false
+	backupsList := &api.MysqlBackupList{}
+	// select all backups with labels recurrent=true and and not completed of the cluster
+	selector := j.backupSelector()
+	selector.MatchingField("status.completed", "false")
+
+	if err := j.c.List(context.TODO(), selector, backupsList); err != nil {
+		log.Error(err, "failed getting backups", "selector", selector)
+		return false
+	}
+
+	if len(backupsList.Items) == 0 {
+		return false
+	}
+
+	log.V(1).Info("at least a backup is running", "backups", backupsList.Items)
+	return true
 }
 
-func (j *job) createBackup() error {
+func (j *job) createBackup() (*api.MysqlBackup, error) {
 	backupName := fmt.Sprintf("%s-auto-%s", j.ClusterName, time.Now().Format("2006-01-02t15-04-05"))
 
 	backup := &api.MysqlBackup{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      backupName,
 			Namespace: j.Namespace,
-			Labels: map[string]string{
-				"recurrent": "true",
-				"cluster":   j.ClusterName,
-			},
+			Labels:    j.recurrentBackupLabels(),
 		},
 		Spec: api.MysqlBackupSpec{
 			ClusterName: j.ClusterName,
 		},
 	}
-	return j.c.Create(context.TODO(), backup)
+	return backup, j.c.Create(context.TODO(), backup)
+}
+
+func (j *job) backupSelector() *client.ListOptions {
+	selector := &client.ListOptions{}
+	return selector.InNamespace(j.Namespace).MatchingLabels(j.recurrentBackupLabels())
+}
+
+func (j *job) recurrentBackupLabels() map[string]string {
+	return map[string]string{
+		"recurrent": "true",
+		"cluster":   j.ClusterName,
+	}
 }
 
 func (j *job) backupGC() {
 	var err error
-
 	backupsList := &api.MysqlBackupList{}
-	selector := &client.ListOptions{}
-	selector = selector.InNamespace(j.Namespace).MatchingLabels(map[string]string{"recurrent": "true"})
-
-	if err = j.c.List(context.TODO(), selector, backupsList); err != nil {
-		log.Error(err, "failed getting backups", "selector", selector)
+	if err = j.c.List(context.TODO(), j.backupSelector(), backupsList); err != nil {
+		log.Error(err, "failed getting backups", "selector", j.backupSelector())
 		return
 	}
 

pkg/controller/mysqlbackupcron/job_backup_test.go

@@ -43,36 +43,52 @@ var _ = Describe("MysqlBackupCron cron job", func() {
 		c client.Client
 		// stop channel for controller manager
 		stop chan struct{}
+
+		clusterName string
+		namespace   string
+		j           *job
 	)
 
 	BeforeEach(func() {
 		mgr, err := manager.New(cfg, manager.Options{})
 		Expect(err).To(Succeed())
 		c = mgr.GetClient()
+
+		// add field indexer for backup
+		// NOTE: add field indexer before starting the manager
+		Expect(addBackupFieldIndexers(mgr)).To(Succeed())
+
 		stop = StartTestManager(mgr)
+
+		clusterName = fmt.Sprintf("cl-%d", rand.Int31())
+		namespace = "default"
+
+		limit := 5
+		j = &job{
+			ClusterName:                    clusterName,
+			Namespace:                      namespace,
+			c:                              c,
+			BackupScheduleJobsHistoryLimit: &limit,
+		}
 	})
 	AfterEach(func() {
 		close(stop)
 	})
 
 	When("more backups are created", func() {
 		var (
-			clusterName string
-			ns          string
-			backups     []api.MysqlBackup
+			backups []api.MysqlBackup
 		)
 
 		BeforeEach(func() {
-			clusterName = fmt.Sprintf("cl-%d", rand.Int31())
-			ns = "default"
-
-			for i := 0; i < 10; i++ {
+			for i := 0; i < (*j.BackupScheduleJobsHistoryLimit + 5); i++ {
 				backup := api.MysqlBackup{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      fmt.Sprintf("bk-%d", i),
-						Namespace: ns,
+						Namespace: namespace,
 						Labels: map[string]string{
 							"recurrent": "true",
+							"cluster":   clusterName,
 						},
 					},
 					Spec: api.MysqlBackupSpec{
@@ -81,7 +97,7 @@ var _ = Describe("MysqlBackupCron cron job", func() {
 				}
 				Expect(c.Create(context.TODO(), &backup)).To(Succeed())
 				backups = append(backups, backup)
-				time.Sleep(time.Second / 3)
+				time.Sleep(time.Second / 6)
 			}
 		})
 
@@ -92,26 +108,51 @@ var _ = Describe("MysqlBackupCron cron job", func() {
 		})
 
 		It("should delete only older backups", func() {
-			limit := len(backups) - 5
-			j := &job{
-				ClusterName:                    clusterName,
-				Namespace:                      ns,
-				c:                              c,
-				BackupScheduleJobsHistoryLimit: &limit,
-			}
+
 			lo := &client.ListOptions{
 				LabelSelector: labels.SelectorFromSet(labels.Set{
 					"recurrent": "true",
+					"cluster":   clusterName,
 				}),
-				Namespace: ns,
+				Namespace: namespace,
 			}
 			Eventually(testutil.ListAllBackupsFn(c, lo)).Should(HaveLen(len(backups)))
 
 			j.backupGC()
 
-			Eventually(testutil.ListAllBackupsFn(c, lo)).Should(HaveLen(limit))
+			Eventually(testutil.ListAllBackupsFn(c, lo)).Should(HaveLen(*j.BackupScheduleJobsHistoryLimit))
 			Eventually(testutil.ListAllBackupsFn(c, lo)).ShouldNot(
 				ContainElement(testutil.BackupWithName("bk-3")))
 		})
 	})
+
+	When("a backup exists", func() {
+		var (
+			backup *api.MysqlBackup
+		)
+
+		BeforeEach(func() {
+			var err error
+			backup, err = j.createBackup()
+			Expect(err).To(Succeed())
+		})
+		AfterEach(func() {
+			c.Delete(context.TODO(), backup)
+		})
+
+		It("should detect the running backup", func() {
+			backup.Status.Completed = false
+			Expect(c.Update(context.TODO(), backup)).To(Succeed())
+			Expect(j.anyScheduledBackupRunning()).To(Equal(true))
+		})
+
+		It("should not detect any running backup", func() {
+			backup.Status.Completed = true
+			Expect(c.Update(context.TODO(), backup)).To(Succeed())
+			Expect(j.anyScheduledBackupRunning()).To(Equal(false))
+
+			backup.Status.Completed = false
+			Expect(c.Update(context.TODO(), backup)).To(Succeed())
+		})
+	})
 })

pkg/controller/mysqlbackupcron/mysqlbackupcron_controller.go

@@ -95,7 +95,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 		return err
 	}
 
-	return nil
+	return addBackupFieldIndexers(mgr)
 }
 
 var _ reconcile.Reconciler = &ReconcileMysqlBackup{}
@@ -202,3 +202,13 @@ func (r *ReconcileMysqlBackup) unregisterCluster(clusterKey types.NamespacedName
 
 	return nil
 }
+
+func addBackupFieldIndexers(mgr manager.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(&mysqlv1alpha1.MysqlBackup{}, "status.completed", func(b runtime.Object) []string {
+		completed := "false"
+		if b.(*mysqlv1alpha1.MysqlBackup).Status.Completed {
+			completed = "true"
+		}
+		return []string{completed}
+	})
+}