Warn users about external changes in sync tag

daemon/loop.go

@@ -60,7 +60,12 @@ func (d *Daemon) Loop(stop chan struct{}, wg *sync.WaitGroup, logger log.Logger)
 	// Ask for a sync, and to poll images, straight away
 	d.AskForSync()
 	d.AskForImagePoll()
+
 	for {
+		var (
+			lastKnownSyncTagRev      string
+			warnedAboutSyncTagChange bool
+		)
 		select {
 		case <-stop:
 			logger.Log("stopping", "true")
@@ -83,7 +88,7 @@ func (d *Daemon) Loop(stop chan struct{}, wg *sync.WaitGroup, logger log.Logger)
 				default:
 				}
 			}
-			if err := d.doSync(logger); err != nil {
+			if err := d.doSync(logger, &lastKnownSyncTagRev, &warnedAboutSyncTagChange); err != nil {
 				logger.Log("err", err)
 			}
 			syncTimer.Reset(d.SyncInterval)
@@ -149,7 +154,7 @@ func (d *LoopVars) AskForImagePoll() {
 
 // -- extra bits the loop needs
 
-func (d *Daemon) doSync(logger log.Logger) (retErr error) {
+func (d *Daemon) doSync(logger log.Logger, lastKnownSyncTagRev *string, warnedAboutSyncTagChange *bool) (retErr error) {
 	started := time.Now().UTC()
 	defer func() {
 		syncDuration.With(
@@ -179,6 +184,14 @@ func (d *Daemon) doSync(logger log.Logger) (retErr error) {
 	if err != nil && !isUnknownRevision(err) {
 		return err
 	}
+	// Check if something other than the current instance of fluxd changed the sync tag.
+	// This is likely to be caused by another fluxd instance using the same tag.
+	// Having multiple instances fighting for the same tag can lead to fluxd missing manifest changes.
+	if *lastKnownSyncTagRev != "" && oldTagRev != *lastKnownSyncTagRev && !*warnedAboutSyncTagChange {
+		logger.Log("warning",
+			"detected external change in git sync tag; the sync tag should not be shared by fluxd instances")
+		*warnedAboutSyncTagChange = true
+	}
 
 	newTagRev, err := working.HeadRevision(ctx)
 	if err != nil {
@@ -419,6 +432,7 @@ func (d *Daemon) doSync(logger log.Logger) (retErr error) {
 			if err != nil {
 				return err
 			}
+			*lastKnownSyncTagRev = newTagRev
 		}
 		logger.Log("tag", d.GitConfig.SyncTag, "old", oldTagRev, "new", newTagRev)
 		{

daemon/loop_test.go

@@ -105,8 +105,12 @@ func TestPullAndSync_InitialSync(t *testing.T) {
 		syncDef = &def
 		return nil
 	}
-
-	d.doSync(log.NewLogfmtLogger(ioutil.Discard))
+	var (
+		logger                   = log.NewLogfmtLogger(ioutil.Discard)
+		lastKnownSyncTagRev      string
+		warnedAboutSyncTagChange bool
+	)
+	d.doSync(logger, &lastKnownSyncTagRev, &warnedAboutSyncTagChange)
 
 	// It applies everything
 	if syncCalled != 1 {
@@ -174,8 +178,12 @@ func TestDoSync_NoNewCommits(t *testing.T) {
 		syncDef = &def
 		return nil
 	}
-
-	if err := d.doSync(log.NewLogfmtLogger(ioutil.Discard)); err != nil {
+	var (
+		logger                   = log.NewLogfmtLogger(ioutil.Discard)
+		lastKnownSyncTagRev      string
+		warnedAboutSyncTagChange bool
+	)
+	if err := d.doSync(logger, &lastKnownSyncTagRev, &warnedAboutSyncTagChange); err != nil {
 		t.Error(err)
 	}
 
@@ -268,8 +276,12 @@ func TestDoSync_WithNewCommit(t *testing.T) {
 		syncDef = &def
 		return nil
 	}
-
-	d.doSync(log.NewLogfmtLogger(ioutil.Discard))
+	var (
+		logger                   = log.NewLogfmtLogger(ioutil.Discard)
+		lastKnownSyncTagRev      string
+		warnedAboutSyncTagChange bool
+	)
+	d.doSync(logger, &lastKnownSyncTagRev, &warnedAboutSyncTagChange)
 
 	// It applies everything
 	if syncCalled != 1 {