diff --git a/ghcide/src/Development/IDE/Core/OfInterest.hs b/ghcide/src/Development/IDE/Core/OfInterest.hs
index b7ff88024a8..32129ec735a 100644
--- a/ghcide/src/Development/IDE/Core/OfInterest.hs
+++ b/ghcide/src/Development/IDE/Core/OfInterest.hs
@@ -77,9 +77,13 @@ getFilesOfInterestUntracked = do
 addFileOfInterest :: IdeState -> NormalizedFilePath -> FileOfInterestStatus -> IO ()
 addFileOfInterest state f v = do
     OfInterestVar var <- getIdeGlobalState state
-    files <- modifyVar' var $ HashMap.insert f v
-    recordDirtyKeys (shakeExtras state) IsFileOfInterest [f]
-    logDebug (ideLogger state) $ "Set files of interest to: " <> T.pack (show files)
+    (prev, files) <- modifyVar var $ \dict -> do
+        let (prev, new) = HashMap.alterF (, Just v) f dict
+        pure (new, (prev, dict))
+    when (prev /= Just v) $
+        recordDirtyKeys (shakeExtras state) IsFileOfInterest [f]
+    logDebug (ideLogger state) $
+        "Set files of interest to: " <> T.pack (show files)
 
 deleteFileOfInterest :: IdeState -> NormalizedFilePath -> IO ()
 deleteFileOfInterest state f = do