Skip to content

Commit 6a8dc22

Browse files
authored
Implement cycle detection in hls-graph (#2756)
1 parent 82c1535 commit 6a8dc22

File tree

21 files changed

+294
-83
lines changed

21 files changed

+294
-83
lines changed

.github/workflows/test.yml

+4
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,10 @@ jobs:
133133
path: "**/.tasty-rerun-log*"
134134
key: v1-${{ runner.os }}-${{ matrix.ghc }}-test-log-${{ github.sha }}
135135

136+
- if: matrix.test
137+
name: Test hls-graph
138+
run: cabal test hls-graph --test-options="$TEST_OPTS"
139+
136140
- if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test
137141
name: Test ghcide
138142
# run the tests without parallelism to avoid running out of memory

ghcide/exe/Main.hs

-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import Development.IDE.Core.OfInterest (kick)
1717
import Development.IDE.Core.Rules (mainRule)
1818
import qualified Development.IDE.Core.Rules as Rules
1919
import Development.IDE.Core.Tracing (withTelemetryLogger)
20-
import Development.IDE.Graph (ShakeOptions (shakeThreads))
2120
import qualified Development.IDE.Main as IDEMain
2221
import qualified Development.IDE.Plugin.HLS.GhcIde as GhcIde
2322
import Development.IDE.Types.Logger (Logger (Logger),
@@ -128,7 +127,6 @@ main = withTelemetryLogger $ \telemetryLogger -> do
128127
in defOptions
129128
{ optShakeProfiling = argsShakeProfiling
130129
, optOTMemoryProfiling = IdeOTMemoryProfiling argsOTMemoryProfiling
131-
, optShakeOptions = (optShakeOptions defOptions){shakeThreads = argsThreads}
132130
, optCheckParents = pure $ checkParents config
133131
, optCheckProject = pure $ checkProject config
134132
, optRunSubset = not argsConservativeChangeTracking

ghcide/src/Development/IDE/Core/Rules.hs

+1-9
Original file line numberDiff line numberDiff line change
@@ -717,15 +717,7 @@ loadGhcSession recorder ghcSessionDepsConfig = do
717717
use_ GetModificationTime nfp
718718
mapM_ addDependency deps
719719

720-
opts <- getIdeOptions
721-
let cutoffHash =
722-
case optShakeFiles opts of
723-
-- optShakeFiles is only set in the DAML case.
724-
-- https://github.com/haskell/ghcide/pull/522#discussion_r428622915
725-
Just {} -> ""
726-
-- Hash the HscEnvEq returned so cutoff if it didn't change
727-
-- from last time
728-
Nothing -> LBS.toStrict $ B.encode (hash (snd val))
720+
let cutoffHash = LBS.toStrict $ B.encode (hash (snd val))
729721
return (Just cutoffHash, val)
730722

731723
defineNoDiagnostics (cmapWithPrio LogShake recorder) $ \(GhcSessionDeps_ fullModSummary) file -> do

ghcide/src/Development/IDE/Core/Shake.hs

+3-6
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ import Development.IDE.Graph hiding (ShakeValue)
135135
import qualified Development.IDE.Graph as Shake
136136
import Development.IDE.Graph.Database (ShakeDatabase,
137137
shakeGetBuildStep,
138-
shakeOpenDatabase,
138+
shakeNewDatabase,
139139
shakeProfileDatabase,
140140
shakeRunDatabaseForKeys)
141141
import Development.IDE.Graph.Rule
@@ -456,7 +456,6 @@ newtype ShakeSession = ShakeSession
456456
data IdeState = IdeState
457457
{shakeDb :: ShakeDatabase
458458
,shakeSession :: MVar ShakeSession
459-
,shakeClose :: IO ()
460459
,shakeExtras :: ShakeExtras
461460
,shakeDatabaseProfile :: ShakeDatabase -> IO (Maybe FilePath)
462461
}
@@ -599,11 +598,10 @@ shakeOpen recorder lspEnv defaultConfig logger debouncer
599598
-- Take one VFS snapshot at the start
600599
vfs <- atomically . newTVar =<< vfsSnapshot lspEnv
601600
pure ShakeExtras{..}
602-
(shakeDbM, shakeClose) <-
603-
shakeOpenDatabase
601+
shakeDb <-
602+
shakeNewDatabase
604603
opts { shakeExtra = newShakeExtra shakeExtras }
605604
rules
606-
shakeDb <- shakeDbM
607605
shakeSession <- newEmptyMVar
608606
shakeDatabaseProfile <- shakeDatabaseProfileIO shakeProfileDir
609607
let ideState = IdeState{..}
@@ -651,7 +649,6 @@ shakeShut IdeState{..} = do
651649
-- request so we first abort that.
652650
for_ runner cancelShakeSession
653651
void $ shakeDatabaseProfile shakeDb
654-
shakeClose
655652
progressStop $ progress shakeExtras
656653

657654

ghcide/src/Development/IDE/Types/Options.hs

+1-11
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ module Development.IDE.Types.Options
1717
, IdeGhcSession(..)
1818
, OptHaddockParse(..)
1919
, ProgressReportingStyle(..)
20-
,optShakeFiles) where
20+
) where
2121

2222
import qualified Data.Text as T
2323
import Data.Typeable
@@ -85,13 +85,6 @@ data IdeOptions = IdeOptions
8585
-- ^ Experimental feature to re-run only the subset of the Shake graph that has changed
8686
}
8787

88-
optShakeFiles :: IdeOptions -> Maybe FilePath
89-
optShakeFiles opts
90-
| value == defValue = Nothing
91-
| otherwise = Just value
92-
where
93-
value = shakeFiles (optShakeOptions opts)
94-
defValue = shakeFiles (optShakeOptions $ defaultIdeOptions undefined)
9588
data OptHaddockParse = HaddockParse | NoHaddockParse
9689
deriving (Eq,Ord,Show,Enum)
9790

@@ -127,9 +120,6 @@ defaultIdeOptions session = IdeOptions
127120
,optExtensions = ["hs", "lhs"]
128121
,optPkgLocationOpts = defaultIdePkgLocationOptions
129122
,optShakeOptions = shakeOptions
130-
{shakeThreads = 0
131-
,shakeFiles = "/dev/null"
132-
}
133123
,optShakeProfiling = Nothing
134124
,optOTMemoryProfiling = IdeOTMemoryProfiling False
135125
,optReportProgress = IdeReportProgress False

hls-graph/hls-graph.cabal

+30
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,33 @@ library
104104
DataKinds
105105
KindSignatures
106106
TypeOperators
107+
108+
test-suite tests
109+
type: exitcode-stdio-1.0
110+
default-language: Haskell2010
111+
hs-source-dirs: test
112+
main-is: Main.hs
113+
other-modules:
114+
ActionSpec
115+
DatabaseSpec
116+
Example
117+
RulesSpec
118+
Spec
119+
120+
ghc-options: -threaded -rtsopts -with-rtsopts=-N -fno-ignore-asserts
121+
build-depends:
122+
, base
123+
, containers
124+
, directory
125+
, extra
126+
, filepath
127+
, hls-graph
128+
, hspec
129+
, stm
130+
, stm-containers
131+
, tasty
132+
, tasty-hspec
133+
, tasty-hunit
134+
, tasty-rerun
135+
, text
136+
build-tool-depends: hspec-discover:hspec-discover -any

hls-graph/src/Development/IDE/Graph.hs

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ module Development.IDE.Graph(
55
Key(..),
66
actionFinally, actionBracket, actionCatch, actionFork,
77
-- * Configuration
8-
ShakeOptions(shakeAllowRedefineRules, shakeThreads, shakeFiles, shakeExtra),
8+
ShakeOptions(shakeAllowRedefineRules, shakeExtra),
99
getShakeExtra, getShakeExtraRules, newShakeExtra,
1010
-- * Explicit parallelism
1111
parallel,

hls-graph/src/Development/IDE/Graph/Database.hs

+4-9
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
module Development.IDE.Graph.Database(
55
ShakeDatabase,
66
ShakeValue,
7-
shakeOpenDatabase,
7+
shakeNewDatabase,
88
shakeRunDatabase,
99
shakeRunDatabaseForKeys,
1010
shakeProfileDatabase,
@@ -23,22 +23,18 @@ import Development.IDE.Graph.Internal.Profile (writeProfile)
2323
import Development.IDE.Graph.Internal.Rules
2424
import Development.IDE.Graph.Internal.Types
2525

26-
data ShakeDatabase = ShakeDatabase !Int [Action ()] Database
2726

2827
-- Placeholder to be the 'extra' if the user doesn't set it
2928
data NonExportedType = NonExportedType
3029

31-
shakeOpenDatabase :: ShakeOptions -> Rules () -> IO (IO ShakeDatabase, IO ())
32-
shakeOpenDatabase opts rules = pure (shakeNewDatabase opts rules, pure ())
33-
3430
shakeNewDatabase :: ShakeOptions -> Rules () -> IO ShakeDatabase
3531
shakeNewDatabase opts rules = do
3632
let extra = fromMaybe (toDyn NonExportedType) $ shakeExtra opts
3733
(theRules, actions) <- runRules extra rules
3834
db <- newDatabase extra theRules
3935
pure $ ShakeDatabase (length actions) actions db
4036

41-
shakeRunDatabase :: ShakeDatabase -> [Action a] -> IO ([a], [IO ()])
37+
shakeRunDatabase :: ShakeDatabase -> [Action a] -> IO [a]
4238
shakeRunDatabase = shakeRunDatabaseForKeys Nothing
4339

4440
-- | Returns the set of dirty keys annotated with their age (in # of builds)
@@ -62,11 +58,10 @@ shakeRunDatabaseForKeys
6258
-- ^ Set of keys changed since last run. 'Nothing' means everything has changed
6359
-> ShakeDatabase
6460
-> [Action a]
65-
-> IO ([a], [IO ()])
61+
-> IO [a]
6662
shakeRunDatabaseForKeys keysChanged (ShakeDatabase lenAs1 as1 db) as2 = do
6763
incDatabase db keysChanged
68-
as <- fmap (drop lenAs1) $ runActions db $ map unvoid as1 ++ as2
69-
return (as, [])
64+
fmap (drop lenAs1) $ runActions db $ map unvoid as1 ++ as2
7065

7166
-- | Given a 'ShakeDatabase', write an HTML profile to the given file about the latest run.
7267
shakeProfileDatabase :: ShakeDatabase -> FilePath -> IO ()

hls-graph/src/Development/IDE/Graph/Internal/Action.hs

+5-3
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ apply1 k = head <$> apply [k]
116116
apply :: (RuleResult key ~ value, ShakeValue key, Typeable value) => [key] -> Action [value]
117117
apply ks = do
118118
db <- Action $ asks actionDatabase
119-
(is, vs) <- liftIO $ build db ks
119+
stack <- Action $ asks actionStack
120+
(is, vs) <- liftIO $ build db stack ks
120121
ref <- Action $ asks actionDeps
121122
liftIO $ modifyIORef ref (ResultDeps is <>)
122123
pure vs
@@ -125,13 +126,14 @@ apply ks = do
125126
applyWithoutDependency :: (RuleResult key ~ value, ShakeValue key, Typeable value) => [key] -> Action [value]
126127
applyWithoutDependency ks = do
127128
db <- Action $ asks actionDatabase
128-
(_, vs) <- liftIO $ build db ks
129+
stack <- Action $ asks actionStack
130+
(_, vs) <- liftIO $ build db stack ks
129131
pure vs
130132

131133
runActions :: Database -> [Action a] -> IO [a]
132134
runActions db xs = do
133135
deps <- newIORef mempty
134-
runReaderT (fromAction $ parallel xs) $ SAction db deps
136+
runReaderT (fromAction $ parallel xs) $ SAction db deps emptyStack
135137

136138
-- | Returns the set of dirty keys annotated with their age (in # of builds)
137139
getDirtySet :: Action [(Key, Int)]

hls-graph/src/Development/IDE/Graph/Internal/Database.hs

+34-29
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,11 @@ updateDirty = Focus.adjust $ \(KeyDetails status rdeps) ->
7777
-- | Unwrap and build a list of keys in parallel
7878
build
7979
:: forall key value . (RuleResult key ~ value, Typeable key, Show key, Hashable key, Eq key, Typeable value)
80-
=> Database -> [key] -> IO ([Key], [value])
81-
build db keys = do
80+
=> Database -> Stack -> [key] -> IO ([Key], [value])
81+
-- build _ st k | traceShow ("build", st, k) False = undefined
82+
build db stack keys = do
8283
(ids, vs) <- runAIO $ fmap unzip $ either return liftIO =<<
83-
builder db (map Key keys)
84+
builder db stack (map Key keys)
8485
pure (ids, map (asV . resultValue) vs)
8586
where
8687
asV :: Value -> value
@@ -90,8 +91,9 @@ build db keys = do
9091
-- If none of the keys are dirty, we can return the results immediately.
9192
-- Otherwise, a blocking computation is returned *which must be evaluated asynchronously* to avoid deadlock.
9293
builder
93-
:: Database -> [Key] -> AIO (Either [(Key, Result)] (IO [(Key, Result)]))
94-
builder db@Database{..} keys = withRunInIO $ \(RunInIO run) -> do
94+
:: Database -> Stack -> [Key] -> AIO (Either [(Key, Result)] (IO [(Key, Result)]))
95+
-- builder _ st kk | traceShow ("builder", st,kk) False = undefined
96+
builder db@Database{..} stack keys = withRunInIO $ \(RunInIO run) -> do
9597
-- Things that I need to force before my results are ready
9698
toForce <- liftIO $ newTVarIO []
9799
current <- liftIO $ readTVarIO databaseStep
@@ -103,11 +105,13 @@ builder db@Database{..} keys = withRunInIO $ \(RunInIO run) -> do
103105
status <- SMap.lookup id databaseValues
104106
val <- case viewDirty current $ maybe (Dirty Nothing) keyStatus status of
105107
Clean r -> pure r
106-
Running _ force val _ -> do
108+
Running _ force val _
109+
| memberStack id stack -> throw $ StackException stack
110+
| otherwise -> do
107111
modifyTVar' toForce (Wait force :)
108112
pure val
109113
Dirty s -> do
110-
let act = run (refresh db id s)
114+
let act = run (refresh db stack id s)
111115
(force, val) = splitIO (join act)
112116
SMap.focus (updateStatus $ Running current force val s) id databaseValues
113117
modifyTVar' toForce (Spawn force:)
@@ -127,32 +131,33 @@ builder db@Database{..} keys = withRunInIO $ \(RunInIO run) -> do
127131
-- * If no dirty dependencies and we have evaluated the key previously, then we refresh it in the current thread.
128132
-- This assumes that the implementation will be a lookup
129133
-- * Otherwise, we spawn a new thread to refresh the dirty deps (if any) and the key itself
130-
refresh :: Database -> Key -> Maybe Result -> AIO (IO Result)
131-
refresh db key result@(Just me@Result{resultDeps = ResultDeps deps}) = do
132-
res <- builder db deps
133-
case res of
134-
Left res ->
135-
if isDirty res
136-
then asyncWithCleanUp $ liftIO $ compute db key RunDependenciesChanged result
137-
else pure $ compute db key RunDependenciesSame result
138-
Right iores -> asyncWithCleanUp $ liftIO $ do
139-
res <- iores
140-
let mode = if isDirty res then RunDependenciesChanged else RunDependenciesSame
141-
compute db key mode result
142-
where
143-
isDirty = any (\(_,dep) -> resultBuilt me < resultChanged dep)
144-
145-
refresh db key result =
146-
asyncWithCleanUp $ liftIO $ compute db key RunDependenciesChanged result
147-
134+
refresh :: Database -> Stack -> Key -> Maybe Result -> AIO (IO Result)
135+
-- refresh _ st k _ | traceShow ("refresh", st, k) False = undefined
136+
refresh db stack key result = case (addStack key stack, result) of
137+
(Left e, _) -> throw e
138+
(Right stack, Just me@Result{resultDeps = ResultDeps deps}) -> do
139+
res <- builder db stack deps
140+
let isDirty = any (\(_,dep) -> resultBuilt me < resultChanged dep)
141+
case res of
142+
Left res ->
143+
if isDirty res
144+
then asyncWithCleanUp $ liftIO $ compute db stack key RunDependenciesChanged result
145+
else pure $ compute db stack key RunDependenciesSame result
146+
Right iores -> asyncWithCleanUp $ liftIO $ do
147+
res <- iores
148+
let mode = if isDirty res then RunDependenciesChanged else RunDependenciesSame
149+
compute db stack key mode result
150+
(Right stack, _) ->
151+
asyncWithCleanUp $ liftIO $ compute db stack key RunDependenciesChanged result
148152

149153
-- | Compute a key.
150-
compute :: Database -> Key -> RunMode -> Maybe Result -> IO Result
151-
compute db@Database{..} key mode result = do
154+
compute :: Database -> Stack -> Key -> RunMode -> Maybe Result -> IO Result
155+
-- compute _ st k _ _ | traceShow ("compute", st, k) False = undefined
156+
compute db@Database{..} stack key mode result = do
152157
let act = runRule databaseRules key (fmap resultData result) mode
153158
deps <- newIORef UnknownDeps
154159
(execution, RunResult{..}) <-
155-
duration $ runReaderT (fromAction act) $ SAction db deps
160+
duration $ runReaderT (fromAction act) $ SAction db deps stack
156161
built <- readTVarIO databaseStep
157162
deps <- readIORef deps
158163
let changed = if runChanged == ChangedRecomputeDiff then built else maybe built resultChanged result
@@ -165,7 +170,7 @@ compute db@Database{..} key mode result = do
165170
deps | not(null deps)
166171
&& runChanged /= ChangedNothing
167172
-> do
168-
void $ forkIO $
173+
void $
169174
updateReverseDeps key db
170175
(getResultDepsDefault [] previousDeps)
171176
(HSet.fromList deps)

hls-graph/src/Development/IDE/Graph/Internal/Options.hs

+1-4
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,13 @@ import Data.Dynamic
55
import Development.IDE.Graph.Internal.Types
66

77
data ShakeOptions = ShakeOptions {
8-
-- | Has no effect, kept only for api compatibility with Shake
9-
shakeThreads :: Int,
10-
shakeFiles :: FilePath,
118
shakeExtra :: Maybe Dynamic,
129
shakeAllowRedefineRules :: Bool,
1310
shakeTimings :: Bool
1411
}
1512

1613
shakeOptions :: ShakeOptions
17-
shakeOptions = ShakeOptions 0 ".shake" Nothing False False
14+
shakeOptions = ShakeOptions Nothing False False
1815

1916
getShakeExtra :: Typeable a => Action (Maybe a)
2017
getShakeExtra = do

hls-graph/src/Development/IDE/Graph/Internal/Rules.hs

-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import Development.IDE.Graph.Classes
2121
import Development.IDE.Graph.Internal.Types
2222

2323
-- | The type mapping between the @key@ or a rule and the resulting @value@.
24-
-- See 'addBuiltinRule' and 'Development.Shake.Rule.apply'.
2524
type family RuleResult key -- = value
2625

2726
action :: Action a -> Rules ()

0 commit comments

Comments
 (0)