Skip to content

Commit aae24c8

Browse files
committed
Rewrite hls-graph to not use the Shake code
1 parent dec47a3 commit aae24c8

File tree

10 files changed

+623
-44
lines changed

10 files changed

+623
-44
lines changed

hls-graph/hls-graph.cabal

+8
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,20 @@ library
3535
Development.IDE.Graph.Internal.Action
3636
Development.IDE.Graph.Internal.Options
3737
Development.IDE.Graph.Internal.Rules
38+
Development.IDE.Graph.Internal.Database
39+
Development.IDE.Graph.Internal.Ids
40+
Development.IDE.Graph.Internal.Intern
41+
Development.IDE.Graph.Internal.Types
3842

3943
hs-source-dirs: src
4044
build-depends:
45+
, async
4146
, base >=4.12 && <5
4247
, bytestring
48+
, extra
49+
, primitive
4350
, shake >= 0.19.4
51+
, transformers
4452
, unordered-containers
4553

4654
ghc-options:

hls-graph/src/Development/IDE/Graph.hs

+1
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@ import qualified Development.Shake as Shake
2323
import Development.IDE.Graph.Internal.Action
2424
import Development.IDE.Graph.Internal.Options
2525
import Development.IDE.Graph.Internal.Rules
26+
import Development.IDE.Graph.Internal.Types
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,49 @@
11

22
module Development.IDE.Graph.Database(
3-
Shake.ShakeDatabase,
3+
ShakeDatabase,
44
shakeOpenDatabase,
55
shakeRunDatabase,
6-
Shake.shakeProfileDatabase,
6+
shakeProfileDatabase,
77
) where
88

9-
import qualified Development.Shake.Database as Shake
109
import Development.IDE.Graph.Internal.Action
1110
import Development.IDE.Graph.Internal.Options
1211
import Development.IDE.Graph.Internal.Rules
12+
import Development.IDE.Graph.Internal.Types
13+
import Data.Maybe
14+
import Data.Dynamic
15+
import Development.IDE.Graph.Internal.Database
16+
import GHC.Conc
17+
import Control.Concurrent.Extra
1318

14-
shakeOpenDatabase :: ShakeOptions -> Rules () -> IO (IO Shake.ShakeDatabase, IO ())
15-
shakeOpenDatabase a b = Shake.shakeOpenDatabase (fromShakeOptions a) (fromRules b)
1619

17-
shakeRunDatabase :: Shake.ShakeDatabase -> [Action a] -> IO ([a], [IO ()])
18-
shakeRunDatabase a b = Shake.shakeRunDatabase a (map fromAction b)
20+
data ShakeDatabase = ShakeDatabase !Int !Int [Action ()] Database
21+
22+
-- Placeholder to be the 'extra' if the user doesn't set it
23+
data NonExportedType = NonExportedType
24+
25+
shakeOpenDatabase :: ShakeOptions -> Rules () -> IO (IO ShakeDatabase, IO ())
26+
shakeOpenDatabase opts rules = pure (shakeNewDatabase opts rules, pure ())
27+
28+
shakeNewDatabase :: ShakeOptions -> Rules () -> IO ShakeDatabase
29+
shakeNewDatabase opts rules = do
30+
let extra = fromMaybe (toDyn NonExportedType) $ shakeExtra opts
31+
(theRules, actions) <- runRules extra rules
32+
db <- newDatabase extra theRules
33+
let threads = shakeThreads opts
34+
threads <- if threads /= 0 then pure threads else getNumProcessors
35+
pure $ ShakeDatabase threads (length actions) actions db
36+
37+
shakeRunDatabase :: ShakeDatabase -> [Action a] -> IO ([a], [IO ()])
38+
shakeRunDatabase (ShakeDatabase threads lenAs1 as1 db) as2 = withNumCapabilities threads $ do
39+
incDatabase db
40+
as <- fmap (drop lenAs1) $ runActions db $ map unvoid as1 ++ as2
41+
return (as, [])
42+
43+
-- Only valid if we never pull on the results, which we don't
44+
unvoid :: Functor m => m () -> m a
45+
unvoid = fmap undefined
46+
47+
-- Noop
48+
shakeProfileDatabase :: ShakeDatabase -> FilePath -> IO ()
49+
shakeProfileDatabase _ file = writeFile file ""
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,100 @@
11
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
22
{-# LANGUAGE TypeFamilies #-}
3+
{-# LANGUAGE ScopedTypeVariables #-}
34

45
module Development.IDE.Graph.Internal.Action where
56

67
import qualified Development.Shake as Shake
7-
import qualified Development.Shake.Rule as Shake
88
import Development.Shake.Classes
99
import Control.Exception
10+
import Control.Concurrent.Async
11+
import System.Exit
1012
import Control.Monad.IO.Class
11-
import Control.Monad.Fail
13+
import Control.Monad.Trans.Reader
14+
import Data.IORef
15+
import Development.IDE.Graph.Internal.Database
16+
import Development.IDE.Graph.Internal.Types
17+
import Control.Monad.Extra
18+
import Control.Monad.Trans.Class
19+
import Control.Concurrent.Async
1220

13-
newtype Action a = Action {fromAction :: Shake.Action a}
14-
deriving (Monad, Applicative, Functor, MonadIO, MonadFail)
1521

1622
alwaysRerun :: Action ()
17-
alwaysRerun = Action Shake.alwaysRerun
23+
alwaysRerun = do
24+
ref <- Action $ asks actionDeps
25+
liftIO $ writeIORef ref Nothing
1826

27+
-- No-op for now
1928
reschedule :: Double -> Action ()
20-
reschedule = Action . Shake.reschedule
29+
reschedule _ = pure ()
2130

2231
parallel :: [Action a] -> Action [a]
23-
parallel = Action . Shake.parallel . map fromAction
32+
parallel [] = pure []
33+
parallel [x] = fmap (:[]) x
34+
parallel xs = do
35+
a <- Action ask
36+
deps <- liftIO $ readIORef $ actionDeps a
37+
case deps of
38+
Nothing ->
39+
-- if we are already in the rerun mode, nothing we do is going to impact our state
40+
liftIO $ mapConcurrently (ignoreState a) xs
41+
Just deps -> do
42+
(newDeps, res) <- liftIO $ unzip <$> mapConcurrently (usingState a) xs
43+
liftIO $ writeIORef (actionDeps a) $ (deps ++) <$> concatMapM id newDeps
44+
pure res
45+
where
46+
ignoreState a x = do
47+
ref <- newIORef Nothing
48+
runReaderT (fromAction x) a{actionDeps=ref}
49+
50+
usingState a x = do
51+
ref <- newIORef $ Just []
52+
res <- runReaderT (fromAction x) a{actionDeps=ref}
53+
deps <- readIORef ref
54+
pure (deps, res)
55+
56+
isAsyncException :: SomeException -> Bool
57+
isAsyncException e
58+
| Just (_ :: AsyncCancelled) <- fromException e = True
59+
| Just (_ :: AsyncException) <- fromException e = True
60+
| Just (_ :: ExitCode) <- fromException e = True
61+
| otherwise = False
62+
2463

2564
actionCatch :: Exception e => Action a -> (e -> Action a) -> Action a
26-
actionCatch a b = Action $ Shake.actionCatch (fromAction a) (fromAction . b)
65+
actionCatch a b = do
66+
v <- Action ask
67+
Action $ lift $ catchJust f (runReaderT (fromAction a) v) (\x -> runReaderT (fromAction (b x)) v)
68+
where
69+
-- Catch only catches exceptions that were caused by this code, not those that
70+
-- are a result of program termination
71+
f e | isAsyncException e = Nothing
72+
| otherwise = fromException e
2773

2874
actionBracket :: IO a -> (a -> IO b) -> (a -> Action c) -> Action c
29-
actionBracket a b c = Action $ Shake.actionBracket a b (fromAction . c)
75+
actionBracket a b c = do
76+
v <- Action ask
77+
Action $ lift $ bracket a b (\x -> runReaderT (fromAction (c x)) v)
3078

3179
actionFinally :: Action a -> IO b -> Action a
32-
actionFinally a b = Action $ Shake.actionFinally (fromAction a) b
80+
actionFinally a b = do
81+
v <- Action ask
82+
Action $ lift $ finally (runReaderT (fromAction a) v) b
3383

3484
apply1 :: (Shake.RuleResult key ~ value, Shake.ShakeValue key, Typeable value) => key -> Action value
35-
apply1 = Action . Shake.apply1
85+
apply1 k = head <$> apply [k]
3686

3787
apply :: (Shake.RuleResult key ~ value, Shake.ShakeValue key, Typeable value) => [key] -> Action [value]
38-
apply = Action . Shake.apply
88+
apply ks = do
89+
db <- Action $ asks actionDatabase
90+
(is, vs) <- liftIO $ build db ks
91+
ref <- Action $ asks actionDeps
92+
deps <- liftIO $ readIORef ref
93+
whenJust deps $ \deps ->
94+
liftIO $ writeIORef ref $ Just $ is ++ deps
95+
pure vs
96+
97+
runActions :: Database -> [Action a] -> IO [a]
98+
runActions db xs = do
99+
deps <- newIORef Nothing
100+
runReaderT (fromAction $ parallel xs) $ SAction db deps
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
-- We deliberately want to ensure the function we add to the rule database
2+
-- has the constraints we need on it when we get it out.
3+
{-# OPTIONS_GHC -Wno-redundant-constraints #-}
4+
{-# LANGUAGE LambdaCase #-}
5+
{-# LANGUAGE ScopedTypeVariables #-}
6+
{-# LANGUAGE RecordWildCards #-}
7+
{-# LANGUAGE TypeFamilies #-}
8+
{-# LANGUAGE TupleSections #-}
9+
{-# LANGUAGE NamedFieldPuns #-}
10+
11+
module Development.IDE.Graph.Internal.Database where
12+
13+
import Development.IDE.Graph.Internal.Intern
14+
import Development.IDE.Graph.Internal.Types
15+
import Data.Dynamic
16+
import qualified Development.IDE.Graph.Internal.Intern as Intern
17+
import qualified Development.IDE.Graph.Internal.Ids as Ids
18+
import Control.Concurrent.Extra
19+
import Data.IORef.Extra
20+
import Control.Monad
21+
import Development.Shake.Classes
22+
import qualified Development.Shake as Shake
23+
import Data.Maybe
24+
import Control.Concurrent.Async
25+
import System.IO.Unsafe
26+
import Development.IDE.Graph.Internal.Rules
27+
import qualified Development.Shake.Rule as Shake
28+
import Control.Exception
29+
import Control.Monad.Trans.Reader
30+
import Data.Tuple.Extra
31+
import Data.Either
32+
33+
newDatabase :: Dynamic -> TheRules -> IO Database
34+
newDatabase databaseExtra databaseRules = do
35+
databaseStep <- newIORef $ Step 0
36+
databaseLock <- newLock
37+
databaseIds <- newIORef Intern.empty
38+
databaseValues <- Ids.empty
39+
pure Database{..}
40+
41+
incDatabase :: Database -> IO ()
42+
incDatabase db = do
43+
modifyIORef' (databaseStep db) $ \(Step i) -> Step $ i + 1
44+
Ids.forMutate (databaseValues db) $ second $ \case
45+
Clean x -> Dirty (Just x)
46+
Dirty x -> Dirty x
47+
Running _ x -> Dirty x
48+
49+
50+
build
51+
:: forall key value . (Shake.RuleResult key ~ value, Typeable key, Show key, Hashable key, Eq key, Typeable value)
52+
=> Database -> [key] -> IO ([Id], [value])
53+
build db keys = do
54+
(ids, vs) <- fmap unzip $ builder db $ map (Right . Key) keys
55+
pure (ids, map (asV . resultValue) vs)
56+
where
57+
asV :: Value -> value
58+
asV (Value x) = unwrapDynamic x
59+
60+
builder
61+
:: Database -> [Either Id Key] -> IO [(Id, Result)]
62+
builder db@Database{..} keys = do
63+
-- Async things that I own and am responsible for killing
64+
ownedAsync <- newIORef []
65+
flip onException (cleanupAsync ownedAsync) $ do
66+
67+
-- Things that I need to force before my results are ready
68+
toForce <- newIORef []
69+
70+
results <- withLock databaseLock $ do
71+
forM keys $ \idKey -> do
72+
id <- case idKey of
73+
Left id -> pure id
74+
Right key -> do
75+
ids <- readIORef databaseIds
76+
case Intern.lookup key ids of
77+
Just v -> pure v
78+
Nothing -> do
79+
(ids, id) <- pure $ Intern.add key ids
80+
writeIORef' databaseIds ids
81+
return id
82+
83+
status <- Ids.lookup databaseValues id
84+
val <- case fromMaybe (fromRight undefined idKey, Dirty Nothing) status of
85+
(_, Clean r) -> pure r
86+
(_, Running act _) -> do
87+
-- we promise to force everything in todo before reading the results
88+
-- so the following unsafePerformIO isn't actually unsafe
89+
let (force, val) = splitIO act
90+
modifyIORef toForce (force:)
91+
pure val
92+
(key, Dirty s) -> do
93+
-- Important we don't lose any Async things we create
94+
act <- uninterruptibleMask $ \restore -> do
95+
-- the child actions should always be spawned unmasked
96+
-- or they can't be killed
97+
async <- async $ restore $ check db key id s
98+
modifyIORef ownedAsync (async:)
99+
pure $ wait async
100+
Ids.insert databaseValues id (key, Running act s)
101+
let (force, val) = splitIO act
102+
modifyIORef toForce (force:)
103+
pure val
104+
105+
pure (id, val)
106+
107+
sequence_ =<< readIORef toForce
108+
pure results
109+
110+
cleanupAsync :: IORef [Async a] -> IO ()
111+
cleanupAsync ref = mapConcurrently_ uninterruptibleCancel =<< readIORef ref
112+
113+
114+
-- Check if we need to run the database.
115+
check :: Database -> Key -> Id -> Maybe Result -> IO Result
116+
check db key id result@(Just me@Result{resultDeps=Just deps}) = do
117+
res <- builder db $ map Left deps
118+
let dirty = all (\(_,dep) -> resultBuilt me < resultChanged dep) res
119+
let mode = if dirty then Shake.RunDependenciesChanged else Shake.RunDependenciesSame
120+
spawn db key id mode result
121+
check db key id result = spawn db key id Shake.RunDependenciesChanged result
122+
123+
124+
-- Spawn a new computation to run the action.
125+
spawn :: Database -> Key -> Id -> Shake.RunMode -> Maybe Result -> IO Result
126+
spawn db@Database{..} key id mode result = do
127+
let act = runRule databaseRules key (fmap resultData result) mode
128+
deps <- newIORef $ Just []
129+
Shake.RunResult{..} <- runReaderT (fromAction act) $ SAction db deps
130+
built <- readIORef databaseStep
131+
deps <- readIORef deps
132+
let changed = if runChanged == Shake.ChangedRecomputeDiff then built else maybe built resultChanged result
133+
let res = Result runValue built changed deps runStore
134+
withLock databaseLock $
135+
Ids.insert databaseValues id (key, Clean res)
136+
pure res
137+
138+
data Box a = Box {fromBox :: a}
139+
140+
splitIO :: IO a -> (IO (), a)
141+
splitIO act = do
142+
let act2 = Box <$> act
143+
let res = unsafePerformIO act2
144+
(void $ evaluate res, fromBox res)

0 commit comments

Comments
 (0)