Benchmark improvements (#1178)

* [test-ci] fix wibble

* [ghcide-bench] fix scrambled output

* [ghcide-bench] add a new experiment: getDefinition after edit

* [ghcide-bench] refine the position used for identifiers

* [ghcide-bench] Support examples with multiple FOIs

* [ghcide-bench] Allow identifierP to be optional

* [ghcide-bench] experiments: do all edits first, then query

* [ghcide-bench] Add examples with multiple FOIs

* [ghcide-bench] add a completions (without edit) experiment

* [ghcide-bench] Fix indentation

* [ghcide-bench] Fix incomplete pattern match

* [ghcide-bench] Include the documentContents setup in the "startup" metric

* [ghcide-bench] fix wibble

* [ghcide-bench] add more verbose output

* [shake-bench] Consolidate -s code

* [bench-hist] Fix: depend on # samples

* [ghcide-bench] cache searchSymbol

* [cabal-bench] --no-clean

.github/workflows/test.yml

@@ -49,7 +49,7 @@ jobs:
  ~/.cabal/store
  key: ${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
  restore-keys: |
- ${{ runner.os }}-${{ matrix.ghc }}-bench-$${ hashFiles('cabal.project') }}
+ ${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
  ${{ runner.os }}-${{ matrix.ghc }}-build-
  ${{ runner.os }}-${{ matrix.ghc }}
 

ghcide/bench/config.yaml

@@ -14,20 +14,23 @@ examples:
  # Medium-sized project without TH
  - name: Cabal
  version: 3.0.0.0
- module: Distribution/Simple.hs
+ modules:
+ - Distribution/Simple.hs
+ - Distribution/Types/Module.hs
  # Small-sized project with TH
- - name: haskell-lsp-types
- version: 0.22.0.0
- module: src/Language/Haskell/LSP/Types/Lens.hs
-# - path: path-to-example
-# module: path-to-module
+ - name: lsp-types
+ version: 1.0.0.1
+ modules:
+ - src/Language/LSP/VFS.hs
+  - src/Language/LSP/Types/Lens.hs
 
 # The set of experiments to execute
 experiments:
- - hover
- - edit
- - getDefinition
+ - "edit"
+ - "hover"
  - "hover after edit"
+ - "getDefinition"
+ - "getDefinition after edit"
  - "completions after edit"
  - "code actions"
  - "code actions after edit"

ghcide/bench/exe/Main.hs

@@ -37,14 +37,23 @@
 import Control.Exception.Safe
 import Experiments
 import Options.Applicative
+import System.IO
+import Control.Monad
+
+optsP :: Parser (Config, Bool)
+optsP = (,) <$> configP <*> switch (long "no-clean")
 
 main :: IO ()
 main = do
- config <- execParser $ info (configP <**> helper) fullDesc
+ hSetBuffering stdout LineBuffering
+ hSetBuffering stderr LineBuffering
+ (config, noClean) <- execParser $ info (optsP <**> helper) fullDesc
  let ?config = config
 
+ hPrint stderr config
+
  output "starting test"
 
  SetupResult{..} <- setup
 
- runBenchmarks experiments `finally` cleanUp
+ runBenchmarks experiments `finally` unless noClean cleanUp

ghcide/bench/hist/Main.hs

@@ -51,6 +51,7 @@ import Experiments.Types (Example, exampleToOptions)
 import qualified Experiments.Types as E
 import GHC.Generics (Generic)
 import Numeric.Natural (Natural)
+import Development.Shake.Classes
 
 
 config :: FilePath
@@ -70,7 +71,7 @@ main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest} $ do
  configStatic <- liftIO $ readConfigIO config
  let build = outputFolder configStatic
  buildRules build ghcideBuildRules
- benchRules build resource (MkBenchRules (benchGhcide $ samples configStatic) "ghcide")
+ benchRules build resource (MkBenchRules (askOracle $ GetSamples ()) benchGhcide "ghcide")
  csvRules build
  svgRules build
  action $ allTargets build
@@ -98,14 +99,18 @@ createBuildSystem userRules = do
 
  _ <- addOracle $ \GetExperiments {} -> experiments <$> readConfig config
  _ <- addOracle $ \GetVersions {} -> versions <$> readConfig config
- _ <- addOracle $ \GetExamples{} -> examples <$> readConfig config
- _ <- addOracle $ \(GetExample name) -> find (\e -> getExampleName e == name) . examples <$> readConfig config
+ _ <- versioned 1 $ addOracle $ \GetExamples{} -> examples <$> readConfig config
+ _ <- versioned 1 $ addOracle $ \(GetExample name) -> find (\e -> getExampleName e == name) . examples <$> readConfig config
  _ <- addOracle $ \GetBuildSystem {} -> buildTool <$> readConfig config
+ _ <- addOracle $ \GetSamples{} -> samples <$> readConfig config
 
  benchResource <- newResource "ghcide-bench" 1
 
  userRules benchResource
 
+newtype GetSamples = GetSamples () deriving newtype (Binary, Eq, Hashable, NFData, Show)
+type instance RuleResult GetSamples = Natural
+
 --------------------------------------------------------------------------------
 
 buildGhcide :: BuildSystem -> [CmdOption] -> FilePath -> Action ()
@@ -130,9 +135,10 @@ buildGhcide Stack args out =
 
 benchGhcide
  :: Natural -> BuildSystem -> [CmdOption] -> BenchProject Example -> Action ()
-benchGhcide samples buildSystem args BenchProject{..} =
+benchGhcide samples buildSystem args BenchProject{..} = do
  command_ args "ghcide-bench" $
  [ "--timeout=3000",
+ "--no-clean",
  "-v",
  "--samples=" <> show samples,
  "--csv=" <> outcsv,