From 84209a0f783b2f9798bb55624cbb0a6a6e358962 Mon Sep 17 00:00:00 2001 From: Kristen Kozak Date: Sat, 14 Dec 2019 16:51:59 -0800 Subject: [PATCH 1/3] Update solver Hackage benchmark for v2-install. Fixes #6417. Changes in this commit: - Set the --lib flag so that the benchmark works for packages with libraries or executables. - Set --ignore-project to prevent cabal projects from affecting the results. - Handle four types of cabal errors that were introduced by v2-install. - Use +nowrap to simplify parsing of cabal errors. --- solver-benchmarks/HackageBenchmark.hs | 44 +++++++++++++++++---------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/solver-benchmarks/HackageBenchmark.hs b/solver-benchmarks/HackageBenchmark.hs index abbf33029de..145b49f9331 100644 --- a/solver-benchmarks/HackageBenchmark.hs +++ b/solver-benchmarks/HackageBenchmark.hs @@ -56,6 +56,10 @@ data CabalResult = Solution | NoInstallPlan | BackjumpLimit + | Unbuildable + | UnbuildableDep + | ComponentCycle + | ModReexpIssue | PkgNotFound | Timeout | Unknown @@ -81,14 +85,14 @@ hackageBenchmarkMain = do -- "trial" or "summary". when argPrintTrials $ putStr $ printf "%-16s " "trial/summary" putStrLn $ - printf "%-*s %-13s %-13s %11s %11s %11s %11s %11s" + printf "%-*s %-14s %-14s %11s %11s %11s %11s %11s" nameColumnWidth "package" "result1" "result2" "mean1" "mean2" "stddev1" "stddev2" "speedup" forM_ pkgs $ \pkg -> do let printTrial msgType result1 result2 time1 time2 = putStrLn $ - printf "%-16s %-*s %-13s %-13s %10.3fs %10.3fs" + printf "%-16s %-*s %-14s %-14s %10.3fs %10.3fs" msgType nameColumnWidth (unPackageName pkg) (show result1) (show result2) (diffTimeToDouble time1) (diffTimeToDouble time2) @@ -125,7 +129,7 @@ hackageBenchmarkMain = do if isSignificantResult result1 result2 || isSignificantTimeDifference argPValue ts1 ts2 then putStrLn $ - printf "%-*s %-13s %-13s %10.3fs %10.3fs %10.3fs %10.3fs %10.3f" + printf "%-*s %-14s %-14s %10.3fs %10.3fs %10.3fs %10.3fs %10.3f" nameColumnWidth (unPackageName pkg) (show result1) (show result2) mean1 mean2 stddev1 stddev2 speedup else when (argPrintTrials || argPrintSkippedPackages) $ @@ -172,7 +176,7 @@ runCabal timeoutSeconds cabal flags pkg = do let timeout = "timeout --foreground -sINT " ++ show timeoutSeconds cabalCmd = unwords $ - [cabal, "install", unPackageName pkg, "--dry-run", "-v0"] ++ flags + [cabal, "install", "--ignore-project", "--lib", unPackageName pkg, "--dry-run", "-vsilent+nowrap"] ++ flags cmd = (shell (timeout ++ " " ++ cabalCmd)) { std_err = CreatePipe } -- TODO: Read stdout and compare the install plans. @@ -182,12 +186,16 @@ runCabal timeoutSeconds cabal flags pkg = do let exhaustiveMsg = "After searching the rest of the dependency tree exhaustively" result - | exitCode == ExitSuccess = Solution - | exitCode == ExitFailure 124 = Timeout - | fromString exhaustiveMsg `B.isInfixOf` err = NoInstallPlan - | fromString "Backjump limit reached" `B.isInfixOf` err = BackjumpLimit - | fromString "There is no package named" `B.isInfixOf` err = PkgNotFound - | otherwise = Unknown + | exitCode == ExitSuccess = Solution + | exitCode == ExitFailure 124 = Timeout + | fromString exhaustiveMsg `B.isInfixOf` err = NoInstallPlan + | fromString "Backjump limit reached" `B.isInfixOf` err = BackjumpLimit + | fromString "none of the components are available to build" `B.isInfixOf` err = Unbuildable + | fromString "Dependency on unbuildable" `B.isInfixOf` err = UnbuildableDep + | fromString "Dependency cycle between the following components" `B.isInfixOf` err = ComponentCycle + | fromString "Problem with module re-exports" `B.isInfixOf` err = ModReexpIssue + | fromString "There is no package named" `B.isInfixOf` err = PkgNotFound + | otherwise = Unknown return (CabalTrial time result) isSampleLargeEnough :: PValue Double -> Int -> Bool @@ -224,12 +232,16 @@ isSignificantResult r1 r2 = r1 /= r2 || not (isExpectedResult r1) -- Is this result expected in a benchmark run on all of Hackage? isExpectedResult :: CabalResult -> Bool -isExpectedResult Solution = True -isExpectedResult NoInstallPlan = True -isExpectedResult BackjumpLimit = True -isExpectedResult Timeout = True -isExpectedResult PkgNotFound = False -isExpectedResult Unknown = False +isExpectedResult Solution = True +isExpectedResult NoInstallPlan = True +isExpectedResult BackjumpLimit = True +isExpectedResult Timeout = True +isExpectedResult Unbuildable = True +isExpectedResult UnbuildableDep = True +isExpectedResult ComponentCycle = True +isExpectedResult ModReexpIssue = True +isExpectedResult PkgNotFound = False +isExpectedResult Unknown = False -- Combine CabalResults from multiple trials. Ignoring timeouts, all results -- should be the same. If they aren't the same, we returns Unknown. From 73cccbbefa29ff7e29186b9f1171c8acfc1aad22 Mon Sep 17 00:00:00 2001 From: Kristen Kozak Date: Mon, 16 Dec 2019 23:17:19 -0800 Subject: [PATCH 2/3] Improve reproducibility of install command used in solver Hackage benchmark. --- solver-benchmarks/HackageBenchmark.hs | 31 ++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/solver-benchmarks/HackageBenchmark.hs b/solver-benchmarks/HackageBenchmark.hs index 145b49f9331..b014e6b8f46 100644 --- a/solver-benchmarks/HackageBenchmark.hs +++ b/solver-benchmarks/HackageBenchmark.hs @@ -174,9 +174,34 @@ runCabal :: Int -> FilePath -> [String] -> PackageName -> IO CabalTrial runCabal timeoutSeconds cabal flags pkg = do ((exitCode, err), time) <- timeEvent $ do let timeout = "timeout --foreground -sINT " ++ show timeoutSeconds - cabalCmd = - unwords $ - [cabal, "install", "--ignore-project", "--lib", unPackageName pkg, "--dry-run", "-vsilent+nowrap"] ++ flags + cabalCmd = unwords $ + [ cabal + + -- A non-existent store directory prevents cabal from reading the + -- store, which would cause the size of the store to affect run + -- time. + , "--store-dir=non-existent-store-dir" + + , "v2-install" + + -- These flags prevent a Cabal project or package environment from + -- affecting the install plan. + , "--ignore-project" + , "--package-env=non-existent-package-env" + + -- --lib allows solving for packages with libraries or + -- executables. + , "--lib" + + , unPackageName pkg + + , "--dry-run" + + -- The test doesn't currently handle stdout, so we suppress it + -- with silent. nowrap simplifies parsing the errors messages. + , "-vsilent+nowrap"] + + ++ flags cmd = (shell (timeout ++ " " ++ cabalCmd)) { std_err = CreatePipe } -- TODO: Read stdout and compare the install plans. From 676ffbd669c03c2053e9a1cfed337c670bd554e3 Mon Sep 17 00:00:00 2001 From: Oleg Grenrus Date: Wed, 18 Dec 2019 18:17:27 +0200 Subject: [PATCH 3/3] Add solver-benchmarks to validate.sh --- .docker/validate-8.8.1.dockerfile | 2 +- cabal.project.validate | 2 +- solver-benchmarks/HackageBenchmark.hs | 6 ++++- solver-benchmarks/solver-benchmarks.cabal | 2 ++ validate.sh | 30 +++++++++++++++++++++-- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.docker/validate-8.8.1.dockerfile b/.docker/validate-8.8.1.dockerfile index ca1fae605c3..0a797ae4a3b 100644 --- a/.docker/validate-8.8.1.dockerfile +++ b/.docker/validate-8.8.1.dockerfile @@ -49,4 +49,4 @@ RUN cabal v2-install -w ghc-8.8.1 --lib \ # Validate WORKDIR /build COPY . /build -RUN sh ./validate.sh -w ghc-8.8.1 -v -D +RUN sh ./validate.sh -w ghc-8.8.1 -v -D -b diff --git a/cabal.project.validate b/cabal.project.validate index a11193fc853..81f99f296c9 100644 --- a/cabal.project.validate +++ b/cabal.project.validate @@ -1,4 +1,4 @@ -packages: Cabal/ cabal-testsuite/ cabal-install/ +packages: Cabal/ cabal-testsuite/ cabal-install/ solver-benchmarks/ write-ghc-environment-files: never diff --git a/solver-benchmarks/HackageBenchmark.hs b/solver-benchmarks/HackageBenchmark.hs index b014e6b8f46..ef5d9efa598 100644 --- a/solver-benchmarks/HackageBenchmark.hs +++ b/solver-benchmarks/HackageBenchmark.hs @@ -27,7 +27,9 @@ import Statistics.Test.MannWhitneyU ( PositionTest(..), TestResult(..) , mannWhitneyUCriticalValue , mannWhitneyUtest) import Statistics.Types (PValue, mkPValue) +import System.Directory (getTemporaryDirectory) import System.Exit (ExitCode(..), exitFailure) +import System.FilePath (()) import System.IO ( BufferMode(LineBuffering), hPutStrLn, hSetBuffering, stderr , stdout) import System.Process ( StdStream(CreatePipe), CreateProcess(..), callProcess @@ -173,6 +175,8 @@ hackageBenchmarkMain = do runCabal :: Int -> FilePath -> [String] -> PackageName -> IO CabalTrial runCabal timeoutSeconds cabal flags pkg = do ((exitCode, err), time) <- timeEvent $ do + tmpDir <- getTemporaryDirectory + let timeout = "timeout --foreground -sINT " ++ show timeoutSeconds cabalCmd = unwords $ [ cabal @@ -180,7 +184,7 @@ runCabal timeoutSeconds cabal flags pkg = do -- A non-existent store directory prevents cabal from reading the -- store, which would cause the size of the store to affect run -- time. - , "--store-dir=non-existent-store-dir" + , "--store-dir=" ++ (tmpDir "non-existent-store-dir") , "v2-install" diff --git a/solver-benchmarks/solver-benchmarks.cabal b/solver-benchmarks/solver-benchmarks.cabal index ab28af0ae75..2046a0b614a 100644 --- a/solver-benchmarks/solver-benchmarks.cabal +++ b/solver-benchmarks/solver-benchmarks.cabal @@ -30,6 +30,8 @@ library base, bytestring, Cabal >= 2.3, + directory, + filepath, optparse-applicative, process, time, diff --git a/validate.sh b/validate.sh index 2c2e3804b11..1ec1ff18c1f 100755 --- a/validate.sh +++ b/validate.sh @@ -14,6 +14,7 @@ CABALSUITETESTS=true CABALONLY=false DEPSONLY=false DOCTEST=false +BENCHMARKS=false VERBOSE=false # Help @@ -23,7 +24,7 @@ show_usage() { cat <