Add support for lcov, as well as an option in config for choosing cov…

…erage formats

CHANGELOG.md

@@ -1,3 +1,8 @@
+## Unreleased
+
+* Added support for lcov coverage report
+* Added a "coverageFormats" config option to choose which coverage reports to make
+
 ## 2.1.1
 
 * Added missing space in ProcessorNotFound message (#977)

lib/Echidna/Config.hs

@@ -19,6 +19,7 @@ import EVM.Types (W256)
 import Echidna.Test
 import Echidna.Types.Campaign
 import Echidna.Mutator.Corpus (defaultMutationConsts)
+import Echidna.Output.Source (CoverageFileType(..))
 import Echidna.Types.Solidity
 import Echidna.Types.Tx (TxConf(TxConf), maxGasPerBlock, defaultTimeDelay, defaultBlockDelay)
 import Echidna.Types.Test (TestConf(..))
@@ -94,7 +95,7 @@ instance FromJSON EConfigWithUsage where
         <*> v ..:? "dictFreq" ..!= 0.40
         <*> v ..:? "corpusDir" ..!= Nothing
         <*> v ..:? "mutConsts" ..!= defaultMutationConsts
-        <*> v ..:? "coverageReport" ..!= True
+        <*> v ..:? "coverageFormats" ..!= [Txt,Html,Lcov]
 
       solConfParser = SolConf
         <$> v ..:? "contractAddr"    ..!= defaultContractAddr

lib/Echidna/Output/Source.hs

@@ -2,13 +2,15 @@ module Echidna.Output.Source where
 
 import Prelude hiding (writeFile)
 
+import Data.Aeson (ToJSON(..), FromJSON(..), withText)
 import Data.Foldable
+import Data.Functor (void)
 import Data.Maybe (fromMaybe, mapMaybe)
 import Data.List (nub, sort)
 import Data.Map qualified as M
 import Data.Sequence qualified as Seq
 import Data.Set qualified as S
-import Data.Text (Text, pack)
+import Data.Text (Text, pack, toLower)
 import Data.Text qualified as T
 import Data.Text.Encoding (decodeUtf8)
 import Data.Text.IO (writeFile)
@@ -27,18 +29,39 @@ import Echidna.Types.Signature (getBytecodeMetadata)
 
 type FilePathText = Text
 
-saveCoverage :: Bool -> Int -> FilePath -> SourceCache -> [SolcContract] -> CoverageMap -> IO ()
-saveCoverage isHtml seed d sc cs s = let extension = if isHtml then ".html" else ".txt"
-                                         fn = d </> "covered." <> show seed <> extension
-                                         cc = ppCoveredCode isHtml sc cs s
-                                     in do
-                                       createDirectoryIfMissing True d
-                                       writeFile fn cc
+data CoverageFileType = Lcov | Html | Txt deriving (Eq, Show)
+
+instance ToJSON CoverageFileType where
+  toJSON = toJSON . show
+
+instance FromJSON CoverageFileType where
+  parseJSON = withText "CoverageFileType" $ readFn . toLower where
+    readFn "lcov" = pure Lcov
+    readFn "html" = pure Html
+    readFn "text" = pure Txt
+    readFn "txt"  = pure Txt
+    readFn _ = fail "could not parse CoverageFileType"
+
+coverageFileExtension :: CoverageFileType -> String
+coverageFileExtension Lcov = ".lcov"
+coverageFileExtension Html = ".html"
+coverageFileExtension Txt = ".txt"
+
+saveCoverages :: [CoverageFileType] -> Int -> FilePath -> SourceCache -> [SolcContract] -> CoverageMap -> IO ()
+saveCoverages fileTypes seed d sc cs s = void $ mapM (\ty -> saveCoverage ty seed d sc cs s) fileTypes
+
+saveCoverage :: CoverageFileType -> Int -> FilePath -> SourceCache -> [SolcContract] -> CoverageMap -> IO ()
+saveCoverage fileType seed d sc cs s = let extension = coverageFileExtension fileType
+                                           fn = d </> "covered." <> show seed <> extension
+                                           cc = ppCoveredCode fileType sc cs s
+                                       in do
+                                         createDirectoryIfMissing True d
+                                         writeFile fn cc
 
 -- | Pretty-print the covered code
-ppCoveredCode :: Bool -> SourceCache -> [SolcContract] -> CoverageMap -> Text
-ppCoveredCode isHtml sc cs s | s == mempty = "Coverage map is empty"
-                             | otherwise   =
+ppCoveredCode :: CoverageFileType -> SourceCache -> [SolcContract] -> CoverageMap -> Text
+ppCoveredCode fileType sc cs s | s == mempty = "Coverage map is empty"
+                               | otherwise   =
   let allFiles = zipWith (\(srcPath, _rawSource) srcLines -> (srcPath, V.map decodeUtf8 srcLines))
                    sc.files
                    sc.lines
@@ -49,47 +72,58 @@ ppCoveredCode isHtml sc cs s | s == mempty = "Coverage map is empty"
       -- ^ Excludes lines such as comments or blanks
       ppFile (srcPath, srcLines) =
         let runtimeLines = fromMaybe mempty $ M.lookup srcPath runtimeLinesMap
-            marked = markLines isHtml srcLines runtimeLines (fromMaybe M.empty (M.lookup srcPath covLines))
+            marked = markLines fileType srcLines runtimeLines (fromMaybe M.empty (M.lookup srcPath covLines))
         in T.unlines (changeFileName srcPath : changeFileLines (V.toList marked))
       -- ^ Pretty print individual file coverage
-      topHeader
-        | isHtml = "<style> code { white-space: pre-wrap; display: block; background-color: #eee; }" <>
-                   ".executed { background-color: #afa; }" <>
-                   ".reverted { background-color: #ffa; }" <>
-                   ".unexecuted { background-color: #faa; }" <>
-                   ".neutral { background-color: #eee; }" <>
-                   "</style>"
-        | otherwise = ""
+      topHeader = case fileType of
+        Lcov -> "TN:\n"
+        Html -> "<style> code { white-space: pre-wrap; display: block; background-color: #eee; }" <>
+                ".executed { background-color: #afa; }" <>
+                ".reverted { background-color: #ffa; }" <>
+                ".unexecuted { background-color: #faa; }" <>
+                ".neutral { background-color: #eee; }" <>
+                "</style>"
+        Txt  -> ""
       -- ^ Text to add to top of the file
-      changeFileName fn
-        | isHtml = "<b>" <> HTML.text fn <> "</b>"
-        | otherwise = fn
+      changeFileName fn = case fileType of
+        Lcov -> "SF:" <> fn
+        Html -> "<b>" <> HTML.text fn <> "</b>"
+        Txt  -> fn
       -- ^ Alter file name, in the case of html turning it into bold text
-      changeFileLines ls
-        | isHtml = "<code>" : ls ++ ["", "</code>","<br />"]
-        | otherwise = ls
+      changeFileLines ls = case fileType of
+        Lcov -> ls ++ ["end_of_record"]
+        Html -> "<code>" : ls ++ ["", "</code>","<br />"]
+        Txt  -> ls
       -- ^ Alter file contents, in the case of html encasing it in <code> and adding a line break
   in topHeader <> T.unlines (map ppFile allFiles)
 
 -- | Mark one particular line, from a list of lines, keeping the order of them
-markLines :: Bool -> V.Vector Text -> S.Set Int -> M.Map Int [TxResult] -> V.Vector Text
-markLines isHtml codeLines runtimeLines resultMap =
-  V.map markLine (V.indexed codeLines)
+markLines :: CoverageFileType -> V.Vector Text -> S.Set Int -> M.Map Int [TxResult] -> V.Vector Text
+markLines fileType codeLines runtimeLines resultMap =
+  V.map markLine . V.filter shouldUseLine $ V.indexed codeLines
   where
+  isLcov = fileType == Lcov
+  isHtml = fileType == Html
+  shouldUseLine (i, _) = case fileType of
+    Lcov -> i + 1 `elem` runtimeLines
+    _ -> True
   markLine (i, codeLine) =
     let n = i + 1
         results  = fromMaybe [] (M.lookup n resultMap)
         markers = sort $ nub $ getMarker <$> results
         wrapLine :: Text -> Text
-        wrapLine line
-          | isHtml = "<span class='" <> cssClass <> "'>" <>
+        wrapLine line = case fileType of
+          Html -> "<span class='" <> cssClass <> "'>" <>
                         HTML.text line <>
                      "</span>"
-          | otherwise = line
+          _ -> line
           where
           cssClass = if n `elem` runtimeLines then getCSSClass markers else "neutral"
+        result = case fileType of
+          Lcov -> pack $ printf "DA:%d,%d" n (length results)
+          _ -> pack $ printf " %*d | %-4s| %s" lineNrSpan n markers (wrapLine codeLine)
 
-    in pack $ printf " %*d | %-4s| %s" lineNrSpan n markers (wrapLine codeLine)
+    in result
   lineNrSpan = length . show $ V.length codeLines + 1
 
 getCSSClass :: String -> Text

lib/Echidna/Types/Campaign.hs

@@ -4,6 +4,7 @@ import Data.Map (Map)
 import Data.Text (Text)
 
 import Echidna.ABI (GenDict, emptyDict)
+import Echidna.Output.Source (CoverageFileType)
 import Echidna.Types
 import Echidna.Types.Corpus
 import Echidna.Types.Coverage (CoverageMap)
@@ -12,30 +13,30 @@ import Echidna.Types.Tx (Tx)
 
 -- | Configuration for running an Echidna 'Campaign'.
 data CampaignConf = CampaignConf
-  { testLimit     :: Int
+  { testLimit       :: Int
     -- ^ Maximum number of function calls to execute while fuzzing
-  , stopOnFail    :: Bool
+  , stopOnFail      :: Bool
     -- ^ Whether to stop the campaign immediately if any property fails
-  , estimateGas   :: Bool
+  , estimateGas     :: Bool
     -- ^ Whether to collect gas usage statistics
-  , seqLen        :: Int
+  , seqLen          :: Int
     -- ^ Number of calls between state resets (e.g. \"every 10 calls,
     -- reset the state to avoid unrecoverable states/save memory\"
-  , shrinkLimit   :: Int
+  , shrinkLimit     :: Int
     -- ^ Maximum number of candidate sequences to evaluate while shrinking
-  , knownCoverage :: Maybe CoverageMap
+  , knownCoverage   :: Maybe CoverageMap
     -- ^ If applicable, initially known coverage. If this is 'Nothing',
     -- Echidna won't collect coverage information (and will go faster)
-  , seed          :: Maybe Int
+  , seed            :: Maybe Int
     -- ^ Seed used for the generation of random transactions
-  , dictFreq      :: Float
+  , dictFreq        :: Float
     -- ^ Frequency for the use of dictionary values in the random transactions
-  , corpusDir     :: Maybe FilePath
+  , corpusDir       :: Maybe FilePath
     -- ^ Directory to load and save lists of transactions
-  , mutConsts     :: MutationConsts Integer
+  , mutConsts       :: MutationConsts Integer
     -- ^ Directory to load and save lists of transactions
-  , coverageReport :: Bool
-    -- ^ Whether or not to generate a coverage report
+  , coverageFormats :: [CoverageFileType]
+    -- ^ List of file formats to save coverage reports
   }
 
 -- | The state of a fuzzing campaign.

src/Main.hs

@@ -135,30 +135,28 @@ main = withUtf8 $ withCP65001 $ do
       -- TODO: We use the corpus dir to save coverage reports which is confusing.
       -- Add config option to pass dir for saving coverage report and decouple it
       -- from corpusDir.
-      when cfg.campaignConf.coverageReport $ do
-        -- We need runId to have a unique directory to save files under so they
-        -- don't collide with the next runs. We use the current time for this
-        -- as it orders the runs chronologically.
-        runId <- fromIntegral . systemSeconds <$> getSystemTime
 
-        -- coverage reports for external contracts, we only support
-        -- Ethereum Mainnet for now
-        when (chainId == Just 1) $ do
-          forM_ (Map.toList contractsCache) $ \(addr, mc) ->
-            case mc of
-              Just contract -> do
-                r <- externalSolcContract addr contract
-                case r of
-                  Just (externalSourceCache, solcContract) -> do
-                    let dir' = dir </> show addr
-                    saveCoverage False runId dir' externalSourceCache [solcContract] campaign.coverage
-                    saveCoverage True  runId dir' externalSourceCache [solcContract] campaign.coverage
-                  Nothing -> pure ()
-              Nothing -> pure ()
+      -- We need runId to have a unique directory to save files under so they
+      -- don't collide with the next runs. We use the current time for this
+      -- as it orders the runs chronologically.
+      runId <- fromIntegral . systemSeconds <$> getSystemTime
 
-        -- save source coverage reports
-        saveCoverage False runId dir sourceCache contracts campaign.coverage
-        saveCoverage True  runId dir sourceCache contracts campaign.coverage
+      -- coverage reports for external contracts, we only support
+      -- Ethereum Mainnet for now
+      when (chainId == Just 1) $ do
+        forM_ (Map.toList contractsCache) $ \(addr, mc) ->
+          case mc of
+            Just contract -> do
+              r <- externalSolcContract addr contract
+              case r of
+                Just (externalSourceCache, solcContract) -> do
+                  let dir' = dir </> show addr
+                  saveCoverages cfg.campaignConf.coverageFormats runId dir' externalSourceCache [solcContract] campaign.coverage
+                Nothing -> pure ()
+            Nothing -> pure ()
+
+      -- save source coverage reports
+      saveCoverages cfg.campaignConf.coverageFormats runId dir sourceCache contracts campaign.coverage
 
   if isSuccessful campaign then exitSuccess else exitWith (ExitFailure 1)
 

src/test/Tests/Seed.hs

@@ -5,6 +5,7 @@ import Test.Tasty.HUnit (testCase, assertBool)
 
 import Common (runContract, overrideQuiet)
 import Data.Function ((&))
+import Echidna.Output.Source (CoverageFileType(..))
 import Echidna.Types.Config (EConfig(..))
 import Echidna.Types.Campaign (Campaign(..), CampaignConf(..))
 import Echidna.Mutator.Corpus (defaultMutationConsts)
@@ -29,7 +30,7 @@ seedTests =
         , dictFreq = 0.15
         , corpusDir = Nothing
         , mutConsts = defaultMutationConsts
-        , coverageReport = False
+        , coverageFormats = [Txt,Html,Lcov]
         }
       }
       & overrideQuiet

tests/solidity/basic/default.yaml

@@ -77,6 +77,8 @@ filterBlacklist: true
 allowFFI: false
 #directory to save the corpus; by default is disabled
 corpusDir: null
+# list of file formats to save coverage reports in; default is all possible formats
+coverageFormats: ["txt","html","lcov"]
 # constants for corpus mutations (for experimentation only)
 mutConsts: [1, 1, 1, 1]
 # maximum value to send to payable functions
@@ -85,5 +87,3 @@ maxValue: 100000000000000000000 # 100 eth
 rpcUrl: null
 # block number to use when fetching over RPC
 rpcBlock: null
-# whether or not to generate a coverage report
-coverageReport: false