Merge pull request #36 from timholy/pull-request/fb2cb863

Improve quality of coverage reports by parsing source files for functions

README.md

@@ -6,7 +6,52 @@ Coverage.jl
 
 **"Take Julia test coverage results and do useful things with them."**
 
-Right now, that is submitting them to [Coveralls.io](https://coveralls.io), a test-coverage tracking tool that integrates with your continuous integration solution (e.g. [TravisCI](https://travis-ci.org/)).
+## Working locally
+
+### Code coverage
+
+Navigate to your test directory, and start julia like this:
+```sh
+julia --code-coverage=user
+```
+or, if you're running julia 0.4 or higher,
+```sh
+julia --code-coverage=user --inline=no
+```
+(Turning off inlining gives substantially more accurate results.)
+
+Then, run your tests (e.g., `include("runtests.jl")`) and quit julia.
+
+Finally, navigate to the top-level directory of your package, restart julia (with no special flags this time), and analyze coverage using
+```julia
+using Coverage
+covered, tot = coverage_folder()  # defaults to src/; alternatively, supply the folder name as a string
+```
+The fraction of total coverage is equal to `covered/tot`.
+
+### Memory allocation
+
+Start julia with
+```sh
+julia --track-allocation=user
+```
+Then:
+- Run whatever commands you wish to test. This first run is to ensure that everything is compiled (because compilation allocates memory).
+- Call `clear_malloc_data()` (or, if running julia 0.4 or higher, `Profile.clear_malloc_data()`)
+- Run your commands again
+- Quit julia
+
+Finally, navigate to the directory holding your source code. Start julia (without command-line flags), and analyze the results using
+```julia
+using Coverage
+analyze_malloc(dirnames)  # could be "." for the current directory, or "src", etc.
+```
+This will return a vector of `MallocInfo` objects, specifying the number of bytes allocated, the file name, and the line number.
+These are sorted in increasing order of allocation size.
+
+## Using Coveralls
+
+[Coveralls.io](https://coveralls.io) is a test-coverage tracking tool that integrates with your continuous integration solution (e.g. [TravisCI](https://travis-ci.org/)).
 
 ## Using Coverage.jl with Coveralls.io?
 

REQUIRE

@@ -1,3 +1,5 @@
 julia 0.3-
 JSON
-Requests
+Requests
+JuliaParser
+Compat

src/Coverage.jl

@@ -5,11 +5,22 @@
 #######################################################################
 module Coverage
 
+    import JuliaParser.Parser
+    using Compat
+
     # process_cov
     # Given a .cov file, return the counts for each line, where the
     # lines that can't be counted are denoted with a -1
-    export process_cov
+    export process_cov, amend_coverage_from_src!, coverage_folder, analyze_malloc
     function process_cov(filename)
+        if !isfile(filename)
+            srcname, ext = splitext(filename)
+            lines = open(srcname) do fp
+                readlines(fp)
+            end
+            coverage = Array(Union(Nothing,Int), length(lines))
+            return fill!(coverage, nothing)
+        end
         fp = open(filename, "r")
         lines = readlines(fp)
         num_lines = length(lines)
@@ -21,12 +32,76 @@ module Coverage
         close(fp)
         return coverage
     end
+    function amend_coverage_from_src!(coverage, srcname)
+        # To make sure things stay in sync, parse the file position corresonding to each new line
+        linepos = Int[]
+        open(srcname) do io
+            while !eof(io)
+                push!(linepos, position(io))
+                readline(io)
+            end
+            push!(linepos, position(io))
+        end
+        open(srcname) do io
+            while !eof(io)
+                pos = position(io)
+                linestart = minimum(searchsorted(linepos, pos))
+                ast = Parser.parse(io)
+                isa(ast, Expr) || continue
+                flines = function_body_lines(ast)
+                if !isempty(flines)
+                    flines += linestart-1
+                    for l in flines
+                        if coverage[l] == nothing
+                            coverage[l] = 0
+                        end
+                    end
+                end
+            end
+        end
+        coverage
+    end
+    function coverage_folder(folder="src")
+        results = Coveralls.process_folder(folder)
+        tot = covered = 0
+        for item in results
+            coverage = item["coverage"]
+            tot += sum(x->x!=nothing, coverage)
+            covered += sum(x->x!=nothing && x>0, coverage)
+        end
+       covered, tot
+    end
+
+    function_body_lines(ast) = function_body_lines!(Int[], ast, false)
+    function_body_lines!(flines, arg, infunction) = flines
+    function function_body_lines!(flines, node::LineNumberNode, infunction)
+        line = node.line
+        if infunction
+            push!(flines, line)
+        end
+        flines
+    end
+    function function_body_lines!(flines, ast::Expr, infunction)
+        if ast.head == :line
+            line = ast.args[1]
+            if infunction
+                push!(flines, line)
+            end
+            return flines
+        end
+        infunction |= Base.Cartesian.isfuncexpr(ast)
+        for arg in ast.args
+            flines = function_body_lines!(flines, arg, infunction)
+        end
+        flines
+    end
 
     export Coveralls
     module Coveralls
         using Requests
         using Coverage
         using JSON
+        using Compat
 
         # coveralls_process_file
         # Given a .jl file, return the Coveralls.io dictionary for this
@@ -40,29 +115,33 @@ module Coverage
         # }
         export process_file
         function process_file(filename)
-            return ["name" => filename,
+            return @compat Dict("name" => filename,
                     "source" => readall(filename),
-                    "coverage" => process_cov(filename*".cov")]
+                    "coverage" => amend_coverage_from_src!(process_cov(filename*".cov"), filename))
         end
 
         # coveralls_process_src
         # Recursively walk through a Julia package's src/ folder
         # and collect coverage statistics
         export process_folder
         function process_folder(folder="src")
-            source_files={}
+            source_files=Any[]
             filelist = readdir(folder)
             for file in filelist
+                _, ext = splitext(file)
+                if ext != ".jl"
+                    continue
+                end
                 fullfile = joinpath(folder,file)
                 println(fullfile)
                 if isfile(fullfile)
                     try
                         new_sf = process_file(fullfile)
                         push!(source_files, new_sf)
                     catch e
-                        if !isa(e,SystemError)
-                            rethrow(e)
-                        end
+#                         if !isa(e,SystemError)
+#                             rethrow(e)
+#                         end
                         # Skip
                         println("Skipped $fullfile")
                     end
@@ -94,17 +173,17 @@ module Coverage
         # }
         export submit, submit_token
         function submit(source_files)
-            data = ["service_job_id" => ENV["TRAVIS_JOB_ID"],
+            data = @compat Dict("service_job_id" => ENV["TRAVIS_JOB_ID"],
                     "service_name" => "travis-ci",
-                    "source_files" => source_files]
+                    "source_files" => source_files)
             r = Requests.post(URI("https://coveralls.io/api/v1/jobs"), files =
                 [FileParam(JSON.json(data),"application/json","json_file","coverage.json")])
             dump(r.data)
         end
 
         function submit_token(source_files)
-            data = ["repo_token" => ENV["REPO_TOKEN"],
-                    "source_files" => source_files]
+            data = @compat Dict("repo_token" => ENV["REPO_TOKEN"],
+                    "source_files" => source_files)
             r = post(URI("https://coveralls.io/api/v1/jobs"), files =
                 [FileParam(JSON.json(data),"application/json","json_file","coverage.json")])
             dump(r.data)

test/data/testparser.jl

@@ -0,0 +1,23 @@
+if VERSION < v"0.4.0-dev"
+    using Docile
+end
+# This line should have no code
+f2(x) = 2x
+if true
+  f3(x) = 3x
+end
+f4(x) = 4x; f5(x) = 5x
+# This line should have no code
+@doc """
+`f6(x)` multiplies `x` by 6
+""" ->
+f6(x) = 6x
+# This line should have no code
+@doc """
+`f7(x)` multiplies `x` by 7
+""" ->
+function f7(x)
+    7x
+end
+
+f8(x) = 8x

test/runtests.jl

@@ -4,7 +4,19 @@
 # https://github.com/IainNZ/Coverage.jl
 #######################################################################
 
-using Coverage
+using Coverage, Base.Test
 
-cd(Pkg.dir("Coverage"))
-j = Coveralls.process_file(joinpath("test","data","Coverage.jl"))
+cd(Pkg.dir("Coverage")) do
+    j = Coveralls.process_file(joinpath("test","data","Coverage.jl"))
+end
+
+srcname = joinpath("data","testparser.jl")
+covname = srcname*".cov"
+isfile(covname) && rm(covname)
+cmdstr = "include(\"$srcname\"); using Base.Test; @test f2(2) == 4"
+run(`julia --code-coverage=user -e $cmdstr`)
+r = Coveralls.process_file(srcname)
+# The next one is the correct one, but julia & JuliaParser don't insert a line number after the 1-line @doc -> test
+# target = [nothing, nothing, nothing, nothing, 1, nothing, 0, nothing, 0, nothing, nothing, nothing, nothing, 0, nothing, nothing, nothing, nothing, nothing, 0, nothing, nothing, 0]
+target = [nothing, nothing, nothing, nothing, 1, nothing, 0, nothing, 0, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing, 0, nothing, nothing, 0]
+@test r["coverage"][1:length(target)] == target