Merge branch 'feature/fastcov' into develop

Makefile

@@ -35,6 +35,7 @@ all:
 	@echo "check-fast - compile and execute test suite (skip long-running tests)"
 	@echo "clean - remove built files"
 	@echo "coverage - create coverage information with lcov"
+	@echo "coverage-fast - create coverage information with fastcov"
 	@echo "cppcheck - analyze code with cppcheck"
 	@echo "cpplint - analyze code with cpplint"
 	@echo "clang_tidy - analyze code with Clang-Tidy"
@@ -82,6 +83,14 @@ coverage:
 	cd build_coverage ; ninja lcov_html
 	open build_coverage/test/html/index.html
 
+coverage-fast:
+	rm -fr build_coverage
+	mkdir build_coverage
+	cd build_coverage ; CXX=$(COMPILER_DIR)/g++ cmake .. -GNinja -DJSON_Coverage=ON -DJSON_MultipleHeaders=ON
+	cd build_coverage ; ninja
+	cd build_coverage ; ctest -E '.*_default' -j10
+	cd build_coverage ; ninja fastcov_html
+	open build_coverage/test/html/index.html
 
 ##########################################################################
 # documentation tests

README.md

@@ -1281,9 +1281,11 @@ The library itself consists of a single header file licensed under the MIT licen
 - [**doctest**](https://github.com/onqtam/doctest) for the unit tests
 - [**Doozer**](https://doozer.io) for [continuous integration](https://doozer.io/nlohmann/json) on Linux (CentOS, Raspbian, Fedora)
 - [**Doxygen**](http://www.stack.nl/~dimitri/doxygen/) to generate [documentation](https://nlohmann.github.io/json/)
+- [**fastcov**](https://github.com/RPGillespie6/fastcov) to process coverage information
 - [**git-update-ghpages**](https://github.com/rstacruz/git-update-ghpages) to upload the documentation to gh-pages
 - [**GitHub Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md)
 - [**Google Benchmark**](https://github.com/google/benchmark) to implement the benchmarks
+- [**lcov**](http://ltp.sourceforge.net/coverage/lcov.php) to process coverage information and create a HTML view
 - [**libFuzzer**](http://llvm.org/docs/LibFuzzer.html) to implement fuzz testing for OSS-Fuzz
 - [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library ([project repository](https://github.com/google/oss-fuzz/tree/master/projects/json))
 - [**Probot**](https://probot.github.io) for automating maintainer tasks such as closing stale issues, requesting missing information, or detecting toxic comments.

test/CMakeLists.txt

@@ -27,17 +27,17 @@ endif()
 
 if(JSON_Coverage)
     message(STATUS "Building test suite with coverage information")
-    #if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-    #    message(FATAL_ERROR "JSON_Coverage requires GCC.")
-    #endif()
+    if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+        message(FATAL_ERROR "JSON_Coverage requires GCC.")
+    endif()
 
     # enable profiling
     set(CMAKE_CXX_FLAGS "--coverage -g -O0 -fprofile-arcs -ftest-coverage")
 
     # from https://github.com/RWTH-HPC/CMake-codecov/blob/master/cmake/FindGcov.cmake
-    #get_filename_component(COMPILER_PATH "${CMAKE_CXX_COMPILER}" PATH)
-    #string(REGEX MATCH "^[0-9]+" GCC_VERSION "${CMAKE_CXX_COMPILER_VERSION}")
-    #find_program(GCOV_BIN NAMES gcov-${GCC_VERSION} gcov HINTS ${COMPILER_PATH})
+    get_filename_component(COMPILER_PATH "${CMAKE_CXX_COMPILER}" PATH)
+    string(REGEX MATCH "^[0-9]+" GCC_VERSION "${CMAKE_CXX_COMPILER_VERSION}")
+    find_program(GCOV_BIN NAMES gcov-${GCC_VERSION} gcov HINTS ${COMPILER_PATH})
 
     # collect all source files from the chosen include dir
     file(GLOB_RECURSE SOURCE_FILES ${NLOHMANN_JSON_INCLUDE_BUILD_DIR}*.hpp)
@@ -46,11 +46,18 @@ if(JSON_Coverage)
     # (filter script from https://stackoverflow.com/a/43726240/266378)
     add_custom_target(lcov_html
         COMMAND lcov --directory . --capture --output-file json.info --rc lcov_branch_coverage=1
-        COMMAND lcov -e json.info ${SOURCE_FILES} --output-file json.info.filtered --rc lcov_branch_coverage=1
+        COMMAND lcov -e json.info ${SOURCE_FILES} --output-file json.info.filtered --gcov-tool ${GCOV_BIN} --rc lcov_branch_coverage=1
         COMMAND ${CMAKE_SOURCE_DIR}/test/thirdparty/imapdl/filterbr.py json.info.filtered > json.info.filtered.noexcept
         COMMAND genhtml --title "JSON for Modern C++" --legend --demangle-cpp --output-directory html --show-details --branch-coverage json.info.filtered.noexcept
         COMMENT "Generating HTML report test/html/index.html"
     )
+
+    add_custom_target(fastcov_html
+        COMMAND ${CMAKE_SOURCE_DIR}/test/thirdparty/fastcov/fastcov.py --branch-coverage --lcov -o json.info --gcov ${GCOV_BIN} --compiler-directory ${CMAKE_BINARY_DIR} --source-files ${SOURCE_FILES}
+        COMMAND ${CMAKE_SOURCE_DIR}/test/thirdparty/imapdl/filterbr.py json.info > json.info.noexcept
+        COMMAND genhtml --title "JSON for Modern C++" --legend --demangle-cpp --output-directory html --show-details --branch-coverage json.info.noexcept
+        COMMENT "Generating HTML report test/html/index.html"
+    )
 endif()
 
 #############################################################################

test/thirdparty/fastcov/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2018-2019 Bryan Gillespie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

test/thirdparty/fastcov/README.md

@@ -0,0 +1,46 @@
+# fastcov
+A massively parallel gcov wrapper for generating intermediate coverage formats *fast*
+
+The goal of fastcov is to generate code coverage intermediate formats *as fast as possible* (ideally < 1 second), even for large projects with hundreds of gcda objects. The intermediate formats may then be consumed by a report generator such as lcov's genhtml, or a dedicated front end such as coveralls. fastcov was originally designed to be a drop-in replacement for lcov (application coverage only, not kernel coverage).
+
+Currently the only intermediate formats supported are gcov json format and lcov info format. Adding support for other formats should require just a few lines of python to transform gcov json format to the desired shape.
+
+In order to achieve the massive speed gains, a few constraints apply:
+
+1. GCC version >= 9.0.0
+
+These versions of GCOV have support for JSON intermediate format as well as streaming report data straight to stdout
+
+2. Object files must be either be built:
+
+- Using absolute paths for all `-I` flags passed to the compiler
+- Invoking the compiler from the same root directory
+
+If you use CMake, you are almost certainly satisfying the second constraint (unless you care about `ExternalProject` coverage).
+
+## Sample Usage:
+```bash
+$ cd build_dir
+$ fastcov.py --zerocounters
+$ <run unit tests>
+$ fastcov.py --exclude /usr/include --lcov -o report.info
+$ genhtml -o code_coverage report.info
+```
+
+## Legacy fastcov
+
+It is possible to reap most of the benefits of fastcov for GCC version < 9.0.0 and >= 7.1.0. However, there will be a *potential* header file loss of correctness.
+
+`fastcov_legacy.py` can be used with pre GCC-9 down to GCC 7.1.0 but with a few penalties due to gcov limitations. This is because running gcov in parallel generates .gcov header reports in parallel which overwrite each other. This isn't a problem unless your header files have actual logic (i.e. header only library) that you want to measure coverage for. Use the `-F` flag to specify which gcda files should not be run in parallel in order to capture accurate header file data just for those. I don't plan on supporting `fastcov_legacy.py` aside from basic bug fixes.
+
+## Benchmarks
+
+Anecdotal testing on my own projects indicate that fastcov is over 100x faster than lcov and over 30x faster than gcovr:
+
+Project Size: ~250 .gcda, ~500 .gcov generated by gcov
+
+Time to process all gcda and parse all gcov:
+
+- fastcov: ~700ms
+- lcov:    ~90s
+- gcovr:   ~30s