From e2eed08c78405e9d5df948fd4fe3aa52dba425b1 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 11 Jul 2022 14:19:53 -0400 Subject: [PATCH 1/8] Collect C-level coverage using llvm-cov --- .github/workflows/c-coverage.yml | 65 ++++++++++++++++++++++++++++++++ Lib/test/test_subprocess.py | 4 +- 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/c-coverage.yml diff --git a/.github/workflows/c-coverage.yml b/.github/workflows/c-coverage.yml new file mode 100644 index 00000000000000..2f3718c515c5f1 --- /dev/null +++ b/.github/workflows/c-coverage.yml @@ -0,0 +1,65 @@ +name: C coverage + +on: + schedule: + # Run this daily at 2400 UTC + - cron: '0 0 * * *' + +permissions: + contents: read + +jobs: + c-coverage: + name: 'C-level coverage (using llvm-cov)' + runs-on: ubuntu-20.04 + env: + OPENSSL_VER: 1.1.1n + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 + - name: Configure clang (with coverage) + run: | + echo "CC=/usr/lib/ccache/clang-12 -fprofile-instr-generate -fcoverage-mapping" >> $GITHUB_ENV + echo "CXX=/usr/lib/ccache/clang++-12 -fprofile-instr-generate -fcoverage-mapping" >> $GITHUB_ENV + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython + run: make -j4 + - name: Collect coverage data + # Specify the LLVM_PROFILE_FILE using %m so multiple shared objects can write + # in parallel. Set the full path to the directory so results aren't written + # into temporary directories created by tests. + # Using "-j 1" is important, or the Github Action runs out of memory + run: LLVM_PROFILE_FILE=${PWD}/python%m.profraw xvfb-run ./python -m test -j 1 + - name: Generate coverage report + run: | + llvm-profdata-12 merge -sparse python*.profraw -o python.profdata + llvm-cov-12 show -format=html -output-dir=cpython-coverage -instr-profile=python.profdata -show-branches=count -show-regions python . + - name: Publish coverage-report + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: cpython-coverage + repository-name: '' # TODO Destination + token: ${{ secrets.COVERAGE_DEPLOY_TOKEN }} # TODO: Use an organization-level token + single-commit: true diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 424a4a93b6f972..72d818e31f0179 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -797,9 +797,11 @@ def is_env_var_to_ignore(n): # This excludes some __CF_* and VERSIONER_* keys MacOS insists # on adding even when the environment in exec is empty. # Gentoo sandboxes also force LD_PRELOAD and SANDBOX_* to exist. + # LLVM coverage adds __LLVM_PROFILE_RT_INIT_ONCE variable. return ('VERSIONER' in n or '__CF' in n or # MacOS n == 'LD_PRELOAD' or n.startswith('SANDBOX') or # Gentoo - n == 'LC_CTYPE') # Locale coercion triggered + n == 'LC_CTYPE' or # Locale coercion triggered + n == '__LLVM_PROFILE_RT_INIT_ONCE') with subprocess.Popen([sys.executable, "-c", 'import os; print(list(os.environ.keys()))'], From 776e423ce7f63655c34f7e6fd5d0220e742992ea Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 13 Jul 2022 14:11:02 -0400 Subject: [PATCH 2/8] Refactor this as a Makefile target --- .github/workflows/c-coverage.yml | 19 +--- .gitignore | 1 + Makefile.pre.in | 44 ++++++++ ...2-07-13-13-24-01.gh-issue-94759.dTLMod.rst | 4 + configure | 100 ++++++++++++++++++ configure.ac | 2 + 6 files changed, 156 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst diff --git a/.github/workflows/c-coverage.yml b/.github/workflows/c-coverage.yml index 2f3718c515c5f1..814e95dfba8a96 100644 --- a/.github/workflows/c-coverage.yml +++ b/.github/workflows/c-coverage.yml @@ -40,26 +40,17 @@ jobs: uses: hendrikmuhs/ccache-action@v1 - name: Configure clang (with coverage) run: | - echo "CC=/usr/lib/ccache/clang-12 -fprofile-instr-generate -fcoverage-mapping" >> $GITHUB_ENV - echo "CXX=/usr/lib/ccache/clang++-12 -fprofile-instr-generate -fcoverage-mapping" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV - name: Configure CPython run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR - - name: Build CPython - run: make -j4 - - name: Collect coverage data - # Specify the LLVM_PROFILE_FILE using %m so multiple shared objects can write - # in parallel. Set the full path to the directory so results aren't written - # into temporary directories created by tests. - # Using "-j 1" is important, or the Github Action runs out of memory - run: LLVM_PROFILE_FILE=${PWD}/python%m.profraw xvfb-run ./python -m test -j 1 - name: Generate coverage report - run: | - llvm-profdata-12 merge -sparse python*.profraw -o python.profdata - llvm-cov-12 show -format=html -output-dir=cpython-coverage -instr-profile=python.profdata -show-branches=count -show-regions python . + # Using "-j1" is important, or the Github Action runs out of memory + run: EXTRATESTOPTS=-j1 xvfb-run make coverage-report-llvm - name: Publish coverage-report uses: JamesIves/github-pages-deploy-action@v4 with: - folder: cpython-coverage + folder: llvm-cov-report repository-name: '' # TODO Destination token: ${{ secrets.COVERAGE_DEPLOY_TOKEN }} # TODO: Use an organization-level token single-commit: true diff --git a/.gitignore b/.gitignore index 6934faa91e9874..4b820174aa5052 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ *.gc?? *.profclang? *.profraw +*.profdata *.dyn .gdb_history .purify diff --git a/Makefile.pre.in b/Makefile.pre.in index 9b35398c4fe767..cb269b91480db2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -50,6 +50,8 @@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ LLVM_PROF_FILE=@LLVM_PROF_FILE@ LLVM_PROF_ERR=@LLVM_PROF_ERR@ +LLVM_PROFDATA=@LLVM_PROFDATA@ +LLVM_COV=@LLVM_COV@ DTRACE= @DTRACE@ DFLAGS= @DFLAGS@ DTRACE_HEADERS= @DTRACE_HEADERS@ @@ -316,6 +318,10 @@ COVERAGE_REPORT=$(abs_builddir)/lcov-report COVERAGE_LCOV_OPTIONS=--rc lcov_branch_coverage=1 COVERAGE_REPORT_OPTIONS=--rc lcov_branch_coverage=1 --branch-coverage --title "CPython $(VERSION) LCOV report [commit $(shell $(GITVERSION))]" +# report files for llvm-cov coverage report +COVERAGE_INFO_LLVM= $(abs_builddir)/coverage.profdata +COVERAGE_REPORT_LLVM=$(abs_builddir)/llvm-cov-report +COVERAGE_REPORT_OPTIONS_LLVM=-show-branches=count -show-regions # === Definitions added by makesetup === @@ -701,6 +707,44 @@ coverage-report: regen-token regen-frozen @ # build lcov report $(MAKE) coverage-lcov +# Compile and calculate coverage with llvm-cov +.PHONY=check-clang coverage-llvm coverage-profdata coverage-report-llvm + +# Check whether the compiler is clang, and if not, error out. +check-clang: + ($(CC) --version | grep clang) || \ + (echo "LLVM coverage only works with clang. Set CC=clang and CXX=clang++ and re-run ./configure"; exit 1) + +coverage-llvm: check-clang + @echo "Building with support for coverage checking:" + $(MAKE) clean + @ # Override CC rather than CFLAGS since these flags must come first + $(MAKE) @DEF_MAKE_RULE@ CC="$(CC) -fprofile-instr-generate -fcoverage-mapping" + +coverage-profdata: + @echo "Creating Coverage HTML report with llvm-profdata/llvm-cov:" + @rm -f $(COVERAGE_INFO_LLVM) + @rm -rf $(COVERAGE_REPORT_LLVM) + @ # Merge coverage results + $(LLVM_PROFDATA) merge -sparse python*.profraw -o $(COVERAGE_INFO_LLVM) + @ # Generate HTML + $(LLVM_COV) show -format=html -output-dir=$(COVERAGE_REPORT_LLVM) -instr-profile=$(COVERAGE_INFO_LLVM) $(COVERAGE_REPORT_OPTIONS_LLVM) python . + @echo + @echo "llvm-cov report at $(COVERAGE_REPORT_LLVM)/index.html" + @echo + +# Force regeneration of parser and importlib +# Specify the LLVM_PROFILE_FILE using %m so multiple shared objects can write +# in parallel. Set the full path to the directory so results aren't written +# into temporary directories created by tests. +coverage-report-llvm: regen-token regen-importlib + @ # build with coverage info + $(MAKE) coverage-llvm + @ # run tests, ignore failures + LLVM_PROFILE_FILE=${PWD}/python%m.profraw $(TESTRUNNER) $(TESTOPTS) || true + @ # build llvm-cov report + $(MAKE) coverage-profdata + # Run "Argument Clinic" over all source files .PHONY=clinic clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c diff --git a/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst new file mode 100644 index 00000000000000..74a17daec2dbf9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst @@ -0,0 +1,4 @@ +A new Makefile target ``coverage-report-llvm`` will use ``clang`` and +``llvm-cov`` to generate a coverage report. This provides more details about +branch coverage and subexpressions than the existing ``gcc`` and ``lcov`` +based ``coverage-report``. diff --git a/configure b/configure index 953c558d6048d4..921b2d78700114 100755 --- a/configure +++ b/configure @@ -883,6 +883,7 @@ CFLAGS_NODIST BASECFLAGS CFLAGS_ALIASING OPT +LLVM_COV LLVM_PROF_FOUND LLVM_PROFDATA LLVM_PROF_ERR @@ -8430,6 +8431,105 @@ case $CC in ;; esac +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}llvm-cov", so it can be a program name with args. +set dummy ${ac_tool_prefix}llvm-cov; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_LLVM_COV+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $LLVM_COV in + [\\/]* | ?:[\\/]*) + ac_cv_path_LLVM_COV="$LLVM_COV" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in ${llvm_path} +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_LLVM_COV="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +LLVM_COV=$ac_cv_path_LLVM_COV +if test -n "$LLVM_COV"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_COV" >&5 +$as_echo "$LLVM_COV" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_LLVM_COV"; then + ac_pt_LLVM_COV=$LLVM_COV + # Extract the first word of "llvm-cov", so it can be a program name with args. +set dummy llvm-cov; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_LLVM_COV+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_LLVM_COV in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_LLVM_COV="$ac_pt_LLVM_COV" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in ${llvm_path} +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_LLVM_COV="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_LLVM_COV=$ac_cv_path_ac_pt_LLVM_COV +if test -n "$ac_pt_LLVM_COV"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_LLVM_COV" >&5 +$as_echo "$ac_pt_LLVM_COV" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_LLVM_COV" = x; then + LLVM_COV="''" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + LLVM_COV=$ac_pt_LLVM_COV + fi +else + LLVM_COV="$ac_cv_path_LLVM_COV" +fi + + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? diff --git a/configure.ac b/configure.ac index 210ce3292cfdf7..a10b408ab76539 100644 --- a/configure.ac +++ b/configure.ac @@ -2055,6 +2055,8 @@ case $CC in LLVM_PROF_FILE="" ;; esac +AC_SUBST(LLVM_COV) +AC_PATH_TOOL(LLVM_COV, llvm-cov, '', ${llvm_path}) # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? From 1ab7870447b70e0e0c2495776aa56cdde0867a55 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 13 Jul 2022 14:56:29 -0400 Subject: [PATCH 3/8] Automatically select the coverage approach based on the compiler --- .github/workflows/c-coverage.yml | 2 +- Makefile.pre.in | 29 ++++++++++++------- ...2-07-13-13-24-01.gh-issue-94759.dTLMod.rst | 6 ++-- configure | 4 +++ configure.ac | 3 ++ 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/c-coverage.yml b/.github/workflows/c-coverage.yml index 814e95dfba8a96..938ca4b186a76e 100644 --- a/.github/workflows/c-coverage.yml +++ b/.github/workflows/c-coverage.yml @@ -46,7 +46,7 @@ jobs: run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR - name: Generate coverage report # Using "-j1" is important, or the Github Action runs out of memory - run: EXTRATESTOPTS=-j1 xvfb-run make coverage-report-llvm + run: EXTRATESTOPTS=-j1 xvfb-run make coverage-report - name: Publish coverage-report uses: JamesIves/github-pages-deploy-action@v4 with: diff --git a/Makefile.pre.in b/Makefile.pre.in index cb269b91480db2..ceaa933fc77d7a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -37,6 +37,7 @@ abs_builddir= @abs_builddir@ CC= @CC@ CXX= @CXX@ LINKCC= @LINKCC@ +CC_NAME=@CC_NAME@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ @@ -661,9 +662,20 @@ bolt-opt: @PREBOLT_RULE@ rm -f $(BUILDPYTHON).bolt_inst mv $(BUILDPYTHON).bolt $(BUILDPYTHON) +.PHONY=coverage-report +coverage-report: + @if [ $(CC_NAME) = "gcc" ]; then \ + $(MAKE) coverage-report-lcov; \ + elif [ $(CC_NAME) = "clang" ]; then \ + $(MAKE) coverage-report-llvm; \ + else \ + echo "Coverage is not supported with the $(CC_NAME) compiler"; \ + exit 1; \ + fi + # Compile and run with gcov -.PHONY=coverage coverage-lcov coverage-report -coverage: +.PHONY=coverage-gcc coverage-lcov coverage-report-lcov +coverage-gcc: @echo "Building with support for coverage checking:" $(MAKE) clean $(MAKE) @DEF_MAKE_RULE@ CFLAGS="$(CFLAGS) -O0 -pg --coverage" LDFLAGS="$(LDFLAGS) --coverage" @@ -699,7 +711,7 @@ coverage-lcov: @echo # Force regeneration of parser and frozen modules -coverage-report: regen-token regen-frozen +coverage-report-lcov: regen-token regen-frozen @ # build with coverage info $(MAKE) coverage @ # run tests, ignore failures @@ -708,14 +720,9 @@ coverage-report: regen-token regen-frozen $(MAKE) coverage-lcov # Compile and calculate coverage with llvm-cov -.PHONY=check-clang coverage-llvm coverage-profdata coverage-report-llvm - -# Check whether the compiler is clang, and if not, error out. -check-clang: - ($(CC) --version | grep clang) || \ - (echo "LLVM coverage only works with clang. Set CC=clang and CXX=clang++ and re-run ./configure"; exit 1) +.PHONY=coverage-clang coverage-profdata coverage-report-llvm -coverage-llvm: check-clang +coverage-clang: @echo "Building with support for coverage checking:" $(MAKE) clean @ # Override CC rather than CFLAGS since these flags must come first @@ -739,7 +746,7 @@ coverage-profdata: # into temporary directories created by tests. coverage-report-llvm: regen-token regen-importlib @ # build with coverage info - $(MAKE) coverage-llvm + $(MAKE) coverage-clang @ # run tests, ignore failures LLVM_PROFILE_FILE=${PWD}/python%m.profraw $(TESTRUNNER) $(TESTOPTS) || true @ # build llvm-cov report diff --git a/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst index 74a17daec2dbf9..b5fe3efb75fd17 100644 --- a/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst +++ b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst @@ -1,4 +1,2 @@ -A new Makefile target ``coverage-report-llvm`` will use ``clang`` and -``llvm-cov`` to generate a coverage report. This provides more details about -branch coverage and subexpressions than the existing ``gcc`` and ``lcov`` -based ``coverage-report``. +The ``coverage-report`` Makefile target will now automatically use ``llvm-cov`` to generate a coverage report when using ``clang``. +This provides more details about branch coverage and subexpressions than the existing ``gcc`` and ``lcov`` based ``coverage-report``. diff --git a/configure b/configure index 921b2d78700114..d160b0e958d008 100755 --- a/configure +++ b/configure @@ -936,6 +936,7 @@ PLATFORM_TRIPLET MULTIARCH ac_ct_CXX CXX +CC_NAME EGREP SED GREP @@ -5371,6 +5372,9 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_name" >&5 $as_echo "$ac_cv_cc_name" >&6; } + +CC_NAME=$ac_cv_cc_name + # checks for UNIX variants that set C preprocessor variables # may set _GNU_SOURCE, __EXTENSIONS__, _POSIX_PTHREAD_SEMANTICS, # _POSIX_SOURCE, _POSIX_1_SOURCE, and more diff --git a/configure.ac b/configure.ac index a10b408ab76539..d5f1719e7c2e10 100644 --- a/configure.ac +++ b/configure.ac @@ -869,6 +869,9 @@ fi rm -f conftest.c conftest.out ]) +AC_SUBST(CC_NAME) +CC_NAME=$ac_cv_cc_name + # checks for UNIX variants that set C preprocessor variables # may set _GNU_SOURCE, __EXTENSIONS__, _POSIX_PTHREAD_SEMANTICS, # _POSIX_SOURCE, _POSIX_1_SOURCE, and more From 80f003913ed32f789d981b3cfcc4efb1aaa70b2c Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 28 Jul 2022 12:31:27 -0400 Subject: [PATCH 4/8] Update configure.ac Co-authored-by: Christian Heimes --- configure | 2 +- configure.ac | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/configure b/configure index d160b0e958d008..dd18b586f2ac24 100755 --- a/configure +++ b/configure @@ -5372,9 +5372,9 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_name" >&5 $as_echo "$ac_cv_cc_name" >&6; } - CC_NAME=$ac_cv_cc_name + # checks for UNIX variants that set C preprocessor variables # may set _GNU_SOURCE, __EXTENSIONS__, _POSIX_PTHREAD_SEMANTICS, # _POSIX_SOURCE, _POSIX_1_SOURCE, and more diff --git a/configure.ac b/configure.ac index d5f1719e7c2e10..42a3cf5c670676 100644 --- a/configure.ac +++ b/configure.ac @@ -869,8 +869,7 @@ fi rm -f conftest.c conftest.out ]) -AC_SUBST(CC_NAME) -CC_NAME=$ac_cv_cc_name +AC_SUBST([CC_NAME], [$ac_cv_cc_name]) # checks for UNIX variants that set C preprocessor variables # may set _GNU_SOURCE, __EXTENSIONS__, _POSIX_PTHREAD_SEMANTICS, From dfa3fcb77a4cb0cd80caadbd6c8895839d74204e Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 8 Aug 2022 10:22:11 -0400 Subject: [PATCH 5/8] Fix lcov coverage --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index ceaa933fc77d7a..2c4efe89274e69 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -713,7 +713,7 @@ coverage-lcov: # Force regeneration of parser and frozen modules coverage-report-lcov: regen-token regen-frozen @ # build with coverage info - $(MAKE) coverage + $(MAKE) coverage-gcc @ # run tests, ignore failures $(TESTRUNNER) $(TESTOPTS) || true @ # build lcov report From 50ba4f44ac32138bedb486b165be951ad606a387 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 24 Aug 2022 13:25:20 -0400 Subject: [PATCH 6/8] Remove unnecessary quotes --- .github/workflows/c-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/c-coverage.yml b/.github/workflows/c-coverage.yml index 938ca4b186a76e..6a5eedbfb9c089 100644 --- a/.github/workflows/c-coverage.yml +++ b/.github/workflows/c-coverage.yml @@ -24,7 +24,7 @@ jobs: echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' + - name: Restore OpenSSL build id: cache-openssl uses: actions/cache@v3 with: From 84f042a1c4fb4f1bd5ba341f2e4e6511fc2a8f09 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 24 Aug 2022 13:27:20 -0400 Subject: [PATCH 7/8] Move more logic into configure --- Makefile.pre.in | 99 ++++++++++++++++--------------------------------- configure | 43 +++++++++++++++++++-- configure.ac | 34 ++++++++++++++++- 3 files changed, 103 insertions(+), 73 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2c4efe89274e69..0daeea9447bd2e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -313,16 +313,15 @@ HOST_GNU_TYPE= @host@ # PROFILE_TASK="-m test --pgo-extended" PROFILE_TASK= @PROFILE_TASK@ -# report files for gcov / lcov coverage report -COVERAGE_INFO= $(abs_builddir)/coverage.info -COVERAGE_REPORT=$(abs_builddir)/lcov-report -COVERAGE_LCOV_OPTIONS=--rc lcov_branch_coverage=1 -COVERAGE_REPORT_OPTIONS=--rc lcov_branch_coverage=1 --branch-coverage --title "CPython $(VERSION) LCOV report [commit $(shell $(GITVERSION))]" - -# report files for llvm-cov coverage report -COVERAGE_INFO_LLVM= $(abs_builddir)/coverage.profdata -COVERAGE_REPORT_LLVM=$(abs_builddir)/llvm-cov-report -COVERAGE_REPORT_OPTIONS_LLVM=-show-branches=count -show-regions +# parameters for coverage +COVERAGE_CC=@COVERAGE_CC@ +COVERAGE_CFLAGS=@COVERAGE_CFLAGS@ +COVERAGE_LDFLAGS=@COVERAGE_LDFLAGS@ +COVERAGE_GEN_TARGET=@COVERAGE_GEN_TARGET@ +COVERAGE_INFO=$(abs_builddir)/@COVERAGE_INFO@ +COVERAGE_REPORT=$(abs_builddir)/coverage-report +COVERAGE_OPTIONS=@COVERAGE_OPTIONS@ +COVERAGE_REPORT_OPTIONS=@COVERAGE_REPORT_OPTIONS@ # === Definitions added by makesetup === @@ -662,36 +661,38 @@ bolt-opt: @PREBOLT_RULE@ rm -f $(BUILDPYTHON).bolt_inst mv $(BUILDPYTHON).bolt $(BUILDPYTHON) -.PHONY=coverage-report +.PHONY=coverage-report coverage coverage-generate-lcov coverage-generate-profdata coverage-report: - @if [ $(CC_NAME) = "gcc" ]; then \ - $(MAKE) coverage-report-lcov; \ - elif [ $(CC_NAME) = "clang" ]; then \ - $(MAKE) coverage-report-llvm; \ - else \ - echo "Coverage is not supported with the $(CC_NAME) compiler"; \ - exit 1; \ - fi + @ # build with coverage info + $(MAKE) coverage + @ # run tests, ignore failures + @ # The LLVM_PROFILE_FILE must specify %m so results can be collected in parallel. + @ # Also, must be an absolute path since test suite changes working directory. + LLVM_PROFILE_FILE=${abs_builddir}/python%m.profraw $(TESTRUNNER) $(TESTOPTS) || true + @ # build lcov report + $(MAKE) $(COVERAGE_GEN_TARGET) + @echo + @echo "coverage report at $(COVERAGE_REPORT)/index.html" + @echo -# Compile and run with gcov -.PHONY=coverage-gcc coverage-lcov coverage-report-lcov -coverage-gcc: +# Compile and run generating coverage +coverage: @echo "Building with support for coverage checking:" $(MAKE) clean - $(MAKE) @DEF_MAKE_RULE@ CFLAGS="$(CFLAGS) -O0 -pg --coverage" LDFLAGS="$(LDFLAGS) --coverage" + $(MAKE) @DEF_MAKE_RULE@ CC="$(COVERAGE_CC)" CFLAGS="$(CFLAGS) $(COVERAGE_CFLAGS)" LDFLAGS="$(LDFLAGS) $(COVERAGE_LDFLAGS)" -coverage-lcov: +coverage-generate-lcov: @echo "Creating Coverage HTML report with LCOV:" @rm -f $(COVERAGE_INFO) @rm -rf $(COVERAGE_REPORT) - @lcov $(COVERAGE_LCOV_OPTIONS) --capture \ + @lcov $(COVERAGE_OPTIONS) --capture \ --directory $(abs_builddir) \ --base-directory $(realpath $(abs_builddir)) \ --path $(realpath $(abs_srcdir)) \ --output-file $(COVERAGE_INFO) @ # remove 3rd party modules, system headers and internal files with @ # debug, test or dummy functions. - @lcov $(COVERAGE_LCOV_OPTIONS) --remove $(COVERAGE_INFO) \ + @lcov $(COVERAGE_OPTIONS) --remove $(COVERAGE_INFO) \ '*/Modules/_blake2/impl/*' \ '*/Modules/_ctypes/libffi*/*' \ '*/Modules/_decimal/libmpdec/*' \ @@ -706,51 +707,15 @@ coverage-lcov: @genhtml $(COVERAGE_INFO) \ --output-directory $(COVERAGE_REPORT) \ $(COVERAGE_REPORT_OPTIONS) - @echo - @echo "lcov report at $(COVERAGE_REPORT)/index.html" - @echo - -# Force regeneration of parser and frozen modules -coverage-report-lcov: regen-token regen-frozen - @ # build with coverage info - $(MAKE) coverage-gcc - @ # run tests, ignore failures - $(TESTRUNNER) $(TESTOPTS) || true - @ # build lcov report - $(MAKE) coverage-lcov -# Compile and calculate coverage with llvm-cov -.PHONY=coverage-clang coverage-profdata coverage-report-llvm - -coverage-clang: - @echo "Building with support for coverage checking:" - $(MAKE) clean - @ # Override CC rather than CFLAGS since these flags must come first - $(MAKE) @DEF_MAKE_RULE@ CC="$(CC) -fprofile-instr-generate -fcoverage-mapping" - -coverage-profdata: +coverage-generate-profdata: @echo "Creating Coverage HTML report with llvm-profdata/llvm-cov:" - @rm -f $(COVERAGE_INFO_LLVM) - @rm -rf $(COVERAGE_REPORT_LLVM) + @rm -f $(COVERAGE_INFO) + @rm -rf $(COVERAGE_REPORT) @ # Merge coverage results - $(LLVM_PROFDATA) merge -sparse python*.profraw -o $(COVERAGE_INFO_LLVM) + $(LLVM_PROFDATA) merge -sparse python*.profraw -o $(COVERAGE_INFO) @ # Generate HTML - $(LLVM_COV) show -format=html -output-dir=$(COVERAGE_REPORT_LLVM) -instr-profile=$(COVERAGE_INFO_LLVM) $(COVERAGE_REPORT_OPTIONS_LLVM) python . - @echo - @echo "llvm-cov report at $(COVERAGE_REPORT_LLVM)/index.html" - @echo - -# Force regeneration of parser and importlib -# Specify the LLVM_PROFILE_FILE using %m so multiple shared objects can write -# in parallel. Set the full path to the directory so results aren't written -# into temporary directories created by tests. -coverage-report-llvm: regen-token regen-importlib - @ # build with coverage info - $(MAKE) coverage-clang - @ # run tests, ignore failures - LLVM_PROFILE_FILE=${PWD}/python%m.profraw $(TESTRUNNER) $(TESTOPTS) || true - @ # build llvm-cov report - $(MAKE) coverage-profdata + $(LLVM_COV) show -format=html -output-dir=$(COVERAGE_REPORT) -instr-profile=$(COVERAGE_INFO) $(COVERAGE_REPORT_OPTIONS) python . # Run "Argument Clinic" over all source files .PHONY=clinic diff --git a/configure b/configure index dd18b586f2ac24..9b84bdeeec3a98 100755 --- a/configure +++ b/configure @@ -884,6 +884,13 @@ BASECFLAGS CFLAGS_ALIASING OPT LLVM_COV +COVERAGE_REPORT_OPTIONS +COVERAGE_OPTIONS +COVERAGE_INFO +COVERAGE_GEN_TARGET +COVERAGE_LDFLAGS +COVERAGE_CFLAGS +COVERAGE_CC LLVM_PROF_FOUND LLVM_PROFDATA LLVM_PROF_ERR @@ -936,7 +943,6 @@ PLATFORM_TRIPLET MULTIARCH ac_ct_CXX CXX -CC_NAME EGREP SED GREP @@ -5372,9 +5378,6 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_name" >&5 $as_echo "$ac_cv_cc_name" >&6; } -CC_NAME=$ac_cv_cc_name - - # checks for UNIX variants that set C preprocessor variables # may set _GNU_SOURCE, __EXTENSIONS__, _POSIX_PTHREAD_SEMANTICS, # _POSIX_SOURCE, _POSIX_1_SOURCE, and more @@ -8435,6 +8438,38 @@ case $CC in ;; esac +# Coverage flags + +case $CC in + *clang*) + COVERAGE_CC="\$(CC) -fprofile-instr-generate -fcoverage-mapping" + COVERAGE_CFLAGS="" + COVERAGE_LDFLAGS="" + COVERAGE_GEN_TARGET="coverage-generate-profdata" + COVERAGE_INFO="coverage.profdata" + COVERAGE_OPTIONS="" + COVERAGE_REPORT_OPTIONS="-show-branches=count -show-regions" + ;; + *gcc*) + COVERAGE_CC="\$(CC)" + COVERAGE_CFLAGS="-O0 -pg --coverage" + COVERAGE_LDFLAGS="--coverage" + COVERAGE_GEN_TARGET="coverage-generate-lcov" + COVERAGE_INFO="coverage.info" + COVERAGE_OPTIONS="--rc lcov_brange_coverage=1" + COVERAGE_REPORT_OPTIONS="\$(COVERAGE_OPTIONS) --branch-coverage --title \"CPython \$(VERSION) LCOV report commit \$(shell \$(GITVERSION))\"" + ;; + *) +esac + + + + + + + + + if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}llvm-cov", so it can be a program name with args. set dummy ${ac_tool_prefix}llvm-cov; ac_word=$2 diff --git a/configure.ac b/configure.ac index 42a3cf5c670676..ecdae76a2bf523 100644 --- a/configure.ac +++ b/configure.ac @@ -869,8 +869,6 @@ fi rm -f conftest.c conftest.out ]) -AC_SUBST([CC_NAME], [$ac_cv_cc_name]) - # checks for UNIX variants that set C preprocessor variables # may set _GNU_SOURCE, __EXTENSIONS__, _POSIX_PTHREAD_SEMANTICS, # _POSIX_SOURCE, _POSIX_1_SOURCE, and more @@ -2057,6 +2055,38 @@ case $CC in LLVM_PROF_FILE="" ;; esac + +# Coverage flags + +case $CC in + *clang*) + COVERAGE_CC="\$(CC) -fprofile-instr-generate -fcoverage-mapping" + COVERAGE_CFLAGS="" + COVERAGE_LDFLAGS="" + COVERAGE_GEN_TARGET="coverage-generate-profdata" + COVERAGE_INFO="coverage.profdata" + COVERAGE_OPTIONS="" + COVERAGE_REPORT_OPTIONS="-show-branches=count -show-regions" + ;; + *gcc*) + COVERAGE_CC="\$(CC)" + COVERAGE_CFLAGS="-O0 -pg --coverage" + COVERAGE_LDFLAGS="--coverage" + COVERAGE_GEN_TARGET="coverage-generate-lcov" + COVERAGE_INFO="coverage.info" + COVERAGE_OPTIONS="--rc lcov_brange_coverage=1" + COVERAGE_REPORT_OPTIONS="\$(COVERAGE_OPTIONS) --branch-coverage --title \"CPython \$(VERSION) LCOV report [commit \$(shell \$(GITVERSION))]\"" + ;; + *) +esac + +AC_SUBST(COVERAGE_CC) +AC_SUBST(COVERAGE_CFLAGS) +AC_SUBST(COVERAGE_LDFLAGS) +AC_SUBST(COVERAGE_GEN_TARGET) +AC_SUBST(COVERAGE_INFO) +AC_SUBST(COVERAGE_OPTIONS) +AC_SUBST(COVERAGE_REPORT_OPTIONS) AC_SUBST(LLVM_COV) AC_PATH_TOOL(LLVM_COV, llvm-cov, '', ${llvm_path}) From ceb32ee3af138a30703930d019761ae063394038 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 24 Aug 2022 13:34:45 -0400 Subject: [PATCH 8/8] Cleanup after recent changes --- .github/workflows/c-coverage.yml | 2 +- Makefile.pre.in | 8 +++++--- .../Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/c-coverage.yml b/.github/workflows/c-coverage.yml index 6a5eedbfb9c089..abcb53cb65925d 100644 --- a/.github/workflows/c-coverage.yml +++ b/.github/workflows/c-coverage.yml @@ -50,7 +50,7 @@ jobs: - name: Publish coverage-report uses: JamesIves/github-pages-deploy-action@v4 with: - folder: llvm-cov-report + folder: coverage-report repository-name: '' # TODO Destination token: ${{ secrets.COVERAGE_DEPLOY_TOKEN }} # TODO: Use an organization-level token single-commit: true diff --git a/Makefile.pre.in b/Makefile.pre.in index 0daeea9447bd2e..f2e1177c1f07d8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -37,7 +37,6 @@ abs_builddir= @abs_builddir@ CC= @CC@ CXX= @CXX@ LINKCC= @LINKCC@ -CC_NAME=@CC_NAME@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ @@ -661,8 +660,9 @@ bolt-opt: @PREBOLT_RULE@ rm -f $(BUILDPYTHON).bolt_inst mv $(BUILDPYTHON).bolt $(BUILDPYTHON) +# Support generating coverage reports .PHONY=coverage-report coverage coverage-generate-lcov coverage-generate-profdata -coverage-report: +coverage-report: regen-token regen-frozen @ # build with coverage info $(MAKE) coverage @ # run tests, ignore failures @@ -679,7 +679,9 @@ coverage-report: coverage: @echo "Building with support for coverage checking:" $(MAKE) clean - $(MAKE) @DEF_MAKE_RULE@ CC="$(COVERAGE_CC)" CFLAGS="$(CFLAGS) $(COVERAGE_CFLAGS)" LDFLAGS="$(LDFLAGS) $(COVERAGE_LDFLAGS)" + $(MAKE) @DEF_MAKE_RULE@ CC="$(COVERAGE_CC)" \ + CFLAGS="$(CFLAGS) $(COVERAGE_CFLAGS)" \ + LDFLAGS="$(LDFLAGS) $(COVERAGE_LDFLAGS)" coverage-generate-lcov: @echo "Creating Coverage HTML report with LCOV:" diff --git a/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst index b5fe3efb75fd17..9327531211d7bb 100644 --- a/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst +++ b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst @@ -1,2 +1 @@ -The ``coverage-report`` Makefile target will now automatically use ``llvm-cov`` to generate a coverage report when using ``clang``. -This provides more details about branch coverage and subexpressions than the existing ``gcc`` and ``lcov`` based ``coverage-report``. +The ``coverage-report`` Makefile now supports both the ``gcc/lcov`` and ``clang/llvm-profdata`` stacks to generate coverage reports, and will select the correct one based on the compiler in use.