diff --git a/.github/workflows/c-coverage.yml b/.github/workflows/c-coverage.yml new file mode 100644 index 00000000000000..abcb53cb65925d --- /dev/null +++ b/.github/workflows/c-coverage.yml @@ -0,0 +1,56 @@ +name: C coverage + +on: + schedule: + # Run this daily at 2400 UTC + - cron: '0 0 * * *' + +permissions: + contents: read + +jobs: + c-coverage: + name: 'C-level coverage (using llvm-cov)' + runs-on: ubuntu-20.04 + env: + OPENSSL_VER: 1.1.1n + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: Restore OpenSSL build + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 + - name: Configure clang (with coverage) + run: | + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Generate coverage report + # Using "-j1" is important, or the Github Action runs out of memory + run: EXTRATESTOPTS=-j1 xvfb-run make coverage-report + - name: Publish coverage-report + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: coverage-report + repository-name: '' # TODO Destination + token: ${{ secrets.COVERAGE_DEPLOY_TOKEN }} # TODO: Use an organization-level token + single-commit: true diff --git a/.gitignore b/.gitignore index 6934faa91e9874..4b820174aa5052 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ *.gc?? *.profclang? *.profraw +*.profdata *.dyn .gdb_history .purify diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 424a4a93b6f972..72d818e31f0179 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -797,9 +797,11 @@ def is_env_var_to_ignore(n): # This excludes some __CF_* and VERSIONER_* keys MacOS insists # on adding even when the environment in exec is empty. # Gentoo sandboxes also force LD_PRELOAD and SANDBOX_* to exist. + # LLVM coverage adds __LLVM_PROFILE_RT_INIT_ONCE variable. return ('VERSIONER' in n or '__CF' in n or # MacOS n == 'LD_PRELOAD' or n.startswith('SANDBOX') or # Gentoo - n == 'LC_CTYPE') # Locale coercion triggered + n == 'LC_CTYPE' or # Locale coercion triggered + n == '__LLVM_PROFILE_RT_INIT_ONCE') with subprocess.Popen([sys.executable, "-c", 'import os; print(list(os.environ.keys()))'], diff --git a/Makefile.pre.in b/Makefile.pre.in index 9b35398c4fe767..f2e1177c1f07d8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -50,6 +50,8 @@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ LLVM_PROF_FILE=@LLVM_PROF_FILE@ LLVM_PROF_ERR=@LLVM_PROF_ERR@ +LLVM_PROFDATA=@LLVM_PROFDATA@ +LLVM_COV=@LLVM_COV@ DTRACE= @DTRACE@ DFLAGS= @DFLAGS@ DTRACE_HEADERS= @DTRACE_HEADERS@ @@ -310,12 +312,15 @@ HOST_GNU_TYPE= @host@ # PROFILE_TASK="-m test --pgo-extended" PROFILE_TASK= @PROFILE_TASK@ -# report files for gcov / lcov coverage report -COVERAGE_INFO= $(abs_builddir)/coverage.info -COVERAGE_REPORT=$(abs_builddir)/lcov-report -COVERAGE_LCOV_OPTIONS=--rc lcov_branch_coverage=1 -COVERAGE_REPORT_OPTIONS=--rc lcov_branch_coverage=1 --branch-coverage --title "CPython $(VERSION) LCOV report [commit $(shell $(GITVERSION))]" - +# parameters for coverage +COVERAGE_CC=@COVERAGE_CC@ +COVERAGE_CFLAGS=@COVERAGE_CFLAGS@ +COVERAGE_LDFLAGS=@COVERAGE_LDFLAGS@ +COVERAGE_GEN_TARGET=@COVERAGE_GEN_TARGET@ +COVERAGE_INFO=$(abs_builddir)/@COVERAGE_INFO@ +COVERAGE_REPORT=$(abs_builddir)/coverage-report +COVERAGE_OPTIONS=@COVERAGE_OPTIONS@ +COVERAGE_REPORT_OPTIONS=@COVERAGE_REPORT_OPTIONS@ # === Definitions added by makesetup === @@ -655,25 +660,41 @@ bolt-opt: @PREBOLT_RULE@ rm -f $(BUILDPYTHON).bolt_inst mv $(BUILDPYTHON).bolt $(BUILDPYTHON) -# Compile and run with gcov -.PHONY=coverage coverage-lcov coverage-report +# Support generating coverage reports +.PHONY=coverage-report coverage coverage-generate-lcov coverage-generate-profdata +coverage-report: regen-token regen-frozen + @ # build with coverage info + $(MAKE) coverage + @ # run tests, ignore failures + @ # The LLVM_PROFILE_FILE must specify %m so results can be collected in parallel. + @ # Also, must be an absolute path since test suite changes working directory. + LLVM_PROFILE_FILE=${abs_builddir}/python%m.profraw $(TESTRUNNER) $(TESTOPTS) || true + @ # build lcov report + $(MAKE) $(COVERAGE_GEN_TARGET) + @echo + @echo "coverage report at $(COVERAGE_REPORT)/index.html" + @echo + +# Compile and run generating coverage coverage: @echo "Building with support for coverage checking:" $(MAKE) clean - $(MAKE) @DEF_MAKE_RULE@ CFLAGS="$(CFLAGS) -O0 -pg --coverage" LDFLAGS="$(LDFLAGS) --coverage" + $(MAKE) @DEF_MAKE_RULE@ CC="$(COVERAGE_CC)" \ + CFLAGS="$(CFLAGS) $(COVERAGE_CFLAGS)" \ + LDFLAGS="$(LDFLAGS) $(COVERAGE_LDFLAGS)" -coverage-lcov: +coverage-generate-lcov: @echo "Creating Coverage HTML report with LCOV:" @rm -f $(COVERAGE_INFO) @rm -rf $(COVERAGE_REPORT) - @lcov $(COVERAGE_LCOV_OPTIONS) --capture \ + @lcov $(COVERAGE_OPTIONS) --capture \ --directory $(abs_builddir) \ --base-directory $(realpath $(abs_builddir)) \ --path $(realpath $(abs_srcdir)) \ --output-file $(COVERAGE_INFO) @ # remove 3rd party modules, system headers and internal files with @ # debug, test or dummy functions. - @lcov $(COVERAGE_LCOV_OPTIONS) --remove $(COVERAGE_INFO) \ + @lcov $(COVERAGE_OPTIONS) --remove $(COVERAGE_INFO) \ '*/Modules/_blake2/impl/*' \ '*/Modules/_ctypes/libffi*/*' \ '*/Modules/_decimal/libmpdec/*' \ @@ -688,18 +709,15 @@ coverage-lcov: @genhtml $(COVERAGE_INFO) \ --output-directory $(COVERAGE_REPORT) \ $(COVERAGE_REPORT_OPTIONS) - @echo - @echo "lcov report at $(COVERAGE_REPORT)/index.html" - @echo -# Force regeneration of parser and frozen modules -coverage-report: regen-token regen-frozen - @ # build with coverage info - $(MAKE) coverage - @ # run tests, ignore failures - $(TESTRUNNER) $(TESTOPTS) || true - @ # build lcov report - $(MAKE) coverage-lcov +coverage-generate-profdata: + @echo "Creating Coverage HTML report with llvm-profdata/llvm-cov:" + @rm -f $(COVERAGE_INFO) + @rm -rf $(COVERAGE_REPORT) + @ # Merge coverage results + $(LLVM_PROFDATA) merge -sparse python*.profraw -o $(COVERAGE_INFO) + @ # Generate HTML + $(LLVM_COV) show -format=html -output-dir=$(COVERAGE_REPORT) -instr-profile=$(COVERAGE_INFO) $(COVERAGE_REPORT_OPTIONS) python . # Run "Argument Clinic" over all source files .PHONY=clinic diff --git a/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst new file mode 100644 index 00000000000000..9327531211d7bb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-07-13-13-24-01.gh-issue-94759.dTLMod.rst @@ -0,0 +1 @@ +The ``coverage-report`` Makefile now supports both the ``gcc/lcov`` and ``clang/llvm-profdata`` stacks to generate coverage reports, and will select the correct one based on the compiler in use. diff --git a/configure b/configure index 953c558d6048d4..9b84bdeeec3a98 100755 --- a/configure +++ b/configure @@ -883,6 +883,14 @@ CFLAGS_NODIST BASECFLAGS CFLAGS_ALIASING OPT +LLVM_COV +COVERAGE_REPORT_OPTIONS +COVERAGE_OPTIONS +COVERAGE_INFO +COVERAGE_GEN_TARGET +COVERAGE_LDFLAGS +COVERAGE_CFLAGS +COVERAGE_CC LLVM_PROF_FOUND LLVM_PROFDATA LLVM_PROF_ERR @@ -8430,6 +8438,137 @@ case $CC in ;; esac +# Coverage flags + +case $CC in + *clang*) + COVERAGE_CC="\$(CC) -fprofile-instr-generate -fcoverage-mapping" + COVERAGE_CFLAGS="" + COVERAGE_LDFLAGS="" + COVERAGE_GEN_TARGET="coverage-generate-profdata" + COVERAGE_INFO="coverage.profdata" + COVERAGE_OPTIONS="" + COVERAGE_REPORT_OPTIONS="-show-branches=count -show-regions" + ;; + *gcc*) + COVERAGE_CC="\$(CC)" + COVERAGE_CFLAGS="-O0 -pg --coverage" + COVERAGE_LDFLAGS="--coverage" + COVERAGE_GEN_TARGET="coverage-generate-lcov" + COVERAGE_INFO="coverage.info" + COVERAGE_OPTIONS="--rc lcov_brange_coverage=1" + COVERAGE_REPORT_OPTIONS="\$(COVERAGE_OPTIONS) --branch-coverage --title \"CPython \$(VERSION) LCOV report commit \$(shell \$(GITVERSION))\"" + ;; + *) +esac + + + + + + + + + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}llvm-cov", so it can be a program name with args. +set dummy ${ac_tool_prefix}llvm-cov; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_LLVM_COV+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $LLVM_COV in + [\\/]* | ?:[\\/]*) + ac_cv_path_LLVM_COV="$LLVM_COV" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in ${llvm_path} +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_LLVM_COV="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +LLVM_COV=$ac_cv_path_LLVM_COV +if test -n "$LLVM_COV"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_COV" >&5 +$as_echo "$LLVM_COV" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_LLVM_COV"; then + ac_pt_LLVM_COV=$LLVM_COV + # Extract the first word of "llvm-cov", so it can be a program name with args. +set dummy llvm-cov; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_LLVM_COV+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_LLVM_COV in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_LLVM_COV="$ac_pt_LLVM_COV" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in ${llvm_path} +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_LLVM_COV="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_LLVM_COV=$ac_cv_path_ac_pt_LLVM_COV +if test -n "$ac_pt_LLVM_COV"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_LLVM_COV" >&5 +$as_echo "$ac_pt_LLVM_COV" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_LLVM_COV" = x; then + LLVM_COV="''" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + LLVM_COV=$ac_pt_LLVM_COV + fi +else + LLVM_COV="$ac_cv_path_LLVM_COV" +fi + + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? diff --git a/configure.ac b/configure.ac index 210ce3292cfdf7..ecdae76a2bf523 100644 --- a/configure.ac +++ b/configure.ac @@ -2056,6 +2056,40 @@ case $CC in ;; esac +# Coverage flags + +case $CC in + *clang*) + COVERAGE_CC="\$(CC) -fprofile-instr-generate -fcoverage-mapping" + COVERAGE_CFLAGS="" + COVERAGE_LDFLAGS="" + COVERAGE_GEN_TARGET="coverage-generate-profdata" + COVERAGE_INFO="coverage.profdata" + COVERAGE_OPTIONS="" + COVERAGE_REPORT_OPTIONS="-show-branches=count -show-regions" + ;; + *gcc*) + COVERAGE_CC="\$(CC)" + COVERAGE_CFLAGS="-O0 -pg --coverage" + COVERAGE_LDFLAGS="--coverage" + COVERAGE_GEN_TARGET="coverage-generate-lcov" + COVERAGE_INFO="coverage.info" + COVERAGE_OPTIONS="--rc lcov_brange_coverage=1" + COVERAGE_REPORT_OPTIONS="\$(COVERAGE_OPTIONS) --branch-coverage --title \"CPython \$(VERSION) LCOV report [commit \$(shell \$(GITVERSION))]\"" + ;; + *) +esac + +AC_SUBST(COVERAGE_CC) +AC_SUBST(COVERAGE_CFLAGS) +AC_SUBST(COVERAGE_LDFLAGS) +AC_SUBST(COVERAGE_GEN_TARGET) +AC_SUBST(COVERAGE_INFO) +AC_SUBST(COVERAGE_OPTIONS) +AC_SUBST(COVERAGE_REPORT_OPTIONS) +AC_SUBST(LLVM_COV) +AC_PATH_TOOL(LLVM_COV, llvm-cov, '', ${llvm_path}) + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code?