Merge pull request #7401 from RenjiSann/coverage

Fix coverage files generation

Author: sylvestre

.config/nextest.toml

@@ -4,3 +4,10 @@ status-level = "all"
 final-status-level = "skip"
 failure-output = "immediate-final"
 fail-fast = false
+
+[profile.coverage]
+retries = 0
+status-level = "all"
+final-status-level = "skip"
+failure-output = "immediate-final"
+fail-fast = false

.github/workflows/CICD.yml

@@ -2,7 +2,7 @@ name: CICD
 
 # spell-checker:ignore (abbrev/names) CICD CodeCOV MacOS MinGW MSVC musl taiki
 # spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS
-# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers dedupe devel
+# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers dedupe devel profdata
 # spell-checker:ignore (people) Peltoche rivy dtolnay Anson dawidd
 # spell-checker:ignore (shell/tools) binutils choco clippy dmake dpkg esac fakeroot fdesc fdescfs gmake grcov halium lcov libclang libfuse libssl limactl mkdir nextest nocross pacman popd printf pushd redoxer rsync rustc rustfmt rustup shopt sccache utmpdump xargs
 # spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos getenforce gnueabihf issuecomment maint manpages msys multisize noconfirm nullglob onexitbegin onexitend pell runtest Swatinem tempfile testsuite toybox uutils
@@ -1001,6 +1001,123 @@ jobs:
         name: toybox-result.json
         path: ${{ steps.vars.outputs.TEST_SUMMARY_FILE }}
 
+  coverage:
+    name: Code Coverage
+    runs-on: ${{ matrix.job.os }}
+    timeout-minutes: 90
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: "sccache"
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - { os: ubuntu-latest  , features: unix, toolchain: nightly }
+          # FIXME: Re-enable macos code coverage
+          # - { os: macos-latest   , features: macos, toolchain: nightly }
+          # FIXME: Re-enable Code Coverage on windows, which currently fails due to "profiler_builtins". See #6686.
+          # - { os: windows-latest , features: windows, toolchain: nightly-x86_64-pc-windows-gnu }
+    steps:
+    - uses: actions/checkout@v4
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ matrix.job.toolchain }}
+        components: rustfmt
+    - uses: taiki-e/install-action@v2
+      with:
+        tool: nextest,grcov@0.8.24
+    - uses: Swatinem/rust-cache@v2
+
+    - name: Run sccache-cache
+      uses: mozilla-actions/sccache-action@v0.0.6
+
+    # - name: Reattach HEAD ## may be needed for accurate code coverage info
+    #   run: git checkout ${{ github.head_ref }}
+
+    - name: Initialize workflow variables
+      id: vars
+      shell: bash
+      run: |
+        ## VARs setup
+        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
+
+        # toolchain
+        TOOLCHAIN="nightly" ## default to "nightly" toolchain (required for certain required unstable compiler flags) ## !maint: refactor when stable channel has needed support
+
+        # * specify gnu-type TOOLCHAIN for windows; `grcov` requires gnu-style code coverage data files
+        case ${{ matrix.job.os }} in windows-*) TOOLCHAIN="$TOOLCHAIN-x86_64-pc-windows-gnu" ;; esac;
+
+        # * use requested TOOLCHAIN if specified
+        if [ -n "${{ matrix.job.toolchain }}" ]; then TOOLCHAIN="${{ matrix.job.toolchain }}" ; fi
+        outputs TOOLCHAIN
+
+        # target-specific options
+
+        # * CARGO_FEATURES_OPTION
+        CARGO_FEATURES_OPTION='--all-features' ;  ## default to '--all-features' for code coverage
+        if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi
+        outputs CARGO_FEATURES_OPTION
+
+        # * CODECOV_FLAGS
+        CODECOV_FLAGS=$( echo "${{ matrix.job.os }}" | sed 's/[^[:alnum:]]/_/g' )
+        outputs CODECOV_FLAGS
+
+    - name: Install/setup prerequisites
+      shell: bash
+      run: |
+        ## Install/setup prerequisites
+        case '${{ matrix.job.os }}' in
+          macos-latest) brew install coreutils ;; # needed for testing
+        esac
+
+        case '${{ matrix.job.os }}' in
+          ubuntu-latest)
+            # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd.
+            # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands. The account also has empty gecos fields.
+            # To work around this for pinky tests, we create a fake login entry for the GH runner account...
+            FAKE_UTMP='[7] [999999] [tty2] [runner] [tty2] [] [0.0.0.0] [2022-02-22T22:22:22,222222+00:00]'
+            # ... by dumping the login records, adding our fake line, then reverse dumping ...
+            (utmpdump /var/run/utmp ; echo $FAKE_UTMP) | sudo utmpdump -r -o /var/run/utmp
+            # ... and add a full name to each account with a gecos field but no full name.
+            sudo sed -i 's/:,/:runner name,/' /etc/passwd
+            # We also create a couple optional files pinky looks for
+            touch /home/runner/.project
+            echo "foo" > /home/runner/.plan
+            ;;
+        esac
+
+        case '${{ matrix.job.os }}' in
+          # Update binutils if MinGW due to https://github.com/rust-lang/rust/issues/112368
+          windows-latest) C:/msys64/usr/bin/pacman.exe -Sy --needed mingw-w64-x86_64-gcc --noconfirm ; echo "C:\msys64\mingw64\bin" >> $GITHUB_PATH ;;
+        esac
+
+        ## Install the llvm-tools component to get access to `llvm-profdata`
+        rustup component add llvm-tools
+
+    - name: Run test and coverage
+      id: run_test_cov
+      run: |
+        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
+
+        # Run the coverage script
+        ./util/build-run-test-coverage-linux.sh
+
+        outputs REPORT_FILE
+      env:
+        COVERAGE_DIR: ${{ github.workspace }}/coverage
+        FEATURES_OPTION: ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }}
+        # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }}
+
+    - name: Upload coverage results (to Codecov.io)
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: ${{ steps.run_test_cov.outputs.report }}
+        ## flags: IntegrationTests, UnitTests, ${{ steps.vars.outputs.CODECOV_FLAGS }}
+        flags: ${{ steps.vars.outputs.CODECOV_FLAGS }}
+        name: codecov-umbrella
+        fail_ci_if_error: false
+
   test_separately:
     name: Separate Builds
     runs-on: ${{ matrix.os }}

.gitignore

@@ -1,6 +1,7 @@
 # spell-checker:ignore (misc) direnv
 
 target/
+coverage/
 /src/*/gen_table
 /build/
 /tmp/

DEVELOPMENT.md

@@ -1,4 +1,4 @@
-<!-- spell-checker:ignore (flags) Ccodegen Coverflow Cpanic Zinstrument Zpanic reimplementing toybox RUNTEST CARGOFLAGS nextest prereq autopoint gettext texinfo automake findutils shellenv libexec gnubin toolchains gsed -->
+<!-- spell-checker:ignore (flags) Ccodegen Coverflow Cpanic Cinstrument Zpanic reimplementing toybox RUNTEST CARGOFLAGS nextest prereq autopoint gettext texinfo automake findutils shellenv libexec gnubin toolchains gsed -->
 
 # Setting up your local development environment
 
@@ -253,13 +253,11 @@ pkg install coreutils gsed
 
 Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov).
 
-### Using Nightly Rust
-
 To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report
 
 ```shell
 export CARGO_INCREMENTAL=0
-export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
+export RUSTFLAGS="-Cinstrument-coverage -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
 export RUSTDOCFLAGS="-Cpanic=abort"
 cargo build <options...> # e.g., --features feat_os_unix
 cargo test <options...> # e.g., --features feat_os_unix test_pathchk
@@ -269,11 +267,6 @@ grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existin
 
 if changes are not reflected in the report then run `cargo clean` and run the above commands.
 
-### Using Stable Rust
-
-If you are using stable version of Rust that doesn't enable code coverage instrumentation by default
-then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above.
-
 ## Tips for setting up on Mac
 
 ### C Compiler and linker

tests/by-util/test_env.rs

@@ -525,14 +525,20 @@ fn test_gnu_e20() {
     let scene = TestScenario::new(util_name!());
 
     let env_bin = String::from(uutests::util::get_tests_binary()) + " " + util_name!();
-
-    let (input, output) = (
-        [
-            String::from("-i"),
-            String::from(r#"-SA="B\_C=D" "#) + env_bin.escape_default().to_string().as_str() + "",
-        ],
-        "A=B C=D\n",
-    );
+    let input = [
+        String::from("-i"),
+        String::from(r#"-SA="B\_C=D" "#) + env_bin.escape_default().to_string().as_str() + "",
+    ];
+
+    let mut output = "A=B C=D\n".to_string();
+
+    // Workaround for the test to pass when coverage is being run.
+    // If enabled, the binary called by env_bin will most probably be
+    // instrumented for coverage, and thus will set the
+    // __LLVM_PROFILE_RT_INIT_ONCE
+    if env::var("__LLVM_PROFILE_RT_INIT_ONCE").is_ok() {
+        output.push_str("__LLVM_PROFILE_RT_INIT_ONCE=__LLVM_PROFILE_RT_INIT_ONCE\n");
+    }
 
     let out = scene.ucmd().args(&input).succeeds();
     assert_eq!(out.stdout_str(), output);

tests/uutests/src/lib/util.rs

@@ -1762,6 +1762,11 @@ impl UCommand {
             }
         }
 
+        // Forward the LLVM_PROFILE_FILE variable to the call, for coverage purposes.
+        if let Some(ld_preload) = env::var_os("LLVM_PROFILE_FILE") {
+            command.env("LLVM_PROFILE_FILE", ld_preload);
+        }
+
         command
             .envs(DEFAULT_ENV)
             .envs(self.env_vars.iter().cloned());