diff --git a/contrib/tools/local-mutation-testing.sh b/contrib/tools/local-mutation-testing.sh deleted file mode 100755 index 11da6810e54..00000000000 --- a/contrib/tools/local-mutation-testing.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Install cargo-mutants -cargo install --version 24.7.1 cargo-mutants --locked - -# Create diff file between current branch and develop branch -git diff origin/develop...HEAD > git.diff - -# Remove git diff files about removed/renamed files -awk ' - /^diff --git/ { - diff_line = $0 - getline - if ($0 !~ /^(deleted file mode|similarity index)/) { - print diff_line - print - } - } - !/^(diff --git|deleted file mode|similarity index|rename from|rename to)/ {print} -' git.diff > processed.diff - -# Extract mutants based on the processed diff -cargo mutants --in-diff processed.diff --list > all_mutants.txt - -# Create a directory for organizing mutants -mkdir -p mutants_by_package - -# Organize mutants into files based on their main folder -while IFS= read -r line; do - package=$(echo "$line" | cut -d'/' -f1) - - case $package in - "stackslib") - echo "$line" >> "mutants_by_package/stackslib.txt" - ;; - "testnet") - echo "$line" >> "mutants_by_package/stacks-node.txt" - ;; - "stacks-signer") - echo "$line" >> "mutants_by_package/stacks-signer.txt" - ;; - *) - echo "$line" >> "mutants_by_package/small-packages.txt" - ;; - esac -done < all_mutants.txt - -# Function to run mutants for a package -run_mutants() { - local package=$1 - local threshold=$2 - local output_dir=$3 - local mutant_file="mutants_by_package/${package}.txt" - - if [ ! -f "$mutant_file" ]; then - echo "No mutants found for $package" - return 0 - fi - - local regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "$mutant_file" | paste -sd'|' -) - local mutant_count=$(cargo mutants -F "$regex_pattern" -E ": replace .{1,2} with .{1,2} in " --list | wc -l) - - if [ "$mutant_count" -gt "$threshold" ]; then - echo "Running mutants for $package ($mutant_count mutants)" - RUST_BACKTRACE=1 BITCOIND_TEST=1 \ - cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ - -F "$regex_pattern" \ - -E ": replace .{1,2} with .{1,2} in " \ - --output "$output_dir" \ - --test-tool=nextest \ - --package "$package" \ - -- --all-targets --test-threads 1 || true - - echo $? > "${output_dir}/exit_code.txt" - else - echo "Skipping $package, only $mutant_count mutants (threshold: $threshold)" - fi - - return 0 -} - -# Run mutants for each wanted package -run_mutants "stacks-signer" 500 "./stacks-signer_mutants" || true -run_mutants "stacks-node" 540 "./stacks-node_mutants" || true -run_mutants "stackslib" 72 "./stackslib_mutants" || true diff --git a/docs/ci-workflow.md b/docs/ci-workflow.md index 0b1ed2b170d..a94f0a74ddb 100644 --- a/docs/ci-workflow.md +++ b/docs/ci-workflow.md @@ -5,7 +5,6 @@ All releases are built via a Github Actions workflow named [`CI`](../.github/wor - Verifying code is formatted correctly - Integration tests - Unit tests -- [Mutation tests](https://en.wikipedia.org/wiki/Mutation_testing) - Creating releases - Building binary archives and calculating checksums - Publishing Docker images @@ -128,100 +127,3 @@ check-tests: jobs: ${{ toJson(needs) }} summary_print: "true" ``` - -## Mutation Testing - -When a new Pull Request (PR) is submitted, this feature evaluates the quality of the tests added or modified in the PR. -It checks the new and altered functions through mutation testing. -Mutation testing involves making small changes (mutations) to the code to check if the tests can detect these changes. - -The mutations are run with or without a [Github Actions matrix](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs). -The matrix is used when there is a large number of mutations to run ([check doc specific cases](https://github.com/stacks-network/actions/blob/main/stacks-core/mutation-testing/check-packages-and-shards/README.md#outputs)). -We utilize a matrix strategy with shards to enable parallel execution in GitHub Actions. -This approach allows for the concurrent execution of multiple jobs across various runners. -The total workload is divided across all shards, effectively reducing the overall duration of a workflow because the time taken is approximately the total time divided by the number of shards (+ initial build & test time). -This is particularly advantageous for large packages that have significant build and test times, as it enhances efficiency and speeds up the process. - -Since mutation testing is directly correlated to the written tests, there are slower packages (due to the quantity or time it takes to run the tests) like `stackslib` or `stacks-node`. -These mutations are run separately from the others, with one or more parallel jobs, depending on the amount of mutations found. - -Once all the jobs have finished testing mutants, the last job collects all the tested mutations from the previous jobs, combines them and outputs them to the `Summary` section of the workflow, at the bottom of the page. -There, you can find all mutants on categories, with links to the function they tested, and a short description on how to fix the issue. -The PR should only be approved/merged after all the mutants tested are in the `Caught` category. - -### Time required to run the workflow based on mutants outcome and packages' size - -- Small packages typically completed in under 30 minutes, aided by the use of shards. -- Large packages like stackslib and stacks-node initially required about 20-25 minutes for build and test processes. - - Each "missed" and "caught" mutant took approximately 15 minutes. Using shards, this meant about 50-55 minutes for processing around 32 mutants (10-16 functions modified). Every additional 8 mutants added another 15 minutes to the runtime. - - "Unviable" mutants, which are functions lacking a Default implementation for their returned struct type, took less than a minute each. - - "Timeout" mutants typically required more time. However, these should be marked to be skipped (by adding a skip flag to their header) since they indicate functions unable to proceed in their test workflow with mutated values, as opposed to the original implementations. - -File: - -- [PR Differences Mutants](../.github/workflows/pr-differences-mutants.yml) - -### Mutant Outcomes - -- caught — A test failed with this mutant applied. - This is a good sign about test coverage. - -- missed — No test failed with this mutation applied, which seems to indicate a gap in test coverage. - Or, it may be that the mutant is undistinguishable from the correct code. - In any case, you may wish to add a better test. - -- unviable — The attempted mutation doesn't compile. - This is inconclusive about test coverage, since the function's return structure may not implement `Default::default()` (one of the mutations applied), hence causing the compile to fail. - It is recommended to add `Default` implementation for the return structures of these functions, only mark that the function should be skipped as a last resort. - -- timeout — The mutation caused the test suite to run for a long time, until it was eventually killed. - You might want to investigate the cause and only mark the function to be skipped if necessary. - -### Skipping Mutations - -Some functions may be inherently hard to cover with tests, for example if: - -- Generated mutants cause tests to hang. -- You've chosen to test the functionality by human inspection or some higher-level integration tests. -- The function has side effects or performance characteristics that are hard to test. -- You've decided that the function is not important to test. - -To mark functions as skipped, so they are not mutated: - -- Add a Cargo dependency of the [mutants](https://crates.io/crates/mutants) crate, version `0.0.3` or later (this must be a regular `dependency`, not a `dev-dependency`, because the annotation will be on non-test code) and mark functions with `#[mutants::skip]`, or - -- You can avoid adding the dependency by using the slightly longer `#[cfg_attr(test, mutants::skip)]`. - -### Example - -```rust -use std::time::{Duration, Instant}; - -/// Returns true if the program should stop -#[cfg_attr(test, mutants::skip)] // Returning false would cause a hang -fn should_stop() -> bool { - true -} - -pub fn controlled_loop() { - let start = Instant::now(); - for i in 0.. { - println!("{}", i); - if should_stop() { - break; - } - if start.elapsed() > Duration::from_secs(60 * 5) { - panic!("timed out"); - } - } -} - -mod test { - #[test] - fn controlled_loop_terminates() { - super::controlled_loop() - } -} -``` - ---- diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md deleted file mode 100644 index 85fcd89a7f6..00000000000 --- a/docs/mutation-testing.md +++ /dev/null @@ -1,146 +0,0 @@ -# Mutation Testing - -This document describes how to run mutation testing locally to mimic the outcome of a PR, without the CI limitation it provides by timing out after 6 hours. -[Here is the script](../contrib/tools/local-mutation-testing.sh) to run the tests locally by running the mutants created by the changes between `HEAD` and develop. -It does automatically all the steps explained below. - -From the root level of the stacks-core repository run -```sh -./contrib/tools/local-mutation-testing.sh -``` - -## Prerequirements - -Install the cargo mutants library -```sh -cargo install --version 24.7.1 cargo-mutants --locked -``` - - -## Steps -1. Be on source branch you would use for the PR. -2. Create diff file comparing this branch with the `develop` branch - ```sh - git diff origin/develop..HEAD > git.diff - ``` -3. Clean up the diff file and create auxiliary files - ```sh - awk ' - /^diff --git/ { - diff_line = $0 - getline - if ($0 !~ /^(deleted file mode|similarity index)/) { - print diff_line - print - } - } - !/^(diff --git|deleted file mode|similarity index|rename from|rename to)/ {print} - ' git.diff > processed.diff - - # Extract mutants based on the processed diff - cargo mutants --in-diff processed.diff --list > all_mutants.txt - - # Create a directory for organizing mutants - mkdir -p mutants_by_package - - # Organize mutants into files based on their main folder - while IFS= read -r line; do - package=$(echo "$line" | cut -d'/' -f1) - - case $package in - "stackslib") - echo "$line" >> "mutants_by_package/stackslib.txt" - ;; - "testnet") - echo "$line" >> "mutants_by_package/stacks-node.txt" - ;; - "stacks-signer") - echo "$line" >> "mutants_by_package/stacks-signer.txt" - ;; - *) - echo "$line" >> "mutants_by_package/small-packages.txt" - ;; - esac - done < all_mutants.txt - ``` -4. Based on the package required to run the mutants for - a. Stackslib package - ```sh - regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stackslib.txt" | paste -sd'|' -) - - RUST_BACKTRACE=1 BITCOIND_TEST=1 \ - cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ - -F "$regex_pattern" \ - -E ": replace .{1,2} with .{1,2} in " \ - --output "./stackslib_mutants" \ - --test-tool=nextest \ - -- --all-targets --test-threads 1 - ``` - b. Stacks-node (testnet) package - ```sh - regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/testnet.txt" | paste -sd'|' -) - - RUST_BACKTRACE=1 BITCOIND_TEST=1 \ - cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ - -F "$regex_pattern" \ - -E ": replace .{1,2} with .{1,2} in " \ - --output "./testnet_mutants" \ - --test-tool=nextest \ - -- --all-targets --test-threads 1 - ``` - c. Stacks-signer - ```sh - regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stacks-signer.txt" | paste -sd'|' -) - - RUST_BACKTRACE=1 BITCOIND_TEST=1 \ - cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ - -F "$regex_pattern" \ - -E ": replace .{1,2} with .{1,2} in " \ - --output "./stacks-signer_mutants" \ - --test-tool=nextest \ - -- --all-targets --test-threads 1 - ``` - d. All other packages combined - ```sh - regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/small-packages.txt" | paste -sd'|' -) - - cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ - -F "$regex_pattern" \ - -E ": replace .{1,2} with .{1,2} in " \ - --output "./small-packages_mutants" \ - --test-tool=nextest \ - -- --all-targets --test-threads 1 - ``` - -## How to run one specific mutant to test it - -Example of output which had a missing mutant -```sh -MISSED stacks-signer/src/runloop.rs:424:9: replace >::run_one_pass -> Option> with None in 3.0s build + 9.3s test -``` - -Example of fix for it -```sh -RUST_BACKTRACE=1 BITCOIND_TEST=1 \ -cargo mutants -vV \ - -F "replace process_stackerdb_event" \ - -E ": replace >::run_one_pass -> Option> with None in " \ - --test-tool=nextest \ - -- \ - --run-ignored all \ - --fail-fast \ - --test-threads 1 -``` - -General command to run -```sh -RUST_BACKTRACE=1 BITCOIND_TEST=1 \ -cargo mutants -vV \ - -F "replace process_stackerdb_event" \ - -E ": replace [modify this] with [modify this] in " \ - --test-tool=nextest \ - -- \ - --run-ignored all \ - --fail-fast \ - --test-threads 1 -```