Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test: Add fail-fast mechanism #45317

Merged
merged 2 commits into from
May 20, 2022
Merged

Test: Add fail-fast mechanism #45317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
226566f
Test: Add fail-fast mechanism
IanButterworth Jan 8, 2022
999cd07
Apply suggestions from code review
IanButterworth May 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ Standard library changes

#### SparseArrays

#### Test
* New fail-fast mode for testsets that will terminate the test run early if a failure or error occurs.
Set either via the `@testset` kwarg `failfast=true` or by setting env var `JULIA_TEST_FAILFAST`
to `"true"` i.e. in CI runs to request the job failure be posted eagerly when issues occur ([#45317])

#### Dates

#### Downloads
Expand Down
43 changes: 39 additions & 4 deletions stdlib/Test/src/Test.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ const DISPLAY_FAILED = (
:contains
)

const FAIL_FAST = Ref{Bool}(false)

#-----------------------------------------------------------------------

# Backtrace utility functions
Expand Down Expand Up @@ -963,8 +965,22 @@ mutable struct DefaultTestSet <: AbstractTestSet
showtiming::Bool
time_start::Float64
time_end::Union{Float64,Nothing}
failfast::Bool
end
DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true) = DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing)
function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing)
if isnothing(failfast)
# pass failfast state into child testsets
parent_ts = get_testset()
if parent_ts isa DefaultTestSet
failfast = parent_ts.failfast
else
failfast = false
end
end
return DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing, failfast)
end

struct FailFastError <: Exception end

# For a broken result, simply store the result
record(ts::DefaultTestSet, t::Broken) = (push!(ts.results, t); t)
Expand All @@ -986,6 +1002,7 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error})
end
end
push!(ts.results, t)
(FAIL_FAST[] || ts.failfast) && throw(FailFastError())
return t
end

Expand Down Expand Up @@ -1262,11 +1279,17 @@ along with a summary of the test results.
Any custom testset type (subtype of `AbstractTestSet`) can be given and it will
also be used for any nested `@testset` invocations. The given options are only
applied to the test set where they are given. The default test set type
accepts two boolean options:
accepts three boolean options:
- `verbose`: if `true`, the result summary of the nested testsets is shown even
when they all pass (the default is `false`).
- `showtiming`: if `true`, the duration of each displayed testset is shown
(the default is `true`).
- `failfast`: if `true`, any test failure or error will cause the testset and any
child testsets to return immediately (the default is `false`). This can also be set
globally via the env var `JULIA_TEST_FAILFAST`.

!!! compat "Julia 1.9"
`failfast` requires at least Julia 1.9.

The description string accepts interpolation from the loop indices.
If no description is provided, one is constructed based on the variables.
Expand Down Expand Up @@ -1310,6 +1333,8 @@ macro testset(args...)
error("Expected function call, begin/end block or for loop as argument to @testset")
end

FAIL_FAST[] = something(tryparse(Bool, get(ENV, "JULIA_TEST_FAILFAST", "false")), false)

if tests.head === :for
return testset_forloop(args, tests, __source__)
else
Expand Down Expand Up @@ -1364,7 +1389,11 @@ function testset_beginend_call(args, tests, source)
# something in the test block threw an error. Count that as an
# error in this test set
trigger_test_failure_break(err)
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
if err isa FailFastError
get_testset_depth() > 1 ? rethrow() : failfast_print()
else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
end
finally
copy!(RNG, oldrng)
Random.set_global_seed!(oldseed)
Expand All @@ -1380,6 +1409,10 @@ function testset_beginend_call(args, tests, source)
return ex
end

function failfast_print()
printstyled("\nFail-fast enabled:"; color = Base.error_color(), bold=true)
printstyled(" Fail or Error occured\n\n"; color = Base.error_color())
end

"""
Generate the code for a `@testset` with a `for` loop argument
Expand Down Expand Up @@ -1440,7 +1473,9 @@ function testset_forloop(args, testloop, source)
# Something in the test block threw an error. Count that as an
# error in this test set
trigger_test_failure_break(err)
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
if !isa(err, FailFastError)
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
end
end
end
quote
Expand Down
105 changes: 105 additions & 0 deletions stdlib/Test/test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1160,6 +1160,111 @@ end
end
end

@testset "failfast option" begin
@testset "non failfast (default)" begin
expected = r"""
Test Summary: | Pass Fail Error Total Time
Foo | 1 2 1 4 \s*\d*.\ds
Bar | 1 1 2 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test

@testset "Foo" begin
@test false
@test error()
@testset "Bar" begin
@test false
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
@testset "failfast" begin
expected = r"""
Test Summary: | Fail Total Time
Foo | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test

@testset "Foo" failfast=true begin
@test false
@test error()
@testset "Bar" begin
@test false
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
@testset "failfast passes to child testsets" begin
expected = r"""
Test Summary: | Fail Total Time
PackageName | 1 1 \s*\d*.\ds
1 | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test

@testset "Foo" failfast=true begin
@testset "1" begin
@test false
end
@testset "2" begin
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
@testset "failfast via env var" begin
expected = r"""
Test Summary: | Fail Total Time
Foo | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test
ENV["JULIA_TEST_FAILFAST"] = true
@testset "Foo" begin
@test false
@test error()
@testset "Bar" begin
@test false
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
end

# Non-booleans in @test (#35888)
struct T35888 end
Base.isequal(::T35888, ::T35888) = T35888()
Expand Down