Skip to content

Commit

Permalink
feat(@test_cases): Added support for a :block terminal expression…
Browse files Browse the repository at this point in the history 
… that will be evaluated at each test case iteration (#34)
  • Loading branch information
@curtd
curtd authored Nov 15, 2023
1 parent eea942b commit 9a1f0c5
Show file tree
Hide file tree
Showing 4 changed files with 174 additions and 85 deletions.
2 changes: 2 additions & 0 deletions Project.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@ PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
TestingUtilitiesDataFramesExt = ["DataFrames", "PrettyTables"]

[compat]
Dates = "1.6"
DataFrames = "1"
MLStyle = "0.4"
OrderedCollections = "1.6"
Preferences = "1"
PrettyTables = "2"
Test = "1.6"
julia = "1.6"

[extras]
Expand Down
17 changes: 8 additions & 9 deletions src/macros/Test.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,19 +153,19 @@ function test_expr_and_init_values(original_ex, failed_test_data_name::Symbol, r
push!(set_failed_test_data_args, :($(QuoteNode(k)) => $(esc(k))))
end
end
initial_values_expr = :(TestingUtilities.OrderedDict{Any,Any}( $( set_failed_test_data_args... )))
initial_values_expr = :($TestingUtilities.OrderedDict{Any,Any}( $( set_failed_test_data_args... )))
return initial_values_expr, test_expr, use_isequals_equality
end

function test_show_values_expr(results_printer_name::Symbol, failed_test_data_sym::Symbol, test_input_data_sym::Symbol; should_set_failed_values)

show_values_func_expr = Base.remove_linenums!(quote
let results_printer=$results_printer_name, failed_test_data=$failed_test_data_sym, test_input_data=$test_input_data_sym, TestingUtilities=$(@__MODULE__)
let results_printer=$results_printer_name, failed_test_data=$failed_test_data_sym, test_input_data=$test_input_data_sym
function()
if !$has_printed(results_printer)
$print_Test_data!(results_printer, failed_test_data, test_input_data)
if !$TestingUtilities.has_printed(results_printer)
$TestingUtilities.print_Test_data!(results_printer, failed_test_data, test_input_data)

$set_failed_values_in_main($test_input_data_sym, $(should_set_failed_values))
$TestingUtilities.set_failed_values_in_main($test_input_data_sym, $(should_set_failed_values))
end
end
end
Expand All @@ -180,17 +180,16 @@ function Test_expr(original_ex; io_expr, should_set_failed_values, _sourceinfo)
show_test_data_expr = Base.remove_linenums!(quote
let results_printer=results_printer, failed_test_data=failed_test_data, test_input_data=test_input_data
function()
if !$has_printed(results_printer)
$print_Test_data!(results_printer, failed_test_data, test_input_data)
if !$TestingUtilities.has_printed(results_printer)
$TestingUtilities.print_Test_data!(results_printer, failed_test_data, test_input_data)

$set_failed_values_in_main(test_input_data, $(should_set_failed_values))
$TestingUtilities.set_failed_values_in_main(test_input_data, $(should_set_failed_values))
end
end
end
end)

output = Base.remove_linenums!(quote
local TestingUtilities = $(@__MODULE__)
local io = $(esc(io_expr))
local results_printer = $TestResultsPrinter(io, $(QuoteNode(original_ex)); use_isequals_equality=$use_isequals_equality)
local test_input_data = $(initial_values_expr)
Expand Down
219 changes: 143 additions & 76 deletions src/macros/test_cases.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,126 @@ function rm_macrocall_linenode!(expr::Expr)
end
end

struct TestExpr
original_expr::Any
end

Base.@kwdef mutable struct EvaluateTestCasesExpr
input_exprs::Vector{Any} = Any[]
num_test_exprs::Int = 0
end
Base.length(e::EvaluateTestCasesExpr) = length(e.input_exprs)

function add_test_expr!(t::EvaluateTestCasesExpr, expr; in_block_expr::Bool)
if Meta.isexpr(expr, :macrocall)
is_test_macro = expr.args[1] in (Symbol("@test"), Symbol("@Test"))

is_test_macro || in_block_expr || error("Only `@test` or `@Test` macros allowed in $(expr)")
if is_test_macro
push!(t.input_exprs, TestExpr(expr.args[end]))
t.num_test_exprs += 1
else
push!(t.input_exprs, expr)
end
return true
elseif in_block_expr
push!(t.input_exprs, expr)
return true
else
return false
end
end

function parse_test_cases(body_args; headers)
test_expr_started = false
evaluate_test_exprs = EvaluateTestCasesExpr()
all_test_case_values = Any[]
for (i, expr) in enumerate(body_args)
expr isa LineNumberNode && continue
if Meta.isexpr(expr, :macrocall)
add_test_expr!(evaluate_test_exprs, expr; in_block_expr=false)
test_expr_started = true
elseif Meta.isexpr(expr, :block)
test_expr_started && error("Expression `$expr` must have either `@test` expressions or a single `:block` expression -- cannot mix the two")
i == length(body_args) || error("`:block` expression must be the last argument")
for arg in expr.args
arg isa LineNumberNode && continue
add_test_expr!(evaluate_test_exprs, arg; in_block_expr=true)
end
else
test_expr_started && error("Cannot have test expressions interspersed with test data in expression $(body)")

if Meta.isexpr(expr, :tuple)
test_data_expr = parse_tuple(expr)
else
test_data_expr = parse_table(expr; is_header=false)
end
length(test_data_expr) == length(headers) || error("Number of test data columns (= $(length(test_data_expr))) in expression (= $(test_data_expr)) must be equal to $(length(headers))")
push!(all_test_case_values, test_data_expr)
end
end

return evaluate_test_exprs, all_test_case_values
end

function test_case_exprs(e::EvaluateTestCasesExpr; source, all_header_names)
data_var = gensym("test_case_data")

run_tests_body = Expr(:block, [Expr(:(=), name, :($data_var.$(name))) for name in all_header_names]...)

show_all_test_data_expr = Expr(:block)

num_test_exprs = 1

for evaluate_test_expr in e.input_exprs
if evaluate_test_expr isa TestExpr
ex = evaluate_test_expr.original_expr
new_test_expr, use_isequals_equality = generate_test_expr(ex, :(local_evaluate_test_data[$num_test_exprs]); escape=false)

push!(run_tests_body.args, Base.remove_linenums!(
quote
empty!(local_evaluate_test_data[$num_test_exprs])
local test_result = try
$(new_test_expr)
$TestingUtilities.Test.Returned(_result, _result, $(source))
catch _e
_e isa InterruptException && rethrow()
$TestingUtilities.Test.Threw(_e, $Base.current_exceptions(), $(source))
end
if $TestingUtilities.test_did_not_succeed(test_result)
testdata_values = local_evaluate_test_data[$num_test_exprs]
current_values = $(Expr(:tuple, Expr(:parameters, all_header_names...)))

current_values_dict = copy(testdata_values)
for (k,v) in pairs(current_values)
if !haskey(current_values_dict,k)
current_values_dict[k] = v
end
end
push!(failed_test_data[$num_test_exprs], current_values_dict)
end
$TestingUtilities.Test.do_test(test_result, $(QuoteNode(ex)))
end)
)

push!(show_all_test_data_expr.args, quote
if !isempty(failed_test_data[$num_test_exprs])
results_printer = $TestingUtilities.TestResultsPrinter(io, $(QuoteNode(ex)); use_isequals_equality=$use_isequals_equality)
$TestingUtilities.print_testcases_data!(results_printer, failed_test_data[$num_test_exprs])
end
end)
num_test_exprs += 1
else
push!(run_tests_body.args, evaluate_test_expr)
end
end

run_tests_expr = Expr(:for, Expr(:(=), data_var, :test_data), run_tests_body)

return run_tests_expr, show_all_test_data_expr
end


"""
@test_cases [io=stderr] begin
[test cases]
Expand Down Expand Up @@ -99,14 +219,28 @@ Note: The `variableᵢ` can involve expressions that refer to `variableⱼ` for
`[test expressions]` must be a series of one or more test evaluation expressions
e.g.,
```julia
@test cond₁
@test cond₂
...
@test condₖ
```
Here, each test condition expression `condᵢ` evalutes to a `Bool` and contains zero or more values from `variable₁, variable₂, ..., variableₙ `.
or a single `begin ... end` expression containing one or more test evaluation expressions, as well as other expressions that will be evaluated for each input data value
e.g.,
```julia
begin
expr₁
@test cond₁
expr₂
@test cond₂
...
end
```
Note, each test condition expression `condᵢ` must evaluate to a `Bool` and contains zero or more values from `variable₁, variable₂, ..., variableₙ `.
"""
macro test_cases(args...)
isempty(args) && error("`@test_cases` must have at least one argument")
Expand All @@ -119,30 +253,10 @@ macro test_cases(args...)
isnothing(idx) && error("Input expression $(body) must not be empty")
headers = parse_table(body.args[idx]; is_header=true)
!isempty(headers) || error("Provided headers cannot be empty")

test_expr_started = false
evaluate_test_exprs = []
all_test_case_values = Any[]
for expr in body.args[idx+1:end]
expr isa LineNumberNode && continue
if Meta.isexpr(expr, :macrocall)
expr.args[1] === Symbol("@test") || error("Only @test macros allowed in $(expr)")
test_expr_started = true
push!(evaluate_test_exprs, expr.args[end])
else
if test_expr_started
error("Cannot have test expressions interspersed with test data in expression $(body)")
end
if Meta.isexpr(expr, :tuple)
test_data_expr = parse_tuple(expr)
else
test_data_expr = parse_table(expr; is_header=false)
end
length(test_data_expr) == length(headers) || error("Number of test data columns (= $(length(test_data_expr))) in expression (= $(test_data_expr)) must be equal to $(length(headers))")
push!(all_test_case_values, test_data_expr)
end
end
isempty(evaluate_test_exprs) && error("No test expressions found input expression $(body)")

evaluate_test_exprs, all_test_case_values = parse_test_cases(body.args[idx+1:end]; headers)
num_test_exprs = evaluate_test_exprs.num_test_exprs
num_test_exprs == 0 && error("No test expressions found input expression $(body)")

normalized_headers = []
all_header_names = []
Expand Down Expand Up @@ -179,59 +293,12 @@ macro test_cases(args...)
push!(output_expr.args, Expr(:tuple, Expr(:parameters, all_header_names...)))
push!(test_data_values_expr.args, output_expr)
end

current_values_expr = Expr(:tuple, Expr(:parameters, all_header_names...))

show_all_test_data_expr = Expr(:block)

source = QuoteNode(__source__)

data_var = gensym("test_case_data")
assign_values_expr = Expr(:block, [Expr(:(=), name, :($data_var.$(name))) for name in all_header_names]...)

run_tests_body = Expr(:block, assign_values_expr)
for (i, evaluate_test_expr) in enumerate(evaluate_test_exprs)

new_test_expr, use_isequals_equality = generate_test_expr(evaluate_test_expr, :(local_evaluate_test_data[$i]); escape=false)
push!(run_tests_body.args, Base.remove_linenums!(
quote
empty!(local_evaluate_test_data[$i])
local test_result = try
$(new_test_expr)
TestingUtilities.Test.Returned(_result, _result, $(source))
catch _e
_e isa InterruptException && rethrow()
TestingUtilities.Test.Threw(_e, Base.current_exceptions(), $(source))
end
if TestingUtilities.test_did_not_succeed(test_result)
testdata_values = local_evaluate_test_data[$i]
current_values = $current_values_expr
current_values_dict = copy(testdata_values)
for (k,v) in pairs(current_values)
if !haskey(current_values_dict,k)
current_values_dict[k] = v
end
end
push!(failed_test_data[$i], current_values_dict)
end
TestingUtilities.Test.do_test(test_result, $(QuoteNode(evaluate_test_expr)))
end)
)

push!(show_all_test_data_expr.args, quote
if !isempty(failed_test_data[$i])
results_printer = TestingUtilities.TestResultsPrinter(io, $(QuoteNode(evaluate_test_expr)); use_isequals_equality=$use_isequals_equality)
TestingUtilities.print_testcases_data!(results_printer, failed_test_data[$i])
end
end)

end

run_tests_expr = Expr(:for, Expr(:(=), data_var, :test_data), run_tests_body)
run_tests_expr, show_all_test_data_expr = test_case_exprs(evaluate_test_exprs; source=QuoteNode(__source__), all_header_names)

out_expr = quote
local TestingUtilities = $(@__MODULE__)
local failed_test_data = [Any[] for i in 1:$(length(evaluate_test_exprs))]
local local_evaluate_test_data = [TestingUtilities.OrderedDict{Any,Any}() for i in 1:$(length(evaluate_test_exprs))]
local failed_test_data = [Any[] for i in 1:$(num_test_exprs)]
local local_evaluate_test_data = [$TestingUtilities.OrderedDict{Any,Any}() for i in 1:$(num_test_exprs)]
local has_set_failed_data = Ref{Bool}(false)

local show_all_test_data = let failed_test_data=failed_test_data, has_set_failed_data=has_set_failed_data, io=$(io_expr)
Expand Down
21 changes: 21 additions & 0 deletions test/test_cases_macro.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,25 @@ append_char(x, c; n::Int) = x * repeat(c, n)
@test message == ref_message
end
@test test_results_match(results, (Test.Fail, Test.Fail, Test.Pass))

results = Test.@testset NoThrowTestSet "BlockExpr tests" begin
io = IOBuffer()
@test_cases io=io begin
a | b | output
0 | 1 | 2
1 | 1 | 3
begin
c = a^2
@test b + c == output
d = c
@test b^2 == d
end
end
message = String(take!(io))
ref_message = """Test `b + c == output` failed:\nValues:\n------\n`b + c` = 1\noutput = 2\na = 0\nb = 1\n------\n`b + c` = 2\noutput = 3\na = 1\nb = 1\nTest `b ^ 2 == d` failed:\nValues:\n------\n`b ^ 2` = 1\nd = 0\na = 0\nb = 1\noutput = 2\n"""
@test message == ref_message
end

@test test_results_match(results, (Test.Fail, Test.Fail, Test.Fail, Test.Pass, Test.Pass))

end

0 comments on commit 9a1f0c5

Please sign in to comment.