From 6b3e9032bedbfe393f985ccbbef9e04300ca4cbc Mon Sep 17 00:00:00 2001
From: JohnnyChen <johnnychen94@hotmail.com>
Date: Sun, 6 Oct 2019 14:09:41 +0800
Subject: [PATCH] dispatch test_reference pipeline according to TestMode
---
src/ReferenceTests.jl | 1 +
src/test_reference.jl | 77 +++++++++++++++++++++++++++++++------------
src/testmode.jl | 47 ++++++++++++++++++++++++++
3 files changed, 104 insertions(+), 21 deletions(-)
create mode 100644 src/testmode.jl
diff --git a/src/ReferenceTests.jl b/src/ReferenceTests.jl
index ef7c750..949ea2d 100644
--- a/src/ReferenceTests.jl
+++ b/src/ReferenceTests.jl
@@ -15,6 +15,7 @@ export
@test_reference,
psnr_equality
+include("testmode.jl")
include("utils.jl")
include("test_reference.jl")
include("fileio.jl")
diff --git a/src/test_reference.jl b/src/test_reference.jl
index d9cdb31..42789de 100644
--- a/src/test_reference.jl
+++ b/src/test_reference.jl
@@ -99,6 +99,7 @@ function test_reference(
path = file.filename
dir, filename = splitdir(path)
+ testmode = TESTMODE()
# infer the default rendermode here
# since `nothing` is always passed to this method from
@@ -112,19 +113,11 @@ function test_reference(
println("Reference file for \"$filename\" does not exist.")
render(rendermode, raw_actual)
- if !isinteractive()
- error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images")
- end
-
- if !input_bool("Create reference file with above content (path: $path)?")
+ continue_test = _create_new_reference(testmode, file, raw_actual)
+ if !continue_test
@test false
- else
- mkpath(dir)
- savefile(file, raw_actual)
- @info("Please run the tests again for any changes to take effect")
+ return nothing
end
-
- return nothing # skip current test case
end
# file exists
@@ -137,21 +130,63 @@ function test_reference(
end
if equiv(reference, actual)
- @test true # to increase test counter if reached
+ @test true
else
# post-processing when test fails
println("Test for \"$filename\" failed.")
render(rendermode, reference, actual)
- if !isinteractive()
- error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
- end
+ increase_fail_count = _update_reference(testmode, file, reference, actual)
+ increase_fail_count && @test false
+ end
+end
- if !input_bool("Replace reference with actual result (path: $path)?")
- @test false
- else
- savefile(file, actual)
- @info("Please run the tests again for any changes to take effect")
- end
+function _create_new_reference(::NonInteractiveMode, file::File, actual)::Bool
+ error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
+
+ # # automatically create new reference file and continue test
+ # path = file.filename
+ # dir, filename = splitdir(path)
+
+ # println("Create new reference file \"$filename\".")
+ # mkpath(dir)
+ # savefile(file, actual)
+
+ # continue_test = true
+ # return continue_test
+end
+
+function _create_new_reference(::InteractiveMode, file::File, actual)::Bool
+ path = file.filename
+ dir = splitdir(path)[1]
+
+ if !input_bool("Create reference file with above content (path: $path)?")
+ continue_test = false
+ else
+ mkpath(dir)
+ savefile(file, actual)
+ continue_test = true
+ end
+ return continue_test
+end
+
+function _update_reference(::NonInteractiveMode, file::File, reference, actual)::Bool
+ error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
+
+ # # Do not update reference
+ # increase_fail_count = true
+ # return increase_fail_count
+end
+
+function _update_reference(::InteractiveMode, file::File, reference, actual)::Bool
+ path = file.filename
+
+ if !input_bool("Replace reference with actual result (path: $path)?")
+ increase_fail_count = true
+ else
+ savefile(file, actual)
+ @info("Please run the tests again for any changes to take effect")
+ increase_fail_count = false
end
+ return increase_fail_count
end
diff --git a/src/testmode.jl b/src/testmode.jl
new file mode 100644
index 0000000..6e621c4
--- /dev/null
+++ b/src/testmode.jl
@@ -0,0 +1,47 @@
+#################################################
+# Test mode
+# This controls how test cases are handled
+abstract type TestMode end
+struct InteractiveMode <: TestMode end
+struct NonInteractiveMode <: TestMode end
+
+"""
+Predefined CI environment variables
+
+# References
+
+* Travis: https://docs.travis-ci.com/user/environment-variables/#default-environment-variables
+* Appveyor: https://www.appveyor.com/docs/environment-variables/
+"""
+const CI_ENVIRONMENTS = Dict(
+ "CI" => "true",
+ "APPVEYOR" => "true",
+ "TRAVIS" => "true",
+ "CONTINUOUS_INTEGRATION" => "true",
+ "DEBIAN_FRONTEND" => "noninteractive"
+)
+
+function TESTMODE()
+ global GLOBAL_TESTMODE
+ if !isdefined(ReferenceTests, :GLOBAL_TESTMODE)
+ # it's only called once in runtime
+ GLOBAL_TESTMODE = _get_testmode()
+ end
+ return GLOBAL_TESTMODE
+end
+
+function _get_testmode()
+ # test if this package is used in a CI environment
+ common_keys = collect(intersect(keys(ENV), keys(CI_ENVIRONMENTS)))
+ matched_envs = map(common_keys) do k
+ # some variables might have different cases in different CI platforms
+ # e.g., in Appveyor, ENV["CI] is "True" in Windows and "true" in Ubuntu.
+ lowercase(ENV[k])==lowercase(CI_ENVIRONMENTS[k])
+ end
+ has_testenv = any(matched_envs)
+ has_testenv && return NonInteractiveMode()
+
+ # fallback
+ @info "You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update references."
+ return isinteractive() ? InteractiveMode() : NonInteractiveMode()
+end