reset GLOBAL_RNG state in/out at-testset for reproducibility

NEWS.md

@@ -439,6 +439,10 @@ Library improvements
     definition relies on `ncodeunits` however, so for optimal performance you may need to
     define a custom method for that function.
 
+  * The global RNG is being re-seeded with its own seed at the beginning of each `@testset`,
+    and have its original state restored at the end ([#24445]). This is breaking for testsets
+    relying implicitly on the global RNG being in a specific state.
+
   * `permutedims(m::AbstractMatrix)` is now short for `permutedims(m, (2,1))`, and is now a
     more convenient way of making a "shallow transpose" of a 2D array. This is the
     recommended approach for manipulating arrays of data, rather than the recursively

stdlib/IterativeEigensolvers/test/runtests.jl

@@ -4,7 +4,7 @@ using IterativeEigensolvers
 using Test
 
 @testset "eigs" begin
-    guardsrand(1234) do
+    srand(1234)
     n = 10
     areal  = sprandn(n,n,0.4)
     breal  = sprandn(n,n,0.4)
@@ -96,14 +96,14 @@ using Test
     end
 
     @testset "Symmetric generalized with singular B" begin
+        srand(127)
         n = 10
         k = 3
         A = randn(n,n); A = A'A
         B = randn(n,k); B = B*B'
         @test sort(eigs(A, B, nev = k, sigma = 1.0)[1]) ≈ sort(eigvals(A, B)[1:k])
     end
 end
-end
 
 # Problematic example from #6965A
 let A6965 = [

stdlib/SuiteSparse/test/cholmod.jl

@@ -723,6 +723,7 @@ end
 end
 
 @testset "Check that Symmetric{SparseMatrixCSC} can be constructed from CHOLMOD.Sparse" begin
+    Int === Int32 && srand(124)
     A = sprandn(10, 10, 0.1)
     B = CHOLMOD.Sparse(A)
     C = B'B

stdlib/Test/src/Test.jl

@@ -922,6 +922,14 @@ this behavior can be customized in other testset types. If a `for` loop is used
 then the macro collects and returns a list of the return values of the `finish`
 method, which by default will return a list of the testset objects used in
 each iteration.
+
+Before the execution of the body of a `@testset`, there is an implicit
+call to `srand(seed)` where `seed` is the current seed of the global RNG.
+Moreover, after the execution of the body, the state of the global RNG is
+restored to what it was before the `@testset`. This is meant to ease
+reproducibility in case of failure, and to allow seamless
+re-arrangements of `@testset`s regardless of their side-effect on the
+global RNG state.
 """
 macro testset(args...)
     isempty(args) && error("No arguments to @testset")
@@ -964,12 +972,20 @@ function testset_beginend(args, tests, source)
         # which is needed for backtrace scrubbing to work correctly.
         while false; end
         push_testset(ts)
+        # we reproduce the logic of guardsrand, but this function
+        # cannot be used as it changes slightly the semantic of @testset,
+        # by wrapping the body in a function
+        oldrng = copy(Base.GLOBAL_RNG)
         try
+            # GLOBAL_RNG is re-seeded with its own seed to ease reproduce a failed test
+            srand(Base.GLOBAL_RNG.seed)
             $(esc(tests))
         catch err
             # something in the test block threw an error. Count that as an
             # error in this test set
             record(ts, Error(:nontest_error, :(), err, catch_backtrace(), $(QuoteNode(source))))
+        finally
+            copy!(Base.GLOBAL_RNG, oldrng)
         end
         pop_testset()
         finish(ts)
@@ -1026,6 +1042,9 @@ function testset_forloop(args, testloop, source)
         if !first_iteration
             pop_testset()
             push!(arr, finish(ts))
+            # it's 1000 times faster to copy from tmprng rather than calling srand
+            copy!(Base.GLOBAL_RNG, tmprng)
+
         end
         ts = $(testsettype)($desc; $options...)
         push_testset(ts)
@@ -1042,6 +1061,9 @@ function testset_forloop(args, testloop, source)
         arr = Vector{Any}()
         local first_iteration = true
         local ts
+        local oldrng = copy(Base.GLOBAL_RNG)
+        srand(Base.GLOBAL_RNG.seed)
+        local tmprng = copy(Base.GLOBAL_RNG)
         try
             $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk))
         finally
@@ -1050,6 +1072,7 @@ function testset_forloop(args, testloop, source)
                 pop_testset()
                 push!(arr, finish(ts))
             end
+            copy!(Base.GLOBAL_RNG, oldrng)
         end
         arr
     end
@@ -1477,7 +1500,7 @@ end
 
 "`guardsrand(f, seed)` is equivalent to running `srand(seed); f()` and
 then restoring the state of the global RNG as it was before."
-guardsrand(f::Function, seed::Integer) = guardsrand() do
+guardsrand(f::Function, seed::Union{Vector{UInt32},Integer}) = guardsrand() do
     srand(seed)
     f()
 end

stdlib/Test/test/runtests.jl

@@ -287,10 +287,8 @@ end
                 end
             end
             @testset "some loops fail" begin
-                guardsrand(123) do
                 @testset for i in 1:5
-                        @test i <= rand(1:10)
-                    end
+                    @test i <= 4
                 end
                 # should add 3 errors and 3 passing tests
                 @testset for i in 1:6
@@ -739,3 +737,22 @@ end
                  be tested in --depwarn=error mode"""
     end
 end
+
+@testset "@testset preserves GLOBAL_RNG's state, and re-seeds it" begin
+    # i.e. it behaves as if it was wrapped in a `guardsrand(GLOBAL_RNG.seed)` block
+    seed = rand(UInt128)
+    srand(seed)
+    a = rand()
+    @testset begin
+        # global RNG must re-seeded at the beginning of @testset
+        @test a == rand()
+    end
+    @testset for i=1:3
+        @test a == rand()
+    end
+    # the @testset's above must have no consequence for rand() below
+    b = rand()
+    srand(seed)
+    @test a == rand()
+    @test b == rand()
+end

test/linalg/lapack.jl

@@ -10,7 +10,7 @@ import Base.LinAlg.BlasInt
 @test_throws ArgumentError Base.LinAlg.LAPACK.chktrans('Z')
 
 @testset "syevr" begin
-    guardsrand(123) do
+    srand(123)
     Ainit = randn(5,5)
     @testset for elty in (Float32, Float64, ComplexF32, ComplexF64)
         if elty == ComplexF32 || elty == ComplexF64
@@ -30,7 +30,6 @@ import Base.LinAlg.BlasInt
         @test_throws DimensionMismatch LAPACK.sygvd!(1,'V','U',copy(Asym),ones(elty,6,6))
     end
 end
-end
 
 @testset "gglse" begin
     let
@@ -207,14 +206,13 @@ end
 
 @testset "gels" begin
     @testset for elty in (Float32, Float64, ComplexF32, ComplexF64)
-        guardsrand(913) do
+        srand(913)
         A = rand(elty,10,10)
         X = rand(elty,10)
         B,Y,z = LAPACK.gels!('N',copy(A),copy(X))
         @test A\X ≈ Y
     end
 end
-end
 
 @testset "getrf/getri" begin
     @testset for elty in (Float32, Float64, ComplexF32, ComplexF64)
@@ -435,7 +433,7 @@ end
 
 @testset "sysv" begin
     @testset for elty in (Float32, Float64, ComplexF32, ComplexF64)
-        guardsrand(123) do
+        srand(123)
         A = rand(elty,10,10)
         A = A + A.' #symmetric!
         b = rand(elty,10)
@@ -445,11 +443,10 @@ end
         @test_throws DimensionMismatch LAPACK.sysv!('U',A,rand(elty,11))
     end
 end
-end
 
 @testset "hesv" begin
     @testset for elty in (ComplexF32, ComplexF64)
-        guardsrand(935) do
+        srand(935)
         A = rand(elty,10,10)
         A = A + A' #hermitian!
         b = rand(elty,10)
@@ -466,7 +463,6 @@ end
         @test_throws DimensionMismatch LAPACK.hesv_rook!('U',A,rand(elty,11))
     end
 end
-end
 
 @testset "ptsv" begin
     @testset for elty in (Float32, Float64, ComplexF32, ComplexF64)

test/linalg/lu.jl

@@ -195,6 +195,7 @@ dimg  = randn(n)/2
 end
 
 @testset "conversion" begin
+    srand(3)
     a = Tridiagonal(rand(9),rand(10),rand(9))
     fa = Array(a)
     falu = lufact(fa)

test/linalg/tridiag.jl

@@ -14,9 +14,9 @@ function test_approx_eq_vecs(a::StridedVecOrMat{S}, b::StridedVecOrMat{T}, error
     end
 end
 
-guardsrand(123) do
-    n = 12 #Size of matrix problem to test
 @testset for elty in (Float32, Float64, ComplexF32, ComplexF64, Int)
+    n = 12 #Size of matrix problem to test
+    srand(123)
     if elty == Int
         srand(61516384)
         d = rand(1:100, n)
@@ -324,7 +324,7 @@ guardsrand(123) do
         end
     end
 end
-end
+
 
 @testset "Issue 12068" begin
     @test SymTridiagonal([1, 2], [0])^3 == [1 0; 0 8]

test/random.jl

@@ -5,8 +5,8 @@ using Main.TestHelpers.OAs
 
 using Base.Random: Sampler, SamplerRangeFast, SamplerRangeInt
 
-# Issue #6573
-guardsrand(0) do
+@testset "Issue #6573" begin
+    srand(0)
     rand()
     x = rand(384)
     @test find(x .== rand()) == []

test/sorting.jl

@@ -226,7 +226,7 @@ function randn_with_nans(n,p)
 end
 
 @testset "advanced sorting" begin
-    guardsrand(0xdeadbeef) do
+    srand(0xdeadbeef)
     for n in [0:10; 100; 101; 1000; 1001]
         local r
         r = -5:5
@@ -292,7 +292,7 @@ end
         end
     end
 end
-end
+
 @testset "sortperm" begin
     inds = [
         1,1,1,2,2,2,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7,8,8,8,9,9,9,10,

test/sparse/sparse.jl

@@ -1691,13 +1691,13 @@ end
 end
 
 @testset "sparse matrix normestinv" begin
-    guardsrand(1234) do
+    srand(1234)
     Ac = sprandn(20,20,.5) + im* sprandn(20,20,.5)
     Aci = ceil.(Int64, 100*sprand(20,20,.5)) + im*ceil.(Int64, sprand(20,20,.5))
     Ar = sprandn(20,20,.5)
     Ari = ceil.(Int64, 100*Ar)
     if Base.USE_GPL_LIBS
-            # NOTE: normestinv is probabilistic, so must be included in the guardsrand block
+        # NOTE: normestinv is probabilistic, so requires a fixed seed (set above in srand(1234))
         @test Base.SparseArrays.normestinv(Ac,3) ≈ norm(inv(Array(Ac)),1) atol=1e-4
         @test Base.SparseArrays.normestinv(Aci,3) ≈ norm(inv(Array(Aci)),1) atol=1e-4
         @test Base.SparseArrays.normestinv(Ar) ≈ norm(inv(Array(Ar)),1) atol=1e-4
@@ -1706,7 +1706,6 @@ end
     end
     @test_throws DimensionMismatch Base.SparseArrays.normestinv(sprand(3,5,.9))
 end
-end
 
 @testset "issue #13008" begin
     @test_throws ArgumentError sparse(collect(1:100), collect(1:100), fill(5,100), 5, 5)
@@ -1745,7 +1744,7 @@ end
 end
 
 @testset "factorization" begin
-    guardsrand(123) do
+    srand(123)
     local A
     A = sparse(Diagonal(rand(5))) + sprandn(5, 5, 0.2) + im*sprandn(5, 5, 0.2)
     A = A + A'
@@ -1769,7 +1768,6 @@ end
     @test_throws ErrorException eig(A)
     @test_throws ErrorException inv(A)
 end
-end
 
 @testset "issue #13792, use sparse triangular solvers for sparse triangular solves" begin
     local A, n, x