reset GLOBAL_RNG state in/out at-testset for reproducibility (#24445)

This is a follow-up on the recently introduced `guardsrand` functionality,
first suggested at
#16940 (comment).
Each `testset` block is now implicitly wrapped in a `guardsrand` block, which
is more user-friendly and more systematic (only few users know about
`guardsrand`).

These are essentially two new features:

1) "in": in base, tests are run with the global RNG randomly seeded,
   but the seed is printed in case of failure to allow reproducing the
   failure; but even if the failure occurs after many tests, when they
   alter the global RNG state, one has to re-run the whole file, which
   can be time-consuming; with this change, at the beginning of each
   `testset`, the global RNG is re-seeded with its own seed: this
   allows to re-run only the failing `testset`, which can be done
   easily in the REPL (the seeding occurs for each loop in a `testset for`;
   this also allows to re-arrange `testset`s in arbitrary order
   w.r.t. the global RNG;

2) "out": a `testset` leaves no tracks of its use of `srand` or `rand`
   (this "feature" should be less and less needed/useful with the
   generalization of the use of `testset` blocks).

Example:
```
@testset begin
   srand(123)
   rand()
   @testset for T in (Int, Float64)
       # here is an implicit srand(123), by 1)
       rand()
   end
   rand() # this value will not be affected if the sub-`testset` block
          # above is removed, or if another rand() call is added therein, by 2)
end
```

Note that guardsrand can't be used directly, as then the testset's body is
wrapped in a function, which causes problem with overwriting loop variable
("outer"), using `using`, defining new methods, etc. So we need to duplicate
guardsrand's logic.

NEWS.md

@@ -439,6 +439,10 @@ Library improvements
     definition relies on `ncodeunits` however, so for optimal performance you may need to
     define a custom method for that function.
 
+  * The global RNG is being re-seeded with its own seed at the beginning of each `@testset`,
+    and have its original state restored at the end ([#24445]). This is breaking for testsets
+    relying implicitly on the global RNG being in a specific state.
+
   * `permutedims(m::AbstractMatrix)` is now short for `permutedims(m, (2,1))`, and is now a
     more convenient way of making a "shallow transpose" of a 2D array. This is the
     recommended approach for manipulating arrays of data, rather than the recursively

stdlib/IterativeEigensolvers/test/runtests.jl

@@ -4,118 +4,118 @@ using IterativeEigensolvers
 using Test
 
 @testset "eigs" begin
-    guardsrand(1234) do
-        n = 10
-        areal  = sprandn(n,n,0.4)
-        breal  = sprandn(n,n,0.4)
-        acmplx = complex.(sprandn(n,n,0.4), sprandn(n,n,0.4))
-        bcmplx = complex.(sprandn(n,n,0.4), sprandn(n,n,0.4))
+    srand(1234)
+    n = 10
+    areal  = sprandn(n,n,0.4)
+    breal  = sprandn(n,n,0.4)
+    acmplx = complex.(sprandn(n,n,0.4), sprandn(n,n,0.4))
+    bcmplx = complex.(sprandn(n,n,0.4), sprandn(n,n,0.4))
 
-        testtol = 1e-6
+    testtol = 1e-6
 
-        @testset for elty in (Float64, ComplexF64)
-            if elty == ComplexF32 || elty == ComplexF64
-                a = acmplx
-                b = bcmplx
-            else
-                a = areal
-                b = breal
-            end
-            a_evs = eigvals(Array(a))
-            a     = convert(SparseMatrixCSC{elty}, a)
-            asym  = a' + a                  # symmetric indefinite
-            apd   = a'*a                    # symmetric positive-definite
+    @testset for elty in (Float64, ComplexF64)
+        if elty == ComplexF32 || elty == ComplexF64
+            a = acmplx
+            b = bcmplx
+        else
+            a = areal
+            b = breal
+        end
+        a_evs = eigvals(Array(a))
+        a     = convert(SparseMatrixCSC{elty}, a)
+        asym  = a' + a                  # symmetric indefinite
+        apd   = a'*a                    # symmetric positive-definite
 
-            b     = convert(SparseMatrixCSC{elty}, b)
-            bsym  = b' + b
-            bpd   = b'*b
+        b     = convert(SparseMatrixCSC{elty}, b)
+        bsym  = b' + b
+        bpd   = b'*b
 
-            (d,v) = eigs(a, nev=3)
-            @test a*v[:,2] ≈ d[2]*v[:,2]
-            @test norm(v) > testtol # eigenvectors cannot be null vectors
-            (d,v) = eigs(a, I, nev=3) # test eigs(A, B; kwargs...)
-            @test a*v[:,2] ≈ d[2]*v[:,2]
-            @test norm(v) > testtol # eigenvectors cannot be null vectors
-            @test_logs (:warn,"Use symbols instead of strings for specifying which eigenvalues to compute") eigs(a, which="LM")
-            @test_logs (:warn,"Adjusting ncv from 1 to 4") eigs(a, ncv=1, nev=2)
-            @test_logs (:warn,"Adjusting nev from $n to $(n-2)") eigs(a, nev=n)
-            # (d,v) = eigs(a, b, nev=3, tol=1e-8) # not handled yet
-            # @test a*v[:,2] ≈ d[2]*b*v[:,2] atol=testtol
-            # @test norm(v) > testtol # eigenvectors cannot be null vectors
-            if elty <: Base.LinAlg.BlasComplex
-                sr_ind = indmin(real.(a_evs))
-                (d, v) = eigs(a, nev=1, which=:SR)
-                @test d[1] ≈ a_evs[sr_ind]
-                si_ind = indmin(imag.(a_evs))
-                (d, v) = eigs(a, nev=1, which=:SI)
-                @test d[1] ≈ a_evs[si_ind]
-                lr_ind = indmax(real.(a_evs))
-                (d, v) = eigs(a, nev=1, which=:LR)
-                @test d[1] ≈ a_evs[lr_ind]
-                li_ind = indmax(imag.(a_evs))
-                (d, v) = eigs(a, nev=1, which=:LI)
-                @test d[1] ≈ a_evs[li_ind]
-            end
+        (d,v) = eigs(a, nev=3)
+        @test a*v[:,2] ≈ d[2]*v[:,2]
+        @test norm(v) > testtol # eigenvectors cannot be null vectors
+        (d,v) = eigs(a, I, nev=3) # test eigs(A, B; kwargs...)
+        @test a*v[:,2] ≈ d[2]*v[:,2]
+        @test norm(v) > testtol # eigenvectors cannot be null vectors
+        @test_logs (:warn,"Use symbols instead of strings for specifying which eigenvalues to compute") eigs(a, which="LM")
+        @test_logs (:warn,"Adjusting ncv from 1 to 4") eigs(a, ncv=1, nev=2)
+        @test_logs (:warn,"Adjusting nev from $n to $(n-2)") eigs(a, nev=n)
+        # (d,v) = eigs(a, b, nev=3, tol=1e-8) # not handled yet
+        # @test a*v[:,2] ≈ d[2]*b*v[:,2] atol=testtol
+        # @test norm(v) > testtol # eigenvectors cannot be null vectors
+        if elty <: Base.LinAlg.BlasComplex
+            sr_ind = indmin(real.(a_evs))
+            (d, v) = eigs(a, nev=1, which=:SR)
+            @test d[1] ≈ a_evs[sr_ind]
+            si_ind = indmin(imag.(a_evs))
+            (d, v) = eigs(a, nev=1, which=:SI)
+            @test d[1] ≈ a_evs[si_ind]
+            lr_ind = indmax(real.(a_evs))
+            (d, v) = eigs(a, nev=1, which=:LR)
+            @test d[1] ≈ a_evs[lr_ind]
+            li_ind = indmax(imag.(a_evs))
+            (d, v) = eigs(a, nev=1, which=:LI)
+            @test d[1] ≈ a_evs[li_ind]
+        end
 
-            (d,v) = eigs(asym, nev=3)
-            @test asym*v[:,1] ≈ d[1]*v[:,1]
-            @test eigs(asym; nev=1, sigma=d[3])[1][1] ≈ d[3]
-            @test norm(v) > testtol # eigenvectors cannot be null vectors
+        (d,v) = eigs(asym, nev=3)
+        @test asym*v[:,1] ≈ d[1]*v[:,1]
+        @test eigs(asym; nev=1, sigma=d[3])[1][1] ≈ d[3]
+        @test norm(v) > testtol # eigenvectors cannot be null vectors
 
-            (d,v) = eigs(apd, nev=3)
-            @test apd*v[:,3] ≈ d[3]*v[:,3]
-            @test eigs(apd; nev=1, sigma=d[3])[1][1] ≈ d[3]
+        (d,v) = eigs(apd, nev=3)
+        @test apd*v[:,3] ≈ d[3]*v[:,3]
+        @test eigs(apd; nev=1, sigma=d[3])[1][1] ≈ d[3]
 
-            (d,v) = eigs(apd, bpd, nev=3, tol=1e-8)
-            @test apd*v[:,2] ≈ d[2]*bpd*v[:,2] atol=testtol
-            @test norm(v) > testtol # eigenvectors cannot be null vectors
+        (d,v) = eigs(apd, bpd, nev=3, tol=1e-8)
+        @test apd*v[:,2] ≈ d[2]*bpd*v[:,2] atol=testtol
+        @test norm(v) > testtol # eigenvectors cannot be null vectors
 
-            @testset "(shift-and-)invert mode" begin
-                (d,v) = eigs(apd, nev=3, sigma=0)
-                @test apd*v[:,3] ≈ d[3]*v[:,3]
-                @test norm(v) > testtol # eigenvectors cannot be null vectors
+        @testset "(shift-and-)invert mode" begin
+            (d,v) = eigs(apd, nev=3, sigma=0)
+            @test apd*v[:,3] ≈ d[3]*v[:,3]
+            @test norm(v) > testtol # eigenvectors cannot be null vectors
 
-                (d,v) = eigs(apd, bpd, nev=3, sigma=0, tol=1e-8)
-                @test apd*v[:,1] ≈ d[1]*bpd*v[:,1] atol=testtol
-                @test norm(v) > testtol # eigenvectors cannot be null vectors
-            end
+            (d,v) = eigs(apd, bpd, nev=3, sigma=0, tol=1e-8)
+            @test apd*v[:,1] ≈ d[1]*bpd*v[:,1] atol=testtol
+            @test norm(v) > testtol # eigenvectors cannot be null vectors
+        end
 
-            @testset "ArgumentErrors" begin
-                @test_throws ArgumentError eigs(rand(elty,2,2))
-                @test_throws ArgumentError eigs(a, nev=-1)
-                @test_throws ArgumentError eigs(a, which=:Z)
-                @test_throws ArgumentError eigs(a, which=:BE)
-                @test_throws DimensionMismatch eigs(a, v0=zeros(elty,n+2))
-                @test_throws ArgumentError eigs(a, v0=zeros(Int,n))
-                if elty == Float64
-                    @test_throws ArgumentError eigs(a+a.',which=:SI)
-                    @test_throws ArgumentError eigs(a+a.',which=:LI)
-                    @test_throws ArgumentError eigs(a,sigma=rand(ComplexF32))
-                end
+        @testset "ArgumentErrors" begin
+            @test_throws ArgumentError eigs(rand(elty,2,2))
+            @test_throws ArgumentError eigs(a, nev=-1)
+            @test_throws ArgumentError eigs(a, which=:Z)
+            @test_throws ArgumentError eigs(a, which=:BE)
+            @test_throws DimensionMismatch eigs(a, v0=zeros(elty,n+2))
+            @test_throws ArgumentError eigs(a, v0=zeros(Int,n))
+            if elty == Float64
+                @test_throws ArgumentError eigs(a+a.',which=:SI)
+                @test_throws ArgumentError eigs(a+a.',which=:LI)
+                @test_throws ArgumentError eigs(a,sigma=rand(ComplexF32))
             end
         end
+    end
 
-        @testset "Symmetric generalized with singular B" begin
-            n = 10
-            k = 3
-            A = randn(n,n); A = A'A
-            B = randn(n,k); B = B*B'
-            @test sort(eigs(A, B, nev = k, sigma = 1.0)[1]) ≈ sort(eigvals(A, B)[1:k])
-        end
+    @testset "Symmetric generalized with singular B" begin
+        srand(127)
+        n = 10
+        k = 3
+        A = randn(n,n); A = A'A
+        B = randn(n,k); B = B*B'
+        @test sort(eigs(A, B, nev = k, sigma = 1.0)[1]) ≈ sort(eigvals(A, B)[1:k])
     end
 end
 
 # Problematic example from #6965A
 let A6965 = [
-         1.0   1.0   1.0   1.0   1.0   1.0   1.0  1.0
-        -1.0   2.0   0.0   0.0   0.0   0.0   0.0  1.0
-        -1.0   0.0   3.0   0.0   0.0   0.0   0.0  1.0
-        -1.0   0.0   0.0   4.0   0.0   0.0   0.0  1.0
-        -1.0   0.0   0.0   0.0   5.0   0.0   0.0  1.0
-        -1.0   0.0   0.0   0.0   0.0   6.0   0.0  1.0
-        -1.0   0.0   0.0   0.0   0.0   0.0   7.0  1.0
-        -1.0  -1.0  -1.0  -1.0  -1.0  -1.0  -1.0  8.0
-       ]
+    1.0   1.0   1.0   1.0   1.0   1.0   1.0  1.0
+    -1.0   2.0   0.0   0.0   0.0   0.0   0.0  1.0
+    -1.0   0.0   3.0   0.0   0.0   0.0   0.0  1.0
+    -1.0   0.0   0.0   4.0   0.0   0.0   0.0  1.0
+    -1.0   0.0   0.0   0.0   5.0   0.0   0.0  1.0
+    -1.0   0.0   0.0   0.0   0.0   6.0   0.0  1.0
+    -1.0   0.0   0.0   0.0   0.0   0.0   7.0  1.0
+    -1.0  -1.0  -1.0  -1.0  -1.0  -1.0  -1.0  8.0
+]
     d, = eigs(A6965,which=:SM,nev=2,ncv=4,tol=eps())
     @test d[1] ≈ 2.5346936860350002
     @test real(d[2]) ≈ 2.6159972444834976

stdlib/SuiteSparse/test/cholmod.jl

@@ -723,6 +723,7 @@ end
 end
 
 @testset "Check that Symmetric{SparseMatrixCSC} can be constructed from CHOLMOD.Sparse" begin
+    Int === Int32 && srand(124)
     A = sprandn(10, 10, 0.1)
     B = CHOLMOD.Sparse(A)
     C = B'B

stdlib/Test/src/Test.jl

@@ -922,6 +922,14 @@ this behavior can be customized in other testset types. If a `for` loop is used
 then the macro collects and returns a list of the return values of the `finish`
 method, which by default will return a list of the testset objects used in
 each iteration.
+
+Before the execution of the body of a `@testset`, there is an implicit
+call to `srand(seed)` where `seed` is the current seed of the global RNG.
+Moreover, after the execution of the body, the state of the global RNG is
+restored to what it was before the `@testset`. This is meant to ease
+reproducibility in case of failure, and to allow seamless
+re-arrangements of `@testset`s regardless of their side-effect on the
+global RNG state.
 """
 macro testset(args...)
     isempty(args) && error("No arguments to @testset")
@@ -964,12 +972,20 @@ function testset_beginend(args, tests, source)
         # which is needed for backtrace scrubbing to work correctly.
         while false; end
         push_testset(ts)
+        # we reproduce the logic of guardsrand, but this function
+        # cannot be used as it changes slightly the semantic of @testset,
+        # by wrapping the body in a function
+        oldrng = copy(Base.GLOBAL_RNG)
         try
+            # GLOBAL_RNG is re-seeded with its own seed to ease reproduce a failed test
+            srand(Base.GLOBAL_RNG.seed)
             $(esc(tests))
         catch err
             # something in the test block threw an error. Count that as an
             # error in this test set
             record(ts, Error(:nontest_error, :(), err, catch_backtrace(), $(QuoteNode(source))))
+        finally
+            copy!(Base.GLOBAL_RNG, oldrng)
         end
         pop_testset()
         finish(ts)
@@ -1026,6 +1042,9 @@ function testset_forloop(args, testloop, source)
         if !first_iteration
             pop_testset()
             push!(arr, finish(ts))
+            # it's 1000 times faster to copy from tmprng rather than calling srand
+            copy!(Base.GLOBAL_RNG, tmprng)
+
         end
         ts = $(testsettype)($desc; $options...)
         push_testset(ts)
@@ -1042,6 +1061,9 @@ function testset_forloop(args, testloop, source)
         arr = Vector{Any}()
         local first_iteration = true
         local ts
+        local oldrng = copy(Base.GLOBAL_RNG)
+        srand(Base.GLOBAL_RNG.seed)
+        local tmprng = copy(Base.GLOBAL_RNG)
         try
             $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk))
         finally
@@ -1050,6 +1072,7 @@ function testset_forloop(args, testloop, source)
                 pop_testset()
                 push!(arr, finish(ts))
             end
+            copy!(Base.GLOBAL_RNG, oldrng)
         end
         arr
     end
@@ -1477,7 +1500,7 @@ end
 
 "`guardsrand(f, seed)` is equivalent to running `srand(seed); f()` and
 then restoring the state of the global RNG as it was before."
-guardsrand(f::Function, seed::Integer) = guardsrand() do
+guardsrand(f::Function, seed::Union{Vector{UInt32},Integer}) = guardsrand() do
     srand(seed)
     f()
 end

stdlib/Test/test/runtests.jl

@@ -287,10 +287,8 @@ end
                 end
             end
             @testset "some loops fail" begin
-                guardsrand(123) do
-                    @testset for i in 1:5
-                        @test i <= rand(1:10)
-                    end
+                @testset for i in 1:5
+                    @test i <= 4
                 end
                 # should add 3 errors and 3 passing tests
                 @testset for i in 1:6
@@ -739,3 +737,22 @@ end
                  be tested in --depwarn=error mode"""
     end
 end
+
+@testset "@testset preserves GLOBAL_RNG's state, and re-seeds it" begin
+    # i.e. it behaves as if it was wrapped in a `guardsrand(GLOBAL_RNG.seed)` block
+    seed = rand(UInt128)
+    srand(seed)
+    a = rand()
+    @testset begin
+        # global RNG must re-seeded at the beginning of @testset
+        @test a == rand()
+    end
+    @testset for i=1:3
+        @test a == rand()
+    end
+    # the @testset's above must have no consequence for rand() below
+    b = rand()
+    srand(seed)
+    @test a == rand()
+    @test b == rand()
+end