Hook for managing any Random instance

hypothesis-python/RELEASE.rst

@@ -0,0 +1,9 @@
+RELEASE_TYPE: minor
+
+This release adds :func:`~hypothesis.register_random`, which registers
+``random.Random`` instances or compatible objects to be seeded and reset
+by Hypothesis to ensure that test cases are deterministic.
+
+We still recommend explicitly passing a ``random.Random`` instance from
+:func:`~hypothesis.strategies.randoms` if possible, but registering a
+framework-global state for Hypothesis to manage is better than flaky tests!

hypothesis-python/docs/details.rst

@@ -479,6 +479,22 @@ If the end user has also specified a custom executor using the
 be applied to the *new* inner test assigned by the test runner.
 
 
+--------------------------------
+Making random code deterministic
+--------------------------------
+
+While Hypothesis' example generation can be used for nondeterministic tests,
+debugging anything nondeterministic is usually a very frustrating excercise.
+To make things worse, our example *shrinking* relies on the same input
+causing the same failure each time - though we show the un-shrunk failure
+and a decent error message if it doesn't.
+
+By default, Hypothesis will handle the global ``random`` and ``numpy.random``
+random number generators for you, and you can register others:
+
+.. autofunction:: hypothesis.register_random
+
+
 -------------------------------
 Using Hypothesis to find values
 -------------------------------

hypothesis-python/src/hypothesis/__init__.py

@@ -27,6 +27,7 @@
 from hypothesis.version import __version_info__, __version__
 from hypothesis.control import assume, note, reject, event
 from hypothesis.core import given, find, example, seed, reproduce_failure, PrintSettings
+from hypothesis.internal.entropy import register_random
 from hypothesis.utils.conventions import infer
 
 
@@ -47,6 +48,7 @@
     "note",
     "event",
     "infer",
+    "register_random",
     "__version__",
     "__version_info__",
 ]

hypothesis-python/src/hypothesis/_strategies.py

@@ -21,7 +21,6 @@
 import enum
 import math
 import operator
-import random
 import string
 import sys
 from decimal import Context, Decimal, localcontext
@@ -55,6 +54,7 @@
     check_sample,
     integer_range,
 )
+from hypothesis.internal.entropy import get_seeder_and_restorer
 from hypothesis.internal.floats import (
     count_between_floats,
     float_of,
@@ -132,6 +132,7 @@
     numpy = None
 
 if False:
+    import random  # noqa
     from types import ModuleType  # noqa
     from typing import Any, Dict, Union, Sequence, Callable, Pattern  # noqa
     from typing import TypeVar, Tuple, List, Set, FrozenSet, overload  # noqa
@@ -1089,26 +1090,23 @@ def __init__(self, seed):
         self.seed = seed
 
     def __repr__(self):
-        return "random.seed(%r)" % (self.seed,)
+        return "RandomSeeder(%r)" % (self.seed,)
 
 
 class RandomModule(SearchStrategy):
     def do_draw(self, data):
         data.can_reproduce_example_from_repr = False
         seed = data.draw(integers(0, 2 ** 32 - 1))
-        state = random.getstate()
-        random.seed(seed)
-        cleanup(lambda: random.setstate(state))
-        if numpy is not None:  # pragma: no cover
-            npstate = numpy.random.get_state()
-            numpy.random.seed(seed)
-            cleanup(lambda: numpy.random.set_state(npstate))
+        seed_all, restore_all = get_seeder_and_restorer(seed)
+        seed_all()
+        cleanup(restore_all)
         return RandomSeeder(seed)
 
 
 @cacheable
 @defines_strategy
 def random_module():
+    # type: () -> SearchStrategy[RandomSeeder]
     """The Hypothesis engine handles PRNG state for the stdlib and Numpy random
     modules internally, always seeding them to zero and restoring the previous
     state after the test.

hypothesis-python/src/hypothesis/internal/entropy.py

@@ -20,10 +20,73 @@
 import contextlib
 import random
 
+from hypothesis.errors import InvalidArgument
+from hypothesis.internal.compat import integer_types
+
+RANDOMS_TO_MANAGE = [random]  # type: list
+
 try:
     import numpy.random as npr
 except ImportError:
-    npr = None
+    pass
+else:
+
+    class NumpyRandomWrapper(object):
+        """A shim to remove those darn underscores."""
+
+        seed = npr.seed
+        getstate = npr.get_state
+        setstate = npr.set_state
+
+    RANDOMS_TO_MANAGE.append(NumpyRandomWrapper)
+
+
+def register_random(r):
+    # type: (random.Random) -> None
+    """Register the given Random instance for management by Hypothesis.
+
+    You can pass ``random.Random`` instances (or other objects with seed,
+    getstate, and setstate methods) to ``register_random(r)`` to have their
+    states seeded and restored in the same way as the global PRNGs from the
+    ``random`` and ``numpy.random`` modules.
+
+    All global PRNGs, from e.g. simulation or scheduling frameworks, should
+    be registered to prevent flaky tests.  Hypothesis will ensure that the
+    PRNG state is consistent for all test runs, or reproducibly varied if you
+    choose to use the :func:`~hypothesis.strategies.random_module` strategy.
+    """
+    if not (hasattr(r, "seed") and hasattr(r, "getstate") and hasattr(r, "setstate")):
+        raise InvalidArgument("r=%r does not have all the required methods" % (r,))
+    if r not in RANDOMS_TO_MANAGE:
+        RANDOMS_TO_MANAGE.append(r)
+
+
+def get_seeder_and_restorer(seed=0):
+    """Return a pair of functions which respectively seed all and restore
+    the state of all registered PRNGs.
+
+    This is used by the core engine via `deterministic_PRNG`, and by users
+    via `register_random`.  We support registration of additional random.Random
+    instances (or other objects with seed, getstate, and setstate methods)
+    to force determinism on simulation or scheduling frameworks which avoid
+    using the global random state.  See e.g. #1709.
+    """
+    assert isinstance(seed, integer_types) and 0 <= seed < 2 ** 32
+    states = []  # type: list
+
+    def seed_all():
+        assert not states
+        for r in RANDOMS_TO_MANAGE:
+            states.append(r.getstate())
+            r.seed(seed)
+
+    def restore_all():
+        assert len(states) == len(RANDOMS_TO_MANAGE)
+        for r, state in zip(RANDOMS_TO_MANAGE, states):
+            r.setstate(state)
+        del states[:]
+
+    return seed_all, restore_all
 
 
 @contextlib.contextmanager
@@ -35,15 +98,9 @@ def deterministic_PRNG():
     bad idea in principle, and breaks all kinds of independence assumptions
     in practice.
     """
-    _random_state = random.getstate()
-    random.seed(0)
-    # These branches are covered by tests/numpy/, not tests/cover/
-    if npr is not None:  # pragma: no cover
-        _npr_state = npr.get_state()
-        npr.seed(0)
+    seed_all, restore_all = get_seeder_and_restorer()
+    seed_all()
     try:
         yield
     finally:
-        random.setstate(_random_state)
-        if npr is not None:  # pragma: no cover
-            npr.set_state(_npr_state)
+        restore_all()

hypothesis-python/tests/cover/test_random_module.py

@@ -22,7 +22,9 @@
 import pytest
 
 import hypothesis.strategies as st
-from hypothesis import given, reporting
+from hypothesis import given, register_random, reporting
+from hypothesis.errors import InvalidArgument
+from hypothesis.internal import entropy
 from tests.common.utils import capture_out
 
 
@@ -36,7 +38,7 @@ def test(r):
                     assert False
 
                 test()
-    assert "random.seed(0)" in out.getvalue()
+    assert "RandomSeeder(0)" in out.getvalue()
 
 
 @given(st.random_module(), st.random_module())
@@ -47,3 +49,57 @@ def test_seed_random_twice(r, r2):
 @given(st.random_module())
 def test_does_not_fail_health_check_if_randomness_is_used(r):
     random.getrandbits(128)
+
+
+def test_cannot_register_non_Random():
+    with pytest.raises(InvalidArgument):
+        register_random("not a Random instance")
+
+
+def test_registering_a_Random_is_idempotent():
+    r = random.Random()
+    register_random(r)
+    register_random(r)
+    assert entropy.RANDOMS_TO_MANAGE.pop() is r
+    assert r not in entropy.RANDOMS_TO_MANAGE
+
+
+def test_manages_registered_Random_instance():
+    r = random.Random()
+    register_random(r)
+    state = r.getstate()
+    result = []
+
+    @given(st.integers())
+    def inner(x):
+        v = r.random()
+        if result:
+            assert v == result[0]
+        else:
+            result.append(v)
+
+    inner()
+    assert state == r.getstate()
+
+    entropy.RANDOMS_TO_MANAGE.remove(r)
+    assert r not in entropy.RANDOMS_TO_MANAGE
+
+
+def test_registered_Random_is_seeded_by_random_module_strategy():
+    r = random.Random()
+    register_random(r)
+    state = r.getstate()
+    results = set()
+    count = [0]
+
+    @given(st.integers())
+    def inner(x):
+        results.add(r.random())
+        count[0] += 1
+
+    inner()
+    assert count[0] > len(results) * 0.9, "too few unique random numbers"
+    assert state == r.getstate()
+
+    entropy.RANDOMS_TO_MANAGE.remove(r)
+    assert r not in entropy.RANDOMS_TO_MANAGE