diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..f85fcc045e --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,9 @@ +RELEASE_TYPE: minor + +This release adds :func:`random_module.register `, +which registers ``random.Random`` instances to be seeded and reset by +Hypothesis to ensure that test cases are deterministic. + +We still recommend explicitly passing a ``random.Random`` instance from +:func:`~hypothesis.strategies.randoms` if possible, but registering a +framework-global state for Hypothesis to manage is better than flaky tests! diff --git a/hypothesis-python/src/hypothesis/_strategies.py b/hypothesis-python/src/hypothesis/_strategies.py index c49360fb2b..790b665191 100644 --- a/hypothesis-python/src/hypothesis/_strategies.py +++ b/hypothesis-python/src/hypothesis/_strategies.py @@ -21,7 +21,6 @@ import enum import math import operator -import random import string import sys from decimal import Context, Decimal, localcontext @@ -56,6 +55,7 @@ choice, integer_range, ) +from hypothesis.internal.entropy import get_seeder_and_restorer, register from hypothesis.internal.floats import ( count_between_floats, float_of, @@ -135,6 +135,7 @@ numpy = None if False: + import random # noqa from types import ModuleType # noqa from typing import Any, Dict, Union, Sequence, Callable, Pattern # noqa from typing import TypeVar, Tuple, List, Set, FrozenSet, overload # noqa @@ -1191,26 +1192,23 @@ def __init__(self, seed): self.seed = seed def __repr__(self): - return "random.seed(%r)" % (self.seed,) + return "RandomSeeder(%r)" % (self.seed,) class RandomModule(SearchStrategy): def do_draw(self, data): data.can_reproduce_example_from_repr = False seed = data.draw(integers(0, 2 ** 32 - 1)) - state = random.getstate() - random.seed(seed) - cleanup(lambda: random.setstate(state)) - if numpy is not None: # pragma: no cover - npstate = numpy.random.get_state() - numpy.random.seed(seed) - cleanup(lambda: numpy.random.set_state(npstate)) + seed_all, restore_all = get_seeder_and_restorer(seed) + seed_all() + cleanup(restore_all) return RandomSeeder(seed) @cacheable @defines_strategy def random_module(): + # type: () -> SearchStrategy[RandomSeeder] """The Hypothesis engine handles PRNG state for the stdlib and Numpy random modules internally, always seeding them to zero and restoring the previous state after the test. @@ -1223,10 +1221,21 @@ def random_module(): If ``numpy.random`` is available, that state is also managed. Examples from these strategy shrink to seeds closer to zero. + + You can pass additional ``random.Random`` instances to + ``random_module.register(r)`` to have their states seeded and restored + in the same way. All global random states, from e.g. simulation or + scheduling frameworks, should be registered to prevent flaky tests. + + Registered Randoms are also zeroed by the engine for any Hypothesis + tests that do not use the ``random_module`` strategy. """ return shared(RandomModule(), "hypothesis.strategies.random_module()") +setattr(random_module, "register", register) + + @cacheable @defines_strategy def builds( diff --git a/hypothesis-python/src/hypothesis/internal/entropy.py b/hypothesis-python/src/hypothesis/internal/entropy.py index 98b12f67d6..f4d84258e1 100644 --- a/hypothesis-python/src/hypothesis/internal/entropy.py +++ b/hypothesis-python/src/hypothesis/internal/entropy.py @@ -20,10 +20,60 @@ import contextlib import random +from hypothesis.internal.compat import integer_types +from hypothesis.internal.validation import check_type + +RANDOMS_TO_MANAGE = [random] # type: list + try: import numpy.random as npr except ImportError: - npr = None + pass +else: + + class NumpyRandomWrapper(object): + """A shim to remove those darn underscores.""" + + seed = npr.seed + getstate = npr.get_state + setstate = npr.set_state + + RANDOMS_TO_MANAGE.append(NumpyRandomWrapper) + + +def register(r): + # type: (random.Random) -> None + """Register the given Random instance for management by Hypothesis.""" + check_type(random.Random, r, "r") + if r not in RANDOMS_TO_MANAGE: + RANDOMS_TO_MANAGE.append(r) + + +def get_seeder_and_restorer(seed=0): + """Return a pair of functions which respectively seed all and restore + the state of all registered PRNGs. + + This is used by the core engine via `deterministic_PRNG`, and by users + via `random_module.register`. We support registration of additional + random.Random instances for simulation or scheduling frameworks which + avoid using the global random state. See e.g. #1709. + """ + assert isinstance(seed, integer_types) and 0 <= seed < 2 ** 32 + states = [] # type: list + + def seed_all(): + assert not states + for r in RANDOMS_TO_MANAGE: + states.append(r.getstate()) + r.seed(seed) + + def restore_all(): + assert len(states) == len(RANDOMS_TO_MANAGE) + for r, state in zip(RANDOMS_TO_MANAGE, states): + r.setstate(state) + del states[:] + + return seed_all, restore_all @contextlib.contextmanager @@ -35,15 +85,9 @@ def deterministic_PRNG(): bad idea in principle, and breaks all kinds of independence assumptions in practice. """ - _random_state = random.getstate() - random.seed(0) - # These branches are covered by tests/numpy/, not tests/cover/ - if npr is not None: # pragma: no cover - _npr_state = npr.get_state() - npr.seed(0) + seed_all, restore_all = get_seeder_and_restorer() + seed_all() try: yield finally: - random.setstate(_random_state) - if npr is not None: # pragma: no cover - npr.set_state(_npr_state) + restore_all() diff --git a/hypothesis-python/tests/cover/test_random_module.py b/hypothesis-python/tests/cover/test_random_module.py index 4da2e505cf..664312c6b4 100644 --- a/hypothesis-python/tests/cover/test_random_module.py +++ b/hypothesis-python/tests/cover/test_random_module.py @@ -23,6 +23,8 @@ import hypothesis.strategies as st from hypothesis import given, reporting +from hypothesis.errors import InvalidArgument +from hypothesis.internal import entropy from tests.common.utils import capture_out @@ -36,7 +38,7 @@ def test(r): assert False test() - assert "random.seed(0)" in out.getvalue() + assert "RandomSeeder(0)" in out.getvalue() @given(st.random_module(), st.random_module()) @@ -47,3 +49,53 @@ def test_seed_random_twice(r, r2): @given(st.random_module()) def test_does_not_fail_health_check_if_randomness_is_used(r): random.getrandbits(128) + + +def test_cannot_register_non_Random(): + with pytest.raises(InvalidArgument): + st.random_module.register("not a Random instance") + + +def test_registering_a_Random_is_idempotent(): + r = random.Random() + st.random_module.register(r) + st.random_module.register(r) + assert entropy.RANDOMS_TO_MANAGE.pop() is r + assert r not in entropy.RANDOMS_TO_MANAGE + + +def test_manages_registered_Random_instance(): + r = random.Random() + st.random_module.register(r) + state = r.getstate() + result = [] + + @given(st.integers()) + def inner(x): + v = r.random() + if result: + assert v == result[0] + else: + result.append(v) + + inner() + entropy.RANDOMS_TO_MANAGE.remove(r) + assert state == r.getstate() + + +def test_registered_Random_is_seeded_by_random_module_strategy(): + r = random.Random() + st.random_module.register(r) + state = r.getstate() + results = set() + count = [0] + + @given(st.integers()) + def inner(x): + results.add(r.random()) + count[0] += 1 + + inner() + assert count[0] > len(results) * 0.9, "too few unique random numbers" + entropy.RANDOMS_TO_MANAGE.remove(r) + assert state == r.getstate()