From 1f6ae21845a8a5f485e3a79725fee604cc77b7b9 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 09:59:42 +0100 Subject: [PATCH 01/11] Disallow .example(), add replacements --- hypothesis-python/tests/common/debug.py | 32 ++++++++++++++++++++++++- hypothesis-python/tests/common/setup.py | 4 ---- pytest.ini | 1 - 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/hypothesis-python/tests/common/debug.py b/hypothesis-python/tests/common/debug.py index 864b54dee1..1759333ca9 100644 --- a/hypothesis-python/tests/common/debug.py +++ b/hypothesis-python/tests/common/debug.py @@ -89,7 +89,7 @@ def assert_no_examples(strategy, condition=lambda _: True): pass -def assert_all_examples(strategy, predicate): +def assert_all_examples(strategy, predicate, settings=None): """Asserts that all examples of the given strategy match the predicate. :param strategy: Hypothesis strategy to check @@ -97,8 +97,38 @@ def assert_all_examples(strategy, predicate): """ @given(strategy) + @Settings(parent=settings) def assert_examples(s): msg = f"Found {s!r} using strategy {strategy} which does not match" assert predicate(s), msg assert_examples() + + +def assert_simple_property(strategy, predicate, settings=None): + """Like assert_all_examples, intended as a self-documenting shortcut for simple constant + properties (`is`, `isinstance`, `==`, ...) that can be adequately verified in just a few + examples. + + For more thorough checking, use assert_all_examples. + """ + + assert_all_examples(strategy, predicate, Settings(parent=settings, max_examples=15)) + + +def check_can_generate_examples(strategy, settings=None): + """Tries to generate a small number of examples from the strategy, to verify that it can + do so without raising. + + Nothing is returned, it only checks that no error is raised. + """ + + assert_simple_property( + strategy, + lambda _: True, + settings=Settings( + parent=settings, + phases=(Phase.generate,), + suppress_health_check=list(HealthCheck), + ), + ) diff --git a/hypothesis-python/tests/common/setup.py b/hypothesis-python/tests/common/setup.py index 193e67b11b..1575feaa48 100644 --- a/hypothesis-python/tests/common/setup.py +++ b/hypothesis-python/tests/common/setup.py @@ -13,7 +13,6 @@ from hypothesis import Phase, Verbosity, settings from hypothesis._settings import not_set -from hypothesis.errors import NonInteractiveExampleWarning from hypothesis.internal.coverage import IN_COVERAGE_TESTS @@ -36,9 +35,6 @@ def run(): category=UserWarning, ) - # User-facing warning which does not apply to our own tests - filterwarnings("ignore", category=NonInteractiveExampleWarning) - # We do a smoke test here before we mess around with settings. x = settings() diff --git a/pytest.ini b/pytest.ini index d6c29c51db..5c9e8030fc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,7 +12,6 @@ addopts = xfail_strict = True filterwarnings = error - ignore::hypothesis.errors.NonInteractiveExampleWarning # https://github.com/pandas-dev/pandas/issues/41199 default:Creating a LegacyVersion has been deprecated and will be removed in the next major release:DeprecationWarning default:distutils Version classes are deprecated\. Use packaging\.version instead:DeprecationWarning From 4731c968fb914a69a00aa484fa033bb6971cd49f Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 10:00:16 +0100 Subject: [PATCH 02/11] Rewrite tests --- .../array_api/test_argument_validation.py | 3 +- .../tests/array_api/test_arrays.py | 25 +++++-- .../tests/array_api/test_partial_adoptors.py | 6 +- .../tests/cover/test_annotations.py | 8 +- .../tests/cover/test_arbitrary_data.py | 5 -- .../tests/cover/test_attrs_inference.py | 16 ++-- .../tests/cover/test_complex_numbers.py | 9 ++- .../tests/cover/test_datetimes.py | 4 +- .../tests/cover/test_deferred_strategies.py | 8 +- .../tests/cover/test_direct_strategies.py | 20 ++--- .../tests/cover/test_draw_example.py | 5 +- hypothesis-python/tests/cover/test_example.py | 75 +------------------ .../tests/cover/test_filter_rewriting.py | 7 +- .../tests/cover/test_functions.py | 4 +- hypothesis-python/tests/cover/test_lookup.py | 74 +++++++++--------- .../tests/cover/test_lookup_py38.py | 12 ++- .../tests/cover/test_lookup_py39.py | 13 +++- .../tests/cover/test_numerics.py | 6 +- .../tests/cover/test_permutations.py | 4 +- .../cover/test_provisional_strategies.py | 10 ++- .../tests/cover/test_recursive.py | 8 +- hypothesis-python/tests/cover/test_regex.py | 13 +++- .../tests/cover/test_sampled_from.py | 19 +++-- .../tests/cover/test_searchstrategy.py | 10 ++- .../tests/cover/test_simple_characters.py | 43 +++++++---- .../tests/cover/test_simple_collections.py | 12 +-- .../tests/cover/test_subnormal_floats.py | 4 +- .../tests/cover/test_type_lookup.py | 63 ++++++++-------- .../cover/test_type_lookup_forward_ref.py | 9 ++- hypothesis-python/tests/cover/test_uuids.py | 8 +- .../tests/cover/test_validation.py | 19 ++--- .../django/toystore/test_given_models.py | 5 +- hypothesis-python/tests/lark/test_grammar.py | 34 +++++---- .../tests/nocover/test_deferred_errors.py | 12 +-- .../tests/nocover/test_drypython_returns.py | 4 +- .../tests/nocover/test_flatmap.py | 4 +- .../tests/nocover/test_type_lookup.py | 10 ++- .../test_type_lookup_future_annotations.py | 4 +- .../tests/numpy/test_argument_validation.py | 5 +- .../tests/numpy/test_deprecation.py | 10 ++- .../tests/numpy/test_from_dtype.py | 6 +- .../tests/numpy/test_from_type.py | 25 ++++--- .../tests/numpy/test_gen_data.py | 6 +- .../tests/pandas/test_argument_validation.py | 10 ++- .../tests/pandas/test_indexes.py | 3 +- .../tests/test_annotated_types.py | 8 +- .../test_backported_types.py | 29 ++++--- 47 files changed, 372 insertions(+), 325 deletions(-) diff --git a/hypothesis-python/tests/array_api/test_argument_validation.py b/hypothesis-python/tests/array_api/test_argument_validation.py index 3a14ee9987..7723bfd326 100644 --- a/hypothesis-python/tests/array_api/test_argument_validation.py +++ b/hypothesis-python/tests/array_api/test_argument_validation.py @@ -16,6 +16,7 @@ from hypothesis.extra.array_api import NominalVersion, make_strategies_namespace from tests.array_api.common import MIN_VER_FOR_COMPLEX +from tests.common.debug import check_can_generate_examples def e(name, *, _min_version: Optional[NominalVersion] = None, **kwargs): @@ -225,7 +226,7 @@ def test_raise_invalid_argument(xp, xps, strat_name, kwargs): strat_func = getattr(xps, strat_name) strat = strat_func(**kwargs) with pytest.raises(InvalidArgument): - strat.example() + check_can_generate_examples(strat) @pytest.mark.parametrize("api_version", [..., "latest", "1970.01", 42]) diff --git a/hypothesis-python/tests/array_api/test_arrays.py b/hypothesis-python/tests/array_api/test_arrays.py index 756576a730..f63b03b9f6 100644 --- a/hypothesis-python/tests/array_api/test_arrays.py +++ b/hypothesis-python/tests/array_api/test_arrays.py @@ -20,7 +20,12 @@ dtype_name_params, flushes_to_zero, ) -from tests.common.debug import assert_all_examples, find_any, minimal +from tests.common.debug import ( + assert_all_examples, + check_can_generate_examples, + find_any, + minimal, +) from tests.common.utils import flaky @@ -222,13 +227,17 @@ def test_cannot_draw_unique_arrays_with_too_small_elements(xp, xps): """Unique strategy with elements strategy range smaller than its size raises helpful error.""" with pytest.raises(InvalidArgument): - xps.arrays(xp.int8, 10, elements=st.integers(0, 5), unique=True).example() + check_can_generate_examples( + xps.arrays(xp.int8, 10, elements=st.integers(0, 5), unique=True) + ) def test_cannot_fill_arrays_with_non_castable_value(xp, xps): """Strategy with fill not castable to dtype raises helpful error.""" with pytest.raises(InvalidArgument): - xps.arrays(xp.int8, 10, fill=st.just("not a castable value")).example() + check_can_generate_examples( + xps.arrays(xp.int8, 10, fill=st.just("not a castable value")) + ) def test_generate_unique_arrays_with_high_collision_elements(xp, xps): @@ -284,7 +293,7 @@ def test_may_not_fill_unique_array_with_non_nan(xp, xps): fill=st.just(0.0), ) with pytest.raises(InvalidArgument): - strat.example() + check_can_generate_examples(strat) @pytest.mark.parametrize( @@ -298,7 +307,7 @@ def test_may_not_use_overflowing_integers(xp, xps, kwargs): """Strategy with elements strategy range outside the dtype's bounds raises helpful error.""" with pytest.raises(InvalidArgument): - xps.arrays(dtype=xp.int8, shape=1, **kwargs).example() + check_can_generate_examples(xps.arrays(dtype=xp.int8, shape=1, **kwargs)) @pytest.mark.parametrize("fill", [False, True]) @@ -321,7 +330,7 @@ def test_may_not_use_unrepresentable_elements(xp, xps, fill, dtype, strat): else: kw = {"elements": strat} with pytest.raises(InvalidArgument): - xps.arrays(dtype=dtype, shape=1, **kw).example() + check_can_generate_examples(xps.arrays(dtype=dtype, shape=1, **kw)) def test_floats_can_be_constrained(xp, xps): @@ -507,6 +516,6 @@ def test_subnormal_elements_validation(xp, xps): strat = xps.arrays(xp.float32, 10, elements=elements) if flushes_to_zero(xp, width=32): with pytest.raises(InvalidArgument, match="Generated subnormal float"): - strat.example() + check_can_generate_examples(strat) else: - strat.example() + check_can_generate_examples(strat) diff --git a/hypothesis-python/tests/array_api/test_partial_adoptors.py b/hypothesis-python/tests/array_api/test_partial_adoptors.py index f1ed146d28..6de5d9dbb8 100644 --- a/hypothesis-python/tests/array_api/test_partial_adoptors.py +++ b/hypothesis-python/tests/array_api/test_partial_adoptors.py @@ -26,6 +26,8 @@ mock_xp, ) +from tests.common.debug import check_can_generate_examples + MOCK_WARN_MSG = f"determine.*{mock_xp.__name__}.*Array API" @@ -56,7 +58,7 @@ def test_error_on_missing_attr(stratname, args, attr): xps = make_strategies_namespace(xp, api_version="draft") func = getattr(xps, stratname) with pytest.raises(InvalidArgument, match=f"{mock_xp.__name__}.*required.*{attr}"): - func(*args).example() + check_can_generate_examples(func(*args)) dtypeless_xp = make_mock_xp(exclude=tuple(DTYPE_NAMES)) @@ -82,7 +84,7 @@ def test_error_on_missing_dtypes(stratname): required dtypes.""" func = getattr(dtypeless_xps, stratname) with pytest.raises(InvalidArgument, match=f"{mock_xp.__name__}.*dtype.*namespace"): - func().example() + check_can_generate_examples(func()) @pytest.mark.filterwarnings(f"ignore:.*{MOCK_WARN_MSG}.*") diff --git a/hypothesis-python/tests/cover/test_annotations.py b/hypothesis-python/tests/cover/test_annotations.py index 57520a154a..dff11f1526 100644 --- a/hypothesis-python/tests/cover/test_annotations.py +++ b/hypothesis-python/tests/cover/test_annotations.py @@ -8,6 +8,7 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import warnings from inspect import Parameter as P, signature import attr @@ -21,6 +22,8 @@ get_pretty_function_description, ) +from tests.common.debug import check_can_generate_examples + @given(st.integers()) def test_has_an_annotation(i: int): @@ -127,5 +130,6 @@ def test_attrs_inference_builds(c): def test_attrs_inference_from_type(): s = st.from_type(Inferrables) - with pytest.warns(SmallSearchSpaceWarning): - s.example() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SmallSearchSpaceWarning) + check_can_generate_examples(s) diff --git a/hypothesis-python/tests/cover/test_arbitrary_data.py b/hypothesis-python/tests/cover/test_arbitrary_data.py index 2054796682..32a2732a1c 100644 --- a/hypothesis-python/tests/cover/test_arbitrary_data.py +++ b/hypothesis-python/tests/cover/test_arbitrary_data.py @@ -75,10 +75,5 @@ def test_errors_when_normal_strategy_functions_are_used(f): getattr(st.data(), f)(lambda x: 1) -def test_errors_when_asked_for_example(): - with raises(InvalidArgument): - st.data().example() - - def test_nice_repr(): assert repr(st.data()) == "data()" diff --git a/hypothesis-python/tests/cover/test_attrs_inference.py b/hypothesis-python/tests/cover/test_attrs_inference.py index c517cea883..09abc24052 100644 --- a/hypothesis-python/tests/cover/test_attrs_inference.py +++ b/hypothesis-python/tests/cover/test_attrs_inference.py @@ -16,6 +16,8 @@ from hypothesis import given, strategies as st from hypothesis.errors import ResolutionFailed +from tests.common.debug import check_can_generate_examples + @attr.s class Inferrables: @@ -83,12 +85,14 @@ def test_attrs_inference_from_type(c): @pytest.mark.parametrize("c", [Required, UnhelpfulConverter]) def test_cannot_infer(c): with pytest.raises(ResolutionFailed): - st.builds(c).example() + check_can_generate_examples(st.builds(c)) def test_cannot_infer_takes_self(): with pytest.raises(ResolutionFailed): - st.builds(Inferrables, has_default_factory_takes_self=...).example() + check_can_generate_examples( + st.builds(Inferrables, has_default_factory_takes_self=...) + ) @attr.s @@ -98,12 +102,12 @@ class HasPrivateAttribute: @pytest.mark.parametrize("s", [st.just(42), ...]) def test_private_attribute(s): - st.builds(HasPrivateAttribute, x=s).example() + check_can_generate_examples(st.builds(HasPrivateAttribute, x=s)) def test_private_attribute_underscore_fails(): with pytest.raises(TypeError, match="unexpected keyword argument '_x'"): - st.builds(HasPrivateAttribute, _x=st.just(42)).example() + check_can_generate_examples(st.builds(HasPrivateAttribute, _x=st.just(42))) def test_private_attribute_underscore_infer_fails(): @@ -112,7 +116,7 @@ def test_private_attribute_underscore_infer_fails(): with pytest.raises( TypeError, match="Unexpected keyword argument _x for attrs class" ): - st.builds(HasPrivateAttribute, _x=...).example() + check_can_generate_examples(st.builds(HasPrivateAttribute, _x=...)) @attr.s @@ -122,4 +126,4 @@ class HasAliasedAttribute: @pytest.mark.parametrize("s", [st.just(42), ...]) def test_aliased_attribute(s): - st.builds(HasAliasedAttribute, crazyname=s).example() + check_can_generate_examples(st.builds(HasAliasedAttribute, crazyname=s)) diff --git a/hypothesis-python/tests/cover/test_complex_numbers.py b/hypothesis-python/tests/cover/test_complex_numbers.py index ba97cb82a5..4af329d98e 100644 --- a/hypothesis-python/tests/cover/test_complex_numbers.py +++ b/hypothesis-python/tests/cover/test_complex_numbers.py @@ -17,7 +17,12 @@ from hypothesis.errors import InvalidArgument from hypothesis.strategies import complex_numbers -from tests.common.debug import assert_no_examples, find_any, minimal +from tests.common.debug import ( + assert_no_examples, + check_can_generate_examples, + find_any, + minimal, +) def test_minimal(): @@ -129,4 +134,4 @@ def test_allow_subnormal(allow_subnormal, min_magnitude, max_magnitude): @pytest.mark.parametrize("allow_subnormal", [1, 0.0, "False"]) def test_allow_subnormal_validation(allow_subnormal): with pytest.raises(InvalidArgument): - complex_numbers(allow_subnormal=allow_subnormal).example() + check_can_generate_examples(complex_numbers(allow_subnormal=allow_subnormal)) diff --git a/hypothesis-python/tests/cover/test_datetimes.py b/hypothesis-python/tests/cover/test_datetimes.py index 9062501d7d..1c9b12499f 100644 --- a/hypothesis-python/tests/cover/test_datetimes.py +++ b/hypothesis-python/tests/cover/test_datetimes.py @@ -15,7 +15,7 @@ from hypothesis import given, settings from hypothesis.strategies import dates, datetimes, timedeltas, times -from tests.common.debug import find_any, minimal +from tests.common.debug import assert_simple_property, find_any, minimal def test_can_find_positive_delta(): @@ -51,7 +51,7 @@ def test_max_value_is_respected(): @given(timedeltas()) def test_single_timedelta(val): - assert find_any(timedeltas(val, val)) is val + assert_simple_property(timedeltas(val, val), lambda v: v is val) def test_simplifies_towards_millenium(): diff --git a/hypothesis-python/tests/cover/test_deferred_strategies.py b/hypothesis-python/tests/cover/test_deferred_strategies.py index ea2ac383cc..adc722fd12 100644 --- a/hypothesis-python/tests/cover/test_deferred_strategies.py +++ b/hypothesis-python/tests/cover/test_deferred_strategies.py @@ -13,7 +13,7 @@ from hypothesis import given, strategies as st from hypothesis.errors import InvalidArgument -from tests.common.debug import assert_no_examples, minimal +from tests.common.debug import assert_no_examples, check_can_generate_examples, minimal def test_binary_tree(): @@ -35,19 +35,19 @@ def test_mutual_recursion(): def test_errors_on_non_function_define(): x = st.deferred(1) with pytest.raises(InvalidArgument): - x.example() + check_can_generate_examples(x) def test_errors_if_define_does_not_return_search_strategy(): x = st.deferred(lambda: 1) with pytest.raises(InvalidArgument): - x.example() + check_can_generate_examples(x) def test_errors_on_definition_as_self(): x = st.deferred(lambda: x) with pytest.raises(InvalidArgument): - x.example() + check_can_generate_examples(x) def test_branches_pass_through_deferred(): diff --git a/hypothesis-python/tests/cover/test_direct_strategies.py b/hypothesis-python/tests/cover/test_direct_strategies.py index 9bb966fee9..a60e2fb49c 100644 --- a/hypothesis-python/tests/cover/test_direct_strategies.py +++ b/hypothesis-python/tests/cover/test_direct_strategies.py @@ -22,7 +22,7 @@ from hypothesis.errors import InvalidArgument from hypothesis.vendor.pretty import pretty -from tests.common.debug import minimal +from tests.common.debug import check_can_generate_examples, minimal # Use `pretty` instead of `repr` for building test names, so that set and dict # parameters print consistently across multiple worker processes with different @@ -214,7 +214,7 @@ def fn_ktest(*fnkwargs): ) def test_validates_keyword_arguments(fn, kwargs): with pytest.raises(InvalidArgument): - fn(**kwargs).example() + check_can_generate_examples(fn(**kwargs)) @fn_ktest( @@ -311,7 +311,7 @@ def test_validates_keyword_arguments(fn, kwargs): (ds.ip_addresses, {"v": 6, "network": IPv6Network("::/64")}), ) def test_produces_valid_examples_from_keyword(fn, kwargs): - fn(**kwargs).example() + check_can_generate_examples(fn(**kwargs)) @fn_test( @@ -321,7 +321,7 @@ def test_produces_valid_examples_from_keyword(fn, kwargs): ) def test_validates_args(fn, args): with pytest.raises(InvalidArgument): - fn(*args).example() + check_can_generate_examples(fn(*args)) @fn_test( @@ -332,23 +332,25 @@ def test_validates_args(fn, args): (ds.builds, (lambda x, y: x + y, ds.integers(), ds.integers())), ) def test_produces_valid_examples_from_args(fn, args): - fn(*args).example() + check_can_generate_examples(fn(*args)) def test_build_class_with_target_kwarg(): NamedTupleWithTargetField = collections.namedtuple("Something", ["target"]) - ds.builds(NamedTupleWithTargetField, target=ds.integers()).example() + check_can_generate_examples( + ds.builds(NamedTupleWithTargetField, target=ds.integers()) + ) def test_builds_raises_with_no_target(): with pytest.raises(TypeError): - ds.builds().example() + check_can_generate_examples(ds.builds()) @pytest.mark.parametrize("non_callable", [1, "abc", ds.integers()]) def test_builds_raises_if_non_callable_as_target_kwarg(non_callable): with pytest.raises(TypeError): - ds.builds(target=non_callable).example() + check_can_generate_examples(ds.builds(target=non_callable)) @pytest.mark.parametrize("non_callable", [1, "abc", ds.integers()]) @@ -356,7 +358,7 @@ def test_builds_raises_if_non_callable_as_first_arg(non_callable): # If there are any positional arguments, then the target (which must be # callable) must be specified as the first one. with pytest.raises(InvalidArgument): - ds.builds(non_callable, target=lambda x: x).example() + check_can_generate_examples(ds.builds(non_callable, target=lambda x: x)) def test_tuples_raise_error_on_bad_kwargs(): diff --git a/hypothesis-python/tests/cover/test_draw_example.py b/hypothesis-python/tests/cover/test_draw_example.py index 8fd5c16672..a417297614 100644 --- a/hypothesis-python/tests/cover/test_draw_example.py +++ b/hypothesis-python/tests/cover/test_draw_example.py @@ -13,13 +13,14 @@ from hypothesis.strategies import lists from tests.common import standard_types +from tests.common.debug import check_can_generate_examples @pytest.mark.parametrize("spec", standard_types, ids=list(map(repr, standard_types))) def test_single_example(spec): - spec.example() + check_can_generate_examples(spec) @pytest.mark.parametrize("spec", standard_types, ids=list(map(repr, standard_types))) def test_list_example(spec): - lists(spec).example() + check_can_generate_examples(lists(spec)) diff --git a/hypothesis-python/tests/cover/test_example.py b/hypothesis-python/tests/cover/test_example.py index 02dc718ab7..1eaedabc61 100644 --- a/hypothesis-python/tests/cover/test_example.py +++ b/hypothesis-python/tests/cover/test_example.py @@ -8,86 +8,15 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -import sys -import warnings -from decimal import Decimal -import pexpect import pytest -from hypothesis import example, find, given, strategies as st -from hypothesis.errors import ( - HypothesisException, - HypothesisWarning, - InvalidArgument, - NonInteractiveExampleWarning, - Unsatisfiable, -) -from hypothesis.internal.compat import WINDOWS +from hypothesis import example, given, strategies as st +from hypothesis.errors import HypothesisWarning, InvalidArgument from tests.common.utils import fails_with -def test_example_of_none_is_none(): - assert st.none().example() is None - - -def test_exception_in_compare_can_still_have_example(): - st.one_of(st.none().map(lambda n: Decimal("snan")), st.just(Decimal(0))).example() - - -def test_does_not_always_give_the_same_example(): - s = st.integers() - assert len({s.example() for _ in range(100)}) >= 10 - - -def test_raises_on_no_examples(): - with pytest.raises(Unsatisfiable): - st.nothing().example() - - -@fails_with(HypothesisException) -@example(False) -@given(st.booleans()) -def test_example_inside_given(b): - st.integers().example() - - -@fails_with(HypothesisException) -def test_example_inside_find(): - find(st.integers(0, 100), lambda x: st.integers().example()) - - -@fails_with(HypothesisException) -def test_example_inside_strategy(): - st.booleans().map(lambda x: st.integers().example()).example() - - -def test_non_interactive_example_emits_warning(): - # We have this warning disabled for most of our tests, because self-testing - # Hypothesis means `.example()` can be a good idea when it never is for users. - with warnings.catch_warnings(): - warnings.simplefilter("always") - with pytest.warns(NonInteractiveExampleWarning): - st.text().example() - - -@pytest.mark.skipif(WINDOWS, reason="pexpect.spawn not supported on Windows") -def test_interactive_example_does_not_emit_warning(): - try: - child = pexpect.spawn(f"{sys.executable} -Werror") - child.expect(">>> ", timeout=10) - except pexpect.exceptions.EOF: - pytest.skip( - "Unable to run python with -Werror. This may be because you are " - "running from an old virtual environment - update your installed " - "copy of `virtualenv` and then create a fresh environment." - ) - child.sendline("from hypothesis.strategies import none") - child.sendline("none().example()") - child.sendline("quit(code=0)") - - def identity(decorator): # The "identity function hack" from https://peps.python.org/pep-0614/ # Method-chaining decorators are otherwise a syntax error in Python <= 3.8 diff --git a/hypothesis-python/tests/cover/test_filter_rewriting.py b/hypothesis-python/tests/cover/test_filter_rewriting.py index 5a77bd533d..42cc60310e 100644 --- a/hypothesis-python/tests/cover/test_filter_rewriting.py +++ b/hypothesis-python/tests/cover/test_filter_rewriting.py @@ -28,6 +28,7 @@ from hypothesis.strategies._internal.strategies import FilteredStrategy from hypothesis.strategies._internal.strings import TextStrategy +from tests.common.debug import check_can_generate_examples from tests.common.utils import fails_with @@ -183,7 +184,7 @@ def test_rewrite_unsatisfiable_filter(s, pred): @pytest.mark.parametrize("s", [st.integers(1, 5), st.floats(1, 5)]) @fails_with(Unsatisfiable) def test_erroring_rewrite_unsatisfiable_filter(s, pred): - s.filter(pred).example() + check_can_generate_examples(s.filter(pred)) @pytest.mark.parametrize( @@ -224,7 +225,7 @@ def test_rewriting_does_not_compare_decimal_snan(): s = st.integers(1, 5).filter(partial(operator.eq, decimal.Decimal("snan"))) s.wrapped_strategy with pytest.raises(decimal.InvalidOperation): - s.example() + check_can_generate_examples(s) @pytest.mark.parametrize("strategy", [st.integers(0, 1), st.floats(0, 1)], ids=repr) @@ -370,7 +371,7 @@ def test_isidentifier_filter_properly_rewritten(al, data): def test_isidentifer_filter_unsatisfiable(al): fs = st.text(alphabet=al).filter(str.isidentifier) with pytest.raises(Unsatisfiable): - fs.example() + check_can_generate_examples(fs) @pytest.mark.parametrize( diff --git a/hypothesis-python/tests/cover/test_functions.py b/hypothesis-python/tests/cover/test_functions.py index e0e3868c3d..2ce9ae700e 100644 --- a/hypothesis-python/tests/cover/test_functions.py +++ b/hypothesis-python/tests/cover/test_functions.py @@ -17,6 +17,8 @@ from hypothesis.reporting import with_reporter from hypothesis.strategies import booleans, functions, integers +from tests.common.debug import check_can_generate_examples + def func_a(): pass @@ -81,7 +83,7 @@ def test_functions_lambda_with_arg(f): ) def test_invalid_arguments(like, returns, pure): with pytest.raises(InvalidArgument): - functions(like=like, returns=returns, pure=pure).example() + check_can_generate_examples(functions(like=like, returns=returns, pure=pure)) def func_returns_str() -> str: diff --git a/hypothesis-python/tests/cover/test_lookup.py b/hypothesis-python/tests/cover/test_lookup.py index 96b92e8dd5..dfaab8f5e0 100644 --- a/hypothesis-python/tests/cover/test_lookup.py +++ b/hypothesis-python/tests/cover/test_lookup.py @@ -39,6 +39,8 @@ from tests.common.debug import ( assert_all_examples, assert_no_examples, + assert_simple_property, + check_can_generate_examples, find_any, minimal, ) @@ -85,7 +87,7 @@ def test_resolve_typing_module(data, typ): @pytest.mark.parametrize("typ", [typing.Any, typing.Union]) def test_does_not_resolve_special_cases(typ): with pytest.raises(InvalidArgument): - from_type(typ).example() + check_can_generate_examples(from_type(typ)) @pytest.mark.parametrize( @@ -99,7 +101,7 @@ def test_specialised_scalar_types(data, typ, instance_of): def test_typing_Type_int(): - assert from_type(typing.Type[int]).example() is int + assert_simple_property(from_type(typing.Type[int]), lambda x: x is int) @given(from_type(typing.Type[typing.Union[str, list]])) @@ -225,7 +227,7 @@ def test_Optional_minimises_to_None(): @pytest.mark.parametrize("n", range(10)) def test_variable_length_tuples(n): type_ = typing.Tuple[int, ...] - from_type(type_).filter(lambda ex: len(ex) == n).example() + check_can_generate_examples(from_type(type_).filter(lambda ex: len(ex) == n)) def test_lookup_overrides_defaults(): @@ -290,7 +292,7 @@ def if_available(name): ids=get_pretty_function_description, ) def test_resolves_weird_types(typ): - from_type(typ).example() + check_can_generate_examples(from_type(typ)) class Foo: @@ -390,11 +392,11 @@ def resolve_type_var(thing): return NotImplemented with temp_registered(typing.TypeVar, resolve_type_var): - assert st.from_type(A).example() is sentinel + assert_simple_property(st.from_type(A), lambda x: x is sentinel) # We've re-defined the default TypeVar resolver, so there is no fallback. # This causes the lookup to fail. with pytest.raises(InvalidArgument): - st.from_type(B).example() + check_can_generate_examples(st.from_type(B)) def annotated_func(a: int, b: int = 2, *, c: int, d: int = 4): @@ -403,7 +405,7 @@ def annotated_func(a: int, b: int = 2, *, c: int, d: int = 4): def test_issue_946_regression(): # Turned type hints into kwargs even if the required posarg was passed - st.builds(annotated_func, st.integers()).example() + check_can_generate_examples(st.builds(annotated_func, st.integers())) @pytest.mark.parametrize( @@ -431,14 +433,14 @@ def non_annotated_func(a, b=2, *, c, d=4): def test_cannot_pass_infer_as_posarg(): with pytest.raises(InvalidArgument): - st.builds(annotated_func, ...).example() + check_can_generate_examples(st.builds(annotated_func, ...)) def test_cannot_force_inference_for_unannotated_arg(): with pytest.raises(InvalidArgument): - st.builds(non_annotated_func, a=..., c=st.none()).example() + check_can_generate_examples(st.builds(non_annotated_func, a=..., c=st.none())) with pytest.raises(InvalidArgument): - st.builds(non_annotated_func, a=st.none(), c=...).example() + check_can_generate_examples(st.builds(non_annotated_func, a=st.none(), c=...)) class UnknownType: @@ -462,9 +464,11 @@ def unknown_annotated_func(a: UnknownType, b=2, *, c: UnknownType, d=4): def test_raises_for_arg_with_unresolvable_annotation(): with pytest.raises(ResolutionFailed): - st.builds(unknown_annotated_func).example() + check_can_generate_examples(st.builds(unknown_annotated_func)) with pytest.raises(ResolutionFailed): - st.builds(unknown_annotated_func, a=st.none(), c=...).example() + check_can_generate_examples( + st.builds(unknown_annotated_func, a=st.none(), c=...) + ) @given(a=..., b=...) @@ -486,9 +490,11 @@ def test_resolves_NewType(): typ = typing.NewType("T", int) nested = typing.NewType("NestedT", typ) uni = typing.NewType("UnionT", typing.Optional[int]) - assert isinstance(from_type(typ).example(), int) - assert isinstance(from_type(nested).example(), int) - assert isinstance(from_type(uni).example(), (int, type(None))) + assert_simple_property(from_type(typ), lambda x: isinstance(x, int)) + assert_simple_property(from_type(nested), lambda x: isinstance(x, int)) + assert_simple_property(from_type(uni), lambda x: isinstance(x, (int, type(None)))) + find_any(from_type(uni), lambda x: isinstance(x, int)) + find_any(from_type(uni), lambda x: isinstance(x, type(None))) @pytest.mark.parametrize("is_handled", [True, False]) @@ -504,9 +510,9 @@ def resolve_custom_strategy(thing): with temp_registered(typ, resolve_custom_strategy): if is_handled: - assert st.from_type(typ).example() is sentinel + assert_simple_property(st.from_type(typ), lambda x: x is sentinel) else: - assert isinstance(st.from_type(typ).example(), int) + assert_simple_property(st.from_type(typ), lambda x: isinstance(x, int)) E = enum.Enum("E", "a b c") @@ -553,9 +559,11 @@ def method(self, a: int, b: int): ) def test_required_args(target, args, kwargs): # Mostly checking that `self` (and only self) is correctly excluded - st.builds( - target, *map(st.just, args), **{k: st.just(v) for k, v in kwargs.items()} - ).example() + check_can_generate_examples( + st.builds( + target, *map(st.just, args), **{k: st.just(v) for k, v in kwargs.items()} + ) + ) class AnnotatedNamedTuple(typing.NamedTuple): @@ -581,7 +589,7 @@ def test_override_args_for_namedtuple(thing): def test_cannot_resolve_bare_forward_reference(thing): t = thing["ConcreteFoo"] with pytest.raises(InvalidArgument): - st.from_type(t).example() + check_can_generate_examples(st.from_type(t)) class Tree: @@ -603,10 +611,10 @@ class TypedTree(typing.TypedDict): def test_resolving_recursive_typeddict(): - tree = st.from_type(TypedTree).example() - assert isinstance(tree, dict) - assert len(tree) == 1 - assert "nxt" in tree + assert_simple_property( + st.from_type(TypedTree), + lambda tree: isinstance(tree, dict) and len(tree) == 1 and "nxt" in tree, + ) class MyList: @@ -743,7 +751,7 @@ def test_resolves_empty_Tuple_issue_1583_regression(ex): def test_can_register_NewType(): Name = typing.NewType("Name", str) st.register_type_strategy(Name, st.just("Eric Idle")) - assert st.from_type(Name).example() == "Eric Idle" + assert_simple_property(st.from_type(Name), lambda x: x == "Eric Idle") @given(st.from_type(typing.Callable)) @@ -1088,7 +1096,7 @@ def really_takes_str(value: int) -> None: ) def test_signature_is_the_most_important_source(thing): """Signature types should take precedence over all other annotations.""" - find_any(st.builds(thing)) + check_can_generate_examples(st.builds(thing)) class AnnotatedAndDefault: @@ -1105,8 +1113,7 @@ def test_from_type_can_be_default_or_annotation(): def test_resolves_builtin_types(t): with warnings.catch_warnings(): warnings.simplefilter("ignore", SmallSearchSpaceWarning) - v = st.from_type(t).example() - assert isinstance(v, t) + assert_simple_property(st.from_type(t), lambda v: isinstance(v, t)) @pytest.mark.parametrize("t", BUILTIN_TYPES, ids=lambda t: t.__name__) @@ -1119,8 +1126,7 @@ def test_resolves_forwardrefs_to_builtin_types(t, data): @pytest.mark.parametrize("t", BUILTIN_TYPES, ids=lambda t: t.__name__) def test_resolves_type_of_builtin_types(t): - v = st.from_type(typing.Type[t.__name__]).example() - assert v is t + assert_simple_property(st.from_type(typing.Type[t.__name__]), lambda v: v is t) @given(st.from_type(typing.Type[typing.Union["str", "int"]])) @@ -1133,9 +1139,9 @@ def test_builds_suggests_from_type(type_): with pytest.raises( InvalidArgument, match=re.escape(f"try using from_type({type_!r})") ): - st.builds(type_).example() + check_can_generate_examples(st.builds(type_)) try: - st.builds(type_, st.just("has an argument")).example() + check_can_generate_examples(st.builds(type_, st.just("has an argument"))) raise AssertionError("Expected strategy to raise an error") except TypeError as err: assert not isinstance(err, InvalidArgument) @@ -1148,7 +1154,7 @@ def f(x: int): msg = "@no_type_check decorator prevented Hypothesis from inferring a strategy" with pytest.raises(TypeError, match=msg): - st.builds(f).example() + check_can_generate_examples(st.builds(f)) class TupleSubtype(tuple): diff --git a/hypothesis-python/tests/cover/test_lookup_py38.py b/hypothesis-python/tests/cover/test_lookup_py38.py index 5b65389998..d46d13c934 100644 --- a/hypothesis-python/tests/cover/test_lookup_py38.py +++ b/hypothesis-python/tests/cover/test_lookup_py38.py @@ -24,7 +24,11 @@ ) from hypothesis.strategies import from_type -from tests.common.debug import find_any +from tests.common.debug import ( + assert_simple_property, + check_can_generate_examples, + find_any, +) from tests.common.utils import fails_with, temp_registered @@ -36,7 +40,7 @@ def test_typing_Final(data): @pytest.mark.parametrize("value", ["dog", b"goldfish", 42, 63.4, -80.5, False]) def test_typing_Literal(value): - assert from_type(typing.Literal[value]).example() == value + assert_simple_property(from_type(typing.Literal[value]), lambda v: v == value) @given(st.data()) @@ -141,7 +145,7 @@ def test_can_resolve_recursive_dataclass(val): def test_can_register_new_type_for_typeddicts(): sentinel = object() with temp_registered(C, st.just(sentinel)): - assert st.from_type(C).example() is sentinel + assert_simple_property(st.from_type(C), lambda v: v is sentinel) @pytest.mark.parametrize( @@ -248,4 +252,4 @@ def test_can_resolve_registered_protocol(data): def test_cannot_resolve_un_registered_protocol(): msg = "Instance and class checks can only be used with @runtime_checkable protocols" with pytest.raises(TypeError, match=msg): - st.from_type(BarProtocol).example() + check_can_generate_examples(st.from_type(BarProtocol)) diff --git a/hypothesis-python/tests/cover/test_lookup_py39.py b/hypothesis-python/tests/cover/test_lookup_py39.py index 0cd538289b..e3bc1a9f3f 100644 --- a/hypothesis-python/tests/cover/test_lookup_py39.py +++ b/hypothesis-python/tests/cover/test_lookup_py39.py @@ -17,7 +17,12 @@ from hypothesis import given, strategies as st from hypothesis.errors import InvalidArgument -from tests.common.debug import assert_all_examples, find_any +from tests.common.debug import ( + assert_all_examples, + assert_simple_property, + check_can_generate_examples, + find_any, +) from tests.common.utils import temp_registered @@ -84,7 +89,7 @@ def test_string_forward_ref_message(): # See https://github.com/HypothesisWorks/hypothesis/issues/3016 s = st.builds(User) with pytest.raises(InvalidArgument, match="`from __future__ import annotations`"): - s.example() + check_can_generate_examples(s) def test_issue_3080(): @@ -155,5 +160,5 @@ def test_lookup_registered_tuple(): sentinel = object() typ = tuple[int] with temp_registered(tuple, st.just(sentinel)): - assert st.from_type(typ).example() is sentinel - assert st.from_type(typ).example() is not sentinel + assert_simple_property(st.from_type(typ), lambda v: v is sentinel) + assert_simple_property(st.from_type(typ), lambda v: v is not sentinel) diff --git a/hypothesis-python/tests/cover/test_numerics.py b/hypothesis-python/tests/cover/test_numerics.py index 66d9899e68..95722754af 100644 --- a/hypothesis-python/tests/cover/test_numerics.py +++ b/hypothesis-python/tests/cover/test_numerics.py @@ -28,7 +28,7 @@ tuples, ) -from tests.common.debug import find_any +from tests.common.debug import check_can_generate_examples, find_any @settings(suppress_health_check=list(HealthCheck)) @@ -167,10 +167,10 @@ def test_issue_739_regression(x): def test_consistent_decimal_error(): bad = "invalid argument to Decimal" with pytest.raises(InvalidArgument) as excinfo: - decimals(bad).example() + check_can_generate_examples(decimals(bad)) with pytest.raises(InvalidArgument) as excinfo2: with decimal.localcontext(decimal.Context(traps=[])): - decimals(bad).example() + check_can_generate_examples(decimals(bad)) assert str(excinfo.value) == str(excinfo2.value) diff --git a/hypothesis-python/tests/cover/test_permutations.py b/hypothesis-python/tests/cover/test_permutations.py index c6712bb56f..47cd5e77e6 100644 --- a/hypothesis-python/tests/cover/test_permutations.py +++ b/hypothesis-python/tests/cover/test_permutations.py @@ -12,7 +12,7 @@ from hypothesis.errors import InvalidArgument from hypothesis.strategies import permutations -from tests.common.debug import minimal +from tests.common.debug import check_can_generate_examples, minimal from tests.common.utils import fails_with @@ -35,4 +35,4 @@ def test_empty_permutations_are_empty(xs): @fails_with(InvalidArgument) def test_cannot_permute_non_sequence_types(): - permutations(set()).example() + check_can_generate_examples(permutations(set())) diff --git a/hypothesis-python/tests/cover/test_provisional_strategies.py b/hypothesis-python/tests/cover/test_provisional_strategies.py index 621db25631..f49f8f6a41 100644 --- a/hypothesis-python/tests/cover/test_provisional_strategies.py +++ b/hypothesis-python/tests/cover/test_provisional_strategies.py @@ -22,7 +22,7 @@ urls, ) -from tests.common.debug import find_any +from tests.common.debug import check_can_generate_examples, find_any @given(urls()) @@ -51,13 +51,17 @@ def test_is_URL(url): @pytest.mark.parametrize("max_element_length", [-1, 0, 4.0, 64, 128]) def test_invalid_domain_arguments(max_length, max_element_length): with pytest.raises(InvalidArgument): - domains(max_length=max_length, max_element_length=max_element_length).example() + check_can_generate_examples( + domains(max_length=max_length, max_element_length=max_element_length) + ) @pytest.mark.parametrize("max_length", [None, 4, 8, 255]) @pytest.mark.parametrize("max_element_length", [None, 1, 2, 4, 8, 63]) def test_valid_domains_arguments(max_length, max_element_length): - domains(max_length=max_length, max_element_length=max_element_length).example() + check_can_generate_examples( + domains(max_length=max_length, max_element_length=max_element_length) + ) @pytest.mark.parametrize("strategy", [domains(), urls()]) diff --git a/hypothesis-python/tests/cover/test_recursive.py b/hypothesis-python/tests/cover/test_recursive.py index 35ecab723b..1335987722 100644 --- a/hypothesis-python/tests/cover/test_recursive.py +++ b/hypothesis-python/tests/cover/test_recursive.py @@ -13,7 +13,7 @@ from hypothesis import given, strategies as st from hypothesis.errors import InvalidArgument -from tests.common.debug import find_any, minimal +from tests.common.debug import check_can_generate_examples, find_any, minimal @given(st.recursive(st.booleans(), st.lists, max_leaves=10)) @@ -38,13 +38,13 @@ def test_can_find_nested(): def test_recursive_call_validates_expand_returns_strategies(): with pytest.raises(InvalidArgument): - st.recursive(st.booleans(), lambda x: 1).example() + check_can_generate_examples(st.recursive(st.booleans(), lambda x: 1)) def test_recursive_call_validates_base_is_strategy(): x = st.recursive(1, lambda x: st.none()) with pytest.raises(InvalidArgument): - x.example() + check_can_generate_examples(x) def test_can_find_exactly_max_leaves(): @@ -87,4 +87,4 @@ def test_issue_1502_regression(s): ) def test_invalid_args(s): with pytest.raises(InvalidArgument): - s.example() + check_can_generate_examples(s) diff --git a/hypothesis-python/tests/cover/test_regex.py b/hypothesis-python/tests/cover/test_regex.py index f11f302687..db5a331ba2 100644 --- a/hypothesis-python/tests/cover/test_regex.py +++ b/hypothesis-python/tests/cover/test_regex.py @@ -28,7 +28,12 @@ regex_strategy, ) -from tests.common.debug import assert_all_examples, assert_no_examples, find_any +from tests.common.debug import ( + assert_all_examples, + assert_no_examples, + check_can_generate_examples, + find_any, +) def is_ascii(s): @@ -262,7 +267,9 @@ def test_can_handle_boundaries_nested(s): def test_groupref_not_shared_between_regex(): # If group references are (incorrectly!) shared between regex, this would # fail as the would only be one reference. - st.tuples(st.from_regex("(a)\\1"), st.from_regex("(b)\\1")).example() + check_can_generate_examples( + st.tuples(st.from_regex("(a)\\1"), st.from_regex("(b)\\1")) + ) @pytest.mark.skipif( @@ -294,7 +301,7 @@ def test_positive_lookbehind(): def test_positive_lookahead(): - st.from_regex("a(?=bc).*").filter(lambda s: s.startswith("abc")).example() + find_any(st.from_regex("a(?=bc).*"), lambda s: s.startswith("abc")) def test_negative_lookbehind(): diff --git a/hypothesis-python/tests/cover/test_sampled_from.py b/hypothesis-python/tests/cover/test_sampled_from.py index f0158f9228..05df230896 100644 --- a/hypothesis-python/tests/cover/test_sampled_from.py +++ b/hypothesis-python/tests/cover/test_sampled_from.py @@ -28,7 +28,11 @@ filter_not_satisfied, ) -from tests.common.debug import assert_all_examples +from tests.common.debug import ( + assert_all_examples, + assert_simple_property, + check_can_generate_examples, +) from tests.common.utils import fails_with an_enum = enum.Enum("A", "a b c") @@ -41,15 +45,15 @@ @fails_with(InvalidArgument) def test_cannot_sample_sets(): - sampled_from(set("abc")).example() + check_can_generate_examples(sampled_from(set("abc"))) def test_can_sample_sequence_without_warning(): - sampled_from([1, 2, 3]).example() + check_can_generate_examples(sampled_from([1, 2, 3])) def test_can_sample_ordereddict_without_warning(): - sampled_from(an_ordereddict).example() + check_can_generate_examples(sampled_from(an_ordereddict)) @pytest.mark.parametrize("enum_class", [an_enum, a_flag, an_empty_flag]) @@ -72,8 +76,9 @@ def test_unsat_filtered_sampling_in_rejection_stage(x): def test_easy_filtered_sampling(): - x = sampled_from(range(100)).filter(lambda x: x == 0).example() - assert x == 0 + assert_simple_property( + sampled_from(range(100)).filter(lambda x: x == 0), lambda x: x == 0 + ) @given(sampled_from(range(100)).filter(lambda x: x == 99)) @@ -193,7 +198,7 @@ class AnnotationsInsteadOfElements(enum.Enum): def test_suggests_elements_instead_of_annotations(): with pytest.raises(InvalidArgument, match="Cannot sample.*annotations.*dataclass"): - st.sampled_from(AnnotationsInsteadOfElements).example() + check_can_generate_examples(st.sampled_from(AnnotationsInsteadOfElements)) class TestErrorNoteBehavior3819: diff --git a/hypothesis-python/tests/cover/test_searchstrategy.py b/hypothesis-python/tests/cover/test_searchstrategy.py index 57543ee501..47d41c5e94 100644 --- a/hypothesis-python/tests/cover/test_searchstrategy.py +++ b/hypothesis-python/tests/cover/test_searchstrategy.py @@ -21,7 +21,11 @@ from hypothesis.strategies import booleans, integers, just, none, tuples from hypothesis.strategies._internal.utils import to_jsonable -from tests.common.debug import assert_no_examples +from tests.common.debug import ( + assert_no_examples, + assert_simple_property, + check_can_generate_examples, +) def test_or_errors_when_given_non_strategy(): @@ -62,7 +66,7 @@ def test_none_strategy_does_not_draw(): def test_can_map(): s = integers().map(pack=lambda t: "foo") - assert s.example() == "foo" + assert_simple_property(s, lambda v: v == "foo") def test_example_raises_unsatisfiable_when_too_filtered(): @@ -88,7 +92,7 @@ def test_can_flatmap_nameless(): def test_flatmap_with_invalid_expand(): with pytest.raises(InvalidArgument): - just(100).flatmap(lambda n: "a").example() + check_can_generate_examples(just(100).flatmap(lambda n: "a")) def test_jsonable(): diff --git a/hypothesis-python/tests/cover/test_simple_characters.py b/hypothesis-python/tests/cover/test_simple_characters.py index 71b81045aa..1410006a75 100644 --- a/hypothesis-python/tests/cover/test_simple_characters.py +++ b/hypothesis-python/tests/cover/test_simple_characters.py @@ -15,32 +15,41 @@ from hypothesis.errors import InvalidArgument from hypothesis.strategies import characters -from tests.common.debug import assert_no_examples, find_any, minimal +from tests.common.debug import ( + assert_no_examples, + check_can_generate_examples, + find_any, + minimal, +) from tests.common.utils import fails_with @fails_with(InvalidArgument) def test_nonexistent_category_argument(): - characters(exclude_categories=["foo"]).example() + check_can_generate_examples(characters(exclude_categories=["foo"])) def test_bad_codepoint_arguments(): with pytest.raises(InvalidArgument): - characters(min_codepoint=42, max_codepoint=24).example() + check_can_generate_examples(characters(min_codepoint=42, max_codepoint=24)) def test_exclude_all_available_range(): with pytest.raises(InvalidArgument): - characters( - min_codepoint=ord("0"), max_codepoint=ord("0"), exclude_characters="0" - ).example() + check_can_generate_examples( + characters( + min_codepoint=ord("0"), max_codepoint=ord("0"), exclude_characters="0" + ) + ) def test_when_nothing_could_be_produced(): with pytest.raises(InvalidArgument): - characters( - categories=["Cc"], min_codepoint=ord("0"), max_codepoint=ord("9") - ).example() + check_can_generate_examples( + characters( + categories=["Cc"], min_codepoint=ord("0"), max_codepoint=ord("9") + ) + ) def test_characters_of_specific_groups(): @@ -90,19 +99,21 @@ def test_find_something_rare(): def test_whitelisted_characters_alone(): with pytest.raises(InvalidArgument): - characters(include_characters="te02тест49st").example() + check_can_generate_examples(characters(include_characters="te02тест49st")) def test_whitelisted_characters_overlap_blacklisted_characters(): good_chars = "te02тест49st" bad_chars = "ts94тсет" with pytest.raises(InvalidArgument) as exc: - characters( - min_codepoint=ord("0"), - max_codepoint=ord("9"), - include_characters=good_chars, - exclude_characters=bad_chars, - ).example() + check_can_generate_examples( + characters( + min_codepoint=ord("0"), + max_codepoint=ord("9"), + include_characters=good_chars, + exclude_characters=bad_chars, + ) + ) assert repr(good_chars) in str(exc) assert repr(bad_chars) in str(exc) diff --git a/hypothesis-python/tests/cover/test_simple_collections.py b/hypothesis-python/tests/cover/test_simple_collections.py index 576f9f9ba2..06c581b115 100644 --- a/hypothesis-python/tests/cover/test_simple_collections.py +++ b/hypothesis-python/tests/cover/test_simple_collections.py @@ -28,7 +28,7 @@ tuples, ) -from tests.common.debug import find_any, minimal +from tests.common.debug import assert_simple_property, find_any, minimal from tests.common.utils import flaky @@ -83,8 +83,10 @@ def test_ordered_dictionaries_preserve_keys(): r = Random() keys = list(range(100)) r.shuffle(keys) - x = fixed_dictionaries(OrderedDict([(k, booleans()) for k in keys])).example() - assert list(x.keys()) == keys + assert_simple_property( + fixed_dictionaries(OrderedDict([(k, booleans()) for k in keys])), + lambda x: list(x.keys()) == keys, + ) @given(fixed_dictionaries({}, optional={0: booleans(), 1: nothing(), 2: booleans()})) @@ -153,11 +155,11 @@ def test_can_find_unique_lists_of_non_set_order(): def test_can_draw_empty_list_from_unsatisfiable_strategy(): - assert find_any(lists(integers().filter(lambda s: False))) == [] + find_any(lists(integers().filter(lambda s: False)), lambda v: v == []) def test_can_draw_empty_set_from_unsatisfiable_strategy(): - assert find_any(sets(integers().filter(lambda s: False))) == set() + find_any(sets(integers().filter(lambda s: False)), lambda v: v == set()) @given(lists(sets(none()), min_size=10)) diff --git a/hypothesis-python/tests/cover/test_subnormal_floats.py b/hypothesis-python/tests/cover/test_subnormal_floats.py index e07207c961..ded75c2788 100644 --- a/hypothesis-python/tests/cover/test_subnormal_floats.py +++ b/hypothesis-python/tests/cover/test_subnormal_floats.py @@ -17,7 +17,7 @@ from hypothesis.strategies import floats from hypothesis.strategies._internal.numbers import next_down_normal, next_up_normal -from tests.common.debug import assert_no_examples, find_any +from tests.common.debug import assert_no_examples, check_can_generate_examples, find_any from tests.common.utils import PYTHON_FTZ pytestmark = [pytest.mark.skipif(PYTHON_FTZ, reason="broken by unsafe compiler flags")] @@ -42,7 +42,7 @@ def kw(marks=(), **kwargs): def test_subnormal_validation(kwargs): strat = floats(**kwargs, allow_subnormal=True) with pytest.raises(InvalidArgument): - strat.example() + check_can_generate_examples(strat) @pytest.mark.parametrize( diff --git a/hypothesis-python/tests/cover/test_type_lookup.py b/hypothesis-python/tests/cover/test_type_lookup.py index 0071718e86..97be789c42 100644 --- a/hypothesis-python/tests/cover/test_type_lookup.py +++ b/hypothesis-python/tests/cover/test_type_lookup.py @@ -20,7 +20,6 @@ HypothesisDeprecationWarning, InvalidArgument, ResolutionFailed, - SmallSearchSpaceWarning, ) from hypothesis.internal.compat import get_type_hints from hypothesis.internal.reflection import get_pretty_function_description @@ -28,7 +27,12 @@ from hypothesis.strategies._internal.types import _global_type_lookup from hypothesis.strategies._internal.utils import _strategies -from tests.common.debug import assert_all_examples, find_any +from tests.common.debug import ( + assert_all_examples, + assert_simple_property, + check_can_generate_examples, + find_any, +) from tests.common.utils import fails_with, temp_registered # Build a set of all types output by core strategies @@ -62,7 +66,7 @@ ): for n in range(3): try: - ex = thing(*([st.nothing()] * n)).example() + ex = check_can_generate_examples(thing(*([st.nothing()] * n))) types_with_core_strat.add(type(ex)) break except (TypeError, InvalidArgument, HypothesisDeprecationWarning): @@ -114,8 +118,8 @@ def test_lookup_values_are_strategies(typ, not_a_strategy): def test_lookup_overrides_defaults(typ): sentinel = object() with temp_registered(typ, st.just(sentinel)): - assert st.from_type(typ).example() is sentinel - assert st.from_type(typ).example() is not sentinel + assert_simple_property(st.from_type(typ), lambda v: v is sentinel) + assert_simple_property(st.from_type(typ), lambda v: v is not sentinel) class ParentUnknownType: @@ -130,30 +134,30 @@ def __init__(self, arg): def test_custom_type_resolution_fails_without_registering(): fails = st.from_type(UnknownType) with pytest.raises(ResolutionFailed): - fails.example() + check_can_generate_examples(fails) def test_custom_type_resolution(): sentinel = object() with temp_registered(UnknownType, st.just(sentinel)): - assert st.from_type(UnknownType).example() is sentinel + assert_simple_property(st.from_type(UnknownType), lambda v: v is sentinel) # Also covered by registration of child class - assert st.from_type(ParentUnknownType).example() is sentinel + assert_simple_property(st.from_type(ParentUnknownType), lambda v: v is sentinel) def test_custom_type_resolution_with_function(): sentinel = object() with temp_registered(UnknownType, lambda _: st.just(sentinel)): - assert st.from_type(UnknownType).example() is sentinel - assert st.from_type(ParentUnknownType).example() is sentinel + assert_simple_property(st.from_type(UnknownType), lambda v: v is sentinel) + assert_simple_property(st.from_type(ParentUnknownType), lambda v: v is sentinel) def test_custom_type_resolution_with_function_non_strategy(): with temp_registered(UnknownType, lambda _: None): with pytest.raises(ResolutionFailed): - st.from_type(UnknownType).example() + check_can_generate_examples(st.from_type(UnknownType)) with pytest.raises(ResolutionFailed): - st.from_type(ParentUnknownType).example() + check_can_generate_examples(st.from_type(ParentUnknownType)) @pytest.mark.parametrize("strategy_returned", [True, False]) @@ -168,20 +172,20 @@ def resolve_strategy(thing): with temp_registered(UnknownType, resolve_strategy): if strategy_returned: - assert st.from_type(UnknownType).example() is sentinel + assert_simple_property(st.from_type(UnknownType), lambda v: v is sentinel) else: with pytest.raises(ResolutionFailed): - st.from_type(UnknownType).example() + check_can_generate_examples(st.from_type(UnknownType)) def test_errors_if_generic_resolves_empty(): with temp_registered(UnknownType, lambda _: st.nothing()): fails_1 = st.from_type(UnknownType) with pytest.raises(ResolutionFailed): - fails_1.example() + check_can_generate_examples(fails_1) fails_2 = st.from_type(ParentUnknownType) with pytest.raises(ResolutionFailed): - fails_2.example() + check_can_generate_examples(fails_2) def test_cannot_register_empty(): @@ -190,15 +194,15 @@ def test_cannot_register_empty(): st.register_type_strategy(UnknownType, st.nothing()) fails = st.from_type(UnknownType) with pytest.raises(ResolutionFailed): - fails.example() + check_can_generate_examples(fails) assert UnknownType not in types._global_type_lookup def test_pulic_interface_works(): - st.from_type(int).example() + check_can_generate_examples(st.from_type(int)) fails = st.from_type("not a type or annotated function") with pytest.raises(InvalidArgument): - fails.example() + check_can_generate_examples(fails) @pytest.mark.parametrize("infer_token", [infer, ...]) @@ -227,13 +231,12 @@ class BrokenClass: def test_uninspectable_builds(): with pytest.raises(TypeError, match="object is not callable"): - st.builds(BrokenClass).example() + check_can_generate_examples(st.builds(BrokenClass)) def test_uninspectable_from_type(): with pytest.raises(TypeError, match="object is not callable"): - with pytest.warns(SmallSearchSpaceWarning): - st.from_type(BrokenClass).example() + check_can_generate_examples(st.from_type(BrokenClass)) def _check_instances(t): @@ -284,7 +287,7 @@ def using_concrete_generic(instance: MyGeneric[int]) -> int: def test_generic_origin_empty(): with pytest.raises(ResolutionFailed): - find_any(st.builds(using_generic)) + check_can_generate_examples(st.builds(using_generic)) def test_issue_2951_regression(): @@ -355,7 +358,7 @@ def test_generic_origin_without_type_args(generic): ) def test_generic_origin_from_type(strat, type_): with temp_registered(MyGeneric, st.builds(MyGeneric)): - find_any(strat(type_)) + check_can_generate_examples(strat(type_)) def test_generic_origin_concrete_builds(): @@ -418,14 +421,14 @@ def test_abstract_resolver_fallback(): # And trying to generate an instance of the abstract type fails, # UNLESS the concrete type is currently resolvable with pytest.raises(ResolutionFailed): - gen_abstractbar.example() + check_can_generate_examples(gen_abstractbar) with temp_registered(ConcreteBar, gen_concretebar): - gen = gen_abstractbar.example() + # which in turn means we resolve to the concrete subtype. + assert_simple_property( + gen_abstractbar, lambda gen: isinstance(gen, ConcreteBar) + ) with pytest.raises(ResolutionFailed): - gen_abstractbar.example() - - # which in turn means we resolve to the concrete subtype. - assert isinstance(gen, ConcreteBar) + check_can_generate_examples(gen_abstractbar) def _one_arg(x: int): diff --git a/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py b/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py index 46e55c92b6..d92ef3ad1e 100644 --- a/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py +++ b/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py @@ -30,6 +30,7 @@ from hypothesis.errors import ResolutionFailed from tests.common import utils +from tests.common.debug import check_can_generate_examples if TYPE_CHECKING: from tests.common.utils import ExcInfo # we just need any type @@ -101,7 +102,7 @@ def test_bound_correct_dot_access_forward_ref(built): def test_bound_missing_dot_access_forward_ref(function): """Resolution of missing type in dot access.""" with pytest.raises(ResolutionFailed): - st.builds(function).example() + check_can_generate_examples(st.builds(function)) # Missing: @@ -116,7 +117,7 @@ def missing_fun(thing: _Missing) -> int: def test_bound_missing_forward_ref(): """We should raise proper errors on missing types.""" with pytest.raises(ResolutionFailed): - st.builds(missing_fun).example() + check_can_generate_examples(st.builds(missing_fun)) # Type checking only: @@ -131,11 +132,11 @@ def typechecking_only_fun(thing: _TypeChecking) -> int: def test_bound_type_cheking_only_forward_ref(): """We should fallback to registering explicit ``ForwardRef`` when we have to.""" with utils.temp_registered(ForwardRef("ExcInfo"), st.just(1)): - st.builds(typechecking_only_fun).example() + check_can_generate_examples(st.builds(typechecking_only_fun)) def test_bound_type_checking_only_forward_ref_wrong_type(): """We should check ``ForwardRef`` parameter name correctly.""" with utils.temp_registered(ForwardRef("WrongType"), st.just(1)): with pytest.raises(ResolutionFailed): - st.builds(typechecking_only_fun).example() + check_can_generate_examples(st.builds(typechecking_only_fun)) diff --git a/hypothesis-python/tests/cover/test_uuids.py b/hypothesis-python/tests/cover/test_uuids.py index 3103439186..6e381e76fb 100644 --- a/hypothesis-python/tests/cover/test_uuids.py +++ b/hypothesis-python/tests/cover/test_uuids.py @@ -15,7 +15,7 @@ from hypothesis import strategies as st from hypothesis.errors import InvalidArgument -from tests.common.debug import assert_no_examples, find_any +from tests.common.debug import assert_no_examples, check_can_generate_examples, find_any def test_no_nil_uuid_by_default(): @@ -27,8 +27,8 @@ def test_can_generate_nil_uuid(): def test_can_only_allow_nil_uuid_with_none_version(): - st.uuids(version=None, allow_nil=True).example() + check_can_generate_examples(st.uuids(version=None, allow_nil=True)) with pytest.raises(InvalidArgument): - st.uuids(version=4, allow_nil=True).example() + check_can_generate_examples(st.uuids(version=4, allow_nil=True)) with pytest.raises(InvalidArgument): - st.uuids(version=None, allow_nil="not a bool").example() + check_can_generate_examples(st.uuids(version=None, allow_nil="not a bool")) diff --git a/hypothesis-python/tests/cover/test_validation.py b/hypothesis-python/tests/cover/test_validation.py index c7c57573ce..fdd88ef641 100644 --- a/hypothesis-python/tests/cover/test_validation.py +++ b/hypothesis-python/tests/cover/test_validation.py @@ -32,6 +32,7 @@ ) from hypothesis.strategies._internal.strategies import check_strategy +from tests.common.debug import check_can_generate_examples from tests.common.utils import fails_with @@ -113,28 +114,28 @@ def foo(x, y, z): def test_float_ranges(): with pytest.raises(InvalidArgument): - floats(float("nan"), 0).example() + check_can_generate_examples(floats(float("nan"), 0)) with pytest.raises(InvalidArgument): - floats(1, -1).example() + check_can_generate_examples(floats(1, -1)) def test_float_range_and_allow_nan_cannot_both_be_enabled(): with pytest.raises(InvalidArgument): - floats(min_value=1, allow_nan=True).example() + check_can_generate_examples(floats(min_value=1, allow_nan=True)) with pytest.raises(InvalidArgument): - floats(max_value=1, allow_nan=True).example() + check_can_generate_examples(floats(max_value=1, allow_nan=True)) def test_float_finite_range_and_allow_infinity_cannot_both_be_enabled(): with pytest.raises(InvalidArgument): - floats(0, 1, allow_infinity=True).example() + check_can_generate_examples(floats(0, 1, allow_infinity=True)) def test_does_not_error_if_min_size_is_bigger_than_default_size(): - lists(integers(), min_size=50).example() - sets(integers(), min_size=50).example() - frozensets(integers(), min_size=50).example() - lists(integers(), min_size=50, unique=True).example() + check_can_generate_examples(lists(integers(), min_size=50)) + check_can_generate_examples(sets(integers(), min_size=50)) + check_can_generate_examples(frozensets(integers(), min_size=50)) + check_can_generate_examples(lists(integers(), min_size=50, unique=True)) def test_list_unique_and_unique_by_cannot_both_be_enabled(): diff --git a/hypothesis-python/tests/django/toystore/test_given_models.py b/hypothesis-python/tests/django/toystore/test_given_models.py index 6a368a7828..f1f42a373b 100644 --- a/hypothesis-python/tests/django/toystore/test_given_models.py +++ b/hypothesis-python/tests/django/toystore/test_given_models.py @@ -26,6 +26,7 @@ from hypothesis.internal.conjecture.data import ConjectureData from hypothesis.strategies import binary, just, lists +from tests.common.debug import check_can_generate_examples from tests.django.toystore.models import ( Car, Company, @@ -104,7 +105,7 @@ def test_mandatory_fields_are_mandatory(self): def test_mandatory_computed_fields_are_mandatory(self): with self.assertRaises(InvalidArgument): - from_model(MandatoryComputed).example() + check_can_generate_examples(from_model(MandatoryComputed)) def test_mandatory_computed_fields_may_not_be_provided(self): mc = from_model(MandatoryComputed, company=from_model(Company)) @@ -163,7 +164,7 @@ def test_foreign_key_primary(self, buf): class TestsNeedingRollback(TransactionTestCase): def test_can_get_examples(self): for _ in range(200): - from_model(Company).example() + check_can_generate_examples(from_model(Company)) class TestRestrictedFields(TestCase): diff --git a/hypothesis-python/tests/lark/test_grammar.py b/hypothesis-python/tests/lark/test_grammar.py index 26da7281df..a491f820fb 100644 --- a/hypothesis-python/tests/lark/test_grammar.py +++ b/hypothesis-python/tests/lark/test_grammar.py @@ -18,7 +18,7 @@ from hypothesis.extra.lark import from_lark from hypothesis.strategies import characters, data, just -from tests.common.debug import find_any +from tests.common.debug import check_can_generate_examples, find_any # Adapted from the official Lark tutorial, with modifications to ensure # that the generated JSON is valid. i.e. no numbers starting with ".", @@ -92,13 +92,15 @@ def test_generation_without_whitespace(): def test_cannot_convert_EBNF_to_strategy_directly(): with pytest.raises(InvalidArgument): # Not a Lark object - from_lark(EBNF_GRAMMAR).example() + check_can_generate_examples(from_lark(EBNF_GRAMMAR)) with pytest.raises(TypeError): # Not even the right number of arguments - from_lark(EBNF_GRAMMAR, start="value").example() + check_can_generate_examples(from_lark(EBNF_GRAMMAR, start="value")) with pytest.raises(InvalidArgument): # Wrong type for explicit_strategies - from_lark(Lark(LIST_GRAMMAR, start="list"), explicit=[]).example() + check_can_generate_examples( + from_lark(Lark(LIST_GRAMMAR, start="list"), explicit=[]) + ) def test_required_undefined_terminals_require_explicit_strategies(): @@ -107,23 +109,27 @@ def test_required_undefined_terminals_require_explicit_strategies(): %declare ELEMENT """ with pytest.raises(InvalidArgument, match=r"%declare"): - from_lark(Lark(elem_grammar, start="list")).example() + check_can_generate_examples(from_lark(Lark(elem_grammar, start="list"))) strategy = {"ELEMENT": just("200")} - from_lark(Lark(elem_grammar, start="list"), explicit=strategy).example() + check_can_generate_examples( + from_lark(Lark(elem_grammar, start="list"), explicit=strategy) + ) def test_cannot_use_explicit_strategies_for_unknown_terminals(): with pytest.raises(InvalidArgument): - from_lark( - Lark(LIST_GRAMMAR, start="list"), explicit={"unused_name": just("")} - ).example() + check_can_generate_examples( + from_lark( + Lark(LIST_GRAMMAR, start="list"), explicit={"unused_name": just("")} + ) + ) def test_non_string_explicit_strategies_are_invalid(): with pytest.raises(InvalidArgument): - from_lark( - Lark(LIST_GRAMMAR, start="list"), explicit={"NUMBER": just(0)} - ).example() + check_can_generate_examples( + from_lark(Lark(LIST_GRAMMAR, start="list"), explicit={"NUMBER": just(0)}) + ) @given( @@ -157,4 +163,6 @@ def test_error_if_alphabet_bans_all_start_rules(): with pytest.raises( InvalidArgument, match=r"No start rule .+ is allowed by alphabet=" ): - from_lark(Lark(LIST_GRAMMAR, start="list"), alphabet="abc").example() + check_can_generate_examples( + from_lark(Lark(LIST_GRAMMAR, start="list"), alphabet="abc") + ) diff --git a/hypothesis-python/tests/nocover/test_deferred_errors.py b/hypothesis-python/tests/nocover/test_deferred_errors.py index 1d871babe2..8333adf1d6 100644 --- a/hypothesis-python/tests/nocover/test_deferred_errors.py +++ b/hypothesis-python/tests/nocover/test_deferred_errors.py @@ -14,6 +14,8 @@ from hypothesis.errors import InvalidArgument from hypothesis.strategies._internal.core import defines_strategy +from tests.common.debug import check_can_generate_examples + def test_does_not_error_on_initial_calculation(): st.floats(max_value=float("nan")) @@ -25,9 +27,9 @@ def test_does_not_error_on_initial_calculation(): def test_errors_each_time(): s = st.integers(max_value=1, min_value=3) with pytest.raises(InvalidArgument): - s.example() + check_can_generate_examples(s) with pytest.raises(InvalidArgument): - s.example() + check_can_generate_examples(s) def test_errors_on_test_invocation(): @@ -48,7 +50,7 @@ def test_errors_on_find(): def test_errors_on_example(): s = st.floats(min_value=2.0, max_value=1.0) with pytest.raises(InvalidArgument): - s.example() + check_can_generate_examples(s) def test_does_not_recalculate_the_strategy(): @@ -61,7 +63,7 @@ def foo(): f = foo() assert calls == [0] - f.example() + check_can_generate_examples(f) assert calls == [1] - f.example() + check_can_generate_examples(f) assert calls == [1] diff --git a/hypothesis-python/tests/nocover/test_drypython_returns.py b/hypothesis-python/tests/nocover/test_drypython_returns.py index 08167667c0..6c1ce3e0a5 100644 --- a/hypothesis-python/tests/nocover/test_drypython_returns.py +++ b/hypothesis-python/tests/nocover/test_drypython_returns.py @@ -15,7 +15,7 @@ from hypothesis import given, strategies as st from hypothesis.errors import ResolutionFailed -from tests.common.debug import find_any +from tests.common.debug import check_can_generate_examples, find_any from tests.common.utils import temp_registered # Primitives: @@ -199,4 +199,4 @@ def wrong_generic_func2(obj: _SecondBase[None, bool]): def test_several_generic_bases_wrong_functions(func): with temp_registered(AllConcrete, st.builds(AllConcrete)): with pytest.raises(ResolutionFailed): - st.builds(func).example() + check_can_generate_examples(st.builds(func)) diff --git a/hypothesis-python/tests/nocover/test_flatmap.py b/hypothesis-python/tests/nocover/test_flatmap.py index 2843ddef02..239d256a8c 100644 --- a/hypothesis-python/tests/nocover/test_flatmap.py +++ b/hypothesis-python/tests/nocover/test_flatmap.py @@ -25,7 +25,7 @@ tuples, ) -from tests.common.debug import minimal +from tests.common.debug import find_any, minimal ConstantLists = integers().flatmap(lambda i: lists(just(i))) @@ -76,7 +76,7 @@ def record_and_test_size(xs): def test_flatmap_does_not_reuse_strategies(): s = builds(list).flatmap(just) - assert s.example() is not s.example() + assert find_any(s) is not find_any(s) def test_flatmap_has_original_strategy_repr(): diff --git a/hypothesis-python/tests/nocover/test_type_lookup.py b/hypothesis-python/tests/nocover/test_type_lookup.py index b5c2bfbd04..c19a246c44 100644 --- a/hypothesis-python/tests/nocover/test_type_lookup.py +++ b/hypothesis-python/tests/nocover/test_type_lookup.py @@ -17,6 +17,8 @@ from hypothesis.internal.compat import Concatenate, ParamSpec from hypothesis.strategies._internal.types import NON_RUNTIME_TYPES +from tests.common.debug import check_can_generate_examples + try: from typing import TypeGuard # new in 3.10 except ImportError: @@ -29,7 +31,7 @@ def test_non_runtime_type_cannot_be_resolved(non_runtime_type): with pytest.raises( InvalidArgument, match="there is no such thing as a runtime instance" ): - strategy.example() + check_can_generate_examples(strategy) @pytest.mark.parametrize("non_runtime_type", NON_RUNTIME_TYPES) @@ -49,7 +51,7 @@ def test_callable_with_concatenate(): InvalidArgument, match="Hypothesis can't yet construct a strategy for instances of a Callable type", ): - strategy.example() + check_can_generate_examples(strategy) with pytest.raises(InvalidArgument, match="Cannot register generic type"): st.register_type_strategy(func_type, st.none()) @@ -64,7 +66,7 @@ def test_callable_with_paramspec(): InvalidArgument, match="Hypothesis can't yet construct a strategy for instances of a Callable type", ): - strategy.example() + check_can_generate_examples(strategy) with pytest.raises(InvalidArgument, match="Cannot register generic type"): st.register_type_strategy(func_type, st.none()) @@ -78,7 +80,7 @@ def test_callable_return_typegard_type(): match="Hypothesis cannot yet construct a strategy for callables " "which are PEP-647 TypeGuards", ): - strategy.example() + check_can_generate_examples(strategy) with pytest.raises(InvalidArgument, match="Cannot register generic type"): st.register_type_strategy(Callable[[], TypeGuard[int]], st.none()) diff --git a/hypothesis-python/tests/nocover/test_type_lookup_future_annotations.py b/hypothesis-python/tests/nocover/test_type_lookup_future_annotations.py index d33550267e..f2ba7739c6 100644 --- a/hypothesis-python/tests/nocover/test_type_lookup_future_annotations.py +++ b/hypothesis-python/tests/nocover/test_type_lookup_future_annotations.py @@ -17,6 +17,8 @@ from hypothesis import given, strategies as st from hypothesis.errors import InvalidArgument +from tests.common.debug import check_can_generate_examples + alias = Union[int, str] @@ -45,4 +47,4 @@ class C(TypedDict): c_strategy = st.from_type(C) with pytest.raises(InvalidArgument): - c_strategy.example() + check_can_generate_examples(c_strategy) diff --git a/hypothesis-python/tests/numpy/test_argument_validation.py b/hypothesis-python/tests/numpy/test_argument_validation.py index 3215945152..1754929680 100644 --- a/hypothesis-python/tests/numpy/test_argument_validation.py +++ b/hypothesis-python/tests/numpy/test_argument_validation.py @@ -15,6 +15,7 @@ from hypothesis.errors import InvalidArgument from hypothesis.extra import numpy as nps +from tests.common.debug import check_can_generate_examples from tests.common.utils import checks_deprecated_behaviour @@ -274,7 +275,7 @@ def e(a, **kwargs): ) def test_raise_invalid_argument(function, kwargs): with pytest.raises(InvalidArgument): - function(**kwargs).example() + check_can_generate_examples(function(**kwargs)) @pytest.mark.parametrize( @@ -287,4 +288,4 @@ def test_raise_invalid_argument(function, kwargs): @checks_deprecated_behaviour def test_raise_invalid_argument_deprecated(function, kwargs): with pytest.raises(InvalidArgument): - function(**kwargs).example() + check_can_generate_examples(function(**kwargs)) diff --git a/hypothesis-python/tests/numpy/test_deprecation.py b/hypothesis-python/tests/numpy/test_deprecation.py index f1338c9518..50f066e110 100644 --- a/hypothesis-python/tests/numpy/test_deprecation.py +++ b/hypothesis-python/tests/numpy/test_deprecation.py @@ -15,20 +15,22 @@ from hypothesis.errors import HypothesisDeprecationWarning, InvalidArgument from hypothesis.extra import numpy as nps +from tests.common.debug import check_can_generate_examples + def test_basic_indices_bad_min_dims_warns(): with pytest.warns(HypothesisDeprecationWarning): with pytest.raises(InvalidArgument): - nps.basic_indices((3, 3, 3), min_dims=4).example() + check_can_generate_examples(nps.basic_indices((3, 3, 3), min_dims=4)) def test_basic_indices_bad_max_dims_warns(): with pytest.warns(HypothesisDeprecationWarning): - nps.basic_indices((3, 3, 3), max_dims=4).example() + check_can_generate_examples(nps.basic_indices((3, 3, 3), max_dims=4)) def test_basic_indices_default_max_dims_does_not_warn(): with catch_warnings(record=True) as record: - nps.basic_indices((3, 3, 3)).example() - nps.basic_indices((3, 3, 3), allow_newaxis=True).example() + check_can_generate_examples(nps.basic_indices((3, 3, 3))) + check_can_generate_examples(nps.basic_indices((3, 3, 3), allow_newaxis=True)) assert len(record) == 0 diff --git a/hypothesis-python/tests/numpy/test_from_dtype.py b/hypothesis-python/tests/numpy/test_from_dtype.py index cd40a7d6d1..b55d2ca48f 100644 --- a/hypothesis-python/tests/numpy/test_from_dtype.py +++ b/hypothesis-python/tests/numpy/test_from_dtype.py @@ -19,7 +19,7 @@ from hypothesis.internal.floats import width_smallest_normals from hypothesis.strategies._internal import SearchStrategy -from tests.common.debug import assert_no_examples, find_any +from tests.common.debug import assert_no_examples, check_can_generate_examples, find_any STANDARD_TYPES = [ np.dtype(t) @@ -183,7 +183,9 @@ def test_arrays_selects_consistent_time_unit(data, dtype_str): def test_arrays_gives_useful_error_on_inconsistent_time_unit(): with pytest.raises(InvalidArgument, match="mismatch of time units in dtypes"): - nps.arrays("m8[Y]", 10, elements=nps.from_dtype(np.dtype("m8[D]"))).example() + check_can_generate_examples( + nps.arrays("m8[Y]", 10, elements=nps.from_dtype(np.dtype("m8[D]"))) + ) @pytest.mark.parametrize( diff --git a/hypothesis-python/tests/numpy/test_from_type.py b/hypothesis-python/tests/numpy/test_from_type.py index a7747b3c37..0ca64098bd 100644 --- a/hypothesis-python/tests/numpy/test_from_type.py +++ b/hypothesis-python/tests/numpy/test_from_type.py @@ -19,7 +19,7 @@ from hypothesis.strategies import builds, from_type from .test_from_dtype import STANDARD_TYPES -from tests.common.debug import find_any +from tests.common.debug import assert_simple_property, find_any STANDARD_TYPES_TYPE = [dtype.type for dtype in STANDARD_TYPES] @@ -49,7 +49,7 @@ def test_resolves_and_varies_numpy_scalar_type(typ): @pytest.mark.parametrize("atype", [np.ndarray, NDArray]) def test_resolves_unspecified_array_type(atype): if atype is not None: - assert isinstance(from_type(atype).example(), np.ndarray) + assert_simple_property(from_type(atype), lambda v: isinstance(v, np.ndarray)) def workaround(dtype): @@ -65,21 +65,24 @@ def workaround(dtype): ) @pytest.mark.parametrize("typ", [workaround(t) for t in STANDARD_TYPES_TYPE]) def test_resolves_specified_ndarray_type(typ): - arr = from_type(np.ndarray[typ]).example() - assert isinstance(arr, np.ndarray) - assert arr.dtype.type == typ + assert_simple_property( + from_type(np.ndarray[typ]), + lambda arr: isinstance(arr, np.ndarray) and arr.dtype.type == typ, + ) - arr = from_type(np.ndarray[typing.Any, typ]).example() - assert isinstance(arr, np.ndarray) - assert arr.dtype.type == typ + assert_simple_property( + from_type(np.ndarray[typing.Any, typ]), + lambda arr: isinstance(arr, np.ndarray) and arr.dtype.type == typ, + ) @pytest.mark.skipif(NDArray is None, **needs_np_typing) @pytest.mark.parametrize("typ", [workaround(t) for t in STANDARD_TYPES_TYPE]) def test_resolves_specified_NDArray_type(typ): - arr = from_type(NDArray[typ]).example() - assert isinstance(arr, np.ndarray) - assert arr.dtype.type == typ + assert_simple_property( + from_type(NDArray[typ]), + lambda arr: isinstance(arr, np.ndarray) and arr.dtype.type == typ, + ) @pytest.mark.skipif(ArrayLike is None, **needs_np_typing) diff --git a/hypothesis-python/tests/numpy/test_gen_data.py b/hypothesis-python/tests/numpy/test_gen_data.py index 05fcd0254c..2b9da13c2e 100644 --- a/hypothesis-python/tests/numpy/test_gen_data.py +++ b/hypothesis-python/tests/numpy/test_gen_data.py @@ -27,7 +27,7 @@ from hypothesis.errors import InvalidArgument, UnsatisfiedAssumption from hypothesis.extra import numpy as nps -from tests.common.debug import find_any, minimal +from tests.common.debug import check_can_generate_examples, find_any, minimal from tests.common.utils import fails_with, flaky ANY_SHAPE = nps.array_shapes(min_dims=0, max_dims=32, min_side=0, max_side=32) @@ -142,7 +142,7 @@ def test_minimise_array_shapes(min_dims, dim_range, min_side, side_range): "kwargs", [{"min_side": 100}, {"min_dims": 15}, {"min_dims": 32}] ) def test_interesting_array_shapes_argument(kwargs): - nps.array_shapes(**kwargs).example() + check_can_generate_examples(nps.array_shapes(**kwargs)) @given(nps.scalar_dtypes()) @@ -257,7 +257,7 @@ def test_array_values_are_unique(arr): def test_cannot_generate_unique_array_of_too_many_elements(): strat = nps.arrays(dtype=int, elements=st.integers(0, 5), shape=10, unique=True) with pytest.raises(InvalidArgument): - strat.example() + check_can_generate_examples(strat) @given( diff --git a/hypothesis-python/tests/pandas/test_argument_validation.py b/hypothesis-python/tests/pandas/test_argument_validation.py index 6ebe474c5a..440badc4a0 100644 --- a/hypothesis-python/tests/pandas/test_argument_validation.py +++ b/hypothesis-python/tests/pandas/test_argument_validation.py @@ -20,7 +20,7 @@ from hypothesis.extra.pandas.impl import IntegerDtype from tests.common.arguments import argument_validation_test, e -from tests.common.debug import find_any +from tests.common.debug import check_can_generate_examples from tests.common.utils import checks_deprecated_behaviour BAD_ARGS = [ @@ -115,7 +115,9 @@ def test_timestamp_as_datetime_bounds(dt): @checks_deprecated_behaviour def test_confusing_object_dtype_aliases(): - pdst.series(elements=st.tuples(st.integers()), dtype=tuple).example() + check_can_generate_examples( + pdst.series(elements=st.tuples(st.integers()), dtype=tuple) + ) @pytest.mark.skipif( @@ -126,7 +128,7 @@ def test_pandas_nullable_types_class(): with pytest.raises( InvalidArgument, match="Otherwise it would be treated as dtype=object" ): - find_any(st, lambda s: s.isna().any()) + check_can_generate_examples(st, lambda s: s.isna().any()) @pytest.mark.parametrize( @@ -140,4 +142,4 @@ def test_pandas_nullable_types_class(): ) def test_invalid_datetime_or_timedelta_dtype_raises_error(dtype_, expected_unit): with pytest.raises(InvalidArgument, match=re.escape(expected_unit)): - pdst.series(dtype=dtype_).example() + check_can_generate_examples(pdst.series(dtype=dtype_)) diff --git a/hypothesis-python/tests/pandas/test_indexes.py b/hypothesis-python/tests/pandas/test_indexes.py index 317d327a2d..465730d565 100644 --- a/hypothesis-python/tests/pandas/test_indexes.py +++ b/hypothesis-python/tests/pandas/test_indexes.py @@ -18,6 +18,7 @@ from hypothesis.errors import Unsatisfiable from hypothesis.extra import numpy as npst, pandas as pdst +from tests.common.debug import check_can_generate_examples from tests.pandas.helpers import supported_by_pandas @@ -41,7 +42,7 @@ def test_gets_right_dtype_for_empty_indices_with_elements(ix): def test_does_not_generate_impossible_conditions(): with pytest.raises(Unsatisfiable): - pdst.indexes(min_size=3, max_size=3, dtype=bool).example() + check_can_generate_examples(pdst.indexes(min_size=3, max_size=3, dtype=bool)) @given(pdst.indexes(dtype=bool, unique=True)) diff --git a/hypothesis-python/tests/test_annotated_types.py b/hypothesis-python/tests/test_annotated_types.py index 17f09b0be1..e1b3103fd2 100644 --- a/hypothesis-python/tests/test_annotated_types.py +++ b/hypothesis-python/tests/test_annotated_types.py @@ -19,6 +19,8 @@ from hypothesis.strategies._internal.lazy import unwrap_strategies from hypothesis.strategies._internal.strategies import FilteredStrategy +from tests.common.debug import check_can_generate_examples + try: from typing import Annotated # new in Python 3.9 @@ -36,7 +38,9 @@ def test_strategy_priority_over_constraints(): def test_invalid_annotated_type(): with pytest.raises(ResolutionFailed): - st.from_type(Annotated[None, "dummy", Annotated[int, "dummy"]]).example() + check_can_generate_examples( + st.from_type(Annotated[None, "dummy", Annotated[int, "dummy"]]) + ) @pytest.mark.parametrize( @@ -57,7 +61,7 @@ def test_unsupported_constraints(unsupported_constraints, message): else: t = Annotated.__class_getitem__((int, *unsupported_constraints)) with pytest.warns(HypothesisWarning, match=re.escape(message)): - st.from_type(t).example() + check_can_generate_examples(st.from_type(t)) @pytest.mark.parametrize( diff --git a/hypothesis-python/tests/typing_extensions/test_backported_types.py b/hypothesis-python/tests/typing_extensions/test_backported_types.py index b1031e39df..74f24e2f6e 100644 --- a/hypothesis-python/tests/typing_extensions/test_backported_types.py +++ b/hypothesis-python/tests/typing_extensions/test_backported_types.py @@ -28,14 +28,21 @@ from hypothesis.strategies import from_type from hypothesis.strategies._internal.types import NON_RUNTIME_TYPES -from tests.common.debug import assert_all_examples, find_any +from tests.common.debug import ( + assert_all_examples, + assert_simple_property, + check_can_generate_examples, + find_any, +) # See also nocover/test_type_lookup.py @pytest.mark.parametrize("value", ["dog", b"goldfish", 42, 63.4, -80.5, False]) def test_typing_extensions_Literal(value): - assert from_type(typing_extensions.Literal[value]).example() == value + assert_simple_property( + from_type(typing_extensions.Literal[value]), lambda v: v == value + ) @given(st.data()) @@ -68,7 +75,7 @@ def test_simple_typeddict(value): def test_typing_extensions_Type_int(): - assert from_type(Type[int]).example() is int + assert_simple_property(from_type(Type[int]), lambda v: v is int) @given(from_type(Union[Type[str], Type[list]])) @@ -80,9 +87,11 @@ def test_resolves_NewType(): typ = NewType("T", int) nested = NewType("NestedT", typ) uni = NewType("UnionT", Union[int, None]) - assert isinstance(from_type(typ).example(), int) - assert isinstance(from_type(nested).example(), int) - assert isinstance(from_type(uni).example(), (int, type(None))) + assert_simple_property(from_type(typ), lambda x: isinstance(x, int)) + assert_simple_property(from_type(nested), lambda x: isinstance(x, int)) + assert_simple_property(from_type(uni), lambda x: isinstance(x, (int, type(None)))) + find_any(from_type(uni), lambda x: isinstance(x, int)) + find_any(from_type(uni), lambda x: isinstance(x, type(None))) @given(from_type(DefaultDict[int, int])) @@ -149,7 +158,7 @@ def test_non_runtime_type_cannot_be_resolved(non_runtime_type): with pytest.raises( InvalidArgument, match="there is no such thing as a runtime instance" ): - strategy.example() + check_can_generate_examples(strategy) @pytest.mark.parametrize("non_runtime_type", NON_RUNTIME_TYPES) @@ -168,7 +177,7 @@ def test_callable_with_concatenate(): InvalidArgument, match="Hypothesis can't yet construct a strategy for instances of a Callable type", ): - strategy.example() + check_can_generate_examples(strategy) with pytest.raises(InvalidArgument, match="Cannot register generic type"): st.register_type_strategy(func_type, st.none()) @@ -182,7 +191,7 @@ def test_callable_with_paramspec(): InvalidArgument, match="Hypothesis can't yet construct a strategy for instances of a Callable type", ): - strategy.example() + check_can_generate_examples(strategy) with pytest.raises(InvalidArgument, match="Cannot register generic type"): st.register_type_strategy(func_type, st.none()) @@ -195,7 +204,7 @@ def test_callable_return_typegard_type(): match="Hypothesis cannot yet construct a strategy for callables " "which are PEP-647 TypeGuards", ): - strategy.example() + check_can_generate_examples(strategy) with pytest.raises(InvalidArgument, match="Cannot register generic type"): st.register_type_strategy(Callable[[], TypeGuard[int]], st.none()) From 89e122b6acf9d1388bb653c8c1611fd57c19dbc6 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 10:06:02 +0100 Subject: [PATCH 03/11] Performance tweaks --- hypothesis-python/tests/common/debug.py | 2 +- hypothesis-python/tests/cover/test_searchstrategy.py | 11 ++++------- hypothesis-python/tests/cover/test_validation.py | 10 +++++----- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/hypothesis-python/tests/common/debug.py b/hypothesis-python/tests/common/debug.py index 1759333ca9..b6002d05f0 100644 --- a/hypothesis-python/tests/common/debug.py +++ b/hypothesis-python/tests/common/debug.py @@ -118,7 +118,7 @@ def assert_simple_property(strategy, predicate, settings=None): def check_can_generate_examples(strategy, settings=None): """Tries to generate a small number of examples from the strategy, to verify that it can - do so without raising. + do so without raising. Nothing is returned, it only checks that no error is raised. """ diff --git a/hypothesis-python/tests/cover/test_searchstrategy.py b/hypothesis-python/tests/cover/test_searchstrategy.py index 47d41c5e94..56cdd736a9 100644 --- a/hypothesis-python/tests/cover/test_searchstrategy.py +++ b/hypothesis-python/tests/cover/test_searchstrategy.py @@ -15,17 +15,13 @@ import attr import pytest -from hypothesis.errors import InvalidArgument +from hypothesis.errors import InvalidArgument, Unsatisfiable from hypothesis.internal.conjecture.data import ConjectureData from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies import booleans, integers, just, none, tuples from hypothesis.strategies._internal.utils import to_jsonable -from tests.common.debug import ( - assert_no_examples, - assert_simple_property, - check_can_generate_examples, -) +from tests.common.debug import assert_simple_property, check_can_generate_examples def test_or_errors_when_given_non_strategy(): @@ -70,7 +66,8 @@ def test_can_map(): def test_example_raises_unsatisfiable_when_too_filtered(): - assert_no_examples(integers().filter(lambda x: False)) + with pytest.raises(Unsatisfiable): + check_can_generate_examples(integers().filter(lambda x: False)) def nameless_const(x): diff --git a/hypothesis-python/tests/cover/test_validation.py b/hypothesis-python/tests/cover/test_validation.py index fdd88ef641..831c199b8e 100644 --- a/hypothesis-python/tests/cover/test_validation.py +++ b/hypothesis-python/tests/cover/test_validation.py @@ -32,7 +32,7 @@ ) from hypothesis.strategies._internal.strategies import check_strategy -from tests.common.debug import check_can_generate_examples +from tests.common.debug import check_can_generate_examples, find_any from tests.common.utils import fails_with @@ -132,10 +132,10 @@ def test_float_finite_range_and_allow_infinity_cannot_both_be_enabled(): def test_does_not_error_if_min_size_is_bigger_than_default_size(): - check_can_generate_examples(lists(integers(), min_size=50)) - check_can_generate_examples(sets(integers(), min_size=50)) - check_can_generate_examples(frozensets(integers(), min_size=50)) - check_can_generate_examples(lists(integers(), min_size=50, unique=True)) + find_any(lists(integers(), min_size=50)) + find_any(sets(integers(), min_size=50)) + find_any(frozensets(integers(), min_size=50)) + find_any(lists(integers(), min_size=50, unique=True)) def test_list_unique_and_unique_by_cannot_both_be_enabled(): From 1c14613674a677fe0e44ca716a1e4ca8dc8e2a04 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 11:02:23 +0100 Subject: [PATCH 04/11] Fix test failures --- hypothesis-python/tests/common/debug.py | 11 +++++++++-- hypothesis-python/tests/cover/test_example.py | 1 - .../tests/django/toystore/test_given_models.py | 9 ++++++--- .../tests/pandas/test_argument_validation.py | 6 +++--- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/hypothesis-python/tests/common/debug.py b/hypothesis-python/tests/common/debug.py index b6002d05f0..5da99cd8b4 100644 --- a/hypothesis-python/tests/common/debug.py +++ b/hypothesis-python/tests/common/debug.py @@ -113,7 +113,15 @@ def assert_simple_property(strategy, predicate, settings=None): For more thorough checking, use assert_all_examples. """ - assert_all_examples(strategy, predicate, Settings(parent=settings, max_examples=15)) + assert_all_examples( + strategy, + predicate, + Settings( + parent=settings, + max_examples=15, + suppress_health_check=list(HealthCheck), + ), + ) def check_can_generate_examples(strategy, settings=None): @@ -129,6 +137,5 @@ def check_can_generate_examples(strategy, settings=None): settings=Settings( parent=settings, phases=(Phase.generate,), - suppress_health_check=list(HealthCheck), ), ) diff --git a/hypothesis-python/tests/cover/test_example.py b/hypothesis-python/tests/cover/test_example.py index 1eaedabc61..818cbe0408 100644 --- a/hypothesis-python/tests/cover/test_example.py +++ b/hypothesis-python/tests/cover/test_example.py @@ -8,7 +8,6 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. - import pytest from hypothesis import example, given, strategies as st diff --git a/hypothesis-python/tests/django/toystore/test_given_models.py b/hypothesis-python/tests/django/toystore/test_given_models.py index f1f42a373b..5fba151c70 100644 --- a/hypothesis-python/tests/django/toystore/test_given_models.py +++ b/hypothesis-python/tests/django/toystore/test_given_models.py @@ -101,15 +101,18 @@ def test_custom_field(self, x): assert x.customish == "a" def test_mandatory_fields_are_mandatory(self): - self.assertRaises(InvalidArgument, from_model(Store).example) + with self.assertRaises(InvalidArgument): + check_can_generate_examples(from_model(Store)) def test_mandatory_computed_fields_are_mandatory(self): with self.assertRaises(InvalidArgument): check_can_generate_examples(from_model(MandatoryComputed)) def test_mandatory_computed_fields_may_not_be_provided(self): - mc = from_model(MandatoryComputed, company=from_model(Company)) - self.assertRaises(RuntimeError, mc.example) + with self.assertRaises(RuntimeError): + check_can_generate_examples( + from_model(MandatoryComputed, company=from_model(Company)) + ) @given(from_model(CustomishDefault, customish=...)) def test_customish_default_overridden_by_infer(self, x): diff --git a/hypothesis-python/tests/pandas/test_argument_validation.py b/hypothesis-python/tests/pandas/test_argument_validation.py index 440badc4a0..1484036676 100644 --- a/hypothesis-python/tests/pandas/test_argument_validation.py +++ b/hypothesis-python/tests/pandas/test_argument_validation.py @@ -20,7 +20,7 @@ from hypothesis.extra.pandas.impl import IntegerDtype from tests.common.arguments import argument_validation_test, e -from tests.common.debug import check_can_generate_examples +from tests.common.debug import check_can_generate_examples, find_any from tests.common.utils import checks_deprecated_behaviour BAD_ARGS = [ @@ -124,11 +124,11 @@ def test_confusing_object_dtype_aliases(): not IntegerDtype, reason="Nullable types not available in this version of Pandas" ) def test_pandas_nullable_types_class(): - st = pdst.series(dtype=pd.core.arrays.integer.Int8Dtype) with pytest.raises( InvalidArgument, match="Otherwise it would be treated as dtype=object" ): - check_can_generate_examples(st, lambda s: s.isna().any()) + st = pdst.series(dtype=pd.core.arrays.integer.Int8Dtype) + find_any(st, lambda s: s.isna().any()) @pytest.mark.parametrize( From 427d8e358dd4c181541dec947e0695b9f0660e15 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 11:33:07 +0100 Subject: [PATCH 05/11] Forgot to add new test file --- .../tests/cover/test_interactive_example.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 hypothesis-python/tests/cover/test_interactive_example.py diff --git a/hypothesis-python/tests/cover/test_interactive_example.py b/hypothesis-python/tests/cover/test_interactive_example.py new file mode 100644 index 0000000000..fabdc4188c --- /dev/null +++ b/hypothesis-python/tests/cover/test_interactive_example.py @@ -0,0 +1,100 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import sys +import warnings +from decimal import Decimal + +import pexpect +import pytest + +from hypothesis import example, find, given, strategies as st +from hypothesis.errors import ( + HypothesisException, + InvalidArgument, + NonInteractiveExampleWarning, + Unsatisfiable, +) +from hypothesis.internal.compat import WINDOWS + +from tests.common.debug import find_any +from tests.common.utils import fails_with + + +# Allow calling .example() without warnings for all tests in this module +@pytest.fixture(scope="function", autouse=True) +def _allow_noninteractive_example(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", NonInteractiveExampleWarning) + yield + + +def test_example_of_none_is_none(): + assert st.none().example() is None + + +def test_exception_in_compare_can_still_have_example(): + st.one_of(st.none().map(lambda n: Decimal("snan")), st.just(Decimal(0))).example() + + +def test_does_not_always_give_the_same_example(): + s = st.integers() + assert len({s.example() for _ in range(100)}) >= 10 + + +def test_raises_on_no_examples(): + with pytest.raises(Unsatisfiable): + st.nothing().example() + + +@fails_with(HypothesisException) +@example(False) +@given(st.booleans()) +def test_example_inside_given(b): + st.integers().example() + + +@fails_with(HypothesisException) +def test_example_inside_find(): + find(st.integers(0, 100), lambda x: st.integers().example()) + + +@fails_with(HypothesisException) +def test_example_inside_strategy(): + find_any(st.booleans().map(lambda x: st.integers().example())) + + +def test_raises_on_arbitrary_data(): + with pytest.raises(InvalidArgument): + st.data().example() + + +def test_non_interactive_example_emits_warning(): + # Revert the effect of the allow_noninteractive_example autouse fixture + with warnings.catch_warnings(): + warnings.simplefilter("always") + with pytest.warns(NonInteractiveExampleWarning): + st.text().example() + + +@pytest.mark.skipif(WINDOWS, reason="pexpect.spawn not supported on Windows") +def test_interactive_example_does_not_emit_warning(): + try: + child = pexpect.spawn(f"{sys.executable} -Werror") + child.expect(">>> ", timeout=10) + except pexpect.exceptions.EOF: + pytest.skip( + "Unable to run python with -Werror. This may be because you are " + "running from an old virtual environment - update your installed " + "copy of `virtualenv` and then create a fresh environment." + ) + child.sendline("from hypothesis.strategies import none") + child.sendline("none().example()") + child.sendline("quit(code=0)") From 031eef070fe0b97ccce2532f5e117bb68e1ec1c0 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 12:53:57 +0100 Subject: [PATCH 06/11] Fix plus a few reverts of unnecessary churn --- hypothesis-python/tests/cover/test_simple_collections.py | 4 ++-- hypothesis-python/tests/cover/test_type_lookup.py | 2 +- hypothesis-python/tests/pandas/test_argument_validation.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hypothesis-python/tests/cover/test_simple_collections.py b/hypothesis-python/tests/cover/test_simple_collections.py index 06c581b115..b811f50c8c 100644 --- a/hypothesis-python/tests/cover/test_simple_collections.py +++ b/hypothesis-python/tests/cover/test_simple_collections.py @@ -155,11 +155,11 @@ def test_can_find_unique_lists_of_non_set_order(): def test_can_draw_empty_list_from_unsatisfiable_strategy(): - find_any(lists(integers().filter(lambda s: False)), lambda v: v == []) + assert find_any(lists(integers().filter(lambda s: False))) == [] def test_can_draw_empty_set_from_unsatisfiable_strategy(): - find_any(sets(integers().filter(lambda s: False)), lambda v: v == set()) + assert find_any(sets(integers().filter(lambda s: False))) == set() @given(lists(sets(none()), min_size=10)) diff --git a/hypothesis-python/tests/cover/test_type_lookup.py b/hypothesis-python/tests/cover/test_type_lookup.py index 97be789c42..bf4a9e5c46 100644 --- a/hypothesis-python/tests/cover/test_type_lookup.py +++ b/hypothesis-python/tests/cover/test_type_lookup.py @@ -66,7 +66,7 @@ ): for n in range(3): try: - ex = check_can_generate_examples(thing(*([st.nothing()] * n))) + ex = find_any(thing(*([st.nothing()] * n))) types_with_core_strat.add(type(ex)) break except (TypeError, InvalidArgument, HypothesisDeprecationWarning): diff --git a/hypothesis-python/tests/pandas/test_argument_validation.py b/hypothesis-python/tests/pandas/test_argument_validation.py index 1484036676..dc13185a37 100644 --- a/hypothesis-python/tests/pandas/test_argument_validation.py +++ b/hypothesis-python/tests/pandas/test_argument_validation.py @@ -124,10 +124,10 @@ def test_confusing_object_dtype_aliases(): not IntegerDtype, reason="Nullable types not available in this version of Pandas" ) def test_pandas_nullable_types_class(): + st = pdst.series(dtype=pd.core.arrays.integer.Int8Dtype) with pytest.raises( InvalidArgument, match="Otherwise it would be treated as dtype=object" ): - st = pdst.series(dtype=pd.core.arrays.integer.Int8Dtype) find_any(st, lambda s: s.isna().any()) From af64e12b4ea5bfddb0f93867105dba14c3470292 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 13:43:43 +0100 Subject: [PATCH 07/11] Add a no-cover - random may be initialized in deterministic_PRNG --- hypothesis-python/src/hypothesis/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 17876c3715..a2a08461a6 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -628,7 +628,7 @@ def get_random_for_wrapped_test(test, wrapped_test): return Random(global_force_seed) else: global _hypothesis_global_random - if _hypothesis_global_random is None: + if _hypothesis_global_random is None: # pragma: no cover _hypothesis_global_random = Random() seed = _hypothesis_global_random.getrandbits(128) wrapped_test._hypothesis_internal_use_generated_seed = seed From de5a04ca544d97ef422b95505446f63a8873e572 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 14:07:48 +0100 Subject: [PATCH 08/11] Needs a RELEASE.rst now --- hypothesis-python/RELEASE.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..f7faef243e --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,3 @@ +RELEASE_TYPE: patch + +Internal test refactoring. From d93ec6d8b31976ede54f56146b1d7c05e2963e97 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Mon, 29 Jan 2024 14:16:27 +0100 Subject: [PATCH 09/11] Possibly fix buffer overrun flake (related to db reuse phase?) --- hypothesis-python/tests/common/debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/tests/common/debug.py b/hypothesis-python/tests/common/debug.py index 5da99cd8b4..fa18627486 100644 --- a/hypothesis-python/tests/common/debug.py +++ b/hypothesis-python/tests/common/debug.py @@ -97,7 +97,7 @@ def assert_all_examples(strategy, predicate, settings=None): """ @given(strategy) - @Settings(parent=settings) + @Settings(parent=settings, database=None) def assert_examples(s): msg = f"Found {s!r} using strategy {strategy} which does not match" assert predicate(s), msg From 2bca570ae84cf0174937fe15305d4a6ec1e1166f Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Tue, 30 Jan 2024 09:17:21 +0100 Subject: [PATCH 10/11] Add note to NonInteracteExampleWarning in selftests --- hypothesis-python/tests/conftest.py | 16 ++++++++++--- .../tests/cover/test_interactive_example.py | 23 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/hypothesis-python/tests/conftest.py b/hypothesis-python/tests/conftest.py index 17c5225376..f5e4eca5ef 100644 --- a/hypothesis-python/tests/conftest.py +++ b/hypothesis-python/tests/conftest.py @@ -17,6 +17,8 @@ import pytest from hypothesis._settings import is_in_ci +from hypothesis.errors import NonInteractiveExampleWarning +from hypothesis.internal.compat import add_note from hypothesis.internal.detection import is_hypothesis_test from tests.common import TIME_INCREMENT @@ -106,20 +108,20 @@ def pytest_runtest_call(item): # This hookwrapper checks for PRNG state leaks from Hypothesis tests. # See: https://github.com/HypothesisWorks/hypothesis/issues/1919 if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)): - yield + outcome = yield elif "pytest_randomly" in sys.modules: # See https://github.com/HypothesisWorks/hypothesis/issues/3041 - this # branch exists to make it easier on external contributors, but should # never run in our CI (because that would disable the check entirely). assert not is_in_ci() - yield + outcome = yield else: # We start by peturbing the state of the PRNG, because repeatedly # leaking PRNG state resets state_after to the (previously leaked) # state_before, and that just shows as "no use of random". random.seed(independent_random.randrange(2**32)) before = random.getstate() - yield + outcome = yield after = random.getstate() if before != after: if after in random_states_after_tests: @@ -129,3 +131,11 @@ def pytest_runtest_call(item): "same global `random.getstate()`; this is probably a nasty bug!" ) random_states_after_tests[after] = item.nodeid + + # Annotate usage of .example() with a hint about alternatives + if isinstance(outcome.exception, NonInteractiveExampleWarning): + add_note( + outcome.exception, + "For hypothesis' own test suite, consider using one of the helper " + "methods in tests.common.debug instead.", + ) diff --git a/hypothesis-python/tests/cover/test_interactive_example.py b/hypothesis-python/tests/cover/test_interactive_example.py index fabdc4188c..c6fbee2269 100644 --- a/hypothesis-python/tests/cover/test_interactive_example.py +++ b/hypothesis-python/tests/cover/test_interactive_example.py @@ -27,6 +27,8 @@ from tests.common.debug import find_any from tests.common.utils import fails_with +pytest_plugins = "pytester" + # Allow calling .example() without warnings for all tests in this module @pytest.fixture(scope="function", autouse=True) @@ -84,6 +86,27 @@ def test_non_interactive_example_emits_warning(): st.text().example() +EXAMPLE_GENERATING_TEST = """ +from hypothesis import strategies as st + +def test_interactive_example(): + st.integers().example() +""" + + +def test_selftests_exception_contains_note(pytester): + # The note is added by a pytest hook, so we need to run it under pytest in a + # subenvironment with (effectively) the same toplevel conftest. + with warnings.catch_warnings(): + warnings.simplefilter("error") + + pytester.makeconftest("from tests.conftest import *") + result = pytester.runpytest_inprocess( + pytester.makepyfile(EXAMPLE_GENERATING_TEST) + ) + assert "helper methods in tests.common.debug" in "\n".join(result.outlines) + + @pytest.mark.skipif(WINDOWS, reason="pexpect.spawn not supported on Windows") def test_interactive_example_does_not_emit_warning(): try: From e20341af6b6874abe592dc41eb07397fe202f160 Mon Sep 17 00:00:00 2001 From: Joachim B Haga Date: Tue, 30 Jan 2024 12:44:27 +0100 Subject: [PATCH 11/11] Fix tests --- hypothesis-python/tests/conftest.py | 2 +- hypothesis-python/tests/cover/test_sideeffect_warnings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hypothesis-python/tests/conftest.py b/hypothesis-python/tests/conftest.py index f5e4eca5ef..858d4eafb1 100644 --- a/hypothesis-python/tests/conftest.py +++ b/hypothesis-python/tests/conftest.py @@ -133,7 +133,7 @@ def pytest_runtest_call(item): random_states_after_tests[after] = item.nodeid # Annotate usage of .example() with a hint about alternatives - if isinstance(outcome.exception, NonInteractiveExampleWarning): + if isinstance(getattr(outcome, "exception", None), NonInteractiveExampleWarning): add_note( outcome.exception, "For hypothesis' own test suite, consider using one of the helper " diff --git a/hypothesis-python/tests/cover/test_sideeffect_warnings.py b/hypothesis-python/tests/cover/test_sideeffect_warnings.py index d3fd6a223f..733df9aeda 100644 --- a/hypothesis-python/tests/cover/test_sideeffect_warnings.py +++ b/hypothesis-python/tests/cover/test_sideeffect_warnings.py @@ -27,7 +27,7 @@ @pytest.fixture def _extend_initialization(monkeypatch): - assert getattr(_hypothesis_globals, IN_INITIALIZATION_ATTR) == 0 + assert getattr(_hypothesis_globals, IN_INITIALIZATION_ATTR) <= 0 monkeypatch.setattr(_hypothesis_globals, IN_INITIALIZATION_ATTR, 1) fs.notice_initialization_restarted(warn=False) assert fs._first_postinit_what is None # validates state as given in comment above