From 88ec4dc3d351f1158c0645cc5df0bd65f39ff8a0 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 25 May 2023 08:56:06 -0700 Subject: [PATCH 1/5] events for more mark_invalid() cases --- AUTHORS.rst | 1 + hypothesis-python/RELEASE.rst | 4 ++++ .../src/hypothesis/internal/conjecture/data.py | 8 +++++--- .../src/hypothesis/internal/conjecture/utils.py | 4 ++-- .../src/hypothesis/strategies/_internal/collections.py | 4 ++-- .../src/hypothesis/strategies/_internal/datetime.py | 5 ++--- .../src/hypothesis/strategies/_internal/strategies.py | 6 ++---- hypothesis-python/tests/conjecture/test_test_data.py | 9 +++++++++ 8 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/AUTHORS.rst b/AUTHORS.rst index cd77860dcb..046203606b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -31,6 +31,7 @@ their individual contributions. * `Bryant Eisenbach `_ * `Buck Evan, copyright Google LLC `_ * `Cameron McGill `_ +* `Carl Meyer `_ * `Charles O'Farrell `_ * `Charlie Tanksley `_ * `Chase Garner `_ (chase@garner.red) diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..842ee56763 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,4 @@ +RELEASE_TYPE: patch + +Hypothesis will now record an event for more cases where data is marked +invalid, including for exceeding the internal depth limit. diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index 29b7143754..29d9025fd1 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -936,10 +936,10 @@ def draw(self, strategy: "SearchStrategy[Ex]", label: Optional[int] = None) -> " strategy.validate() if strategy.is_empty: - self.mark_invalid() + self.mark_invalid("strategy is empty") if self.depth >= MAX_DEPTH: - self.mark_invalid() + self.mark_invalid("max depth exceeded") if label is None: assert isinstance(strategy.label, int) @@ -1132,7 +1132,9 @@ def mark_interesting( ) -> None: self.conclude_test(Status.INTERESTING, interesting_origin) - def mark_invalid(self): + def mark_invalid(self, why: str | None = None): + if why is not None: + self.note_event(why) self.conclude_test(Status.INVALID) def mark_overrun(self): diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py index a8df183751..4aabb1a45d 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py @@ -452,7 +452,7 @@ def more(self) -> bool: self.data.stop_example() return False - def reject(self): + def reject(self, why: str | None = None): """Reject the last example (i.e. don't count it towards our budget of elements because it's not going to go in the final collection).""" assert self.count > 0 @@ -463,7 +463,7 @@ def reject(self): # failing too fast when we reject the first draw. if self.rejections > max(3, 2 * self.count): if self.count < self.min_size: - self.data.mark_invalid() + self.data.mark_invalid(why) else: self.force_stop = True diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py index 5476239ad7..99173f97ca 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py @@ -234,7 +234,7 @@ def not_yet_in_unique_list(val): while elements.more(): value = filtered.do_filtered_draw(data) if value is filter_not_satisfied: - elements.reject() + elements.reject("Aborted test because unable to satisfy {filtered!r}") else: for key, seen in zip(self.keys, seen_sets): seen.add(key(value)) @@ -274,7 +274,7 @@ def do_draw(self, data): value = (value,) + data.draw(self.tuple_suffixes) result.append(value) else: - should_draw.reject() + should_draw.reject("UniqueSampledListStrategy filter not satisfied or value already seen") assert self.max_size >= len(result) >= self.min_size return result diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py b/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py index 5ec85ec858..c3501d1506 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py @@ -156,7 +156,7 @@ def do_draw(self, data): # If we happened to end up with a disallowed imaginary time, reject it. if (not self.allow_imaginary) and datetime_does_not_exist(result): - data.mark_invalid() + data.mark_invalid("nonexistent datetime") return result def draw_naive_datetime_and_combine(self, data, tz): @@ -165,8 +165,7 @@ def draw_naive_datetime_and_combine(self, data, tz): return replace_tzinfo(dt.datetime(**result), timezone=tz) except (ValueError, OverflowError): msg = "Failed to draw a datetime between %r and %r with timezone from %r." - data.note_event(msg % (self.min_value, self.max_value, self.tz_strat)) - data.mark_invalid() + data.mark_invalid(msg % (self.min_value, self.max_value, self.tz_strat)) @defines_strategy(force_reusable_values=True) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py index 712b573ed8..9c73fadf2c 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py @@ -529,8 +529,7 @@ def _transform(self, element): def do_draw(self, data): result = self.do_filtered_draw(data) if result is filter_not_satisfied: - data.note_event(f"Aborted test because unable to satisfy {self!r}") - data.mark_invalid() + data.mark_invalid(f"Aborted test because unable to satisfy {self!r}") return result def get_element(self, i): @@ -944,8 +943,7 @@ def do_draw(self, data: ConjectureData) -> Ex: if result is not filter_not_satisfied: return result - data.note_event(f"Aborted test because unable to satisfy {self!r}") - data.mark_invalid() + data.mark_invalid(f"Aborted test because unable to satisfy {self!r}") raise NotImplementedError("Unreachable, for Mypy") def note_retried(self, data): diff --git a/hypothesis-python/tests/conjecture/test_test_data.py b/hypothesis-python/tests/conjecture/test_test_data.py index 346063452c..0efd0f129d 100644 --- a/hypothesis-python/tests/conjecture/test_test_data.py +++ b/hypothesis-python/tests/conjecture/test_test_data.py @@ -90,6 +90,15 @@ def test_can_mark_invalid(): assert x.status == Status.INVALID +def test_can_mark_invalid_with_why(): + x = ConjectureData.for_buffer(b"") + with pytest.raises(StopTest): + x.mark_invalid("some reason") + assert x.frozen + assert x.status == Status.INVALID + assert x.events == {"some reason"} + + class BoomStrategy(SearchStrategy): def do_draw(self, data): data.draw_bytes(1) From 724e30861e496d7379bd4fafeddf59d19c9997ec Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 25 May 2023 09:07:36 -0700 Subject: [PATCH 2/5] fix formatting --- .../src/hypothesis/strategies/_internal/collections.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py index 99173f97ca..125054f4b0 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py @@ -274,7 +274,9 @@ def do_draw(self, data): value = (value,) + data.draw(self.tuple_suffixes) result.append(value) else: - should_draw.reject("UniqueSampledListStrategy filter not satisfied or value already seen") + should_draw.reject( + "UniqueSampledListStrategy filter not satisfied or value already seen" + ) assert self.max_size >= len(result) >= self.min_size return result From 37bbf42e764eb880b7d89d96574816984b381d1d Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 25 May 2023 09:12:18 -0700 Subject: [PATCH 3/5] use Optional instead of | None --- hypothesis-python/src/hypothesis/internal/conjecture/data.py | 2 +- hypothesis-python/src/hypothesis/internal/conjecture/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index 29d9025fd1..95e75d1b43 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -1132,7 +1132,7 @@ def mark_interesting( ) -> None: self.conclude_test(Status.INTERESTING, interesting_origin) - def mark_invalid(self, why: str | None = None): + def mark_invalid(self, why: Optional[str] = None): if why is not None: self.note_event(why) self.conclude_test(Status.INVALID) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py index 4aabb1a45d..4afbda63aa 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py @@ -452,7 +452,7 @@ def more(self) -> bool: self.data.stop_example() return False - def reject(self, why: str | None = None): + def reject(self, why: Optional[str] = None): """Reject the last example (i.e. don't count it towards our budget of elements because it's not going to go in the final collection).""" assert self.count > 0 From e8f3ed309617bbf7dd8900645c96b8bd518a6de3 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 25 May 2023 11:16:06 -0700 Subject: [PATCH 4/5] add some return type annotations --- .../src/hypothesis/internal/conjecture/data.py | 11 ++++++----- .../src/hypothesis/internal/conjecture/utils.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index 95e75d1b43..fd4f100da5 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -21,6 +21,7 @@ Iterable, Iterator, List, + NoReturn, Optional, Sequence, Set, @@ -726,7 +727,7 @@ def as_result(self) -> "_Overrun": global_test_counter = 0 -MAX_DEPTH = 100 +MAX_DEPTH = 200 class DataObserver: @@ -1119,7 +1120,7 @@ def conclude_test( self, status: Status, interesting_origin: Optional[InterestingOrigin] = None, - ) -> None: + ) -> NoReturn: assert (interesting_origin is None) or (status == Status.INTERESTING) self.__assert_not_frozen("conclude_test") self.interesting_origin = interesting_origin @@ -1129,15 +1130,15 @@ def conclude_test( def mark_interesting( self, interesting_origin: Optional[InterestingOrigin] = None - ) -> None: + ) -> NoReturn: self.conclude_test(Status.INTERESTING, interesting_origin) - def mark_invalid(self, why: Optional[str] = None): + def mark_invalid(self, why: Optional[str] = None) -> NoReturn: if why is not None: self.note_event(why) self.conclude_test(Status.INVALID) - def mark_overrun(self): + def mark_overrun(self) -> NoReturn: self.conclude_test(Status.OVERRUN) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py index 4afbda63aa..84589edc4e 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py @@ -452,7 +452,7 @@ def more(self) -> bool: self.data.stop_example() return False - def reject(self, why: Optional[str] = None): + def reject(self, why: Optional[str] = None) -> None: """Reject the last example (i.e. don't count it towards our budget of elements because it's not going to go in the final collection).""" assert self.count > 0 From fff3915cf719ac062825a2118b9de24c3157d5d0 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 25 May 2023 11:17:34 -0700 Subject: [PATCH 5/5] revert accidental MAX_DEPTH change --- hypothesis-python/src/hypothesis/internal/conjecture/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index fd4f100da5..0b43d9c1b1 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -727,7 +727,7 @@ def as_result(self) -> "_Overrun": global_test_counter = 0 -MAX_DEPTH = 200 +MAX_DEPTH = 100 class DataObserver: