Merge pull request #3654 from carljm/invalidwhy

AUTHORS.rst

@@ -31,6 +31,7 @@ their individual contributions.
 * `Bryant Eisenbach <https://github.com/fubuloubu>`_
 * `Buck Evan, copyright Google LLC <https://github.com/bukzor>`_
 * `Cameron McGill <https://www.github.com/Cameron-JM>`_
+* `Carl Meyer <https://www.github.com/carljm>`_
 * `Charles O'Farrell <https://www.github.com/charleso>`_
 * `Charlie Tanksley <https://www.github.com/charlietanksley>`_
 * `Chase Garner <https://www.github.com/chasegarner>`_ (chase@garner.red)

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: patch
+
+Hypothesis will now record an event for more cases where data is marked
+invalid, including for exceeding the internal depth limit.

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -21,6 +21,7 @@
  Iterable,
  Iterator,
  List,
+ NoReturn,
  Optional,
  Sequence,
  Set,
@@ -936,10 +937,10 @@ def draw(self, strategy: "SearchStrategy[Ex]", label: Optional[int] = None) -> "
  strategy.validate()
 
  if strategy.is_empty:
- self.mark_invalid()
+ self.mark_invalid("strategy is empty")
 
  if self.depth >= MAX_DEPTH:
- self.mark_invalid()
+ self.mark_invalid("max depth exceeded")
 
  if label is None:
  assert isinstance(strategy.label, int)
@@ -1119,7 +1120,7 @@ def conclude_test(
  self,
  status: Status,
  interesting_origin: Optional[InterestingOrigin] = None,
- ) -> None:
+ ) -> NoReturn:
  assert (interesting_origin is None) or (status == Status.INTERESTING)
  self.__assert_not_frozen("conclude_test")
  self.interesting_origin = interesting_origin
@@ -1129,13 +1130,15 @@ def conclude_test(
 
  def mark_interesting(
  self, interesting_origin: Optional[InterestingOrigin] = None
- ) -> None:
+ ) -> NoReturn:
  self.conclude_test(Status.INTERESTING, interesting_origin)
 
- def mark_invalid(self):
+ def mark_invalid(self, why: Optional[str] = None) -> NoReturn:
+ if why is not None:
+ self.note_event(why)
  self.conclude_test(Status.INVALID)
 
- def mark_overrun(self):
+ def mark_overrun(self) -> NoReturn:
  self.conclude_test(Status.OVERRUN)
 
 

hypothesis-python/src/hypothesis/internal/conjecture/utils.py

@@ -452,7 +452,7 @@ def more(self) -> bool:
  self.data.stop_example()
  return False
 
- def reject(self):
+ def reject(self, why: Optional[str] = None) -> None:
  """Reject the last example (i.e. don't count it towards our budget of
  elements because it's not going to go in the final collection)."""
  assert self.count > 0
@@ -463,7 +463,7 @@ def reject(self):
  # failing too fast when we reject the first draw.
  if self.rejections > max(3, 2 * self.count):
  if self.count < self.min_size:
- self.data.mark_invalid()
+ self.data.mark_invalid(why)
  else:
  self.force_stop = True
 

hypothesis-python/src/hypothesis/strategies/_internal/collections.py

@@ -234,7 +234,7 @@ def not_yet_in_unique_list(val):
  while elements.more():
  value = filtered.do_filtered_draw(data)
  if value is filter_not_satisfied:
- elements.reject()
+ elements.reject("Aborted test because unable to satisfy {filtered!r}")
  else:
  for key, seen in zip(self.keys, seen_sets):
  seen.add(key(value))
@@ -274,7 +274,9 @@ def do_draw(self, data):
  value = (value,) + data.draw(self.tuple_suffixes)
  result.append(value)
  else:
- should_draw.reject()
+ should_draw.reject(
+ "UniqueSampledListStrategy filter not satisfied or value already seen"
+ )
  assert self.max_size >= len(result) >= self.min_size
  return result
 

hypothesis-python/src/hypothesis/strategies/_internal/datetime.py

@@ -156,7 +156,7 @@ def do_draw(self, data):
 
  # If we happened to end up with a disallowed imaginary time, reject it.
  if (not self.allow_imaginary) and datetime_does_not_exist(result):
- data.mark_invalid()
+ data.mark_invalid("nonexistent datetime")
  return result
 
  def draw_naive_datetime_and_combine(self, data, tz):
@@ -165,8 +165,7 @@ def draw_naive_datetime_and_combine(self, data, tz):
  return replace_tzinfo(dt.datetime(**result), timezone=tz)
  except (ValueError, OverflowError):
  msg = "Failed to draw a datetime between %r and %r with timezone from %r."
- data.note_event(msg % (self.min_value, self.max_value, self.tz_strat))
- data.mark_invalid()
+ data.mark_invalid(msg % (self.min_value, self.max_value, self.tz_strat))
 
 
 @defines_strategy(force_reusable_values=True)

hypothesis-python/src/hypothesis/strategies/_internal/strategies.py

@@ -529,8 +529,7 @@ def _transform(self, element):
  def do_draw(self, data):
  result = self.do_filtered_draw(data)
  if result is filter_not_satisfied:
- data.note_event(f"Aborted test because unable to satisfy {self!r}")
- data.mark_invalid()
+ data.mark_invalid(f"Aborted test because unable to satisfy {self!r}")
  return result
 
  def get_element(self, i):
@@ -944,8 +943,7 @@ def do_draw(self, data: ConjectureData) -> Ex:
  if result is not filter_not_satisfied:
  return result
 
- data.note_event(f"Aborted test because unable to satisfy {self!r}")
- data.mark_invalid()
+ data.mark_invalid(f"Aborted test because unable to satisfy {self!r}")
  raise NotImplementedError("Unreachable, for Mypy")
 
  def note_retried(self, data):

hypothesis-python/tests/conjecture/test_test_data.py

@@ -90,6 +90,15 @@ def test_can_mark_invalid():
  assert x.status == Status.INVALID
 
 
+def test_can_mark_invalid_with_why():
+ x = ConjectureData.for_buffer(b"")
+ with pytest.raises(StopTest):
+ x.mark_invalid("some reason")
+ assert x.frozen
+ assert x.status == Status.INVALID
+ assert x.events == {"some reason"}
+
+
 class BoomStrategy(SearchStrategy):
  def do_draw(self, data):
  data.draw_bytes(1)