BUG: handle setup_cache correctly in asv check

asv/benchmark.py

@@ -358,7 +358,10 @@ def _get_sourceline_info(obj, basedir):
  return ""
 
 
-def check_num_args(root, benchmark_name, func, num_args):
+def check_num_args(root, benchmark_name, func, min_num_args, max_num_args=None):
+ if max_num_args is None:
+ max_num_args = min_num_args
+
  try:
  if sys.version_info[0] >= 3:
  info = inspect.getfullargspec(func)
@@ -382,15 +385,19 @@ def check_num_args(root, benchmark_name, func, num_args):
  if info.varargs is not None:
  max_args = float('inf')
 
- ok = min_args <= num_args <= max_args
+ ok = (min_args <= max_num_args) and (min_num_args <= max_args)
  if not ok:
  if min_args == max_args:
  args_str = min_args
  else:
  args_str = "{}-{}".format(min_args, max_args)
+ if min_num_args == max_num_args:
+ num_args_str = min_num_args
+ else:
+ num_args_str = "{}-{}".format(min_num_args, max_num_args)
  print("{!s}: wrong number of arguments (for {!r}{!s}): expected {}, has {}".format(
  benchmark_name, func, _get_sourceline_info(func, root),
- num_args, args_str))
+ num_args_str, args_str))
 
  return ok
 
@@ -481,23 +488,24 @@ def check(self, root):
  if self._params:
  self.set_param_idx(0)
 
+ min_num_args = len(self._current_params)
+ max_num_args = min_num_args
+
  if self.setup_cache_key is not None:
- self.insert_param(None)
  ok = ok and check_num_args(root, self.name + ": setup_cache",
  self._setup_cache, 0)
-
- num_args = len(self._current_params)
+ max_num_args += 1
 
  for setup in self._setups:
  ok = ok and check_num_args(root, self.name + ": setup",
- setup, num_args)
+ setup, min_num_args, max_num_args)
 
  ok = ok and check_num_args(root, self.name + ": call",
- self.func, num_args)
+ self.func, min_num_args, max_num_args)
 
  for teardown in self._teardowns:
  ok = ok and check_num_args(root, self.name + ": teardown",
- teardown, num_args)
+ teardown, min_num_args, max_num_args)
 
  return ok
 

test/benchmark/cache_examples.py

@@ -86,4 +86,3 @@ def time_fail_second_run(data):
 time_fail_second_run.repeat = 1
 time_fail_second_run.warmup_time = 0
 time_fail_second_run.processes = 2
-

test/benchmark/params_examples.py

@@ -97,3 +97,9 @@ def track_bytes():
  return 1000000
 
 track_bytes.unit = "bytes"
+
+
+def track_wrong_number_of_args(a, b):
+ return 0
+
+track_wrong_number_of_args.params = [[1, 2]]

test/test_benchmarks.py

@@ -73,7 +73,7 @@ def test_discover_benchmarks(benchmarks_fixture):
  b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
  regex='example')
  conf.branches = old_branches
- assert len(b) == 29
+ assert len(b) == 30
 
  b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
  regex='time_example_benchmark_1')
@@ -106,7 +106,7 @@ def test_discover_benchmarks(benchmarks_fixture):
  assert b._benchmark_selection['params_examples.track_param_selection'] == [0, 1, 2, 3]
 
  b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
- assert len(b) == 40
+ assert len(b) == 41
 
  assert 'named.OtherSuite.track_some_func' in b
 

test/test_check.py

@@ -23,5 +23,6 @@ def test_check(capsys, basic_conf):
 
  text, err = capsys.readouterr()
 
- assert re.search(r"cache_examples\.[A-Za-z]+\.track_[a-z]+: call: wrong number of arguments", text)
+ assert re.search(r"params_examples\.track_wrong_number_of_args: call: "
+ r"wrong number of arguments.*: expected 1, has 2", text)
  assert text.count("wrong number of arguments") == 1