Skip to content

Commit

Permalink
Initial data collection and --exhaust option
Browse files Browse the repository at this point in the history
  • Loading branch information
@hannelita @fruffy
hannelita authored and fruffy committed Jan 19, 2023
1 parent a86d317 commit 019ac58
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 13 deletions.
12 changes: 12 additions & 0 deletions backends/p4tools/benchmarks/max_results_pins_middleblock1.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
seed,max_tests_input,DFS Coverage,max_cov_on_test,time (ms),Random Access Stack,max_cov_on_test,time (ms),Random Access Max Cov,max_cov_on_test,time (ms)
7189,1000,0.88961 ,999,39292 ,1 ,92,43436 ,1 ,669,53209
9086,1000,0.896104 ,999,34573 ,1 ,138,52027 ,0.993506 ,999,50950
2878,1000,0.909091 ,999,43856 ,0.980519 ,999,48694 ,0.987013 ,999,50755
1728,1000,0.88961 ,999,36824 ,0.980519 ,999,46906 ,0.993506 ,999,56864
1203,1000,0.896104 ,999,41891 ,1 ,204,41926 ,0.928571 ,999,51926
3985,1000,0.896104 ,999,44374 ,1 ,262,51266 ,1 ,858,45431
2004,1000,0.896104 ,999,32599 ,0.987013 ,999,48789 ,0.941558 ,999,54808
7057,1000,0.896104 ,999,39935 ,1 ,143,53224 ,0.928571 ,999,56407
3058,1000,0.896104 ,999,45602 ,0.980519 ,999,47407 ,1 ,173,55572
2838,1000,0.896104 ,999,35944 ,1 ,204,49342 ,1 ,409,54213
4254,1000,0.87013 ,999,37310 ,1 ,361,43384 ,1 ,309,57149
45 changes: 45 additions & 0 deletions backends/p4tools/benchmarks/results_pins_middleblock1.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
seed,max_tests_input,DFS Coverage,max_cov_on_test,time (ms),Random Access Stack,max_cov_on_test,time (ms),Random Access Max Cov,max_cov_on_test,time (ms)
7189,10,0.662338 ,9,301 ,0.746753 ,9,414 ,0.746753 ,9,458
7189,20,0.688312 ,19,626 ,0.857143 ,19,824 ,0.88961 ,19,863
7189,30,0.688312 ,29,857 ,0.909091 ,29,1170 ,0.948052 ,29,1324
7189,100,0.694805 ,99,2700 ,1 ,92,4190 ,0.993506 ,99,4586
9086,10,0.649351 ,9,312 ,0.707792 ,9,425 ,0.811688 ,9,459
9086,20,0.675325 ,19,683 ,0.844156 ,19,984 ,0.857143 ,19,812
9086,30,0.675325 ,29,1027 ,0.896104 ,29,1405 ,0.857143 ,29,1123
9086,100,0.694805 ,99,2715 ,0.980519 ,99,4958 ,0.922078 ,99,4199
2878,10,0.681818 ,9,290 ,0.474026 ,9,327 ,0.694805 ,9,428
2878,20,0.688312 ,19,590 ,0.824675 ,19,920 ,0.831169 ,19,843
2878,30,0.688312 ,29,875 ,0.850649 ,29,1303 ,0.88961 ,29,1227
2878,100,0.694805 ,99,2864 ,0.974026 ,99,4540 ,0.915584 ,99,4261
1728,10,0.701299 ,9,423 ,0.727273 ,9,517 ,0.779221 ,9,564
1728,20,0.772727 ,19,781 ,0.831169 ,19,882 ,0.785714 ,19,983
1728,30,0.772727 ,29,1141 ,0.850649 ,29,1261 ,0.87013 ,29,1415
1728,100,0.837662 ,99,3656 ,0.967532 ,99,3940 ,0.974026 ,99,4959
1203,10,0.675325 ,9,271 ,0.668831 ,9,276 ,0.616883 ,9,412
1203,20,0.688312 ,19,535 ,0.701299 ,19,738 ,0.74026 ,19,828
1203,30,0.688312 ,29,1035 ,0.701299 ,29,1007 ,0.74026 ,29,1144
1203,100,0.688312 ,99,2626 ,0.824675 ,99,3673 ,0.928571 ,99,4101
3985,10,0.62987 ,9,335 ,0.688312 ,9,428 ,0.720779 ,9,381
3985,20,0.688312 ,19,604 ,0.902597 ,19,912 ,0.87013 ,19,834
3985,30,0.688312 ,29,881 ,0.961039 ,29,1420 ,0.88961 ,29,1223
3985,100,0.694805 ,99,2829 ,0.974026 ,99,4299 ,0.902597 ,99,3965
2004,10,0.675325 ,9,303 ,0.681818 ,9,311 ,0.798701 ,9,447
2004,20,0.675325 ,19,605 ,0.688312 ,19,626 ,0.922078 ,19,1019
2004,30,0.675325 ,29,800 ,0.688312 ,29,802 ,0.928571 ,29,1565
2004,100,0.688312 ,99,2603 ,0.915584 ,99,3217 ,0.941558 ,99,4651
7057,10,0.668831 ,9,396 ,0.61039 ,9,457 ,0.694805 ,9,473
7057,20,0.668831 ,19,806 ,0.694805 ,19,826 ,0.707792 ,19,749
7057,30,0.668831 ,29,1268 ,0.701299 ,29,1318 ,0.88961 ,29,1488
7057,100,0.688312 ,99,4167 ,0.987013 ,99,4888 ,0.909091 ,99,4718
3058,10,0.655844 ,9,315 ,0.824675 ,9,477 ,0.727273 ,9,446
3058,20,0.681818 ,19,591 ,0.863636 ,19,886 ,0.772727 ,19,836
3058,30,0.681818 ,29,817 ,0.896104 ,29,1348 ,0.824675 ,29,1202
3058,100,0.694805 ,99,3214 ,0.974026 ,99,4995 ,0.928571 ,99,4138
2838,10,0.733766 ,9,341 ,0.792208 ,9,501 ,0.876623 ,9,583
2838,20,0.811688 ,19,677 ,0.922078 ,19,1077 ,0.88961 ,19,1074
2838,30,0.837662 ,29,1035 ,0.928571 ,29,1475 ,0.902597 ,29,1594
2838,100,0.837662 ,99,3531 ,0.993506 ,99,4926 ,0.915584 ,99,5185
4254,10,0.655844 ,9,274 ,0.681818 ,9,335 ,0.714286 ,9,414
4254,20,0.675325 ,19,562 ,0.785714 ,19,769 ,0.909091 ,19,971
4254,30,0.675325 ,29,765 ,0.798701 ,29,1121 ,0.909091 ,29,1521
4254,100,0.688312 ,99,2513 ,0.941558 ,99,3767 ,0.935065 ,99,4880
54 changes: 41 additions & 13 deletions backends/p4tools/benchmarks/test_coverage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import csv
import dateutil
import pandas as pd
import argparse

# generate random seeds, increase the number for extra sampling
ITERATIONS = 10

ITERATIONS = 1
# 7189 is an example of a good seed, which gets cov 1 with less than 100 tests
# in random access stack.
seeds = [7189]
Expand All @@ -29,24 +29,34 @@
out_dir = f"/p4/p4c/build/testgen/testgen-p4c-bmv2/{p4_program_name}.out"

includes = "/p4/p4c/build/p4include"

parser = argparse.ArgumentParser()
parser.add_argument('--exhaust', action='store_true')

extras = ""
extra_val = ""

# csv results file path
results_path = f"/results_{p4_program_name}.csv"
results_path_max = f"/max_results_{p4_program_name}.csv"
results_path_exhaust = f"/exhaust_results_{p4_program_name}.csv"

header = ["seed", "max_tests_input", "DFS Coverage", "max_cov_on_test", "time (ms)", "Random Access Stack",
"max_cov_on_test",
"time (ms)", "Random Access Max Cov", "max_cov_on_test", "time (ms)"]


args = parser.parse_args()

def run_strategies_for_max_tests():
data_row = [seed, max_tests]

commands = [p4testgen, "--target", "bmv2", "--arch", "v1model", "--std", "p4-16", "-I",
includes, "--test-backend", "STF", "--strict", "--print-traces", "--print-performance-report",
"--seed", str(seed), "--max-tests", str(max_tests), "--out-dir", out_dir, p4_program, extras, extra_val]
commands = [c for c in commands if c]
# DFS
proc = subprocess.run(
[p4testgen, "--target", "bmv2", "--arch", "v1model", "--std", "p4-16", "-I",
includes, "--test-backend", "STF", "--strict", "--print-traces", "--print-performance-report",
"--seed", str(seed), "--max-tests", str(max_tests), "--out-dir", out_dir, p4_program],
commands,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
Expand All @@ -65,12 +75,13 @@ def run_strategies_for_max_tests():
data_row.append(time)

# random access stack; pop-level fixed at 3

proc = subprocess.run(
[p4testgen, "--target", "bmv2", "--arch", "v1model", "--std", "p4-16", "-I",
commands = [p4testgen, "--target", "bmv2", "--arch", "v1model", "--std", "p4-16", "-I",
includes, "--test-backend", "STF", "--strict", "--print-traces", "--print-performance-report",
"--seed", str(seed), "--max-tests", str(max_tests), "--out-dir", out_dir, p4_program,
"--exploration-strategy", "RANDOM_ACCESS_STACK", "--pop-level", "3"],
"--exploration-strategy", "RANDOM_ACCESS_STACK", "--pop-level", "3", extras, extra_val]
commands = [c for c in commands if c]
proc = subprocess.run(
commands,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
Expand All @@ -89,11 +100,14 @@ def run_strategies_for_max_tests():
data_row.append(time)

# random access max cov, saddle-point fixed at 2
proc = subprocess.run(
[p4testgen, "--target", "bmv2", "--arch", "v1model", "--std", "p4-16", "-I",
commands = [p4testgen, "--target", "bmv2", "--arch", "v1model", "--std", "p4-16", "-I",
includes, "--test-backend", "STF", "--strict", "--print-traces", "--print-performance-report",
"--seed", str(seed), "--max-tests", str(max_tests), "--out-dir", out_dir, p4_program,
"--exploration-strategy", "RANDOM_ACCESS_MAX_COVERAGE", "--saddle-point", "3"],
"--exploration-strategy", "RANDOM_ACCESS_MAX_COVERAGE", "--saddle-point", "3"]
commands = [c for c in commands if c]

proc = subprocess.run(
commands,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
Expand All @@ -114,6 +128,7 @@ def run_strategies_for_max_tests():


with open(results_path, 'w') as f:
print("Generating basic metrics up to 100 tests")
writer = csv.writer(f)
writer.writerow(header)
for seed in seeds:
Expand All @@ -134,9 +149,22 @@ def run_strategies_for_max_tests():
writer.writerow(data_row)

with open(results_path_max, 'w') as f:
print("Generating metrics up to 1000 tests")
writer = csv.writer(f)
writer.writerow(header)
for seed in seeds:
max_tests = 1000
data_row = run_strategies_for_max_tests()
writer.writerow(data_row)

if args.exhaust:
print("Exhaust")
with open(results_path_exhaust, 'w') as f:
writer = csv.writer(f)
writer.writerow(header)
for seed in seeds:
max_tests = 0
extras = "--stop-metric"
extra_val = "MAX_STATEMENT_COVERAGE"
data_row = run_strategies_for_max_tests()
writer.writerow(data_row)

0 comments on commit 019ac58

Please sign in to comment.