From 6c6a12d4e23c7ef552dc8077c2101c6b0d740f7b Mon Sep 17 00:00:00 2001 From: Marc Pfetsch Date: Sat, 27 Apr 2024 13:48:20 +0200 Subject: [PATCH 1/7] do not print Wilcoxon statistic with less than 20 items --- check/cmpres.awk | 68 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/check/cmpres.awk b/check/cmpres.awk index 52c14097d2..f36fabcc9e 100755 --- a/check/cmpres.awk +++ b/check/cmpres.awk @@ -1,4 +1,3 @@ - #!/usr/bin/awk -f #* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * #* * @@ -1697,20 +1696,28 @@ END { { s = printorder[o]; - parse_time(ref_array,solver_array,time,o,printorder,probidx,problistlen); + parse_time(ref_array, solver_array, time, o, printorder, probidx, problistlen); n = filter(ref_array, solver_array, problistlen, 0.01, 0.01); factorize(ref_array, solver_array, n, timelimit[s]) - z = wilcoxon(ref_array, solver_array, n, timelimit[s]); - printf(" z %8.4f",z); - z_to_p(z); + # do not print test if number of items is less than 20, because the test is not reliable there + if( n > 20 ) + { + z = wilcoxon(ref_array, solver_array, n, timelimit[s]); + printf(" z %8.4f", z); + z_to_p(z); + } + else + { + printf(" %8s", ""); + printf(" -- (N = %2d <= 20)", n); + } if( printconfs ) printf("%18s", ""); } printf("\n"); - # compute and print result for Wilcoxon signed rank test for time to first solution w.r.t. reference setting if( printsoltimes ) { @@ -1724,13 +1731,22 @@ END { { s = printorder[o]; - parse_time(ref_array,solver_array,timetofirst,o,printorder,probidx,problistlen); + parse_time(ref_array, solver_array, timetofirst, o, printorder, probidx, problistlen); n = filter(ref_array, solver_array, problistlen, 0.01, 0.01); factorize(ref_array, solver_array, n, timelimit[s]) - z = wilcoxon(ref_array, solver_array, n, timelimit[s]); - printf(" z %8.4f",z); - z_to_p(z); + # do not print test if number of items is less than 20, because the test is not reliable there + if( n > 20 ) + { + z = wilcoxon(ref_array, solver_array, n, timelimit[s]); + printf(" z %8.4f", z); + z_to_p(z); + } + else + { + printf(" %8s", ""); + printf(" -- (N = %2d <= 20)", n); + } if( printconfs ) printf("%18s", ""); @@ -1751,9 +1767,18 @@ END { n = filter(ref_array, solver_array, problistlen, 0.01, 0.01); factorize(ref_array, solver_array, n, infinity) - z = wilcoxon(ref_array, solver_array, n, infinity); - printf(" z %8.4f",z); - z_to_p(z); + # do not print test if number of items is less than 20, because the test is not reliable there + if( n > 20 ) + { + z = wilcoxon(ref_array, solver_array, n, infinity); + printf(" z %8.4f", z); + z_to_p(z); + } + else + { + printf(" %8s", ""); + printf(" -- (N = %2d <= 20)", n); + } if( printconfs ) printf("%18s", ""); @@ -1768,13 +1793,22 @@ END { { s = printorder[o]; - parse_time(ref_array,solver_array,confs,o,printorder,probidx,problistlen); + parse_time(ref_array, solver_array, confs, o, printorder, probidx, problistlen); n = filter(ref_array, solver_array, problistlen, 0.01, 0.01); factorize(ref_array, solver_array, n, timelimit[s]) - z = wilcoxon(ref_array, solver_array, n, timelimit[s]); - printf(" z %8.4f", z ); - z_to_p(z); + # do not print test if number of items is less than 20, because the test is not reliable there + if( n > 20 ) + { + z = wilcoxon(ref_array, solver_array, n, timelimit[s]); + printf(" z %8.4f", z); + z_to_p(z); + } + else + { + printf(" %8s", ""); + printf(" -- (N = %2d <= 20)", n); + } if( printconfs ) printf("%18s", ""); From 06e84ed7cc66526f3b5eb6b656e8a3483dea5df7 Mon Sep 17 00:00:00 2001 From: Marc Pfetsch Date: Sun, 28 Apr 2024 09:53:35 +0200 Subject: [PATCH 2/7] fix lengths of strings --- check/cmpres.awk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/check/cmpres.awk b/check/cmpres.awk index f36fabcc9e..1104082c6d 100755 --- a/check/cmpres.awk +++ b/check/cmpres.awk @@ -795,21 +795,21 @@ END { if ( length(sname) <= 58 ) printf(" %58s |", sname); else - printf(" *%57s |", substr(sname, length(sname)-58)); + printf(" *%57s |", substr(sname, length(sname) - 57)); } else if( printsoltimes == 0 && printconfs ) { if ( length(sname) <= 55 ) printf(" %55s |", sname); else - printf(" *%54s |", substr(sname, length(sname)-54)); + printf(" *%54s |", substr(sname, length(sname) - 54)); } else { if ( length(sname) <= 39 ) printf(" %39s |", sname) else - printf(" *%38s |", substr(sname, length(sname)-39)); + printf(" *%38s |", substr(sname, length(sname) - 38)); } } else @@ -819,28 +819,28 @@ END { if( length(sname) <= 19 ) printf("%19s |", sname); else - printf("*%16s |", substr(sname, length(sname)-17)); + printf("*%18s |", substr(sname, length(sname) - 18)); } else if( printsoltimes && printconfs == 0 ) { if( length(sname) <= 47 ) printf("%47s |", sname); else - printf("*%46s |", substr(sname, length(sname)-47)); + printf("*%46s |", substr(sname, length(sname) - 46)); } else if( printsoltimes == 0 && printconfs ) { if( length(sname) <= 48 ) printf("%48s |", sname); else - printf("*%47s |", substr(sname, length(sname)-48)); + printf("*%47s |", substr(sname, length(sname) - 47)); } else { if( length(sname) <= 33 ) printf("%31s |", sname); else - printf("*%30s |", substr(sname, length(sname)-31)); + printf("*%30s |", substr(sname, length(sname) - 30)); } } } From d62ecac59ad2fbfb71be36866b4e586e8284b171 Mon Sep 17 00:00:00 2001 From: Marc Pfetsch Date: Sun, 28 Apr 2024 16:01:26 +0200 Subject: [PATCH 3/7] add parameter for length of setting names --- check/cmpres.awk | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/check/cmpres.awk b/check/cmpres.awk index 1104082c6d..a363a5e77e 100755 --- a/check/cmpres.awk +++ b/check/cmpres.awk @@ -463,6 +463,7 @@ BEGIN { printconfs = 0; printgap = 0; # if timeout, then print absolute gap at termination in time column, if gap is finite printsoltimes = !short && printsoltimes; # short deactivates the detailed solution times output + setnamelen = 50; # display length of test sets infinity = 1e+20; timegeomshift = 1.0; nodegeomshift = 100.0; @@ -1915,8 +1916,8 @@ END { header = (cat == -1 ? "optimal" : (cat == 0 ? "all" : (cat == 1 ? "diff" : (cat == 2 ? "equal" : "timeout")))); printf("\n"); - printf("%-7s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n", - header); + printf("%-*s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n", + setnamelen, header); for( o = 0; o < nsolver; ++o ) { @@ -1931,11 +1932,12 @@ END { } if( (o > 0 || cat == 0 || cat == -1) && nevalprobs[s,cat] > 0 ) { - if ( length(sname) <= 50 ) - printf("%-50s %4d %4d %4d %4d %4d %4d", sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], + if ( length(sname) <= setnamelen ) + printf("%-*s %4d %4d %4d %4d %4d %4d", setnamelen, sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); else - printf("*%-49s %4d %4d %4d %4d %4d %4d", substr(sname, length(sname)-48), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], + printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen - 1), + nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); printf(" %4d %4d", better[s,cat], worse[s,cat]); @@ -1949,7 +1951,7 @@ END { } if( cat == 0 ) { - printf("%-50s %4d %4d %4d %4s", "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, ""); + printf("%-*s %4d %4d %4d %4s", setnamelen, "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, ""); printf(" %4d %4s", bestbetter, ""); printf(" %4d %4s %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7s\n", bestbetterobj, "", bestfeasibles, @@ -1962,7 +1964,8 @@ END { # output the all optimal case header = "all optimal"; printf("\n"); - printf("%-11s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n", header); + printf("%-*s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n", + setnamelen, header); cat = -1; for( o = 0; o < nsolver; ++o ) { @@ -1977,11 +1980,12 @@ END { } if( (o > 0 || cat == 0 || cat == -1) && nevalprobs[s,cat] > 0 ) { - if ( length(sname) <= 50 ) - printf("%-50s %4d %4d %4d %4d %4d %4d", sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], + if ( length(sname) <= setnamelen ) + printf("%-*s %4d %4d %4d %4d %4d %4d", setnamelen, sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); else - printf("*%-49s %4d %4d %4d %4d %4d %4d", substr(sname, length(sname)-48), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], + printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen - 1), + nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); printf(" %4d %4d", better[s,cat], worse[s,cat]); printf(" %4d %4d %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7.2f\n", @@ -1994,7 +1998,7 @@ END { } if( cat == 0 ) { - printf("%-50s %4d %4d %4d %4s", "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, ""); + printf("%-*s %4d %4d %4d %4s", setnamelen, "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, ""); printf(" %4d %4s", bestbetter, ""); printf(" %4d %4s %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7s\n", bestbetterobj, "", bestfeasibles, From c1302e3a751e01654fc316a2858c5e222bd61c73 Mon Sep 17 00:00:00 2001 From: Marc Pfetsch Date: Sun, 28 Apr 2024 16:36:51 +0200 Subject: [PATCH 4/7] fix width specification --- check/cmpres.awk | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/check/cmpres.awk b/check/cmpres.awk index a363a5e77e..a010e09e23 100755 --- a/check/cmpres.awk +++ b/check/cmpres.awk @@ -796,21 +796,21 @@ END { if ( length(sname) <= 58 ) printf(" %58s |", sname); else - printf(" *%57s |", substr(sname, length(sname) - 57)); + printf(" *%57s |", substr(sname, length(sname) - 56)); } else if( printsoltimes == 0 && printconfs ) { if ( length(sname) <= 55 ) printf(" %55s |", sname); else - printf(" *%54s |", substr(sname, length(sname) - 54)); + printf(" *%54s |", substr(sname, length(sname) - 53)); } else { if ( length(sname) <= 39 ) printf(" %39s |", sname) else - printf(" *%38s |", substr(sname, length(sname) - 38)); + printf(" *%38s |", substr(sname, length(sname) - 37)); } } else @@ -820,28 +820,28 @@ END { if( length(sname) <= 19 ) printf("%19s |", sname); else - printf("*%18s |", substr(sname, length(sname) - 18)); + printf("*%18s |", substr(sname, length(sname) - 17)); } else if( printsoltimes && printconfs == 0 ) { if( length(sname) <= 47 ) printf("%47s |", sname); else - printf("*%46s |", substr(sname, length(sname) - 46)); + printf("*%46s |", substr(sname, length(sname) - 45)); } else if( printsoltimes == 0 && printconfs ) { if( length(sname) <= 48 ) printf("%48s |", sname); else - printf("*%47s |", substr(sname, length(sname) - 47)); + printf("*%47s |", substr(sname, length(sname) - 46)); } else { if( length(sname) <= 33 ) printf("%31s |", sname); else - printf("*%30s |", substr(sname, length(sname) - 30)); + printf("*%30s |", substr(sname, length(sname) - 29)); } } } @@ -1933,10 +1933,11 @@ END { if( (o > 0 || cat == 0 || cat == -1) && nevalprobs[s,cat] > 0 ) { if ( length(sname) <= setnamelen ) - printf("%-*s %4d %4d %4d %4d %4d %4d", setnamelen, sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], + printf("%-*s %4d %4d %4d %4d %4d %4d", setnamelen, sname, + nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); else - printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen - 1), + printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen + 2), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); @@ -1984,7 +1985,7 @@ END { printf("%-*s %4d %4d %4d %4d %4d %4d", setnamelen, sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); else - printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen - 1), + printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen + 2), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat], ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]); printf(" %4d %4d", better[s,cat], worse[s,cat]); From d9bea4fc737ce87244f71c804b27477ba7116ff6 Mon Sep 17 00:00:00 2001 From: Marc Pfetsch Date: Sun, 28 Apr 2024 16:38:23 +0200 Subject: [PATCH 5/7] correct inconsistent width --- check/cmpres.awk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/cmpres.awk b/check/cmpres.awk index a010e09e23..c23c269f40 100755 --- a/check/cmpres.awk +++ b/check/cmpres.awk @@ -839,9 +839,9 @@ END { else { if( length(sname) <= 33 ) - printf("%31s |", sname); + printf("%33s |", sname); else - printf("*%30s |", substr(sname, length(sname) - 29)); + printf("*%32s |", substr(sname, length(sname) - 31)); } } } From 6f012aeb6a730461d402711fb8c32c6a897bba2c Mon Sep 17 00:00:00 2001 From: Marc Pfetsch Date: Sun, 28 Apr 2024 16:42:40 +0200 Subject: [PATCH 6/7] remove space for first set --- check/cmpres.awk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/check/cmpres.awk b/check/cmpres.awk index c23c269f40..0417713303 100755 --- a/check/cmpres.awk +++ b/check/cmpres.awk @@ -794,23 +794,23 @@ END { if( printsoltimes && printconfs == 0 ) { if ( length(sname) <= 58 ) - printf(" %58s |", sname); + printf("%58s |", sname); else - printf(" *%57s |", substr(sname, length(sname) - 56)); + printf("*%57s |", substr(sname, length(sname) - 56)); } else if( printsoltimes == 0 && printconfs ) { if ( length(sname) <= 55 ) - printf(" %55s |", sname); + printf("%55s |", sname); else - printf(" *%54s |", substr(sname, length(sname) - 53)); + printf("*%54s |", substr(sname, length(sname) - 53)); } else { if ( length(sname) <= 39 ) - printf(" %39s |", sname) + printf("%39s |", sname) else - printf(" *%38s |", substr(sname, length(sname) - 37)); + printf("*%38s |", substr(sname, length(sname) - 37)); } } else From 74a10d7bdcd80906fef41e1f406d547f1f2257a4 Mon Sep 17 00:00:00 2001 From: Marc Pfetsch Date: Sun, 28 Apr 2024 16:49:59 +0200 Subject: [PATCH 7/7] correct width --- check/cmpres.awk | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/check/cmpres.awk b/check/cmpres.awk index 0417713303..5e063749c3 100755 --- a/check/cmpres.awk +++ b/check/cmpres.awk @@ -793,24 +793,24 @@ END { { if( printsoltimes && printconfs == 0 ) { - if ( length(sname) <= 58 ) - printf("%58s |", sname); + if ( length(sname) <= 59 ) + printf("%59s |", sname); else - printf("*%57s |", substr(sname, length(sname) - 56)); + printf("*%58s |", substr(sname, length(sname) - 57)); } else if( printsoltimes == 0 && printconfs ) { - if ( length(sname) <= 55 ) - printf("%55s |", sname); + if ( length(sname) <= 56 ) + printf("%56s |", sname); else - printf("*%54s |", substr(sname, length(sname) - 53)); + printf("*%55s |", substr(sname, length(sname) - 54)); } else { - if ( length(sname) <= 39 ) - printf("%39s |", sname) + if ( length(sname) <= 40 ) + printf("%40s |", sname) else - printf("*%38s |", substr(sname, length(sname) - 37)); + printf("*%39s |", substr(sname, length(sname) - 38)); } } else