Include custom results in results report

Previously, results would only be included in the total number of result
items if the status for that result was one of the built-in result
statuses (passed, failed, skipped, etc). With the addition of manual
results reporting, if a user used other custom statuses, these would not
be included in the default `results` output and it would look like
Sonobuoy had processed no results.

This commit modifies the logic for the default `results` output to count
all items, not just those which match the built-in statuses. Custom
statuses with their count are also included in the output. The overall
counts for the built-in statuses are still included as before, but now
we also iterate over all other statuses and include them in the output.

Signed-off-by: Bridget McErlean <bmcerlean@vmware.com>

cmd/sonobuoy/app/results.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -248,44 +249,66 @@ func printResultsDetails(treePath []string, o *results.Item, input resultsInput)
 }
 
 func printResultsSummary(o *results.Item) error {
-	p, f, s, failList := walkForSummary(o, 0, 0, 0, []string{})
+	statusCounts := map[string]int{}
+	var failedList []string
 
-	fmt.Printf(`Plugin: %v
-Status: %v
-Total: %v
-Passed: %v
-Failed: %v
-Skipped: %v
-`, o.Name, o.Status, p+f+s, p, f, s)
+	statusCounts, failedList = walkForSummary(o, statusCounts, failedList)
 
-	if len(failList) > 0 {
+	total := 0
+	for _, v := range statusCounts {
+		total += v
+	}
+
+	fmt.Println("Plugin:", o.Name)
+	fmt.Println("Status:", o.Status)
+	fmt.Println("Total:", total)
+
+	// We want to print the built-in status type results first before printing any custom statuses, so print first then delete.
+	fmt.Println("Passed:", statusCounts[results.StatusPassed])
+	fmt.Println("Failed:", statusCounts[results.StatusFailed]+statusCounts[results.StatusTimeout])
+	fmt.Println("Skipped:", statusCounts[results.StatusSkipped])
+
+	delete(statusCounts, results.StatusPassed)
+	delete(statusCounts, results.StatusFailed)
+	delete(statusCounts, results.StatusTimeout)
+	delete(statusCounts, results.StatusSkipped)
+
+	// We want the custom statuses to always be printed in order so sort them before proceeding
+	keys := []string{}
+	for k := range statusCounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		fmt.Printf("%v: %v\n", k, statusCounts[k])
+	}
+
+	if len(failedList) > 0 {
 		fmt.Print("\nFailed tests:\n")
-		fmt.Print(strings.Join(failList, "\n"))
+		fmt.Print(strings.Join(failedList, "\n"))
 		fmt.Println()
 	}
 
 	return nil
 }
 
-func walkForSummary(o *results.Item, p, f, s int, failList []string) (numPassed, numFailed, numSkipped int, failed []string) {
-	if len(o.Items) > 0 {
-		for _, v := range o.Items {
-			p, f, s, failList = walkForSummary(&v, p, f, s, failList)
+func walkForSummary(result *results.Item, statusCounts map[string]int, failList []string) (map[string]int, []string) {
+	if len(result.Items) > 0 {
+		for _, item := range result.Items {
+			statusCounts, failList = walkForSummary(&item, statusCounts, failList)
 		}
-		return p, f, s, failList
+		return statusCounts, failList
 	}
 
-	switch o.Status {
-	case results.StatusPassed:
-		p++
+	statusCounts[result.Status]++
+
+	switch result.Status {
 	case results.StatusFailed, results.StatusTimeout:
-		f++
-		failList = append(failList, o.Name)
-	case results.StatusSkipped:
-		s++
+		failList = append(failList, result.Name)
 	}
 
-	return p, f, s, failList
+	return statusCounts, failList
 }
 
 func getFileFromMeta(m map[string]string) string {

cmd/sonobuoy/app/results_test.go

@@ -38,6 +38,27 @@ func ExampleNewCmdResults() {
 	// [sig-storage] CSI Volumes CSI Topology test using GCE PD driver [Serial] should fail to schedule a pod with a zone missing from AllowedTopologies; PD is provisioned with immediate volume binding
 }
 
+func ExampleNewCmdResults_custom() {
+	cmd := NewCmdResults()
+	cmd.SetArgs([]string{
+		filepath.Join("testdata", "testResultsOutputCustomStatuses.tar.gz"),
+		"--plugin=custom-status",
+	})
+	cmd.Execute()
+	// Output:
+	// Plugin: custom-status
+	// Status: custom-overall-status
+	// Total: 7
+	// Passed: 1
+	// Failed: 1
+	// Skipped: 1
+	// complete: 2
+	// custom: 2
+	//
+	// Failed tests:
+	// [sig-storage] CSI Volumes CSI Topology test using GCE PD driver [Serial] should fail to schedule a pod with a zone missing from AllowedTopologies; PD is provisioned with immediate volume binding
+}
+
 func ExampleNewCmdResults_detailed() {
 	cmd := NewCmdResults()
 	cmd.SetArgs([]string{