Skip to content

Commit

Permalink
feat: conformance report summary field
Browse files Browse the repository at this point in the history
Signed-off-by: Mattia Lavacca <lavacca.mattia@gmail.com>
  • Loading branch information
mlavacca committed Feb 22, 2024
1 parent 8a58569 commit f5ff512
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 6 deletions.
8 changes: 4 additions & 4 deletions conformance/apis/v1alpha1/profilereport.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ type ProfileReport struct {
// "TLS", "Mesh", e.t.c.).
Name string `json:"name"`

// Summary is a human-readable message intended for end-users to understand
// the overall status at a glance.
Summary string `json:"summary"`

// Core indicates the core support level which includes the set of tests
// which are the minimum the implementation must pass to be considered at
// all conformant.
Expand Down Expand Up @@ -51,10 +55,6 @@ type ExtendedStatus struct {
type Status struct {
Result `json:"result"`

// Summary is a human-readable message intended for end-users to understand
// the overall status at a glance.
Summary string `json:"summary"`

// Statistics includes numerical statistics on the result of the test run.
Statistics `json:"statistics"`

Expand Down
4 changes: 2 additions & 2 deletions conformance/experimental_conformance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/yaml"

v1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
"sigs.k8s.io/gateway-api/apis/v1alpha2"
"sigs.k8s.io/gateway-api/apis/v1beta1"
confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
Expand Down Expand Up @@ -66,7 +66,7 @@ func TestExperimentalConformance(t *testing.T) {

v1alpha2.AddToScheme(mgrClient.Scheme())
v1beta1.AddToScheme(mgrClient.Scheme())
v1.AddToScheme(mgrClient.Scheme())
gatewayv1.AddToScheme(mgrClient.Scheme())

// standard conformance flags
supportedFeatures = suite.ParseSupportedFeatures(*flags.SupportedFeatures)
Expand Down
24 changes: 24 additions & 0 deletions conformance/utils/suite/experimental_reports.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package suite

import (
"fmt"
"sort"

"k8s.io/apimachinery/pkg/util/sets"
Expand Down Expand Up @@ -128,6 +129,7 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr
report.Extended.Result = confv1a1.Success
}
}
report.Summary = buildSummary(report)
p[key] = report

supportedFeatures := supportedFeaturesMap[ConformanceProfileName(report.Name)]
Expand Down Expand Up @@ -183,3 +185,25 @@ func isTestExtended(profile ConformanceProfile, test ConformanceTest) bool {
}
return false
}

// buildSummary creates a human-readable message about each profile's test outcomes.
func buildSummary(report confv1a1.ProfileReport) string {
reportMessage := fmt.Sprintf("Core tests %s", buildReportMessage(report.Core))
if report.Extended != nil {
reportMessage = fmt.Sprintf("%s. Extended tests %s", reportMessage, buildReportMessage(report.Extended.Status))
}
return fmt.Sprintf("%s.", reportMessage)
}

func buildReportMessage(status confv1a1.Status) string {
var message string
switch status.Result {
case confv1a1.Success:
message = "succeded"
case confv1a1.Partial:
message = fmt.Sprintf("partially succeded with %d test skips", status.Statistics.Skipped)
case confv1a1.Failure:
message = fmt.Sprintf("failed with %d test failures", status.Statistics.Failed)
}
return message
}
121 changes: 121 additions & 0 deletions conformance/utils/suite/experimental_reports_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package suite

import (
"testing"

"github.com/stretchr/testify/require"
confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
)

func TestBuildSummary(t *testing.T) {
testCases := []struct {
name string
report confv1a1.ProfileReport
expectedSummary string
}{
{
name: "core tests failed, no extended tests",
report: confv1a1.ProfileReport{
Name: string(HTTPConformanceProfileName),
Core: confv1a1.Status{
Result: confv1a1.Failure,
Statistics: confv1a1.Statistics{
Passed: 5,
Failed: 3,
},
},
},
expectedSummary: "Core tests failed with 3 test failures.",
},
{
name: "core tests succeded, extended tests failed",
report: confv1a1.ProfileReport{
Name: string(HTTPConformanceProfileName),
Core: confv1a1.Status{
Result: confv1a1.Success,
Statistics: confv1a1.Statistics{
Passed: 8,
},
},
Extended: &confv1a1.ExtendedStatus{
Status: confv1a1.Status{
Result: confv1a1.Failure,
Statistics: confv1a1.Statistics{
Passed: 2,
Failed: 1,
},
},
},
},
expectedSummary: "Core tests succeded. Extended tests failed with 1 test failures.",
},
{
name: "core tests partially succeded, extended tests succeded",
report: confv1a1.ProfileReport{
Name: string(HTTPConformanceProfileName),
Core: confv1a1.Status{
Result: confv1a1.Partial,
Statistics: confv1a1.Statistics{
Passed: 6,
Skipped: 2,
},
},
Extended: &confv1a1.ExtendedStatus{
Status: confv1a1.Status{
Result: confv1a1.Success,
Statistics: confv1a1.Statistics{
Passed: 2,
},
},
},
},
expectedSummary: "Core tests partially succeded with 2 test skips. Extended tests succeded.",
},
{
name: "core tests succeded, extended tests partially succeded",
report: confv1a1.ProfileReport{
Name: string(HTTPConformanceProfileName),
Core: confv1a1.Status{
Result: confv1a1.Success,
Statistics: confv1a1.Statistics{
Passed: 8,
},
},
Extended: &confv1a1.ExtendedStatus{
Status: confv1a1.Status{
Result: confv1a1.Partial,
Statistics: confv1a1.Statistics{
Passed: 2,
Skipped: 1,
},
},
},
},
expectedSummary: "Core tests succeded. Extended tests partially succeded with 1 test skips.",
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
tc := tc
summary := buildSummary(tc.report)
require.Equal(t, tc.expectedSummary, summary)
})
}
}

0 comments on commit f5ff512

Please sign in to comment.