From e20f560425232247d47bffb9210d54d0f82a06be Mon Sep 17 00:00:00 2001 From: Mattia Lavacca Date: Thu, 22 Feb 2024 14:29:59 +0100 Subject: [PATCH 1/2] feat: conformance report summary field Signed-off-by: Mattia Lavacca --- conformance/apis/v1alpha1/profilereport.go | 8 +- conformance/experimental_conformance_test.go | 4 +- .../utils/suite/experimental_reports.go | 24 ++++ .../utils/suite/experimental_reports_test.go | 122 ++++++++++++++++++ 4 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 conformance/utils/suite/experimental_reports_test.go diff --git a/conformance/apis/v1alpha1/profilereport.go b/conformance/apis/v1alpha1/profilereport.go index 865a9d4aba..6e377ffddf 100644 --- a/conformance/apis/v1alpha1/profilereport.go +++ b/conformance/apis/v1alpha1/profilereport.go @@ -23,6 +23,10 @@ type ProfileReport struct { // "TLS", "Mesh", e.t.c.). Name string `json:"name"` + // Summary is a human-readable message intended for end-users to understand + // the overall status at a glance. + Summary string `json:"summary"` + // Core indicates the core support level which includes the set of tests // which are the minimum the implementation must pass to be considered at // all conformant. @@ -51,10 +55,6 @@ type ExtendedStatus struct { type Status struct { Result `json:"result"` - // Summary is a human-readable message intended for end-users to understand - // the overall status at a glance. - Summary string `json:"summary"` - // Statistics includes numerical statistics on the result of the test run. Statistics `json:"statistics"` diff --git a/conformance/experimental_conformance_test.go b/conformance/experimental_conformance_test.go index 4d4de988e1..a70681758d 100644 --- a/conformance/experimental_conformance_test.go +++ b/conformance/experimental_conformance_test.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/yaml" - v1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" @@ -66,7 +66,7 @@ func TestExperimentalConformance(t *testing.T) { v1alpha2.AddToScheme(mgrClient.Scheme()) v1beta1.AddToScheme(mgrClient.Scheme()) - v1.AddToScheme(mgrClient.Scheme()) + gatewayv1.AddToScheme(mgrClient.Scheme()) // standard conformance flags supportedFeatures = suite.ParseSupportedFeatures(*flags.SupportedFeatures) diff --git a/conformance/utils/suite/experimental_reports.go b/conformance/utils/suite/experimental_reports.go index d158a7cd73..42a5cd9236 100644 --- a/conformance/utils/suite/experimental_reports.go +++ b/conformance/utils/suite/experimental_reports.go @@ -17,6 +17,7 @@ limitations under the License. package suite import ( + "fmt" "sort" "k8s.io/apimachinery/pkg/util/sets" @@ -128,6 +129,7 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr report.Extended.Result = confv1a1.Success } } + report.Summary = buildSummary(report) p[key] = report supportedFeatures := supportedFeaturesMap[ConformanceProfileName(report.Name)] @@ -183,3 +185,25 @@ func isTestExtended(profile ConformanceProfile, test ConformanceTest) bool { } return false } + +// buildSummary creates a human-readable message about each profile's test outcomes. +func buildSummary(report confv1a1.ProfileReport) string { + reportMessage := fmt.Sprintf("Core tests %s", buildReportMessage(report.Core)) + if report.Extended != nil { + reportMessage = fmt.Sprintf("%s. Extended tests %s", reportMessage, buildReportMessage(report.Extended.Status)) + } + return fmt.Sprintf("%s.", reportMessage) +} + +func buildReportMessage(status confv1a1.Status) string { + var message string + switch status.Result { + case confv1a1.Success: + message = "succedeed" + case confv1a1.Partial: + message = fmt.Sprintf("partially succedeed with %d test skips", status.Statistics.Skipped) + case confv1a1.Failure: + message = fmt.Sprintf("failed with %d test failures", status.Statistics.Failed) + } + return message +} diff --git a/conformance/utils/suite/experimental_reports_test.go b/conformance/utils/suite/experimental_reports_test.go new file mode 100644 index 0000000000..15179e2b8f --- /dev/null +++ b/conformance/utils/suite/experimental_reports_test.go @@ -0,0 +1,122 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package suite + +import ( + "testing" + + "github.com/stretchr/testify/require" + + confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" +) + +func TestBuildSummary(t *testing.T) { + testCases := []struct { + name string + report confv1a1.ProfileReport + expectedSummary string + }{ + { + name: "core tests failed, no extended tests", + report: confv1a1.ProfileReport{ + Name: string(HTTPConformanceProfileName), + Core: confv1a1.Status{ + Result: confv1a1.Failure, + Statistics: confv1a1.Statistics{ + Passed: 5, + Failed: 3, + }, + }, + }, + expectedSummary: "Core tests failed with 3 test failures.", + }, + { + name: "core tests succedeed, extended tests failed", + report: confv1a1.ProfileReport{ + Name: string(HTTPConformanceProfileName), + Core: confv1a1.Status{ + Result: confv1a1.Success, + Statistics: confv1a1.Statistics{ + Passed: 8, + }, + }, + Extended: &confv1a1.ExtendedStatus{ + Status: confv1a1.Status{ + Result: confv1a1.Failure, + Statistics: confv1a1.Statistics{ + Passed: 2, + Failed: 1, + }, + }, + }, + }, + expectedSummary: "Core tests succedeed. Extended tests failed with 1 test failures.", + }, + { + name: "core tests partially succedeed, extended tests succedeed", + report: confv1a1.ProfileReport{ + Name: string(HTTPConformanceProfileName), + Core: confv1a1.Status{ + Result: confv1a1.Partial, + Statistics: confv1a1.Statistics{ + Passed: 6, + Skipped: 2, + }, + }, + Extended: &confv1a1.ExtendedStatus{ + Status: confv1a1.Status{ + Result: confv1a1.Success, + Statistics: confv1a1.Statistics{ + Passed: 2, + }, + }, + }, + }, + expectedSummary: "Core tests partially succedeed with 2 test skips. Extended tests succedeed.", + }, + { + name: "core tests succedeed, extended tests partially succedeed", + report: confv1a1.ProfileReport{ + Name: string(HTTPConformanceProfileName), + Core: confv1a1.Status{ + Result: confv1a1.Success, + Statistics: confv1a1.Statistics{ + Passed: 8, + }, + }, + Extended: &confv1a1.ExtendedStatus{ + Status: confv1a1.Status{ + Result: confv1a1.Partial, + Statistics: confv1a1.Statistics{ + Passed: 2, + Skipped: 1, + }, + }, + }, + }, + expectedSummary: "Core tests succedeed. Extended tests partially succedeed with 1 test skips.", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc := tc + summary := buildSummary(tc.report) + require.Equal(t, tc.expectedSummary, summary) + }) + } +} From b91df55766cefd366758ca445daa68e187e1209f Mon Sep 17 00:00:00 2001 From: Mattia Lavacca Date: Thu, 22 Feb 2024 19:03:34 +0100 Subject: [PATCH 2/2] address review's comments Signed-off-by: Mattia Lavacca --- conformance/utils/suite/experimental_reports.go | 14 +++++++------- .../utils/suite/experimental_reports_test.go | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/conformance/utils/suite/experimental_reports.go b/conformance/utils/suite/experimental_reports.go index 42a5cd9236..5b9b1c773a 100644 --- a/conformance/utils/suite/experimental_reports.go +++ b/conformance/utils/suite/experimental_reports.go @@ -187,21 +187,21 @@ func isTestExtended(profile ConformanceProfile, test ConformanceTest) bool { } // buildSummary creates a human-readable message about each profile's test outcomes. -func buildSummary(report confv1a1.ProfileReport) string { - reportMessage := fmt.Sprintf("Core tests %s", buildReportMessage(report.Core)) +func buildSummary(report confv1a1.ProfileReport) (reportSummary string) { + reportSummary = fmt.Sprintf("Core tests %s", buildReportSummary(report.Core)) if report.Extended != nil { - reportMessage = fmt.Sprintf("%s. Extended tests %s", reportMessage, buildReportMessage(report.Extended.Status)) + reportSummary = fmt.Sprintf("%s. Extended tests %s", reportSummary, buildReportSummary(report.Extended.Status)) } - return fmt.Sprintf("%s.", reportMessage) + return fmt.Sprintf("%s.", reportSummary) } -func buildReportMessage(status confv1a1.Status) string { +func buildReportSummary(status confv1a1.Status) string { var message string switch status.Result { case confv1a1.Success: - message = "succedeed" + message = "succeeded" case confv1a1.Partial: - message = fmt.Sprintf("partially succedeed with %d test skips", status.Statistics.Skipped) + message = fmt.Sprintf("partially succeeded with %d test skips", status.Statistics.Skipped) case confv1a1.Failure: message = fmt.Sprintf("failed with %d test failures", status.Statistics.Failed) } diff --git a/conformance/utils/suite/experimental_reports_test.go b/conformance/utils/suite/experimental_reports_test.go index 15179e2b8f..311bf5bffd 100644 --- a/conformance/utils/suite/experimental_reports_test.go +++ b/conformance/utils/suite/experimental_reports_test.go @@ -45,7 +45,7 @@ func TestBuildSummary(t *testing.T) { expectedSummary: "Core tests failed with 3 test failures.", }, { - name: "core tests succedeed, extended tests failed", + name: "core tests succeeded, extended tests failed", report: confv1a1.ProfileReport{ Name: string(HTTPConformanceProfileName), Core: confv1a1.Status{ @@ -64,10 +64,10 @@ func TestBuildSummary(t *testing.T) { }, }, }, - expectedSummary: "Core tests succedeed. Extended tests failed with 1 test failures.", + expectedSummary: "Core tests succeeded. Extended tests failed with 1 test failures.", }, { - name: "core tests partially succedeed, extended tests succedeed", + name: "core tests partially succeeded, extended tests succeeded", report: confv1a1.ProfileReport{ Name: string(HTTPConformanceProfileName), Core: confv1a1.Status{ @@ -86,10 +86,10 @@ func TestBuildSummary(t *testing.T) { }, }, }, - expectedSummary: "Core tests partially succedeed with 2 test skips. Extended tests succedeed.", + expectedSummary: "Core tests partially succeeded with 2 test skips. Extended tests succeeded.", }, { - name: "core tests succedeed, extended tests partially succedeed", + name: "core tests succeeded, extended tests partially succeeded", report: confv1a1.ProfileReport{ Name: string(HTTPConformanceProfileName), Core: confv1a1.Status{ @@ -108,7 +108,7 @@ func TestBuildSummary(t *testing.T) { }, }, }, - expectedSummary: "Core tests succedeed. Extended tests partially succedeed with 1 test skips.", + expectedSummary: "Core tests succeeded. Extended tests partially succeeded with 1 test skips.", }, }