use FieldValidation only if the k8s version is above 1.25

Signed-off-by: clyang82 <chuyang@redhat.com>

controllers/configurationpolicy_controller.go

@@ -18,6 +18,7 @@ import (
 	gocmp "github.com/google/go-cmp/cmp"
 	"github.com/prometheus/client_golang/prometheus"
 	templates "github.com/stolostron/go-template-utils/v3/pkg/templates"
+	"golang.org/x/mod/semver"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	extensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -29,12 +30,17 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	apimachineryerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/tools/record"
+	kubeopenapivalidation "k8s.io/kube-openapi/pkg/util/proto/validation"
+	"k8s.io/kubectl/pkg/util/openapi"
+	openapivalidation "k8s.io/kubectl/pkg/util/openapi/validation"
+	"k8s.io/kubectl/pkg/validation"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -90,6 +96,7 @@ type cachedEncryptionKey struct {
 type discoveryInfo struct {
 	apiResourceList        []*metav1.APIResourceList
 	apiGroups              []*restmapper.APIGroupResources
+	serverVersion          string
 	discoveryLastRefreshed time.Time
 }
 
@@ -113,6 +120,8 @@ type ConfigurationPolicyReconciler struct {
 	// Whether custom metrics collection is enabled
 	EnableMetrics bool
 	discoveryInfo
+	// This is used to fetch and parse OpenAPI documents to perform client-side validation of object definitions.
+	openAPIParser *openapi.CachedOpenAPIParser
 	// A lock when performing actions that are not thread safe (i.e. reassigning object properties).
 	lock sync.RWMutex
 }
@@ -290,6 +299,13 @@ func (r *ConfigurationPolicyReconciler) refreshDiscoveryInfo() error {
 
 	dd := r.TargetK8sClient.Discovery()
 
+	serverVersion, err := dd.ServerVersion()
+	if err != nil {
+		log.Error(err, "Could not get the server version")
+	}
+
+	r.serverVersion = serverVersion.String()
+
 	_, apiResourceList, resourceErr := dd.ServerGroupsAndResources()
 	if resourceErr != nil {
 		log.Error(resourceErr, "Could not get the full API resource list")
@@ -310,6 +326,8 @@ func (r *ConfigurationPolicyReconciler) refreshDiscoveryInfo() error {
 		r.discoveryInfo.apiGroups = apiGroups
 	}
 
+	// Reset the OpenAPI cache in case the CRDs were updated since the last fetch.
+	r.openAPIParser = openapi.NewOpenAPIParser(dd)
 	r.discoveryInfo.discoveryLastRefreshed = time.Now().UTC()
 
 	if resourceErr != nil {
@@ -2178,6 +2196,14 @@ func (r *ConfigurationPolicyReconciler) createObject(
 	objLog := log.WithValues("name", unstruct.GetName(), "namespace", unstruct.GetNamespace())
 	objLog.V(2).Info("Entered createObject", "unstruct", unstruct)
 
+	// FieldValidation is supported in k8s 1.25 as beta release
+	// so if the version is below 1.25, we need to use client side validation to validate the object
+	if semver.Compare(r.serverVersion, "v1.25.0") < 0 {
+		if err := r.validateObject(&unstruct); err != nil {
+			return nil, err
+		}
+	}
+
 	object, err = res.Create(context.TODO(), &unstruct, metav1.CreateOptions{
 		FieldValidation: metav1.FieldValidationStrict,
 	})
@@ -2525,6 +2551,53 @@ func handleSingleKey(
 	return "", updateNeeded, mergedValue, false
 }
 
+// validateObject performs client-side validation of the input object using the server's OpenAPI definitions that are
+// cached. An error is returned if the input object is invalid or the OpenAPI data could not be fetched.
+func (r *ConfigurationPolicyReconciler) validateObject(object *unstructured.Unstructured) error {
+	// Parse() handles caching of the OpenAPI data.
+	r.lock.RLock()
+	openAPIResources, err := r.openAPIParser.Parse()
+	r.lock.RUnlock()
+
+	if err != nil {
+		return fmt.Errorf("failed to retrieve the OpenAPI data from the Kubernetes API: %w", err)
+	}
+
+	schema := validation.ConjunctiveSchema{
+		openapivalidation.NewSchemaValidation(openAPIResources),
+		validation.NoDoubleKeySchema{},
+	}
+
+	objectJSON, err := object.MarshalJSON()
+	if err != nil {
+		return fmt.Errorf("failed to marshal the object to JSON: %w", err)
+	}
+
+	schemaErr := schema.ValidateBytes(objectJSON)
+
+	// Filter out errors due to missing fields in the status since those are ignored when enforcing the policy. This
+	// allows a user to switch a policy between inform and enforce without having to remove the status check.
+	return apimachineryerrors.FilterOut(
+		schemaErr,
+		func(err error) bool {
+			var validationErr kubeopenapivalidation.ValidationError
+			if !errors.As(err, &validationErr) {
+				return false
+			}
+
+			// Path is in the format of Pod.status.conditions[0].
+			pathParts := strings.SplitN(validationErr.Path, ".", 3)
+			if len(pathParts) < 2 || pathParts[1] != "status" {
+				return false
+			}
+
+			var missingFieldErr kubeopenapivalidation.MissingRequiredFieldError
+
+			return errors.As(validationErr.Err, &missingFieldErr)
+		},
+	)
+}
+
 // checkAndUpdateResource checks each individual key of a resource and passes it to handleKeys to see if it
 // matches the template and update it if the remediationAction is enforce. UpdateNeeded indicates whether the
 // function tried to update the child object and updateSucceeded indicates whether the update was applied
@@ -2611,6 +2684,16 @@ func (r *ConfigurationPolicyReconciler) checkAndUpdateResource(
 	if updateNeeded {
 		log.V(2).Info("Updating the object based on the template definition")
 
+		// FieldValidation is supported in k8s 1.25 as beta release
+		// so if the version is below 1.25, we need to use client side validation to validate the object
+		if semver.Compare(r.serverVersion, "v1.25.0") < 0 {
+			if err := r.validateObject(obj.object); err != nil {
+				message := fmt.Sprintf("Error validating the object %s, the error is `%v`", obj.name, err)
+
+				return false, message, true, updateNeeded, false
+			}
+		}
+
 		_, err = res.Update(context.TODO(), obj.object, metav1.UpdateOptions{
 			FieldValidation: metav1.FieldValidationStrict,
 		})

go.mod

@@ -13,12 +13,15 @@ require (
 	github.com/stolostron/go-log-utils v0.1.2
 	github.com/stolostron/go-template-utils/v3 v3.2.1
 	github.com/stretchr/testify v1.8.1
+	golang.org/x/mod v0.10.0
 	k8s.io/api v0.27.1
 	k8s.io/apiextensions-apiserver v0.27.1
 	k8s.io/apimachinery v0.27.1
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.100.1
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f
+	k8s.io/kubectl v0.0.0-00010101000000-000000000000
 	open-cluster-management.io/addon-framework v0.6.1
 	sigs.k8s.io/controller-runtime v0.14.6
 	sigs.k8s.io/yaml v1.3.0
@@ -34,7 +37,9 @@ require (
 	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -46,6 +51,7 @@ require (
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
@@ -58,6 +64,7 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -67,6 +74,8 @@ require (
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/stolostron/kubernetes-dependency-watches v0.2.1 // indirect
+	github.com/xlab/treeprint v1.1.0 // indirect
+	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
@@ -85,10 +94,12 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/cli-runtime v0.26.4 // indirect
 	k8s.io/component-base v0.27.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.12.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )