[Feature] Integrated OpenDataHub Centralized Custom Certificate

controllers/constants/constants.go

@@ -35,3 +35,14 @@ const (
 	ModelRegistryModelVersionIdLabel     = "modelregistry.opendatahub.io/model-version-id"
 	ModelRegistryRegisteredModelIdLabel  = "modelregistry.opendatahub.io/registered-model-id"
 )
+
+const (
+	KServeCACertFileName       = "cabundle.crt"
+	KServeCACertConfigMapName  = "odh-kserve-custom-ca-bundle"
+	ODHGlobalCertConfigMapName = "odh-trusted-ca-bundle"
+	ODHCustomCACertFileName    = "odh-ca-bundle.crt"
+)
+
+const (
+	DefaultStorageConfig = "storage-config"
+)

controllers/kserve_customcacert_controller.go

@@ -0,0 +1,227 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/go-logr/logr"
+	"github.com/opendatahub-io/odh-model-controller/controllers/constants"
+	"github.com/opendatahub-io/odh-model-controller/controllers/utils"
+	corev1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+const (
+	kserveCustomCACertConfigMapName = constants.KServeCACertConfigMapName
+	kserveCustomCACertFileName      = constants.KServeCACertFileName
+	odhGlobalCACertConfigMapName    = constants.ODHGlobalCertConfigMapName
+)
+
+type KServeCustomCACertReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+	Log    logr.Logger
+}
+
+// reconcileConfigMap watch odh global ca cert and it will create/update/delete kserve custom cert configmap
+func (r *KServeCustomCACertReconciler) reconcileConfigMap(configmap *corev1.ConfigMap, ctx context.Context, log logr.Logger) error {
+
+	var odhCustomCertData string
+	// If kserve custom cert configmap changed, rollback it
+	if configmap.Name == kserveCustomCACertConfigMapName {
+		odhCustomCertConfigMap := &corev1.ConfigMap{}
+		err := r.Get(ctx, types.NamespacedName{Name: odhGlobalCACertConfigMapName, Namespace: configmap.Namespace}, odhCustomCertConfigMap)
+		if err != nil {
+			return err
+		}
+		configmap = odhCustomCertConfigMap
+	}
+	odhCustomCertData = configmap.Data[constants.ODHCustomCACertFileName]
+
+	// Create Desired resource
+	configData := map[string]string{kserveCustomCACertFileName: odhCustomCertData}
+	desiredResource := getDesiredCaCertConfigMapForKServe(kserveCustomCACertConfigMapName, configmap.Namespace, configData)
+
+	// Get Existing resource
+	existingResource := &corev1.ConfigMap{}
+	err := r.Get(ctx, types.NamespacedName{Name: kserveCustomCACertConfigMapName, Namespace: configmap.Namespace}, existingResource)
+	if err != nil {
+		if apierrs.IsNotFound(err) {
+			existingResource = nil
+		} else {
+			return err
+		}
+	}
+	// Process Delta
+	if err = r.processDelta(ctx, log, desiredResource, existingResource); err != nil {
+		return err
+	}
+	return nil
+
+}
+
+func checkOpenDataHubGlobalCertCAConfigMapName(objectName string) bool {
+	return objectName == odhGlobalCACertConfigMapName || objectName == kserveCustomCACertConfigMapName
+}
+
+// reconcileOpenDataHubGlobalCertConfigMap filters out all ConfigMaps that are not the OpenDataHub global certificate ConfigMap.
+func reconcileOpenDataHubGlobalCACertConfigMap() predicate.Predicate {
+	return predicate.Funcs{
+		CreateFunc: func(e event.CreateEvent) bool {
+			objectName := e.Object.GetName()
+			return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+		},
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			objectName := e.Object.GetName()
+			return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+		},		
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			objectName := e.ObjectNew.GetName()
+			return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+		},
+	}
+}
+
+func (r *KServeCustomCACertReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Initialize logger format
+	log := r.Log.WithValues("ConfigMap", req.Name, "namespace", req.Namespace)
+
+	configmap := &corev1.ConfigMap{}
+	err := r.Get(ctx, req.NamespacedName, configmap)
+	if err != nil && apierrs.IsNotFound(err) {
+		log.Info("Opendatahub global cert ConfigMap not found")
+		configmap.Namespace = req.Namespace
+	} else if err != nil {
+		log.Error(err, "Unable to fetch the ConfigMap")
+		return ctrl.Result{}, err
+	}
+
+	err = r.reconcileConfigMap(configmap, ctx, log)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *KServeCustomCACertReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	// Create a builder that only watch OpenDataHub global certificate ConfigMap
+	builder := ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.ConfigMap{}).
+		WithEventFilter(reconcileOpenDataHubGlobalCACertConfigMap())
+	err := builder.Complete(r)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getDesiredCaCertConfigMapForKServe(configmapName string, namespace string, caCertData map[string]string) *corev1.ConfigMap {
+	desiredConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configmapName,
+			Namespace: namespace,
+			Labels:    map[string]string{"opendatahub.io/managed": "true"},
+		},
+		Data: caCertData,
+	}
+
+	return desiredConfigMap
+}
+
+func (r *KServeCustomCACertReconciler) processDelta(ctx context.Context, log logr.Logger, desiredConfigMap *corev1.ConfigMap, existingConfigMap *corev1.ConfigMap) (err error) {
+	hasChanged := false
+
+	if isAdded(desiredConfigMap, existingConfigMap) {
+		hasChanged = true
+		log.V(1).Info("Delta found", "create", desiredConfigMap.GetName())
+		if err = r.Create(ctx, desiredConfigMap); err != nil {
+			return err
+		}
+	}
+
+	if isUpdated(desiredConfigMap, existingConfigMap) {
+		hasChanged = true
+		log.V(1).Info("Delta found", "update", existingConfigMap.GetName())
+		rp := desiredConfigMap.DeepCopy()
+		rp.Labels = existingConfigMap.Labels
+
+		if err = r.Update(ctx, rp); err != nil {
+			return err
+		}
+	}
+
+	if isRemoved(desiredConfigMap, existingConfigMap) {
+		hasChanged = true
+		log.V(1).Info("Delta found", "delete", existingConfigMap.GetName())
+		if err = r.Delete(ctx, existingConfigMap); err != nil {
+			return err
+		}
+	}
+
+	if !hasChanged {
+		log.V(1).Info("No delta found")
+		return nil
+	}
+
+	if err := r.deleteStorageSecret(ctx, desiredConfigMap.Namespace); err != nil {
+		log.Error(err, "Failed to delete the storage-config secret to update the custom cert")
+		return err
+	}
+	log.V(1).Info("Deleted the storage-config Secret to update the custom cert")
+
+	return nil
+}
+
+// This section is intended for regenerating StorageSecret using new data.
+func (r *KServeCustomCACertReconciler) deleteStorageSecret(ctx context.Context, namespace string) error {
+	foundStorageSecret := &corev1.Secret{}
+
+	err := r.Get(ctx, types.NamespacedName{
+		Name:      constants.DefaultStorageConfig,
+		Namespace: namespace,
+	}, foundStorageSecret)
+
+	if err == nil && foundStorageSecret.Labels["opendatahub.io/managed"] == "true" {
+		err = r.Delete(ctx, foundStorageSecret)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func isAdded(desiredConfigMap *corev1.ConfigMap, existingConfigMap *corev1.ConfigMap) bool {
+	return desiredConfigMap.Data[kserveCustomCACertFileName] != "" && utils.IsNil(existingConfigMap)
+}
+
+func isUpdated(desiredConfigMap *corev1.ConfigMap, existingConfigMap *corev1.ConfigMap) bool {
+	return utils.IsNotNil(existingConfigMap) && desiredConfigMap.Data[kserveCustomCACertFileName] != "" && !reflect.DeepEqual(desiredConfigMap.Data, existingConfigMap.Data)
+}
+
+func isRemoved(desiredConfigMap *corev1.ConfigMap, existingConfigMap *corev1.ConfigMap) bool {
+	return utils.IsNotNil(existingConfigMap) && desiredConfigMap.Data[kserveCustomCACertFileName] == ""
+}

controllers/kserve_customcacert_controller_test.go

@@ -0,0 +1,100 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"reflect"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/opendatahub-io/odh-model-controller/controllers/constants"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	odhtrustedcabundleConfigMapUpdatedPath = "./testdata/configmaps/odh-trusted-ca-bundle-configmap-updated.yaml"
+	kservecustomcacertConfigMapUpdatedPath = "./testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml"
+)
+
+var _ = Describe("KServe Custom CA Cert ConfigMap controller", func() {
+	ctx := context.Background()
+
+	Context("when a configmap 'odh-trusted-ca-bundle' exists", func() {
+		It("should create a configmap that is for kserve custom ca cert", func() {
+			By("creating odh-trusted-ca-bundle configmap")
+			odhtrustedcacertConfigMap := &corev1.ConfigMap{}
+			err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed())
+
+			_, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
+	Context("when a configmap 'odh-trusted-ca-bundle' updated", func() {
+		It("should update kserve custom cert configmap", func() {
+			By("creating odh-trusted-ca-bundle configmap")
+			odhtrustedcacertConfigMap := &corev1.ConfigMap{}
+			err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed())
+
+			_, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("updating odh-trusted-ca-bundle configmap")
+			updatedOdhtrustedcacertConfigMap := &corev1.ConfigMap{}
+			err = convertToStructuredResource(odhtrustedcabundleConfigMapUpdatedPath, updatedOdhtrustedcacertConfigMap)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(cli.Update(ctx, updatedOdhtrustedcacertConfigMap)).Should(Succeed())
+
+			// Wait for updating ConfigMap
+			time.Sleep(1 * time.Second)
+			kserveCACertConfigmap, err := waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+			expectedKserveCACertConfigmap := &corev1.ConfigMap{}
+			err = convertToStructuredResource(kservecustomcacertConfigMapUpdatedPath, expectedKserveCACertConfigmap)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(compareConfigMap(kserveCACertConfigmap, expectedKserveCACertConfigmap)).Should((BeTrue()))
+		})
+	})
+})
+
+func waitForConfigMap(cli client.Client, namespace, configmapName string, timeout time.Duration) (*corev1.ConfigMap, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	for {
+		configmap := &corev1.ConfigMap{}
+		err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configmapName}, configmap)
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		return configmap, nil
+	}
+}
+
+// compareConfigMap checks if two ConfigMap data are equal, if not return false
+func compareConfigMap(s1 *corev1.ConfigMap, s2 *corev1.ConfigMap) bool {
+	// Two ConfigMap will be equal if the data is identical
+	return reflect.DeepEqual(s1.Data, s2.Data)
+}