[Feature] Integrated OpenDataHub Centralized Custom Certificate

controllers/constants/constants.go

@@ -35,3 +35,14 @@ const (
 	ModelRegistryModelVersionIdLabel     = "modelregistry.opendatahub.io/model-version-id"
 	ModelRegistryRegisteredModelIdLabel  = "modelregistry.opendatahub.io/registered-model-id"
 )
+
+const (
+	KServeCACertFileName       = "cabundle.crt"
+	KServeCACertConfigMapName  = "odh-kserve-custom-ca-bundle"
+	ODHGlobalCertConfigMapName = "odh-trusted-ca-bundle"
+	ODHCustomCACertFileName    = "odh-ca-bundle.crt"
+)
+
+const (
+	DefaultStorageConfig = "storage-config"
+)

controllers/kserve_customcacert_controller.go

@@ -0,0 +1,203 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	"github.com/opendatahub-io/odh-model-controller/controllers/constants"
+	corev1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+const (
+	kserveCustomCACertConfigMapName = constants.KServeCACertConfigMapName
+	odhGlobalCACertConfigMapName    = constants.ODHGlobalCertConfigMapName
+)
+
+type KServeCustomCACertReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+	Log    logr.Logger
+}
+
+// reconcileConfigMap watch odh global ca cert and it will create/update/delete kserve custom cert configmap
+func (r *KServeCustomCACertReconciler) reconcileConfigMap(configmap *corev1.ConfigMap,
+	ctx context.Context) error {
+	// Initialize logger format
+	log := r.Log
+
+	odhCustomCertData := configmap.Data[constants.ODHCustomCACertFileName]
+	if odhCustomCertData != "" {
+		configData := map[string]string{
+			constants.KServeCACertFileName: odhCustomCertData,
+		}
+		newCaCertConfigMap := &corev1.ConfigMap{}
+		newCaCertConfigMap = getDesiredCaCertConfigMapForKServe(kserveCustomCACertConfigMapName, configmap.Namespace, configData)
+
+		existingOdhCustomCertData := ""
+		kserveCustomCertConfigMap := &corev1.ConfigMap{}
+		err := r.Get(ctx, types.NamespacedName{
+			Name:      kserveCustomCACertConfigMapName,
+			Namespace: configmap.Namespace,
+		}, kserveCustomCertConfigMap)
+		if err == nil {
+			log.Info("Checking if the data in KServe custom cert ConfigMap differs from the data in the opendatahub global CA cert ConfigMap")
+			existingOdhCustomCertData = kserveCustomCertConfigMap.Data[constants.KServeCACertFileName]
+			if existingOdhCustomCertData == odhCustomCertData {
+				log.Info(fmt.Sprintf("No updates required for KServe custom cert ConfigMap (%s) as the data matches the opendatahub global cert ConfigMap (%s)", kserveCustomCACertConfigMapName, odhGlobalCACertConfigMapName))
+				return nil
+			} else {
+				log.Info(fmt.Sprintf("Updating KServe custom cert ConfigMap due to changes in the opendatahub global cert ConfigMap (%s)", odhGlobalCACertConfigMapName))
+				if err := r.Update(context.TODO(), newCaCertConfigMap); err != nil {
+					return err
+				}
+				err = r.deleteStorageSecret(ctx, configmap.Namespace)
+				if err != nil {
+					log.Error(err, "Failed to delete the storage-config secret to update the custom cert")
+					return err
+				}
+				log.V(1).Info("Deleted the storage-config Secret to update the custom cert")
+			}
+		} else {
+			if apierrs.IsNotFound(err) {
+				log.Info(fmt.Sprintf("Creating KServe custom cert ConfigMap (%s) because it detected the creation of the opendatahub global cert ConfigMap (%s)", kserveCustomCACertConfigMapName, odhGlobalCACertConfigMapName))
+				if err := r.Create(context.TODO(), newCaCertConfigMap); err != nil {
+					log.Error(err, "Failed to create KServe custom cert ConfigMap")
+					return err
+				}
+			} else {
+				return err
+			}
+		}
+	} else {
+		log.Info(fmt.Sprintf("Detected opendatahub global cert ConfigMap (%s), but custom cert is not set\n", odhGlobalCACertConfigMapName))
+		kserveCustomCertConfigMap := &corev1.ConfigMap{}
+		err := r.Get(ctx, types.NamespacedName{
+			Name:      kserveCustomCACertConfigMapName,
+			Namespace: configmap.Namespace,
+		}, kserveCustomCertConfigMap)
+		if err == nil {
+			log.Info(fmt.Sprintf("Deleting KServe custom cert ConfigMap (%s)", kserveCustomCACertConfigMapName))
+			err := r.Delete(ctx, kserveCustomCertConfigMap)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// This section is intended for regenerating StorageSecret using new data.
+func (r *KServeCustomCACertReconciler) deleteStorageSecret(ctx context.Context, namespace string) error {
+	foundStorageSecret := &corev1.Secret{}
+
+	err := r.Get(ctx, types.NamespacedName{
+		Name:      constants.DefaultStorageConfig,
+		Namespace: namespace,
+	}, foundStorageSecret)
+
+	if err == nil {
+		err = r.Delete(ctx, foundStorageSecret)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func checkOpenDataHubGlobalCertCAConfigMapName(objectName string) bool {
+	return objectName == odhGlobalCACertConfigMapName
+}
+
+// reconcileOpenDataHubGlobalCertConfigMap filters out all ConfigMaps that are not the OpenDataHub global certificate ConfigMap.
+func reconcileOpenDataHubGlobalCACertConfigMap() predicate.Predicate {
+	return predicate.Funcs{
+		CreateFunc: func(e event.CreateEvent) bool {
+			objectName := e.Object.GetName()
+			return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+		},
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			objectName := e.Object.GetName()
+			return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+		},
+		GenericFunc: func(e event.GenericEvent) bool {
+			objectName := e.Object.GetName()
+			return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+		},
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			objectName := e.ObjectNew.GetName()
+			return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+		},
+	}
+}
+
+func (r *KServeCustomCACertReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Initialize logger format
+	log := r.Log.WithValues("ConfigMap", req.Name, "namespace", req.Namespace)
+
+	configmap := &corev1.ConfigMap{}
+	err := r.Get(ctx, req.NamespacedName, configmap)
+	if err != nil && apierrs.IsNotFound(err) {
+		log.Info("Opendatahub global cert ConfigMap not found")
+		configmap.Namespace = req.Namespace
+	} else if err != nil {
+		log.Error(err, "Unable to fetch the ConfigMap")
+		return ctrl.Result{}, err
+	}
+
+	err = r.reconcileConfigMap(configmap, ctx)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *KServeCustomCACertReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	// Create a builder that only watch OpenDataHub global certificate ConfigMap
+	builder := ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.ConfigMap{}).
+		WithEventFilter(reconcileOpenDataHubGlobalCACertConfigMap())
+	err := builder.Complete(r)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getDesiredCaCertConfigMapForKServe(configmapName string, namespace string, caCertData map[string]string) *corev1.ConfigMap {
+	desiredConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configmapName,
+			Namespace: namespace,
+		},
+		Data: caCertData,
+	}
+
+	return desiredConfigMap
+}

controllers/kserve_customcacert_controller_test.go

@@ -0,0 +1,99 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/opendatahub-io/odh-model-controller/controllers/constants"
+	corev1 "k8s.io/api/core/v1"
+	"reflect"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"time"
+)
+
+const (
+	odhtrustedcabundleConfigMapUpdatedPath = "./testdata/configmaps/odh-trusted-ca-bundle-configmap-updated.yaml"
+	kservecustomcacertConfigMapUpdatedPath = "./testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml"
+)
+
+var _ = Describe("KServe Custom CA Cert ConfigMap controller", func() {
+	ctx := context.Background()
+
+	Context("when a configmap 'odh-trusted-ca-bundle' exists", func() {
+		It("should create a configmap that is for kserve custom ca cert", func() {
+			By("creating odh-trusted-ca-bundle configmap")
+			odhtrustedcacertConfigMap := &corev1.ConfigMap{}
+			err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed())
+
+			_, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
+	Context("when a configmap 'odh-trusted-ca-bundle' updated", func() {
+		It("should update kserve custom cert configmap", func() {
+			By("creating odh-trusted-ca-bundle configmap")
+			odhtrustedcacertConfigMap := &corev1.ConfigMap{}
+			err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed())
+
+			_, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("updating odh-trusted-ca-bundle configmap")
+			updatedOdhtrustedcacertConfigMap := &corev1.ConfigMap{}
+			err = convertToStructuredResource(odhtrustedcabundleConfigMapUpdatedPath, updatedOdhtrustedcacertConfigMap)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(cli.Update(ctx, updatedOdhtrustedcacertConfigMap)).Should(Succeed())
+
+			// Wait for updating ConfigMap
+			time.Sleep(1 * time.Second)
+			kserveCACertConfigmap, err := waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+			expectedKserveCACertConfigmap := &corev1.ConfigMap{}
+			err = convertToStructuredResource(kservecustomcacertConfigMapUpdatedPath, expectedKserveCACertConfigmap)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(compareConfigMap(kserveCACertConfigmap, expectedKserveCACertConfigmap)).Should((BeTrue()))
+		})
+	})
+})
+
+func waitForConfigMap(cli client.Client, namespace, configmapName string, timeout time.Duration) (*corev1.ConfigMap, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	for {
+		configmap := &corev1.ConfigMap{}
+		err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configmapName}, configmap)
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		return configmap, nil
+	}
+}
+
+// compareConfigMap checks if two ConfigMap data are equal, if not return false
+func compareConfigMap(s1 *corev1.ConfigMap, s2 *corev1.ConfigMap) bool {
+	// Two ConfigMap will be equal if the data is identical
+	return reflect.DeepEqual(s1.Data, s2.Data)
+}

controllers/storageconfig_controller.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 
 	"github.com/go-logr/logr"
+	"github.com/opendatahub-io/odh-model-controller/controllers/constants"
 	corev1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -33,7 +34,7 @@ import (
 )
 
 const (
-	storageSecretName = "storage-config"
+	storageSecretName = constants.DefaultStorageConfig
 )
 
 type StorageSecretReconciler struct {
@@ -44,7 +45,7 @@ type StorageSecretReconciler struct {
 
 // newStorageSecret takes a list of data connection secrets and generates a single storage config secret
 // https://github.com/kserve/modelmesh-serving/blob/main/docs/predictors/setup-storage.md
-func newStorageSecret(dataConnectionSecretsList *corev1.SecretList) *corev1.Secret {
+func newStorageSecret(dataConnectionSecretsList *corev1.SecretList, odhCustomCertData string) *corev1.Secret {
 	desiredSecret := &corev1.Secret{}
 	desiredSecret.Data = map[string][]byte{}
 	dataConnectionElement := map[string]string{}
@@ -59,8 +60,10 @@ func newStorageSecret(dataConnectionSecretsList *corev1.SecretList) *corev1.Secr
 		dataConnectionElement["default_bucket"] = string(secret.Data["AWS_S3_BUCKET"])
 		dataConnectionElement["bucket"] = string(secret.Data["AWS_S3_BUCKET"])
 		dataConnectionElement["region"] = string(secret.Data["AWS_DEFAULT_REGION"])
-		if secret.Data["AWS_CA_BUNDLE"] != nil {
-			dataConnectionElement["certificate"] = string(secret.Data["AWS_CA_BUNDLE"])
+
+		if odhCustomCertData != "" {
+			dataConnectionElement["certificate"] = odhCustomCertData
+			dataConnectionElement["cabundle_configmap"] = constants.KServeCACertConfigMapName
 		}
 		jsonBytes, _ := json.Marshal(dataConnectionElement)
 		storageByteData[secret.Name] = jsonBytes
@@ -77,7 +80,7 @@ func CompareStorageSecrets(s1 corev1.Secret, s2 corev1.Secret) bool {
 // reconcileSecret grabs all data connection secrets in the triggering namespace and
 // creates/updates the storage config secret
 func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret,
-	ctx context.Context, newStorageSecret func(dataConnectionSecretsList *corev1.SecretList) *corev1.Secret) error {
+	ctx context.Context, newStorageSecret func(dataConnectionSecretsList *corev1.SecretList, odhCustomCertData string) *corev1.Secret) error {
 	// Initialize logger format
 	log := r.Log.WithValues("secret", secret.Name, "namespace", secret.Namespace)
 
@@ -91,11 +94,23 @@ func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret,
 	if err != nil {
 		if apierrs.IsNotFound(err) {
 			log.Info("No data connections found in namespace ", secret.Namespace)
+			return nil
 		}
 	}
 
+	odhCustomCertData := ""
+	odhGlobalCertConfigMap := &corev1.ConfigMap{}
+	err = r.Get(ctx, types.NamespacedName{
+		Name:      constants.ODHGlobalCertConfigMapName,
+		Namespace: secret.Namespace,
+	}, odhGlobalCertConfigMap)
+
+	if err == nil {
+		odhCustomCertData = odhGlobalCertConfigMap.Data[constants.ODHCustomCACertFileName]
+	}
+
 	// Generate desire Storage Config Secret
-	desiredStorageSecret := newStorageSecret(dataConnectionSecretsList)
+	desiredStorageSecret := newStorageSecret(dataConnectionSecretsList, odhCustomCertData)
 	desiredStorageSecret.Name = storageSecretName
 	desiredStorageSecret.Namespace = secret.Namespace
 	desiredStorageSecret.Labels = map[string]string{}