[Feature] Integrated OpenDataHub Centralized Custom Certificate

controllers/constants/constants.go

@@ -35,3 +35,14 @@ const (
  ModelRegistryModelVersionIdLabel = "modelregistry.opendatahub.io/model-version-id"
  ModelRegistryRegisteredModelIdLabel = "modelregistry.opendatahub.io/registered-model-id"
 )
+
+const (
+ KServeCACertFileName = "cabundle.crt"
+ KServeCACertConfigMapName = "odh-kserve-custom-ca-bundle"
+ ODHGlobalCertConfigMapName = "odh-trusted-ca-bundle"
+ ODHCustomCACertFileName = "odh-ca-bundle.crt"
+)
+
+const (
+ DefaultStorageConfig = "storage-config"
+)

controllers/kserve_customcacert_controller.go

@@ -0,0 +1,190 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/go-logr/logr"
+ "github.com/opendatahub-io/odh-model-controller/controllers/constants"
+ corev1 "k8s.io/api/core/v1"
+ apierrs "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/event"
+ "sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+const (
+ kserveCustomCACertConfigMapName = constants.KServeCACertConfigMapName
+ odhGlobalCACertConfigMapName = constants.ODHGlobalCertConfigMapName
+)
+
+type KServeCustomCACertReconciler struct {
+ client.Client
+ Scheme *runtime.Scheme
+ Log logr.Logger
+}
+
+// reconcileConfigMap watch odh global ca cert and it will create/update/delete kserve custom cert configmap
+func (r *KServeCustomCACertReconciler) reconcileConfigMap(configmap *corev1.ConfigMap, ctx context.Context) error {
+ // Initialize logger format
+ log := r.Log
+
+ odhCustomCertData := configmap.Data[constants.ODHCustomCACertFileName]
+ if odhCustomCertData == "" {
+ log.Info(fmt.Sprintf("Detected opendatahub global cert ConfigMap (%s), but custom cert is not set\n", odhGlobalCACertConfigMapName))
+ kserveCustomCertConfigMap := &corev1.ConfigMap{}
+ err := r.Get(ctx, types.NamespacedName{Name: kserveCustomCACertConfigMapName, Namespace: configmap.Namespace}, kserveCustomCertConfigMap)
+ if err == nil {
+ log.Info(fmt.Sprintf("Deleting KServe custom cert ConfigMap (%s)", kserveCustomCACertConfigMapName))
+ if err := r.Delete(ctx, kserveCustomCertConfigMap); err != nil {
+ return err
+ }
+ } else if !apierrs.IsNotFound(err) {
+ return err
+ }
+ return nil
+ }
+
+ configData := map[string]string{constants.KServeCACertFileName: odhCustomCertData}
+ newCaCertConfigMap := getDesiredCaCertConfigMapForKServe(kserveCustomCACertConfigMapName, configmap.Namespace, configData)
+
+ kserveCustomCertConfigMap := &corev1.ConfigMap{}
+ if err := r.Get(ctx, types.NamespacedName{Name: kserveCustomCACertConfigMapName, Namespace: configmap.Namespace}, kserveCustomCertConfigMap); err != nil {
+ if !apierrs.IsNotFound(err) {
+ return err
+ }
+ log.Info(fmt.Sprintf("Creating KServe custom cert ConfigMap (%s) because it detected the creation of the opendatahub global cert ConfigMap (%s)", kserveCustomCACertConfigMapName, odhGlobalCACertConfigMapName))
+ if err := r.Create(ctx, newCaCertConfigMap); err != nil {
+ log.Error(err, "Failed to create KServe custom cert ConfigMap")
+ return err
+ }
+ } else {
+ log.Info("Checking if the data in KServe custom cert ConfigMap differs from the data in the opendatahub global CA cert ConfigMap")
+ existingOdhCustomCertData := kserveCustomCertConfigMap.Data[constants.KServeCACertFileName]
+ if existingOdhCustomCertData == odhCustomCertData {
+ log.Info(fmt.Sprintf("No updates required for KServe custom cert ConfigMap (%s) as the data matches the opendatahub global cert ConfigMap (%s)", kserveCustomCACertConfigMapName, odhGlobalCACertConfigMapName))
+ return nil
+ }
+ log.Info(fmt.Sprintf("Updating KServe custom cert ConfigMap due to changes in the opendatahub global cert ConfigMap (%s)", odhGlobalCACertConfigMapName))
+ if err := r.Update(ctx, newCaCertConfigMap); err != nil {
+ return err
+ }
+ if err := r.deleteStorageSecret(ctx, configmap.Namespace); err != nil {
+ log.Error(err, "Failed to delete the storage-config secret to update the custom cert")
+ return err
+ }
+ log.V(1).Info("Deleted the storage-config Secret to update the custom cert")
+ }
+
+ return nil
+}
+
+// This section is intended for regenerating StorageSecret using new data.
+func (r *KServeCustomCACertReconciler) deleteStorageSecret(ctx context.Context, namespace string) error {
+ foundStorageSecret := &corev1.Secret{}
+
+ err := r.Get(ctx, types.NamespacedName{
+ Name: constants.DefaultStorageConfig,
+ Namespace: namespace,
+ }, foundStorageSecret)
+
+ if err == nil {
+ err = r.Delete(ctx, foundStorageSecret)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func checkOpenDataHubGlobalCertCAConfigMapName(objectName string) bool {
+ return objectName == odhGlobalCACertConfigMapName
+}
+
+// reconcileOpenDataHubGlobalCertConfigMap filters out all ConfigMaps that are not the OpenDataHub global certificate ConfigMap.
+func reconcileOpenDataHubGlobalCACertConfigMap() predicate.Predicate {
+ return predicate.Funcs{
+ CreateFunc: func(e event.CreateEvent) bool {
+ objectName := e.Object.GetName()
+ return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+ },
+ DeleteFunc: func(e event.DeleteEvent) bool {
+ objectName := e.Object.GetName()
+ return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+ },
+ GenericFunc: func(e event.GenericEvent) bool {
+ objectName := e.Object.GetName()
+ return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+ },
+ UpdateFunc: func(e event.UpdateEvent) bool {
+ objectName := e.ObjectNew.GetName()
+ return checkOpenDataHubGlobalCertCAConfigMapName(objectName)
+ },
+ }
+}
+
+func (r *KServeCustomCACertReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+ // Initialize logger format
+ log := r.Log.WithValues("ConfigMap", req.Name, "namespace", req.Namespace)
+
+ configmap := &corev1.ConfigMap{}
+ err := r.Get(ctx, req.NamespacedName, configmap)
+ if err != nil && apierrs.IsNotFound(err) {
+ log.Info("Opendatahub global cert ConfigMap not found")
+ configmap.Namespace = req.Namespace
+ } else if err != nil {
+ log.Error(err, "Unable to fetch the ConfigMap")
+ return ctrl.Result{}, err
+ }
+
+ err = r.reconcileConfigMap(configmap, ctx)
+ if err != nil {
+ return ctrl.Result{}, err
+ }
+ return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *KServeCustomCACertReconciler) SetupWithManager(mgr ctrl.Manager) error {
+ // Create a builder that only watch OpenDataHub global certificate ConfigMap
+ builder := ctrl.NewControllerManagedBy(mgr).
+ For(&corev1.ConfigMap{}).
+ WithEventFilter(reconcileOpenDataHubGlobalCACertConfigMap())
+ err := builder.Complete(r)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func getDesiredCaCertConfigMapForKServe(configmapName string, namespace string, caCertData map[string]string) *corev1.ConfigMap {
+ desiredConfigMap := &corev1.ConfigMap{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: configmapName,
+ Namespace: namespace,
+ },
+ Data: caCertData,
+ }
+
+ return desiredConfigMap
+}

controllers/kserve_customcacert_controller_test.go

@@ -0,0 +1,99 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+ "context"
+ . "github.com/onsi/ginkgo"
+ . "github.com/onsi/gomega"
+ "github.com/opendatahub-io/odh-model-controller/controllers/constants"
+ corev1 "k8s.io/api/core/v1"
+ "reflect"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "time"
+)
+
+const (
+ odhtrustedcabundleConfigMapUpdatedPath = "./testdata/configmaps/odh-trusted-ca-bundle-configmap-updated.yaml"
+ kservecustomcacertConfigMapUpdatedPath = "./testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml"
+)
+
+var _ = Describe("KServe Custom CA Cert ConfigMap controller", func() {
+ ctx := context.Background()
+
+ Context("when a configmap 'odh-trusted-ca-bundle' exists", func() {
+ It("should create a configmap that is for kserve custom ca cert", func() {
+ By("creating odh-trusted-ca-bundle configmap")
+ odhtrustedcacertConfigMap := &corev1.ConfigMap{}
+ err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed())
+
+ _, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+ Expect(err).NotTo(HaveOccurred())
+ })
+ })
+
+ Context("when a configmap 'odh-trusted-ca-bundle' updated", func() {
+ It("should update kserve custom cert configmap", func() {
+ By("creating odh-trusted-ca-bundle configmap")
+ odhtrustedcacertConfigMap := &corev1.ConfigMap{}
+ err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed())
+
+ _, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+ Expect(err).NotTo(HaveOccurred())
+
+ By("updating odh-trusted-ca-bundle configmap")
+ updatedOdhtrustedcacertConfigMap := &corev1.ConfigMap{}
+ err = convertToStructuredResource(odhtrustedcabundleConfigMapUpdatedPath, updatedOdhtrustedcacertConfigMap)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(cli.Update(ctx, updatedOdhtrustedcacertConfigMap)).Should(Succeed())
+
+ // Wait for updating ConfigMap
+ time.Sleep(1 * time.Second)
+ kserveCACertConfigmap, err := waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30*time.Second)
+ Expect(err).NotTo(HaveOccurred())
+ expectedKserveCACertConfigmap := &corev1.ConfigMap{}
+ err = convertToStructuredResource(kservecustomcacertConfigMapUpdatedPath, expectedKserveCACertConfigmap)
+ Expect(err).NotTo(HaveOccurred())
+
+ Expect(compareConfigMap(kserveCACertConfigmap, expectedKserveCACertConfigmap)).Should((BeTrue()))
+ })
+ })
+})
+
+func waitForConfigMap(cli client.Client, namespace, configmapName string, timeout time.Duration) (*corev1.ConfigMap, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+
+ for {
+ configmap := &corev1.ConfigMap{}
+ err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configmapName}, configmap)
+ if err != nil {
+ time.Sleep(1 * time.Second)
+ continue
+ }
+ return configmap, nil
+ }
+}
+
+// compareConfigMap checks if two ConfigMap data are equal, if not return false
+func compareConfigMap(s1 *corev1.ConfigMap, s2 *corev1.ConfigMap) bool {
+ // Two ConfigMap will be equal if the data is identical
+ return reflect.DeepEqual(s1.Data, s2.Data)
+}

controllers/storageconfig_controller.go

@@ -21,6 +21,7 @@ import (
  "reflect"
 
  "github.com/go-logr/logr"
+ "github.com/opendatahub-io/odh-model-controller/controllers/constants"
  corev1 "k8s.io/api/core/v1"
  apierrs "k8s.io/apimachinery/pkg/api/errors"
  "k8s.io/apimachinery/pkg/runtime"
@@ -33,7 +34,7 @@ import (
 )
 
 const (
- storageSecretName = "storage-config"
+ storageSecretName = constants.DefaultStorageConfig
 )
 
 type StorageSecretReconciler struct {
@@ -44,7 +45,7 @@ type StorageSecretReconciler struct {
 
 // newStorageSecret takes a list of data connection secrets and generates a single storage config secret
 // https://github.com/kserve/modelmesh-serving/blob/main/docs/predictors/setup-storage.md
-func newStorageSecret(dataConnectionSecretsList *corev1.SecretList) *corev1.Secret {
+func newStorageSecret(dataConnectionSecretsList *corev1.SecretList, odhCustomCertData string) *corev1.Secret {
  desiredSecret := &corev1.Secret{}
  desiredSecret.Data = map[string][]byte{}
  dataConnectionElement := map[string]string{}
@@ -59,8 +60,10 @@ func newStorageSecret(dataConnectionSecretsList *corev1.SecretList) *corev1.Secr
  dataConnectionElement["default_bucket"] = string(secret.Data["AWS_S3_BUCKET"])
  dataConnectionElement["bucket"] = string(secret.Data["AWS_S3_BUCKET"])
  dataConnectionElement["region"] = string(secret.Data["AWS_DEFAULT_REGION"])
- if secret.Data["AWS_CA_BUNDLE"] != nil {
- dataConnectionElement["certificate"] = string(secret.Data["AWS_CA_BUNDLE"])
+
+ if odhCustomCertData != "" {
+ dataConnectionElement["certificate"] = odhCustomCertData
+ dataConnectionElement["cabundle_configmap"] = constants.KServeCACertConfigMapName
  }
  jsonBytes, _ := json.Marshal(dataConnectionElement)
  storageByteData[secret.Name] = jsonBytes
@@ -77,7 +80,7 @@ func CompareStorageSecrets(s1 corev1.Secret, s2 corev1.Secret) bool {
 // reconcileSecret grabs all data connection secrets in the triggering namespace and
 // creates/updates the storage config secret
 func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret,
- ctx context.Context, newStorageSecret func(dataConnectionSecretsList *corev1.SecretList) *corev1.Secret) error {
+ ctx context.Context, newStorageSecret func(dataConnectionSecretsList *corev1.SecretList, odhCustomCertData string) *corev1.Secret) error {
  // Initialize logger format
  log := r.Log.WithValues("secret", secret.Name, "namespace", secret.Namespace)
 
@@ -91,11 +94,23 @@ func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret,
  if err != nil {
  if apierrs.IsNotFound(err) {
  log.Info("No data connections found in namespace ", secret.Namespace)
+ return nil
  }
  }
 
+ odhCustomCertData := ""
+ odhGlobalCertConfigMap := &corev1.ConfigMap{}
+ err = r.Get(ctx, types.NamespacedName{
+ Name: constants.ODHGlobalCertConfigMapName,
+ Namespace: secret.Namespace,
+ }, odhGlobalCertConfigMap)
+
+ if err == nil {
+ odhCustomCertData = odhGlobalCertConfigMap.Data[constants.ODHCustomCACertFileName]
+ }
+
  // Generate desire Storage Config Secret
- desiredStorageSecret := newStorageSecret(dataConnectionSecretsList)
+ desiredStorageSecret := newStorageSecret(dataConnectionSecretsList, odhCustomCertData)
  desiredStorageSecret.Name = storageSecretName
  desiredStorageSecret.Namespace = secret.Namespace
  desiredStorageSecret.Labels = map[string]string{}