Support customizing install ns

Users are able to configure the addonDeploymentConfig to specify a different install namespace for the addon

Ref: https://issues.redhat.com/browse/ACM-11331
Signed-off-by: yiraeChristineKim <yikim@redhat.com>

go.mod

@@ -5,16 +5,16 @@ go 1.21
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/onsi/ginkgo/v2 v2.17.1
-	github.com/onsi/gomega v1.30.0
+	github.com/onsi/gomega v1.31.1
 	github.com/openshift/library-go v0.0.0-20240116081341-964bcb3f545c
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.64.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stolostron/kubernetes-dependency-watches v0.5.2
-	k8s.io/apimachinery v0.29.1
-	k8s.io/client-go v0.29.1
+	k8s.io/apimachinery v0.29.2
+	k8s.io/client-go v0.29.2
 	k8s.io/component-base v0.29.1
-	k8s.io/klog/v2 v2.110.1
+	k8s.io/klog/v2 v2.120.1
 	open-cluster-management.io/addon-framework v0.9.1-0.20240321030609-bffb77d5d895
 	open-cluster-management.io/api v0.13.0
 	sigs.k8s.io/controller-runtime v0.17.2
@@ -57,7 +57,7 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
@@ -101,37 +101,37 @@ require (
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
 	golang.org/x/net v0.22.0 // indirect
-	golang.org/x/oauth2 v0.13.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.19.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
-	google.golang.org/grpc v1.58.3 // indirect
+	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.14.2 // indirect; CVE-2024-26147: >=v3.14.2
-	k8s.io/api v0.29.1 // indirect
+	k8s.io/api v0.29.2 // indirect
 	k8s.io/apiextensions-apiserver v0.29.0 // indirect
 	k8s.io/apiserver v0.29.0 // indirect
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/kms v0.29.0 // indirect
 	k8s.io/kube-aggregator v0.29.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
-	open-cluster-management.io/sdk-go v0.13.0 // indirect
+	k8s.io/utils v0.0.0-20240310230437-4693a0247e57 // indirect
+	open-cluster-management.io/sdk-go v0.13.1-0.20240416062924-20307e6fe090 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect

pkg/addon/common.go

@@ -229,3 +229,34 @@ func IsOldKubernetes(cluster *clusterv1.ManagedCluster) bool {
 
 	return false
 }
+
+func CommonAgentInstallNamespaceFromDeploymentConfigFunc(
+	adcgetter utils.AddOnDeploymentConfigGetter,
+) func(*addonapiv1alpha1.ManagedClusterAddOn) string {
+	return func(addon *addonapiv1alpha1.ManagedClusterAddOn) string {
+		if addon == nil {
+			log.Info("failed to get addon install namespace, addon is nil")
+
+			return ""
+		}
+
+		hostingClusterName := addon.Annotations["addon.open-cluster-management.io/hosting-cluster-name"]
+		// Check it is hosted mode
+		if hostingClusterName != "" && addon.Spec.InstallNamespace != "" {
+			return addon.Spec.InstallNamespace
+		}
+
+		config, err := utils.GetDesiredAddOnDeploymentConfig(addon, adcgetter)
+		if err != nil {
+			log.Error(err, fmt.Sprintf("failed to get deployment config for addon %s: ", addon.Name))
+
+			return ""
+		}
+
+		if config == nil {
+			return ""
+		}
+
+		return config.Spec.AgentInstallNamespace
+	}
+}

pkg/addon/configpolicy/agent_addon.go

@@ -246,6 +246,10 @@ func GetAgentAddon(ctx context.Context, controllerContext *controllercmd.Control
 		).
 		WithManagedClusterClient(clusterClient).
 		WithAgentRegistrationOption(registrationOption).
+		WithAgentInstallNamespace(
+			policyaddon.
+				CommonAgentInstallNamespaceFromDeploymentConfigFunc(utils.NewAddOnDeploymentConfigGetter(addonClient)),
+		).
 		WithScheme(policyaddon.Scheme).
 		WithAgentHostedModeEnabledOption().
 		BuildHelmAgentAddon()

pkg/addon/policyframework/agent_addon.go

@@ -260,6 +260,10 @@ func GetAgentAddon(ctx context.Context, controllerContext *controllercmd.Control
 		).
 		WithManagedClusterClient(clusterClient).
 		WithAgentRegistrationOption(registrationOption).
+		WithAgentInstallNamespace(
+			policyaddon.
+				CommonAgentInstallNamespaceFromDeploymentConfigFunc(utils.NewAddOnDeploymentConfigGetter(addonClient)),
+		).
 		WithScheme(policyaddon.Scheme).
 		WithAgentHostedModeEnabledOption().
 		BuildHelmAgentAddon()

test/e2e/case1_framework_deployment_test.go

@@ -20,6 +20,7 @@ const (
 	case1ManagedClusterAddOnCR           string = "../resources/framework_addon_cr.yaml"
 	case1ClusterManagementAddOnCRDefault string = "../resources/framework_clustermanagementaddon.yaml"
 	case1ClusterManagementAddOnCR        string = "../resources/framework_clustermanagementaddon_config.yaml"
+	case1CMAAddonWithInstallNs           string = "../resources/framework_cma_config_agentInstallNs.yaml"
 	case1hubAnnotationMCAOCR             string = "../resources/framework_hub_annotation_addon_cr.yaml"
 	case1hubValuesMCAOCR                 string = "../resources/framework_hub_values_addon_cr.yaml"
 	case1DeploymentName                  string = "governance-policy-framework"
@@ -43,6 +44,48 @@ var _ = Describe("Test framework deployment", Ordered, func() {
 		Kubectl("delete", "-f", case1ClusterManagementAddOnCRDefault)
 	})
 
+	It("should create the framework deployment in hosted mode in user's custom namespace", func() {
+		By("Creating the AddOnDeploymentConfig")
+		Kubectl("apply", "-f", addOnDeploymentConfigWithAgentInstallNs)
+		DeferCleanup(func() {
+			By("Delete the AddOnDeploymentConfig")
+			Kubectl("delete", "-f", addOnDeploymentConfigWithAgentInstallNs)
+		})
+
+		By("Applying the framework ClusterManagementAddOn to use the AddOnDeploymentConfig")
+		Kubectl("apply", "-f", case1CMAAddonWithInstallNs)
+		DeferCleanup(func() {
+			By("Apply Default ClusterManagementAdd")
+			Kubectl("apply", "-f", case1ClusterManagementAddOnCRDefault)
+		})
+
+		for _, cluster := range managedClusterList[1:] {
+			logPrefix := cluster.clusterType + " " + cluster.clusterName + ": "
+			By(logPrefix + "deploying the default framework managedclusteraddon")
+			Kubectl("apply", "-n", cluster.clusterName, "-f", case1ManagedClusterAddOnCR)
+
+			By("Addon should be installed in " + agentInstallNs)
+			deploy := GetWithTimeout(
+				cluster.clusterClient, gvrDeployment, case1DeploymentName, agentInstallNs, true, 60,
+			)
+			Expect(deploy).NotTo(BeNil())
+
+			By(logPrefix +
+				"removing the framework deployment when the ManagedClusterAddOn CR is removed")
+			Kubectl("delete", "-n", cluster.clusterName, "-f", case1ManagedClusterAddOnCR, "--timeout=90s")
+			deploy = GetWithTimeout(
+				cluster.clusterClient, gvrDeployment, case1DeploymentName, agentInstallNs, false, 180,
+			)
+			Expect(deploy).To(BeNil())
+
+			opts := metav1.ListOptions{
+				LabelSelector: case1PodSelector,
+			}
+			pods := ListWithTimeoutByNamespace(cluster.clusterClient, gvrPod, opts, agentInstallNs, 0, false, 180)
+			Expect(pods).To(BeNil())
+		}
+	})
+
 	It("should create the default framework deployment on separate managed clusters", func(ctx context.Context) {
 		hubClusterConfig := managedClusterList[0]
 		hubClient := hubClusterConfig.clusterClient

test/e2e/case2_config_deployment_test.go

@@ -20,6 +20,7 @@ const (
 	case2ManagedClusterAddOnCR           string = "../resources/config_policy_addon_cr.yaml"
 	case2ClusterManagementAddOnCRDefault string = "../resources/config_policy_clustermanagementaddon.yaml"
 	case2ClusterManagementAddOnCR        string = "../resources/config_policy_clustermanagementaddon_config.yaml"
+	case2CMAAddonWithInstallNs           string = "../resources/config_policy_cma_config_agentInstallNs.yaml"
 	case2DeploymentName                  string = "config-policy-controller"
 	case2PodSelector                     string = "app=config-policy-controller"
 	case2OpenShiftClusterClaim           string = "../resources/openshift_cluster_claim.yaml"
@@ -101,6 +102,44 @@ var _ = Describe("Test config-policy-controller deployment", Ordered, func() {
 		By("Deleting the default config-policy-controller ClusterManagementAddon from the hub cluster")
 		Kubectl("delete", "-f", case2ClusterManagementAddOnCRDefault)
 	})
+	It("should create the config-policy-controller deployment in hosted mode in user's custom namespace", func() {
+		By("Creating the AddOnDeploymentConfig")
+		Kubectl("apply", "-f", addOnDeploymentConfigWithAgentInstallNs)
+		DeferCleanup(func() {
+			By("Delete the AddOnDeploymentConfig")
+			Kubectl("delete", "-f", addOnDeploymentConfigWithAgentInstallNs)
+		})
+
+		By("Applying the config-policy-controller ClusterManagementAddOn to use the AddOnDeploymentConfig")
+		Kubectl("apply", "-f", case2CMAAddonWithInstallNs)
+		DeferCleanup(func() {
+			By("Apply Default ClusterManagementAdd")
+			Kubectl("apply", "-f", case2ClusterManagementAddOnCRDefault)
+		})
+
+		for i, cluster := range managedClusterList[1:] {
+			logPrefix := cluster.clusterType + " " + cluster.clusterName + ": "
+			By(logPrefix + "deploying the default config-policy-controller managedclusteraddon")
+			Kubectl("apply", "-n", cluster.clusterName, "-f", case2ManagedClusterAddOnCR)
+			By("Addon should be installed in " + agentInstallNs)
+			// Use i+1 since the for loop ranges over a slice skipping first index
+			verifyConfigPolicyDeployment(logPrefix, cluster.clusterClient, cluster.clusterName, agentInstallNs, i+1)
+
+			By(logPrefix +
+				"removing the config-policy-controller deployment when the ManagedClusterAddOn CR is removed")
+			Kubectl("delete", "-n", cluster.clusterName, "-f", case2ManagedClusterAddOnCR, "--timeout=90s")
+			deploy := GetWithTimeout(
+				cluster.clusterClient, gvrDeployment, case2DeploymentName, agentInstallNs, false, 180,
+			)
+			Expect(deploy).To(BeNil())
+
+			opts := metav1.ListOptions{
+				LabelSelector: case2PodSelector,
+			}
+			pods := ListWithTimeoutByNamespace(cluster.clusterClient, gvrPod, opts, agentInstallNs, 0, false, 180)
+			Expect(pods).To(BeNil())
+		}
+	})
 
 	It("should create the default config-policy-controller deployment on the managed cluster", func() {
 		for i, cluster := range managedClusterList {
@@ -608,5 +647,8 @@ func setupClusterSecretForHostedMode(
 	_, err = client.Resource(gvrSecret).Namespace(installNamespace).Create(
 		context.TODO(), &secret, metav1.CreateOptions{},
 	)
-	Expect(err).ToNot(HaveOccurred())
+
+	if !errors.IsAlreadyExists(err) {
+		Expect(err).ToNot(HaveOccurred())
+	}
 }

test/e2e/e2e_suite_test.go

@@ -21,16 +21,18 @@ import (
 )
 
 const (
-	addonNamespace                        string = "open-cluster-management-agent-addon"
-	controllerNamespace                   string = "open-cluster-management"
-	kubeconfigFilename                    string = "../../kubeconfig_cluster"
-	loggingLevelAnnotation                string = "log-level=8"
-	evaluationConcurrencyAnnotation       string = "policy-evaluation-concurrency=5"
-	clientQPSAnnotation                   string = "client-qps=50"
-	prometheusEnabledAnnotation           string = "prometheus-metrics-enabled=true"
-	opPolicyEnabledAnnotation             string = "operator-policy-disabled=false"
-	addOnDeploymentConfigCR               string = "../resources/addondeploymentconfig.yaml"
-	addOnDeploymentConfigWithCustomVarsCR string = "../resources/addondeploymentconfig_customvars.yaml"
+	addonNamespace                          string = "open-cluster-management-agent-addon"
+	controllerNamespace                     string = "open-cluster-management"
+	kubeconfigFilename                      string = "../../kubeconfig_cluster"
+	loggingLevelAnnotation                  string = "log-level=8"
+	evaluationConcurrencyAnnotation         string = "policy-evaluation-concurrency=5"
+	clientQPSAnnotation                     string = "client-qps=50"
+	prometheusEnabledAnnotation             string = "prometheus-metrics-enabled=true"
+	opPolicyEnabledAnnotation               string = "operator-policy-disabled=false"
+	addOnDeploymentConfigCR                 string = "../resources/addondeploymentconfig.yaml"
+	addOnDeploymentConfigWithCustomVarsCR   string = "../resources/addondeploymentconfig_customvars.yaml"
+	addOnDeploymentConfigWithAgentInstallNs string = "../resources/addondeploymentconfig_agentInstallNs.yaml"
+	agentInstallNs                          string = "test-install-ns"
 )
 
 var (

test/resources/addondeploymentconfig_agentInstallNs.yaml

@@ -0,0 +1,7 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: AddOnDeploymentConfig
+metadata:
+  name: addon-namespace-placement
+  namespace: open-cluster-management
+spec:
+  agentInstallNamespace: test-install-ns

test/resources/config_policy_cma_config_agentInstallNs.yaml

@@ -0,0 +1,11 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: ClusterManagementAddOn
+metadata:
+  name: config-policy-controller
+spec:
+  supportedConfigs:
+  - group: addon.open-cluster-management.io
+    resource: addondeploymentconfigs
+    defaultConfig:
+      name: addon-namespace-placement
+      namespace: open-cluster-management

test/resources/framework_cma_config_agentInstallNs.yaml

@@ -0,0 +1,11 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: ClusterManagementAddOn
+metadata:
+  name: governance-policy-framework
+spec:
+  supportedConfigs:
+  - group: addon.open-cluster-management.io
+    resource: addondeploymentconfigs
+    defaultConfig:
+      name: addon-namespace-placement
+      namespace: open-cluster-management