feat: support set custom domain ingress class separately

apiserver/paasng/paas_wl/workloads/networking/ingress/managers/domain.py

@@ -16,8 +16,9 @@
 # to the current version of the project delivered to anyone in the future.
 
 import logging
-from typing import List, Set
+from typing import Dict, List, Set
 
+from django.conf import settings
 from django.db import transaction
 
 from paas_wl.bk_app.applications.models import WlApp
@@ -122,6 +123,15 @@ def list_desired_domains(self) -> List[PIngressDomain]:
             )
         ]
 
+    def get_annotations(self) -> Dict:
+        """update annotations if custom domain ingress class is set"""
+        annotations = super().get_annotations()
+
+        if settings.CUSTOM_DOMAIN_INGRESS_CLASS is not None:
+            annotations["kubernetes.io/ingress.class"] = settings.CUSTOM_DOMAIN_INGRESS_CLASS
+
+        return annotations
+
 
 class IngressDomainFactory:
     """A factory class creates `PIngressDomain` objects"""

apiserver/paasng/paas_wl/workloads/networking/ingress/managers/subpath.py

@@ -16,9 +16,7 @@
 # to the current version of the project delivered to anyone in the future.
 
 import logging
-from typing import Dict, List, Set
-
-from django.conf import settings
+from typing import List, Set
 
 from paas_wl.bk_app.applications.models import WlApp
 from paas_wl.infras.cluster.utils import get_cluster_by_app
@@ -95,15 +93,6 @@ def list_desired_domains(self) -> List[PIngressDomain]:
 
         return [self.create_ingress_domain(domain.name, paths, domain.https_enabled) for domain in domains]
 
-    def get_annotations(self) -> Dict:
-        annotations = {}
-
-        # 当有多个 ingress controller 存在时，可以指定需要使用的链路
-        if settings.APP_INGRESS_CLASS is not None:
-            annotations["kubernetes.io/ingress.class"] = settings.APP_INGRESS_CLASS
-
-        return annotations
-
     def create_ingress_domain(self, host: str, path_prefix_list: List[str], https_enabled: bool) -> PIngressDomain:
         """Create a domain object, will create HTTPS related Secret resource on demand"""
         if not https_enabled:

apiserver/paasng/paasng/platform/applications/views.py

@@ -37,7 +37,7 @@
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from paas_wl.bk_app.cnative.specs.resource import delete_bkapp
+from paas_wl.bk_app.cnative.specs.resource import delete_bkapp, delete_networking
 from paas_wl.infras.cluster.constants import ClusterFeatureFlag
 from paas_wl.infras.cluster.shim import RegionClusterService
 from paas_wl.infras.cluster.utils import get_cluster_by_app
@@ -332,6 +332,7 @@ def destroy(self, request, code):
                 envs = module.envs.all()
                 for env in envs:
                     delete_bkapp(env)
+                    delete_networking(env)
 
         # 审计记录在事务外创建, 避免由于数据库回滚而丢失
         pre_delete_application.send(sender=Application, application=application, operator=self.request.user.pk)

apiserver/paasng/paasng/platform/engine/deploy/archive/operator.py

@@ -19,7 +19,7 @@
 
 from blue_krill.async_utils.poll_task import CallbackHandler
 
-from paas_wl.bk_app.cnative.specs.resource import delete_bkapp
+from paas_wl.bk_app.cnative.specs.resource import delete_bkapp, delete_networking
 from paasng.platform.engine.deploy.archive.base import BaseArchiveManager
 from paasng.platform.engine.deploy.bg_wait.wait_deployment import wait_for_all_stopped
 from paasng.platform.engine.models.offline import OfflineOperation
@@ -31,6 +31,7 @@ def perform_implement(self, offline_operation: OfflineOperation, result_handler:
         op_id = str(offline_operation.pk)
         # 清理 BkApp crd
         delete_bkapp(self.env)
+        delete_networking(self.env)
         # 清理进程配置
         # 与普通应用不同, 由于云原生应用可由用户直接选择 plan, 重新部署后可恢复分配的 plan 记录
         # 因此下架应用可直接删除 ProcessSpec 数据, 无需担心在管理端后台分配的 plan 记录被清理

apiserver/paasng/paasng/settings/workloads.py

@@ -155,6 +155,9 @@
 
 # 不指定则使用默认，可以指定为 bk-ingress-nginx
 APP_INGRESS_CLASS = settings.get("APP_INGRESS_CLASS")
+# 独立域名的 ingress 使用的 ingress class
+# 在蓝鲸私有化版本的部署架构中, 存在两层 nginx controller, 而独立域名需要在最外层的 nginx controller 中生效
+CUSTOM_DOMAIN_INGRESS_CLASS = settings.get("CUSTOM_DOMAIN_INGRESS_CLASS")
 
 # 控制 ingress 资源路径是否严格匹配末尾斜杆, 如某个 ingress 路径设置成 "/foo/", 开启严格匹配将无法通过 "/foo" 访问应用
 # 如果希望通过 "/foo" 也能访问, 则需要设置 APP_INGRESS_EXT_V1BETA1_PATH_TRAILING_SLASH、APP_INGRESS_V1_PATH_TRAILING_SLASH 为 False

apiserver/paasng/tests/paas_wl/workloads/networking/ingress/managers/test_domain.py

@@ -18,6 +18,7 @@
 from unittest.mock import patch
 
 import pytest
+from django.test.utils import override_settings
 from django_dynamic_fixture import G
 
 from paas_wl.infras.resources.kube_res.exceptions import AppEntityNotFound
@@ -277,6 +278,24 @@ def test_normal_delete(self, bk_stag_env, bk_stag_wl_app):
         with pytest.raises(AppEntityNotFound):
             ingress_kmodel.get(bk_stag_wl_app, mgr.make_ingress_name())
 
+    def test_get_ingress_class(self, bk_stag_env, bk_stag_wl_app):
+        domain = G(
+            Domain,
+            name="foo.example.com",
+            module_id=bk_stag_env.module.id,
+            environment_id=bk_stag_env.id,
+        )
+        mgr = CustomDomainIngressMgr(domain)
+        assert mgr.get_annotations().get("kubernetes.io/ingress.class") is None
+
+        with override_settings(APP_INGRESS_CLASS="test-ingress"):
+            mgr = CustomDomainIngressMgr(domain)
+            assert mgr.get_annotations()["kubernetes.io/ingress.class"] == "test-ingress"
+
+        with override_settings(CUSTOM_DOMAIN_INGRESS_CLASS="test-custom-ingress"):
+            mgr = CustomDomainIngressMgr(domain)
+            assert mgr.get_annotations()["kubernetes.io/ingress.class"] == "test-custom-ingress"
+
 
 @pytest.mark.auto_create_ns()
 class TestIntegratedDomains:

apiserver/paasng/tests/paas_wl/workloads/networking/ingress/managers/test_subpath.py

@@ -16,6 +16,7 @@
 # to the current version of the project delivered to anyone in the future.
 
 import pytest
+from django.test.utils import override_settings
 
 from paas_wl.infras.resources.kube_res.exceptions import AppEntityNotFound
 from paas_wl.workloads.networking.ingress.managers.subpath import SubPathAppIngressMgr, assign_subpaths
@@ -96,3 +97,8 @@ def test_subpath_transfer_fully(self, bk_stag_wl_app, bk_prod_wl_app):
         ingress = SubPathAppIngressMgr(bk_prod_wl_app).get()
         assert len(ingress.domains) == 1
         assert ingress.domains[0].path_prefix_list == paths
+
+    def test_get_ingress_class(self, bk_stag_wl_app):
+        with override_settings(APP_INGRESS_CLASS="test-foo-ingress"):
+            ingress_mgr = SubPathAppIngressMgr(bk_stag_wl_app)
+            assert ingress_mgr.get_annotations()["kubernetes.io/ingress.class"] == "test-foo-ingress"

operator/api/v1alpha1/projectconfig_types.go

@@ -34,6 +34,10 @@ type PlatformConfig struct {
 	SentryDSN string `json:"sentryDSN"`
 	// if ingressClassName configured, kubernetes.io/ingress.class=$value will be added to ingress's annotations
 	IngressClassName string `json:"ingressClassName"`
+	// CustomDomainIngressClassName works the same as IngressClassName, but only for ingress with custom domains.
+	// In the deployment architecture of BK, there are two layers of nginx controllers, and the custom domain needs to
+	// take effect in the outermost nginx controller.
+	CustomDomainIngressClassName string `json:"customDomainIngressClassName"`
 }
 
 // IngressPluginConfig contains the config for controlling ingress config
@@ -166,6 +170,15 @@ func (p *ProjectConfig) GetIngressClassName() string {
 	return p.Platform.IngressClassName
 }
 
+// GetCustomDomainIngressClassName returns the ingress class name for custom domain ingress.
+// if not set, return the common ingress class name
+func (p *ProjectConfig) GetCustomDomainIngressClassName() string {
+	if p.Platform.CustomDomainIngressClassName != "" {
+		return p.Platform.CustomDomainIngressClassName
+	}
+	return p.GetIngressClassName()
+}
+
 // IsAutoscalingEnabled returns whether autoscaling is enabled
 func (p *ProjectConfig) IsAutoscalingEnabled() bool {
 	return p.Autoscaling.Enabled

operator/api/v1alpha1/projectconfig_types_test.go

@@ -76,5 +76,58 @@ platform:
 		Expect(projCfg.Platform.BkAPIGatewayURL).To(Equal("https://example.com"))
 		Expect(projCfg.Platform.SentryDSN).To(Equal("https://sentry.example.com"))
 		Expect(projCfg.Platform.IngressClassName).To(Equal("nginx"))
+		Expect(projCfg.GetIngressClassName()).To(Equal("nginx"))
+		Expect(projCfg.Platform.CustomDomainIngressClassName).To(Equal(""))
+		Expect(projCfg.GetCustomDomainIngressClassName()).To(Equal("nginx"))
+	})
+
+	It("test load from file with customDomainIngressClassName", func() {
+		var projCfg ProjectConfig
+		configContent := `
+apiVersion: paas.bk.tencent.com/v1alpha1
+kind: ProjectConfig
+metadata:
+  name: projectconfig-sample
+platform:
+  bkAppCode: "foo"
+  bkAppSecret: "bar"
+  ingressClassName: bk-nginx
+  customDomainIngressClassName: nginx
+`
+		file, err := os.CreateTemp("", "")
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = file.Write([]byte(configContent))
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = ctrl.ConfigFile().AtPath(file.Name()).OfKind(&projCfg).Complete()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(projCfg.GetIngressClassName()).To(Equal("bk-nginx"))
+		Expect(projCfg.GetCustomDomainIngressClassName()).To(Equal("nginx"))
+	})
+
+	It("test load from file without any ingressClassName", func() {
+		var projCfg ProjectConfig
+		configContent := `
+apiVersion: paas.bk.tencent.com/v1alpha1
+kind: ProjectConfig
+metadata:
+  name: projectconfig-sample
+platform:
+  bkAppCode: "foo"
+  bkAppSecret: "bar"
+`
+		file, err := os.CreateTemp("", "")
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = file.Write([]byte(configContent))
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = ctrl.ConfigFile().AtPath(file.Name()).OfKind(&projCfg).Complete()
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(projCfg.GetIngressClassName()).To(Equal(""))
+		Expect(projCfg.GetCustomDomainIngressClassName()).To(Equal(""))
 	})
 })

operator/config/crd/bases/paas.bk.tencent.com_projectconfigs.yaml

@@ -193,6 +193,12 @@ spec:
                 type: string
               bkAppSecret:
                 type: string
+              customDomainIngressClassName:
+                description: CustomDomainIngressClassName works the same as IngressClassName,
+                  but only for ingress with custom domains. In the deployment architecture
+                  of BK, there are two layers of nginx controllers, and the custom
+                  domain needs to take effect in the outermost nginx controller.
+                type: string
               ingressClassName:
                 description: if ingressClassName configured, kubernetes.io/ingress.class=$value
                   will be added to ingress's annotations
@@ -205,6 +211,7 @@ spec:
             - bkAPIGatewayURL
             - bkAppCode
             - bkAppSecret
+            - customDomainIngressClassName
             - ingressClassName
             - sentryDSN
             type: object

operator/main.go

@@ -139,7 +139,7 @@ func main() {
 	mgrCli := kubeutil.NewTracedClient(mgr.GetClient())
 	mgrScheme := mgr.GetScheme()
 
-	bkappMgrOpts := genGroupKindMgrOpts(paasv1alpha1.GroupKindBkApp, projConf.Controller)
+	bkappMgrOpts := genGroupKindMgrOpts(paasv1alpha2.GroupKindBkApp, projConf.Controller)
 	if err = controllers.NewBkAppReconciler(mgrCli, mgrScheme).
 		SetupWithManager(setupCtx, mgr, bkappMgrOpts); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "BkApp")

operator/pkg/config/config.go

@@ -31,6 +31,8 @@ type ProjectConfigReader interface {
 
 	// Platform related methods
 	GetIngressClassName() string
+	// GetCustomDomainIngressClassName returns the ingress class name for custom domain ingress
+	GetCustomDomainIngressClassName() string
 	IsAutoscalingEnabled() bool
 }
 
@@ -66,6 +68,11 @@ func (d defaultConfig) GetIngressClassName() string {
 	return "nginx"
 }
 
+// GetCustomDomainIngressClassName returns the ingress class name for custom domain ingress
+func (d defaultConfig) GetCustomDomainIngressClassName() string {
+	return d.GetIngressClassName()
+}
+
 func (d defaultConfig) IsAutoscalingEnabled() bool {
 	return false
 }

operator/pkg/controllers/dgroupmapping/ingress/builder.go

@@ -195,6 +195,7 @@ func (builder MonoIngressBuilder) Build(domains []Domain) ([]*networkingv1.Ingre
 	}
 	// 如果已配置 ingressClassName，则使用
 	if ingClassName := config.Global.GetIngressClassName(); ingClassName != "" {
+		// WARNING: kubernetes.io/ingress.class annotation will be deprecated in the future
 		ingress.Annotations[paasv1alpha2.IngressClassAnnoKey] = ingClassName
 	}
 	results = append(results, &ingress)
@@ -243,6 +244,12 @@ func (builder CustomIngressBuilder) Build(domains []Domain) ([]*networkingv1.Ing
 				TLS:   makeTLS([]Domain{d}),
 			},
 		}
+
+		if ingClassName := config.Global.GetCustomDomainIngressClassName(); ingClassName != "" {
+			// WARNING: kubernetes.io/ingress.class annotation will be deprecated in the future
+			val.Annotations[paasv1alpha2.IngressClassAnnoKey] = ingClassName
+		}
+
 		results = append(results, &val)
 
 	}

operator/pkg/controllers/dgroupmapping/ingress/builder_test.go

@@ -109,6 +109,7 @@ var _ = Describe("Test ingresses.go", func() {
 			Expect(ingresses[0].Name).To(Equal("demo-subdomain"))
 			Expect(ingresses[0].Spec.Rules[0].Host).To(Equal("foo.example.com"))
 			Expect(ingresses[0].Annotations[SkipFilterCLBAnnoKey]).To(Equal("true"))
+			Expect(ingresses[0].Annotations[paasv1alpha2.IngressClassAnnoKey]).To(Equal("nginx"))
 		})
 	})
 
@@ -132,6 +133,7 @@ var _ = Describe("Test ingresses.go", func() {
 			Expect(ingresses[0].Spec.Rules[0].Host).To(Equal("foo.example.com"))
 			Expect(ingresses[1].Spec.Rules[0].Host).To(Equal("with-name.example.com"))
 			Expect(ingresses[1].Annotations[SkipFilterCLBAnnoKey]).To(Equal("true"))
+			Expect(ingresses[0].Annotations[paasv1alpha2.IngressClassAnnoKey]).To(Equal("nginx"))
 		})
 	})