feat: add process services to expose process network explicitly

apiserver/paasng/paas_wl/bk_app/cnative/specs/constants.py

@@ -59,6 +59,12 @@
 # 轮询云原生应用的部署状态时，如果获取到失败状态的次数超过最大容忍次数，就认为部署失败
 CNATIVE_DEPLOY_STATUS_POLLING_FAILURE_LIMITS = 3
 
+# PROC_SERVICES_ENABLED_ANNOTATION_KEY 注解表示是否启用 process services 特性, 可选值为 "true" 或 "false".
+# true 表示 operator 将根据 process services 的配置来创建和关联 k8s service.
+# 该注解实际为了向后兼容 spec_version: 2 而设计, 当版本 <= spec_version: 2 时, 设置值为 "false", 否则设置为 "true".
+# 说明: 未设置该注解值的 BkApp 均是线上存量资源, 为了兼容, operator 会继续按照旧逻辑调和.
+PROC_SERVICES_ENABLED_ANNOTATION_KEY = "bkapp.paas.bk.tencent.com/proc-services-feature-enabled"
+
 
 class ApiVersion(str, StructuredEnum):
     """Kubernetes CRD API versions"""

apiserver/paasng/paas_wl/bk_app/cnative/specs/crd/bk_app.py

@@ -25,7 +25,13 @@
 
 from pydantic import BaseModel, Field, validator
 
-from paas_wl.bk_app.cnative.specs.constants import ApiVersion, MResPhaseType, ResQuotaPlan
+from paas_wl.bk_app.cnative.specs.constants import (
+    PROC_SERVICES_ENABLED_ANNOTATION_KEY,
+    ApiVersion,
+    MResPhaseType,
+    ResQuotaPlan,
+)
+from paas_wl.workloads.networking.constants import ExposedTypeName
 from paas_wl.workloads.release_controller.constants import ImagePullPolicy
 from paasng.utils.structure import register
 
@@ -111,6 +117,34 @@ class ProbeSet(BaseModel):
     startup: Optional[Probe] = None
 
 
+class ExposedType(BaseModel):
+    """ExposedType is the exposed type of the ProcService
+
+    :param name: the name of the exposed type
+    """
+
+    name: Literal[ExposedTypeName.BK_HTTP] = ExposedTypeName.BK_HTTP
+
+
+class ProcService(BaseModel):
+    """ProcService is a process service which used to expose network
+
+    :param name: the name of the service
+    :param targetPort: the target port of the service
+    :param protocol: the protocol of the service
+    :param exposedType: the exposed type of the service. If not specified, the service can only
+        be accessed within the cluster, not from outside.
+    :param port: the port that will be exposed by this service. If not specified, the value of
+        the 'targetPort' field is used.
+    """
+
+    name: str
+    targetPort: int
+    protocol: Literal["TCP", "UDP"] = "TCP"
+    exposedType: Optional[ExposedType] = None
+    port: Optional[int] = None
+
+
 class BkAppProcess(BaseModel):
     """Process resource"""
 
@@ -123,6 +157,7 @@ class BkAppProcess(BaseModel):
     resQuotaPlan: Optional[ResQuotaPlan] = None
     autoscaling: Optional[AutoscalingSpec] = None
     probes: Optional[ProbeSet] = None
+    services: Optional[List[ProcService]] = None
 
 
 class Hook(BaseModel):
@@ -346,3 +381,14 @@ def to_deployable(self) -> Dict:
         result = self.dict(exclude_none=True, exclude={"status"})
         result["metadata"].pop("generation", None)
         return result
+
+    def set_proc_services_annotation(self, enabled: str):
+        """set proc services feature annotation
+
+        :param enabled: "true" or "false". "true" 表示启用, "false" 表示不启用
+        """
+        self.metadata.annotations[PROC_SERVICES_ENABLED_ANNOTATION_KEY] = enabled
+
+    def get_proc_services_annotation(self) -> Optional[str]:
+        """get proc services feature annotation"""
+        return self.metadata.annotations.get(PROC_SERVICES_ENABLED_ANNOTATION_KEY)

apiserver/paasng/paas_wl/bk_app/cnative/specs/resource.py

@@ -39,8 +39,8 @@
 from paas_wl.infras.resources.base.exceptions import ResourceMissing
 from paas_wl.infras.resources.utils.basic import get_client_by_app
 from paas_wl.workloads.images.kres_entities import ImageCredentials
+from paas_wl.workloads.networking.constants import ExposedTypeName
 from paasng.platform.applications.models import ModuleEnvironment
-from paasng.platform.bkapp_model.models import ModuleProcessSpec
 
 logger = logging.getLogger(__name__)
 
@@ -107,16 +107,21 @@ def deploy(env: ModuleEnvironment, manifest: Dict) -> Dict:
         # 创建或更新 BkApp
         bkapp = create_or_update_bkapp_with_retries(client, env, manifest)
 
-    # Deploy other dependencies
-    deploy_networking(env)
+    sync_networking(env, BkAppResource(**bkapp))
     return bkapp.to_dict()
 
 
+def sync_networking(env: ModuleEnvironment, res: BkAppResource) -> None:
+    """Sync the networking related resources for env, such as Ingress etc."""
+
+    if _need_exposed_services(res):
+        deploy_networking(env)
+    else:
+        delete_networking(env)
+
+
 def deploy_networking(env: ModuleEnvironment) -> None:
     """Deploy the networking related resources for env, such as Ingress etc."""
-    if not _has_web_process(env):
-        return
-
     save_addresses(env)
     mapping = AddrResourceManager(env).build_mapping()
     wl_app = WlApp.objects.get(pk=env.engine_app_id)
@@ -144,9 +149,6 @@ def delete_bkapp(env: ModuleEnvironment):
 
 def delete_networking(env: ModuleEnvironment):
     """Delete network group mapping in cluster"""
-    if not _has_web_process(env):
-        return
-
     mapping = AddrResourceManager(env).build_mapping()
     wl_app = env.wl_app
     with get_client_by_app(wl_app) as client:
@@ -206,6 +208,19 @@ def _find_condition(self, type_: MResConditionType) -> Optional[MetaV1Condition]
         return None
 
 
-def _has_web_process(env: ModuleEnvironment) -> bool:
-    """Check if the module has web process"""
-    return ModuleProcessSpec.objects.filter(module=env.module, name="web").exists()
+def _need_exposed_services(res: BkAppResource) -> bool:
+    """
+    _need_exposed_services checks if the bkapp needs to expose services outside the cluster
+    """
+    enabled = res.get_proc_services_annotation()
+    # bkapp.paas.bk.tencent.com/proc-services-feature-enabled: false 时, 表示版本低于 specVersion: 3, 因此向后兼容, 需要向集群外暴露服务
+    if enabled == "false":
+        return True
+
+    # bkapp.paas.bk.tencent.com/proc-services-feature-enabled: true 时, 设置了 exposedType 为 bk/http 才需要向集群外暴露服务
+    for proc in res.spec.processes:
+        for svc in proc.services or []:
+            if svc.exposedType and svc.exposedType.name == ExposedTypeName.BK_HTTP:
+                return True
+
+    return False

apiserver/paasng/paas_wl/workloads/networking/constants.py

@@ -20,3 +20,7 @@
 class NetworkProtocol(str, StructuredEnum):
     TCP = EnumField("TCP", label="TCP")
     UDP = EnumField("UDP", label="UDP")
+
+
+class ExposedTypeName(str, StructuredEnum):
+    BK_HTTP = "bk/http"

apiserver/paasng/paasng/platform/bkapp_model/constants.py

@@ -22,6 +22,23 @@
 DEFAULT_SLUG_RUNNER_ENTRYPOINT = ["bash", "/runner/init"]
 
 
+class ExposedTypeName(str, StructuredEnum):
+    """与 paas_wl.workloads.networking.constants.ExposedTypeName 重复定义
+    # TODO 将 paasng 和 paas_wl 中重复定义的一些常量, 合并放到更底层的模块中, 避免破坏当前 importlinter 的依赖规则?
+    """
+
+    BK_HTTP = "bk/http"
+
+
+class NetworkProtocol(str, StructuredEnum):
+    """与 paas_wl.workloads.networking.constants.NetworkProtocol 重复定义
+    # TODO 将 paasng 和 paas_wl 中重复定义的一些常量, 合并放到更底层的模块中, 避免破坏当前 importlinter 的依赖规则?
+    """
+
+    TCP = EnumField("TCP", label="TCP")
+    UDP = EnumField("UDP", label="UDP")
+
+
 class ImagePullPolicy(str, StructuredEnum):
     """duplicated from paas_wl.workloads.release_controller.constants.ImagePullPolicy to decouple dependencies
     TODO 统一放置到一个独立于 paas_wl 和 paasng 的模块下?

apiserver/paasng/paasng/platform/bkapp_model/entities/__init__.py

@@ -35,6 +35,7 @@
 from .mounts import ConfigMapSource, Mount, MountOverlay, PersistentStorage, VolumeSource
 from .probes import ExecAction, HTTPGetAction, HTTPHeader, Probe, ProbeHandler, ProbeSet, TCPSocketAction
 from .proc_env_overlays import AutoscalingOverlay, ReplicasOverlay, ResQuotaOverlay
+from .proc_service import ProcService
 from .processes import Process
 from .scaling_config import AutoscalingConfig
 from .svc_discovery import SvcDiscConfig, SvcDiscEntryBkSaaS
@@ -69,4 +70,5 @@
     "ReplicasOverlay",
     "ResQuotaOverlay",
     "MountOverlay",
+    "ProcService",
 ]

apiserver/paasng/paasng/platform/bkapp_model/entities/proc_service.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+
+from typing import Literal, Optional
+
+from pydantic import BaseModel
+
+from paasng.platform.bkapp_model.constants import ExposedTypeName
+from paasng.utils.structure import prepare_json_field
+
+
+class ExposedType(BaseModel):
+    """ExposedType is the exposed type of the ProcService
+
+    :param name: name of the exposed type. Default is bk/http
+    """
+
+    name: Literal[ExposedTypeName.BK_HTTP] = ExposedTypeName.BK_HTTP
+
+
+@prepare_json_field
+class ProcService(BaseModel):
+    """
+    ProcService is a process service which used to expose network
+
+    :param name: service name
+    :param target_port: number of the port to access on the pods(container) targeted by the service
+    :param protocol: protocol of the service. Default is TCP
+    :param exposed_type: exposed type of the service. If not specified, the service can only be accessed within the
+        cluster, not from outside
+    :param port: number of the port that will be exposed by this service
+    """
+
+    name: str
+    target_port: int
+    protocol: Literal["TCP", "UDP"] = "TCP"
+    exposed_type: Optional[ExposedType] = None
+    port: Optional[int] = None

apiserver/paasng/paasng/platform/bkapp_model/entities/processes.py

@@ -24,6 +24,7 @@
 from paasng.utils.procfile import generate_bash_command_with_tokens
 
 from .probes import ProbeSet
+from .proc_service import ProcService
 from .scaling_config import AutoscalingConfig
 
 
@@ -35,6 +36,7 @@ class Process(BaseModel):
     :param command: 进程启动命令
     :param args: 进程启动参数
     :param proc_command: 单行脚本命令, 与 command/args 二选一, 优先于 command/args, 用于设置 Procfile 文件中进程 command
+    :param services: 暴露进程网路服务的 service 列表
     :param target_port: 监听端口
     :param replicas: 进程副本数. `None` value means the replicas is not specified.
     :param res_quota_plan: 资源配额套餐名
@@ -48,6 +50,7 @@ class Process(BaseModel):
     args: Optional[List[str]] = Field(default_factory=list)
     proc_command: Optional[str] = None
 
+    services: Optional[List[ProcService]] = None
     target_port: Optional[int] = None
 
     replicas: Optional[int] = None

apiserver/paasng/paasng/platform/bkapp_model/entities_syncer/processes.py

@@ -49,6 +49,7 @@ def sync_processes(module: Module, processes: List[Process]) -> CommonSyncResult
             "port": process.target_port,
             "plan_name": process.res_quota_plan or ResQuotaPlan.P_DEFAULT,
             "probes": process.probes,
+            "services": process.services,
         }
 
         # When the replicas value is None, only update the data if the process appears

apiserver/paasng/paasng/platform/bkapp_model/manager.py

@@ -74,6 +74,7 @@ def process_spec_builder(process: ProcessTmpl) -> ModuleProcessSpec:
                 target_replicas=process.replicas or PROC_DEFAULT_REPLICAS,
                 plan_name=process.plan or ResQuotaPlan.P_DEFAULT,
                 probes=process.probes,
+                services=process.services,
             )
 
         self.bulk_create_procs(proc_creator=process_spec_builder, adding_procs=adding_procs)
@@ -91,14 +92,19 @@ def process_spec_updator(process: ProcessTmpl) -> Tuple[bool, ModuleProcessSpec]
                 recorder.setattr("plan_name", process.plan)
             if process.replicas and process_spec.target_replicas != process.replicas:
                 recorder.setattr("target_replicas", process.replicas)
+
             if process.probes and process_spec.probes != process.probes:
                 recorder.setattr("probes", process.probes)
+
+            if process.services and process_spec.services != process.services:
+                recorder.setattr("services", process.services)
+
             return recorder.changed, process_spec
 
         self.bulk_update_procs(
             proc_updator=process_spec_updator,
             updating_procs=updating_procs,
-            updated_fields=["proc_command", "target_replicas", "plan_name", "probes", "updated"],
+            updated_fields=["proc_command", "target_replicas", "plan_name", "services", "probes", "updated"],
         )
         # update spec objects end
 

apiserver/paasng/paasng/platform/bkapp_model/manifest.py

@@ -36,6 +36,7 @@
     LOG_COLLECTOR_TYPE_ANNO_KEY,
     MODULE_NAME_ANNO_KEY,
     PA_SITE_ID_ANNO_KEY,
+    PROC_SERVICES_ENABLED_ANNOTATION_KEY,
     USE_CNB_ANNO_KEY,
     WLAPP_NAME_ANNO_KEY,
     ApiVersion,
@@ -65,6 +66,7 @@
     merge_env_vars_overlay,
     override_env_vars_overlay,
 )
+from paasng.platform.declarative.models import DeploymentDescription
 from paasng.platform.engine.configurations.config_var import get_env_variables
 from paasng.platform.engine.constants import AppEnvName, ConfigVarEnvName, RuntimeType
 from paasng.platform.engine.models import Deployment
@@ -187,6 +189,7 @@ def apply_to(self, model_res: crd.BkAppResource, module: Module):
                 res_quota_plan=self.get_quota_plan(process_spec.plan_name),
                 autoscaling=process_spec.scaling_config,
                 probes=process_spec.probes,
+                services=process_spec.services,
             )
             processes.append(crd.BkAppProcess(**dict_to_camel(process_entity.dict())))
 
@@ -482,6 +485,12 @@ def get_bkapp_resource_for_deploy(
     # such as: if log collector type is set to "ELK", the operator should mount app logs to host path
     model_res.metadata.annotations[LOG_COLLECTOR_TYPE_ANNO_KEY] = get_log_collector_type(env)
 
+    # 设置 bkapp.paas.bk.tencent.com/proc-services-feature-enabled 注解值
+    proc_svc_enabled = "true"
+    if deployment and (desc_obj := DeploymentDescription.objects.first(deployment=deployment)):
+        proc_svc_enabled = desc_obj.runtime.get(PROC_SERVICES_ENABLED_ANNOTATION_KEY, "true")
+    model_res.set_proc_services_annotation(proc_svc_enabled)
+
     # Apply other changes to the resource
     apply_env_annots(model_res, env, deploy_id=deploy_id)
     apply_builtin_env_vars(model_res, env)

apiserver/paasng/paasng/platform/bkapp_model/migrations/0013_moduleprocessspec_services.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-07-18 12:51
+
+from django.db import migrations
+import paasng.platform.bkapp_model.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bkapp_model', '0012_moduleprocessspec_probes'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='moduleprocessspec',
+            name='services',
+            field=paasng.platform.bkapp_model.models.ProcServicesField(default=None, help_text='进程服务列表', null=True),
+        ),
+    ]