Ft header rewrite

src/dashboard/apigateway/apigateway/apps/plugin/constants.py

@@ -32,6 +32,7 @@ class PluginTypeEnum(StructuredEnum):
 class PluginTypeCodeEnum(StructuredEnum):
     BK_RATE_LIMIT = EnumField("bk-rate-limit", label=_("频率控制"))
     BK_CORS = EnumField("bk-cors", label="CORS")
+    BK_HEADER_REWRITE = EnumField("bk-header-rewrite", label=_("Header 转换"))
     BK_IP_RESTRICTION = EnumField("bk-ip-restriction", label="ip-restriction")
 
 

src/dashboard/apigateway/apigateway/apps/plugin/management/commands/migrate_header_rewrite.py

@@ -0,0 +1,99 @@
+#
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+#
+import logging
+
+from django.core.management.base import BaseCommand
+from django.core.paginator import Paginator
+
+from apigateway.apigateway.apps.plugin.constants import PluginBindingScopeEnum
+from apigateway.apigateway.common.plugin.header_rewrite import HeaderRewriteConvertor
+from apigateway.apigateway.core.constants import ContextScopeTypeEnum, ContextTypeEnum
+from apigateway.core.models import Context, Proxy, Stage
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """将stage/resource的proxy请求头配置迁移成bk-header-rewrite插件配置"""
+
+    def handle(self, *args, **options):
+        # 遍历stage, 迁移proxy请求头
+        qs = Stage.objects.all()
+
+        logger.info("start migrate stage header rewrite plugin config, all stage count %s", qs.count())
+
+        paginator = Paginator(qs, 100)
+        for i in paginator.page_range:
+            logger.info("migrate stage count %s", (i + 1) * 100)
+
+            for stage in paginator.page(i):
+                context = Context.objects.filter(
+                    scope_type=ContextScopeTypeEnum.STAGE.value,
+                    scope_id=stage.id,
+                    type=ContextTypeEnum.STAGE_PROXY_HTTP.value,
+                ).first()
+
+                if not context:
+                    continue
+
+                config = context.config
+                if "transform_headers" not in config:
+                    continue
+
+                # 迁移stage的proxy请求头
+                stage_transform_headers = context.config.get("transform_headers")
+                stage_config = HeaderRewriteConvertor.transform_headers_to_plugin_config(stage_transform_headers)
+                HeaderRewriteConvertor.alter_plugin(
+                    stage.api_id, PluginBindingScopeEnum.STAGE.value, stage.id, stage_config
+                )
+
+                # 迁移后清理 transform_headers
+                config.pop("transform_headers")
+                context.config = config
+                context.save()
+
+        logger.info("finish migrate stage header rewrite plugin config")
+
+        # 迁移resource的proxy请求头
+        qs = Proxy.objects.prefetch_related("resource").all()
+
+        logger.info("start migrate resource header rewrite plugin config, all stage count %s", qs.count())
+
+        paginator = Paginator(qs, 100)
+        for i in paginator.page_range:
+            logger.info("migrate resource count %s", (i + 1) * 100)
+
+            for proxy in paginator.page(i):
+                config = proxy.config
+
+                if "transform_headers" not in config:
+                    continue
+
+                resource_transform_headers = config.get("transform_headers")
+                resource_config = HeaderRewriteConvertor.transform_headers_to_plugin_config(resource_transform_headers)
+
+                HeaderRewriteConvertor.alter_plugin(
+                    proxy.resource.api_id, PluginBindingScopeEnum.RESOURCE.value, proxy.resource.id, resource_config
+                )
+
+                # 迁移后清理 transform_headers
+                config.pop("transform_headers")
+                proxy.config = config
+                proxy.save()
+
+        logger.info("finish migrate resource header rewrite plugin config")

src/dashboard/apigateway/apigateway/apps/plugin/models.py

@@ -22,7 +22,6 @@
 from django.db.models.signals import post_delete, pre_save
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
-from jsonschema import validate
 from tencent_apigateway_common.i18n.field import I18nProperty
 
 from apigateway.apps.plugin.constants import PluginBindingScopeEnum, PluginStyleEnum, PluginTypeEnum
@@ -154,20 +153,8 @@ def config(self) -> Dict[str, Any]:
 
     @config.setter
     def config(self, yaml_: str):
-        loaded_config = yaml_loads(yaml_)
-        self._validate_config(loaded_config)
         self.yaml = yaml_
 
-    def _validate_config(self, config: Dict[str, Any]):
-        if not isinstance(config, dict):
-            raise ValueError("config must be a dict")
-
-        schema = self.type and self.type.schema
-        if not schema:
-            return
-
-        validate(config, schema=schema.schema)
-
     def __str__(self) -> str:
         return f"<PluginConfig {self.name}({self.pk})>"
 

src/dashboard/apigateway/apigateway/apps/plugin/plugin/checker.py

@@ -88,6 +88,21 @@ def _check_duplicate_items(self, data: List[str], key: str):
             raise ValueError(_("{} 存在重复的元素：{}。").format(key, ", ".join(duplicate_items)))
 
 
+class HeaderRewriteChecker(BaseChecker):
+    def check(self, yaml_: str):
+        loaded_data = yaml_loads(yaml_)
+
+        set_keys = [item["key"] for item in loaded_data["set"]]
+        set_duplicate_keys = [key for key, count in Counter(set_keys).items() if count >= 2]
+        if set_duplicate_keys:
+            raise ValueError(_("set 存在重复的元素：{}。").format(", ".join(set_duplicate_keys)))
+
+        remove_keys = [item["key"] for item in loaded_data["remove"]]
+        remove_duplicate_keys = [key for key, count in Counter(remove_keys).items() if count >= 2]
+        if remove_duplicate_keys:
+            raise ValueError(_("remove 存在重复的元素：{}。").format(", ".join(remove_duplicate_keys)))
+
+
 class BkIPRestrictionChecker(BaseChecker):
     def _check_ip_content(self, ip_content: str):
         """check each line is a valid ipv4/ipv6 or ipv4 cidr/ipv6 cidr
@@ -132,6 +147,7 @@ def check(self, payload: str):
 class PluginConfigYamlChecker:
     type_code_to_checker: ClassVar[Dict[str, BaseChecker]] = {
         PluginTypeCodeEnum.BK_CORS.value: BkCorsChecker(),
+        PluginTypeCodeEnum.BK_HEADER_REWRITE.value: HeaderRewriteChecker(),
         PluginTypeCodeEnum.BK_IP_RESTRICTION.value: BkIPRestrictionChecker(),
     }
 

src/dashboard/apigateway/apigateway/apps/plugin/plugin/serializers.py

@@ -21,6 +21,7 @@
 from django.utils.translation import gettext as _
 from django.utils.translation import gettext_lazy
 from jsonschema import ValidationError as SchemaValidationError
+from jsonschema import validate
 from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 from rest_framework.settings import api_settings
@@ -31,6 +32,7 @@
 from apigateway.apps.plugin.plugin.checker import PluginConfigYamlChecker
 from apigateway.apps.plugin.plugin.convertor import PluginConfigYamlConvertor
 from apigateway.common.fields import CurrentGatewayDefault
+from apigateway.controller.crds.release_data.plugin import PluginConvertorFactory
 
 
 class PluginConfigSLZ(serializers.ModelSerializer):
@@ -94,6 +96,12 @@ def _update_plugin(self, plugin: PluginConfig, validated_data: Dict[str, Any]):
 
         try:
             plugin.config = validated_data["yaml"]
+            # 转换数据, 校验apisix schema
+            schema = plugin.type and plugin.type.schema
+            if schema:
+                convertor = PluginConvertorFactory.get_convertor(plugin.type.code)
+                _data = convertor.convert(plugin)
+                validate(_data, schema=schema.schema)
         except SchemaValidationError as err:
             raise ValidationError(
                 {api_settings.NON_FIELD_ERRORS_KEY: f"{err.message}, path {list(err.absolute_path)}"}

src/dashboard/apigateway/apigateway/common/plugin/__init__.py

@@ -0,0 +1,17 @@
+#
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+#

src/dashboard/apigateway/apigateway/common/plugin/header_rewrite.py

@@ -0,0 +1,85 @@
+#
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+#
+from typing import Optional
+
+from apigateway.apps.plugin.constants import PluginTypeCodeEnum
+from apigateway.apps.plugin.models import PluginBinding, PluginConfig, PluginType
+from apigateway.utils.yaml import yaml_dumps
+
+
+class HeaderRewriteConvertor:
+    @staticmethod
+    def transform_headers_to_plugin_config(transform_headers: dict) -> Optional[dict]:
+        # both set and delete empty
+        if not transform_headers or (not transform_headers.get("set") and not transform_headers.get("delete")):
+            return None
+
+        return {
+            "set": [{"key": key, "value": value} for key, value in (transform_headers.get("set") or {}).items()],
+            "remove": [{"key": key} for key in (transform_headers.get("delete") or [])],
+        }
+
+    @staticmethod
+    def alter_plugin(gateway_id: int, scope_type: str, scope_id: int, plugin_config: Optional[dict]):
+        # 1. 判断resource是否已经绑定header rewrite插件
+        binding = (
+            PluginBinding.objects.filter(
+                scope_type=scope_type,
+                scope_id=scope_id,
+                config__type__code=PluginTypeCodeEnum.BK_HEADER_REWRITE.value,
+            )
+            .prefetch_related("config")
+            .first()
+        )
+
+        if not binding and not plugin_config:
+            return
+
+        if binding:
+            if plugin_config:
+                # 如果已经绑定, 更新插件配置
+                config = binding.config
+                config.yaml = yaml_dumps(plugin_config)
+                # NOTE: 用bulk_update避免触发信号
+                PluginConfig.objects.bulk_update([config], ["yaml"])
+                return
+
+            # 插件配置为空, 清理数据
+            config = binding.config
+            # NOTE: 用bulk_delete避免触发信号
+            PluginBinding.objects.bulk_delete([binding])
+            PluginConfig.objects.bulk_delete([config])
+            return
+
+        # 如果没有绑定, 新建插件配置, 并绑定到stage
+        if plugin_config:
+            config = PluginConfig(
+                api_id=gateway_id,
+                name=f"{scope_type} [{scope_id}] header rewrite",
+                type=PluginType.objects.get(code=PluginTypeCodeEnum.BK_HEADER_REWRITE.value),
+                yaml=yaml_dumps(plugin_config),
+            )
+            config.save()
+            binding = PluginBinding(
+                api_id=gateway_id,
+                scope_type=scope_type,
+                scope_id=scope_id,
+                config=config,
+            )
+            # NOTE: 用bulk_create避免触发信号
+            PluginBinding.objects.bulk_create([binding])

src/dashboard/apigateway/apigateway/controller/crds/release_data/base.py

@@ -28,6 +28,8 @@ class PluginData:
     _type_code_to_name: ClassVar[Dict[str, str]] = {
         "bk-rate-limit:stage": "bk-stage-rate-limit",
         "bk-rate-limit:resource": "bk-resource-rate-limit",
+        "bk-header-rewrite:stage": "bk-stage-header-rewrite",
+        "bk-header-rewrite:resource": "bk-resource-header-rewrite",
     }
 
     @property

src/dashboard/apigateway/apigateway/controller/crds/release_data/plugin.py

@@ -19,6 +19,7 @@
 # PluginConfig 中前端表单数据，转换成 apisix 插件配置
 
 from abc import ABC, abstractmethod
+
 from typing import Any, ClassVar, Dict, List, Union
 
 from apigateway.apps.plugin.constants import PluginTypeCodeEnum
@@ -44,6 +45,18 @@ def convert(self, plugin_config: PluginConfig) -> Dict[str, Any]:
         return plugin_config.config
 
 
+class HeaderWriteConvertor(PluginConvertor):
+    plugin_type_code: ClassVar[str] = PluginTypeCodeEnum.BK_HEADER_REWRITE.value
+
+    def convert(self, plugin_config: PluginConfig) -> Dict[str, Any]:
+        config = plugin_config.config
+
+        return {
+            "set": {item["key"]: item["value"] for item in config["set"]},
+            "remove": [item["key"] for item in config["remove"]],
+        }
+
+
 class IPRestrictionConvertor(PluginConvertor):
     plugin_type_code: ClassVar[str] = PluginTypeCodeEnum.BK_IP_RESTRICTION.value
 
@@ -83,6 +96,7 @@ class PluginConvertorFactory:
     plugin_convertors: ClassVar[Dict[PluginTypeCodeEnum, PluginConvertor]] = {
         c.plugin_type_code: c  # type: ignore
         for c in [
+            HeaderWriteConvertor(),
             IPRestrictionConvertor(),
         ]
     }