From 6b49e9b8687bb1b6668c41aec9b46d910f2f4aa3 Mon Sep 17 00:00:00 2001 From: piglei Date: Fri, 1 Nov 2024 17:24:22 +0800 Subject: [PATCH] feat: drop the support for app description version 1(used by legacy s-mart apps) (#1693) --- .../paasng/platform/declarative/constants.py | 10 +- .../paasng/platform/declarative/exceptions.py | 4 + .../paasng/platform/declarative/handlers.py | 155 +++++++++----- .../platform/declarative/serializers.py | 137 +----------- .../engine/configurations/source_file.py | 19 +- .../paasng/platform/engine/utils/source.py | 9 +- .../platform/smart_app/services/detector.py | 63 ++---- .../platform/smart_app/services/dispatch.py | 10 +- .../platform/smart_app/services/patcher.py | 23 -- .../platform/sourcectl/controllers/package.py | 2 +- .../paasng/platform/sourcectl/exceptions.py | 4 +- .../paasng/paasng/platform/sourcectl/views.py | 2 +- .../paasng/tests/api/apigw/test_lesscode.py | 2 +- .../declarative/handlers/test_handlers.py | 98 +++++++++ .../platform/declarative/handlers/test_v1.py | 146 ------------- .../handlers/v2/test_deployment.py | 8 +- .../paasng/platform/smart_app/conftest.py | 4 +- .../platform/smart_app/test_detector.py | 201 ++++++++---------- .../paasng/platform/smart_app/test_patcher.py | 161 ++------------ .../platform/sourcectl/packages/test_utils.py | 12 +- .../platform/sourcectl/packages/utils.py | 14 +- 21 files changed, 399 insertions(+), 685 deletions(-) create mode 100644 apiserver/paasng/tests/paasng/platform/declarative/handlers/test_handlers.py delete mode 100644 apiserver/paasng/tests/paasng/platform/declarative/handlers/test_v1.py diff --git a/apiserver/paasng/paasng/platform/declarative/constants.py b/apiserver/paasng/paasng/platform/declarative/constants.py index 12d7631072..501a8d13ed 100644 --- a/apiserver/paasng/paasng/platform/declarative/constants.py +++ b/apiserver/paasng/paasng/platform/declarative/constants.py @@ -44,16 +44,20 @@ def dict(self, *args, **kwargs): class AppSpecVersion(IntStructuredEnum): - # VER_1 meaning app.yaml is provided by S-Mart App + # VER_1 meaning app.yaml is provided by legacy S-Mart App, + # **this version is not supported anymore** VER_1 = 1 + VER_2 = 2 - # VER_3 meaning cnative app + # VER_3 means cnative app VER_3 = 3 + # UNSPECIFIED means the version is not specified + UNSPECIFIED = -1 + class AppDescPluginType(StrStructuredEnum): APP_VERSION = EnumField("app_version", label="应用版本") - APP_LIBRARIES = EnumField("app_libraries", label="应用依赖库") class DiffType(StrStructuredEnum): diff --git a/apiserver/paasng/paasng/platform/declarative/exceptions.py b/apiserver/paasng/paasng/platform/declarative/exceptions.py index d6963af77c..f88050190b 100644 --- a/apiserver/paasng/paasng/platform/declarative/exceptions.py +++ b/apiserver/paasng/paasng/platform/declarative/exceptions.py @@ -53,6 +53,10 @@ def from_validation_error(cls, error: ValidationError) -> "DescriptionValidation return cls(error.detail, err_messages[0]) +class UnsupportedSpecVer(Exception): + """Raised if the app spec data is using an unsupported version""" + + class ControllerError(Exception): """An error occurred when controller is processing the input data""" diff --git a/apiserver/paasng/paasng/platform/declarative/handlers.py b/apiserver/paasng/paasng/platform/declarative/handlers.py index 263b825206..028e54afbc 100644 --- a/apiserver/paasng/paasng/platform/declarative/handlers.py +++ b/apiserver/paasng/paasng/platform/declarative/handlers.py @@ -39,9 +39,8 @@ from paasng.platform.declarative.deployment.resources import DeploymentDesc from paasng.platform.declarative.deployment.validations import v2 as deploy_spec_v2 from paasng.platform.declarative.deployment.validations import v3 as deploy_spec_v3 -from paasng.platform.declarative.exceptions import DescriptionValidationError +from paasng.platform.declarative.exceptions import DescriptionValidationError, UnsupportedSpecVer from paasng.platform.declarative.serializers import ( - SMartV1DescriptionSLZ, UniConfigSLZ, validate_desc, validate_procfile_procs, @@ -58,21 +57,39 @@ def get_desc_handler(json_data: Dict) -> "DescriptionHandler": :param json_data: The description data in dict format. """ - spec_version = detect_spec_version(json_data) - # TODO 删除 SMartDescriptionHandler 分支. VER_1 存量版本基本不再支持 - if spec_version == AppSpecVersion.VER_1: - return SMartDescriptionHandler(json_data) - elif spec_version == AppSpecVersion.VER_2: - return AppDescriptionHandler(json_data) - else: - # 对应 AppSpecVersion.VER_3 - return CNativeAppDescriptionHandler(json_data) + try: + spec_version = detect_spec_version(json_data) + except ValueError as e: + return UnsupportedVerDescriptionHandler(version=str(e)) + + match spec_version: + case AppSpecVersion.VER_2: + return AppDescriptionHandler(json_data) + case AppSpecVersion.VER_3: + return CNativeAppDescriptionHandler(json_data) + case AppSpecVersion.UNSPECIFIED: + return NoVerDescriptionHandler() + case _: + return UnsupportedVerDescriptionHandler(version=str(spec_version.value)) def detect_spec_version(json_data: Dict) -> AppSpecVersion: + """Detect the spec version from the input data. + + :return: The version. + :raise ValueError: When the version is specified but it's value is invalid. + """ if spec_version := json_data.get("spec_version") or json_data.get("specVersion"): - return AppSpecVersion(spec_version) - return AppSpecVersion.VER_1 + try: + return AppSpecVersion(spec_version) + except ValueError: + raise ValueError(spec_version) + + # The spec ver "1" use no version field while the "app_code" field is always presented. + if "app_code" in json_data: + return AppSpecVersion.VER_1 + + return AppSpecVersion.UNSPECIFIED class DescriptionHandler(Protocol): @@ -176,34 +193,31 @@ def handle_app(self, user: User, source_origin: Optional[SourceOrigin] = None) - return controller.perform_action(self.app_desc) -class SMartDescriptionHandler: - """A handler to process S-Mart app description file""" - - @classmethod - def from_file(cls, fp: TextIO): - return cls(yaml.safe_load(fp)) +class UnsupportedVerDescriptionHandler: + """A special handler, raise error if the version is not supported.""" - def __init__(self, json_data: Dict): - self.json_data = json_data + def __init__(self, version: str): + self.message = f'App spec version "{version}" is not supported, please use a valid version like "3".' @property def app_desc(self) -> ApplicationDesc: - """Turn json data into application description object - - :raises: DescriptionValidationError when input is invalid - """ - instance = get_application(self.json_data, "app_code") - # S-mart application always perform a full update by using partial=False - app_desc, _ = validate_desc(SMartV1DescriptionSLZ, self.json_data, instance, partial=False) - return app_desc + raise DescriptionValidationError(self.message) def handle_app(self, user: User, source_origin: Optional[SourceOrigin] = None) -> Application: - """Handle a app config + raise DescriptionValidationError(self.message) - :param user: User to perform actions as - """ - controller = AppDeclarativeController(user) - return controller.perform_action(self.app_desc) + +class NoVerDescriptionHandler: + """A special handler, raise error if no version is specified.""" + + message = "No spec version is specified, please set the spec version to a valid value." + + @property + def app_desc(self) -> ApplicationDesc: + raise DescriptionValidationError(self.message) + + def handle_app(self, user: User, source_origin: Optional[SourceOrigin] = None) -> Application: + raise DescriptionValidationError(self.message) ### @@ -218,15 +232,27 @@ def get_deploy_desc_handler( :param desc_data: The description data in dict format, optional :param procfile_data: The "Procfile" data in dict format, format: {: } + :raise ValueError: When the input data is invalid for creating a handler. """ if not (desc_data or procfile_data): - raise ValueError("json_data and procfile_data can't be both None") + raise ValueError("the app desc and procfile data can't be both empty") if not desc_data: # Only procfile data is provided, use the handler to handle processes data only assert procfile_data return ProcfileOnlyDeployDescHandler(procfile_data) - return DefaultDeployDescHandler(desc_data, procfile_data, get_desc_getter_func(desc_data)) + + try: + _func = get_desc_getter_func(desc_data) + except UnsupportedSpecVer as e: + # When the spec version is not supported and no procfile data is provided, + # raise an error to inform the user. + if not procfile_data: + raise ValueError("the procfile data is empty and the app desc data is invalid, detail: {}".format(str(e))) + else: + return ProcfileOnlyDeployDescHandler(procfile_data) + + return DefaultDeployDescHandler(desc_data, procfile_data, _func) def get_deploy_desc_by_module(desc_data: Dict, module_name: str) -> DeploymentDesc: @@ -234,8 +260,28 @@ def get_deploy_desc_by_module(desc_data: Dict, module_name: str) -> DeploymentDe :param desc_data: The description data in dict format, may contains multiple modules :param module_name: The module name + :raise DescriptionValidationError: When the input data is invalid """ - return get_desc_getter_func(desc_data)(desc_data, module_name) + try: + _func = get_desc_getter_func(desc_data) + except UnsupportedSpecVer as e: + raise DescriptionValidationError(str(e)) + return _func(desc_data, module_name) + + +def get_source_dir_from_desc(desc_data: Dict, module_name: str) -> str: + """Get the source directory specified in the description data by module name. + + :param desc_data: The description data in dict format, may contains multiple modules + :param module_name: The module name + :return: The source directory + """ + try: + _func = get_desc_getter_func(desc_data) + except UnsupportedSpecVer: + # When the spec version is not supported, use the default value + return "" + return _func(desc_data, module_name).source_dir class DeployDescHandler(Protocol): @@ -252,15 +298,24 @@ def handle(self, deployment: Deployment) -> DeployHandleResult: def get_desc_getter_func(desc_data: Dict) -> DescGetterFunc: - """Get the description getter function by current desc data.""" - spec_version = detect_spec_version(desc_data) - # TODO: 删除此分支，VER_1 存量版本基本不再支持 - if spec_version == AppSpecVersion.VER_1: - return deploy_desc_getter_v1 - elif spec_version == AppSpecVersion.VER_2: - return deploy_desc_getter_v2 - else: - return deploy_desc_getter_v3 + """Get the description getter function by current desc data. + + :raise UnsupportedSpecVer: When the spec version is not supported. + """ + try: + spec_version = detect_spec_version(desc_data) + except ValueError as e: + raise UnsupportedSpecVer(f'app spec version "{str(e)}" is not supported') + + match spec_version: + case AppSpecVersion.VER_2: + return deploy_desc_getter_v2 + case AppSpecVersion.VER_3: + return deploy_desc_getter_v3 + case AppSpecVersion.UNSPECIFIED: + raise UnsupportedSpecVer("no spec version is specified") + case _: + raise UnsupportedSpecVer(f'app spec version "{spec_version.value}" is not supported') class DefaultDeployDescHandler: @@ -296,14 +351,6 @@ def handle(self, deployment: Deployment) -> DeployHandleResult: return handle_procfile_procs(deployment, procfile_procs) -def deploy_desc_getter_v1(json_data: Dict, module_name: str) -> DeploymentDesc: - """Get the deployment desc object, spec ver 1.""" - instance = get_application(json_data, "app_code") - # S-mart application always perform a full update by using partial=False - _, deploy_desc = validate_desc(SMartV1DescriptionSLZ, json_data, instance, partial=False) - return deploy_desc - - def deploy_desc_getter_v2(json_data: Dict, module_name: str) -> DeploymentDesc: """Get the deployment desc object, spec ver 2.""" validate_desc(UniConfigSLZ, json_data) diff --git a/apiserver/paasng/paasng/platform/declarative/serializers.py b/apiserver/paasng/paasng/platform/declarative/serializers.py index cba750b447..52fe08086a 100644 --- a/apiserver/paasng/paasng/platform/declarative/serializers.py +++ b/apiserver/paasng/paasng/platform/declarative/serializers.py @@ -15,8 +15,7 @@ # We undertake not to change the open source license (MIT license) applicable # to the current version of the project delivered to anyone in the future. -import shlex -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, List, Optional, Type import cattr from django.conf import settings @@ -24,17 +23,10 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from paasng.accessories.publish.market.serializers import ProductTagByNameField from paasng.platform.applications.constants import AppLanguage -from paasng.platform.applications.serializers import AppIDSMartField, AppNameField -from paasng.platform.bkapp_model.entities import Addon, v1alpha2 -from paasng.platform.declarative import constants -from paasng.platform.declarative.application.resources import ApplicationDesc, DisplayOptions, MarketDesc -from paasng.platform.declarative.deployment.resources import DeploymentDesc, ProcfileProc +from paasng.platform.declarative.application.resources import DisplayOptions +from paasng.platform.declarative.deployment.resources import ProcfileProc from paasng.platform.declarative.exceptions import DescriptionValidationError -from paasng.platform.declarative.utils import get_quota_plan -from paasng.utils.i18n.serializers import I18NExtend, i18n -from paasng.utils.serializers import Base64FileField from paasng.utils.validators import PROC_TYPE_MAX_LENGTH, PROC_TYPE_PATTERN @@ -141,129 +133,6 @@ class LegacyEnvVariableSLZ(serializers.Serializer): value = serializers.CharField(required=True, max_length=1000) -@i18n -class SMartV1DescriptionSLZ(serializers.Serializer): - """Serializer for parsing the origin version of S-Mart application description""" - - # For some reason, the max length for the `app_code` field uses the legacy value of 16 - # in the v1 schema, the value is changed to 20 in later versions. - app_code = AppIDSMartField(max_length=16) - app_name = I18NExtend(AppNameField()) - version = serializers.RegexField(r"^([0-9]+)\.([0-9]+)\.([0-9]+)$", required=True, help_text="版本") - # Celery 相关 - is_use_celery = serializers.BooleanField(required=True, help_text="是否启用 celery") - is_use_celery_with_gevent = serializers.BooleanField( - required=False, help_text="是否启用 celery (gevent)模式", default=False - ) - is_use_celery_beat = serializers.BooleanField(required=False, help_text="是否启用 celery beat", default=False) - author = serializers.CharField(required=True, help_text="应用作者") - introduction = I18NExtend(serializers.CharField(required=True, help_text="简介")) - - # Not required fields - category = ProductTagByNameField(required=False, source="tag") - language = serializers.CharField( - required=False, help_text="开发语言", default=AppLanguage.PYTHON.value, validators=[validate_language] - ) - desktop = DesktopOptionsSLZ(required=False, default=DesktopOptionsSLZ.gen_default_value, help_text="桌面展示选项") - env = serializers.ListField(child=LegacyEnvVariableSLZ(), required=False, default=list) - container = ContainerSpecSLZ(required=False, allow_null=True, source="package_plan") - libraries = serializers.ListField(child=LibrarySLZ(), required=False, default=list) - logo_b64data = Base64FileField(required=False, help_text="logo", source="logo") - - def to_internal_value(self, data) -> Tuple[ApplicationDesc, DeploymentDesc]: - attrs = super().to_internal_value(data) - market_desc = MarketDesc( - introduction_en=attrs["introduction_en"], - introduction_zh_cn=attrs["introduction_zh_cn"], - display_options=attrs.get("desktop"), - logo=attrs.get("logo", constants.OMITTED_VALUE), - ) - if attrs.get("tag"): - market_desc.tag_id = attrs["tag"].id - - package_plan = get_quota_plan(attrs.get("package_plan")) if attrs.get("package_plan") else None - addons = [Addon(name=service) for service in settings.SMART_APP_DEFAULT_SERVICES_CONFIG] - processes = [ - { - "name": "web", - "args": shlex.split(constants.WEB_PROCESS), - "res_quota_plan": package_plan, - "replicas": 1, - "proc_command": constants.WEB_PROCESS, - } - ] - is_use_celery = False - if attrs["is_use_celery"]: - is_use_celery = True - addons.append(Addon(name="rabbitmq")) - processes.append( - { - "name": "celery", - "args": shlex.split(constants.CELERY_PROCESS), - "res_quota_plan": package_plan, - "replicas": 1, - "proc_command": constants.CELERY_PROCESS, - } - ) - elif attrs["is_use_celery_with_gevent"]: - is_use_celery = True - addons.append(Addon(name="rabbitmq")) - processes.append( - { - "name": "celery", - "args": shlex.split(constants.CELERY_PROCESS_WITH_GEVENT), - "res_quota_plan": package_plan, - "replicas": 1, - "proc_command": constants.CELERY_PROCESS_WITH_GEVENT, - } - ) - - if attrs["is_use_celery_beat"]: - if not is_use_celery: - raise ValueError("Can't use celery beat but not use celery.") - processes.append( - { - "name": "beat", - "args": shlex.split(constants.CELERY_BEAT_PROCESS), - "res_quota_plan": package_plan, - "replicas": 1, - "proc_command": constants.CELERY_BEAT_PROCESS, - } - ) - - plugins = [ - dict(type=constants.AppDescPluginType.APP_VERSION, data=attrs["version"]), - dict(type=constants.AppDescPluginType.APP_LIBRARIES, data=attrs["libraries"]), - ] - - spec = v1alpha2.BkAppSpec( - processes=processes, - configuration={"env": [{"name": item["key"], "value": item["value"]} for item in attrs.get("env", [])]}, - addons=addons, - ) - application_desc = ApplicationDesc( - spec_version=constants.AppSpecVersion.VER_1, - code=attrs["app_code"], - name_en=attrs["app_name_en"], - name_zh_cn=attrs["app_name_zh_cn"], - market=market_desc, - modules={ - "default": { - "name": "default", - "is_default": True, - "services": [{"name": addon.name} for addon in addons], - } - }, - plugins=plugins, - instance_existed=bool(self.instance), - ) - deployment_desc = cattr.structure( - {"spec": spec, "language": attrs.get("language"), "spec_version": constants.AppSpecVersion.VER_1}, - DeploymentDesc, - ) - return application_desc, deployment_desc - - def validate_procfile_procs(data: Dict[str, str]) -> List[ProcfileProc]: """Validate process data which was read from procfile. diff --git a/apiserver/paasng/paasng/platform/engine/configurations/source_file.py b/apiserver/paasng/paasng/platform/engine/configurations/source_file.py index 796973cf42..ecf8754649 100644 --- a/apiserver/paasng/paasng/platform/engine/configurations/source_file.py +++ b/apiserver/paasng/paasng/platform/engine/configurations/source_file.py @@ -49,7 +49,10 @@ def get_procfile(self, version_info: VersionInfo) -> Dict[str, str]: """ def get_app_desc(self, version_info: VersionInfo) -> Dict: - """Read app.yaml/app_desc.yaml from repository + """Read app_desc.yaml from repository + + NOTE: The platform used to support "app.yaml" using the version 1 spec, support + has been dropped because no applications use it anymore. :raises: exceptions.GetAppYamlError """ @@ -100,11 +103,11 @@ def get_procfile(self, version_info: VersionInfo) -> Dict[str, str]: return procfile def get_app_desc(self, version_info: VersionInfo) -> Dict: - """Read app.yaml/app_desc.yaml from repository + """Read app_desc.yaml from repository :raises: exceptions.GetAppYamlError """ - possible_keys = ["app_desc.yaml", "app_desc.yml", "app.yml", "app.yaml"] + possible_keys = ["app_desc.yaml", "app_desc.yml"] if self.source_dir != Path("."): # Note: 为了保证不影响源码包部署的应用, 优先从根目录读取 app_desc.yaml, 随后再尝试从 source_dir 目录读取 possible_keys = [ @@ -112,10 +115,6 @@ def get_app_desc(self, version_info: VersionInfo) -> Dict: "app_desc.yml", str(self.source_dir / "app_desc.yaml"), str(self.source_dir / "app_desc.yml"), - "app.yml", - "app.yaml", - str(self.source_dir / "app.yml"), - str(self.source_dir / "app.yaml"), ] content = None @@ -134,9 +133,9 @@ def get_app_desc(self, version_info: VersionInfo) -> Dict: try: app_description = yaml.full_load(content) except Exception as e: - raise exceptions.GetAppYamlFormatError('file "app.yaml"\'s format is not YAML') from e + raise exceptions.GetAppYamlFormatError('file "app_desc.yaml"\'s format is not YAML') from e if not isinstance(app_description, dict): - raise exceptions.GetAppYamlFormatError('file "app.yaml" must be dict type') + raise exceptions.GetAppYamlFormatError('file "app_desc.yaml" must be dict type') return app_description def get_dockerignore(self, version_info: VersionInfo) -> str: @@ -220,7 +219,7 @@ def get_procfile(self, version_info: VersionInfo) -> Dict[str, str]: return super().get_procfile(version_info) def get_app_desc(self, version_info: VersionInfo) -> Dict: - """Read app.yaml/app_desc.yaml from SourcePackage.meta_data(the field stored app_desc) or repository""" + """Read app_desc.yaml from SourcePackage.meta_data(the field stored app_desc) or repository""" _, version = self.extract_version_info(version_info) package_storage = self.module.packages.get(version=version) if package_storage.meta_info: diff --git a/apiserver/paasng/paasng/platform/engine/utils/source.py b/apiserver/paasng/paasng/platform/engine/utils/source.py index b9c813ff5f..16ca3f92ab 100644 --- a/apiserver/paasng/paasng/platform/engine/utils/source.py +++ b/apiserver/paasng/paasng/platform/engine/utils/source.py @@ -29,7 +29,7 @@ from paasng.accessories.smart_advisor.tagging import dig_tags_local_repo from paasng.platform.applications.constants import AppFeatureFlag, ApplicationType from paasng.platform.applications.models import Application -from paasng.platform.declarative.handlers import DeployDescHandler, get_deploy_desc_by_module, get_deploy_desc_handler +from paasng.platform.declarative.handlers import DeployDescHandler, get_deploy_desc_handler, get_source_dir_from_desc from paasng.platform.engine.configurations.building import get_dockerfile_path from paasng.platform.engine.configurations.source_file import get_metadata_reader from paasng.platform.engine.exceptions import InitDeployDescHandlerError, SkipPatchCode @@ -134,7 +134,7 @@ def get_source_dir(module: Module, operator: str, version_info: VersionInfo) -> desc_data = get_desc_data_by_version(module, operator, version_info) if not desc_data: return "" - return get_deploy_desc_by_module(desc_data, module.name).source_dir + return get_source_dir_from_desc(desc_data, module.name) _current_path = Path(".") @@ -216,7 +216,10 @@ def get_deploy_desc_handler_by_version( msg.append(f"[Procfile] {procfile_exc}") raise InitDeployDescHandlerError("; ".join(msg)) - return get_deploy_desc_handler(app_desc, procfile_data) + try: + return get_deploy_desc_handler(app_desc, procfile_data) + except ValueError as e: + raise InitDeployDescHandlerError(str(e)) def get_source_package_path(deployment: Deployment) -> str: diff --git a/apiserver/paasng/paasng/platform/smart_app/services/detector.py b/apiserver/paasng/paasng/platform/smart_app/services/detector.py index b61de13255..4566ee6345 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/detector.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/detector.py @@ -20,8 +20,6 @@ import logging import re import zipfile -from dataclasses import dataclass -from itertools import product from os import PathLike from pathlib import Path from typing import Dict, Optional, Tuple, Union @@ -33,7 +31,7 @@ from yaml import YAMLError from paasng.platform.declarative.application.resources import ApplicationDesc -from paasng.platform.declarative.constants import AppDescPluginType, AppSpecVersion +from paasng.platform.declarative.constants import AppDescPluginType from paasng.platform.declarative.exceptions import DescriptionValidationError from paasng.platform.declarative.handlers import get_desc_handler from paasng.platform.smart_app.services.path import PathProtocol @@ -48,34 +46,22 @@ logger = logging.getLogger(__name__) -@dataclass -class DetectResult: +def relative_path_of_app_desc(filepath: str) -> Optional[str]: + """Get the relative path of the app description file, if the given path is not + a app description file, return None. """ - :param str relative_path: relative_path of the app description file and the version of meta file - :param AppSpecVersion version: the version of app description file - """ - - relative_path: str - version: AppSpecVersion - - -class AppYamlDetector: - _regexes = { - AppSpecVersion.VER_1: r"(^(?<=[/\\\\])?|(?<=[/\\\\]))app\.ya?ml$", - AppSpecVersion.VER_2: r"(^(?<=[/\\\\])?|(?<=[/\\\\]))app_desc\.ya?ml$", - } - - @classmethod - def detect(cls, filepath: str, spec_version: AppSpecVersion = AppSpecVersion.VER_2) -> Optional[DetectResult]: - """Detect whether the file path meets the specification of `app.yaml` - - if matched, return DetectResult, else, return None - """ - regex = cls._regexes[spec_version] - result = re.split(regex, filepath) - if len(result) > 1: - return DetectResult(relative_path=result[0], version=spec_version) - return None + # The pattern acts as a delimiter to help split the relative path of the app + # description file. It uses a lookbehind to match the path delimiter before the + # filename and the result of re.split() can still have the delimiter. + # + # Example of split(): + # - /path/to/app_desc.yaml -> ['/path/to/', '', ''] + # + desc_pattern = re.compile(r"(^(?<=[/\\\\])?|(?<=[/\\\\]))app_desc\.ya?ml$") + parts = desc_pattern.split(filepath) + if len(parts) > 1: + return parts[0] + return None class SourcePackageStatReader: @@ -92,10 +78,10 @@ def accessor(self): def get_meta_info(self) -> Tuple[str, Dict]: """ - Get package's meta info which was stored in file 'app.yaml' + Get package's meta info which was stored in file 'app_desc.yaml' :returns: Tuple[str, Dict] - - the relative path of app.yaml (to the root dir in the tar file), "./" is returned by default + - the relative path of app_desc.yaml (to the root dir in the tar file), "./" is returned by default - the raw meta info of source package, `{}` is returned by default :raises InvalidPackageFileFormatError: The file is not valid, it's content might be corrupt. :raises ValidationError: The file content is not valid YAML. @@ -111,11 +97,10 @@ def get_meta_info(self) -> Tuple[str, Dict]: logger.warning("Unable to list contents in the package file, path: %s.", self.path) return relative_path, {} - for spec_version, filename in product([AppSpecVersion.VER_2, AppSpecVersion.VER_1], existed_filenames): - result = AppYamlDetector.detect(filename, spec_version) - if result is not None: - app_filename = filename - relative_path = result.relative_path + for filepath in existed_filenames: + if (p := relative_path_of_app_desc(filepath)) is not None: + app_filename = filepath + relative_path = p break else: # If not description file can be found, return empty info @@ -244,12 +229,10 @@ def detect_procfile(self) -> str: raise KeyError("Procfile not found.") def detect_app_desc(self) -> str: - """探测源码包中的 app.yaml 的路径""" + """探测源码包中的 app_desc.yaml 的路径""" possible_keys = [ self.package_root / self.relative_path / "app_desc.yaml", self.package_root / self.relative_path / "app_desc.yml", - self.package_root / self.relative_path / "app.yaml", - self.package_root / self.relative_path / "app.yml", ] for key in possible_keys: if key.exists(): diff --git a/apiserver/paasng/paasng/platform/smart_app/services/dispatch.py b/apiserver/paasng/paasng/platform/smart_app/services/dispatch.py index fc82991d8b..eb98045806 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/dispatch.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/dispatch.py @@ -26,7 +26,7 @@ from paasng.infras.accounts.models import User from paasng.platform.applications.models import Application -from paasng.platform.declarative.handlers import get_deploy_desc_by_module +from paasng.platform.declarative.handlers import get_source_dir_from_desc from paasng.platform.modules.models import Module from paasng.platform.smart_app.conf import bksmart_settings from paasng.platform.smart_app.constants import SMartPackageBuilderVersionFlag @@ -86,7 +86,7 @@ def patch_and_store_package(module: Module, tarball_filepath: Path, stat: SPStat """Patch an uncompressed package and upload compressed tarball one blobstore, then bind the package to provided module. - [deprecated] `patch_and_store_package` is a handler for legacy pacakge which only contain source code. + [deprecated] `patch_and_store_package` is a handler for legacy package which only contain source code. """ logger.debug("Patching module for module '%s'", module.name) with generate_temp_dir() as workplace: @@ -110,10 +110,10 @@ def dispatch_slug_image_to_registry(module: Module, workplace: Path, stat: SPSta """ logger.debug("dispatching slug-image for module '%s', working at '%s'", module.name, workplace) - deploy_desc = get_deploy_desc_by_module(stat.meta_info, module.name) + source_dir = get_source_dir_from_desc(stat.meta_info, module.name) - layer_path = workplace / stat.relative_path / deploy_desc.source_dir / "layer.tar.gz" - procfile_path = workplace / stat.relative_path / deploy_desc.source_dir / f"{module.name}.Procfile.tar.gz" + layer_path = workplace / stat.relative_path / source_dir / "layer.tar.gz" + procfile_path = workplace / stat.relative_path / source_dir / f"{module.name}.Procfile.tar.gz" mgr = SMartImageManager(module) base_image = mgr.get_slugrunner_image_info() diff --git a/apiserver/paasng/paasng/platform/smart_app/services/patcher.py b/apiserver/paasng/paasng/platform/smart_app/services/patcher.py index 2addf345cb..4b6c1bc4df 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/patcher.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/patcher.py @@ -22,8 +22,6 @@ import yaml from django.utils.functional import cached_property -from paasng.platform.applications.constants import AppLanguage -from paasng.platform.declarative.constants import AppDescPluginType from paasng.platform.declarative.handlers import get_deploy_desc_by_module, get_desc_handler from paasng.platform.modules.constants import SourceOrigin from paasng.platform.modules.specs import ModuleSpecs @@ -62,8 +60,6 @@ def patch_source_dir(cls, module: "Module", source_dir: Path, dest: Path, stat: ) # 尝试添加 Procfile patcher.add_procfile() - # 尝试添加 requirements.txt - patcher.add_requirements_txt() # 尝试添加 manifest.yaml patcher.add_manifest() # 重新压缩源码包 @@ -128,25 +124,6 @@ def add_procfile(self): key.write_text(yaml.safe_dump(procfile)) - def add_requirements_txt(self): - """尝试往 Python 类型的应用源码目录创建 requirements.txt 文件, 如果源码已加密, 则注入至应用描述文件目录下""" - key = self._make_key("requirements.txt") - plugin = self.app_description.get_plugin(AppDescPluginType.APP_LIBRARIES) - libraries = (plugin or {}).get("data", []) - - if self.deploy_description.language != AppLanguage.PYTHON: - logger.debug("Only handle python app.") - return - - if not libraries: - logger.warning("Undefined libraries, skip add `requirements.txt`.") - return - - if key.exists(): - logger.warning(f"file<{key}> in package will be overwrite!.") - - key.write_text("\n".join([f"{lib['name']}=={lib['version']}" for lib in libraries])) - def add_manifest(self): """尝试往源码根目录添加 manifest""" if self.module.get_source_origin() != SourceOrigin.S_MART: diff --git a/apiserver/paasng/paasng/platform/sourcectl/controllers/package.py b/apiserver/paasng/paasng/platform/sourcectl/controllers/package.py index bb924872fb..ae0c1d5ac9 100644 --- a/apiserver/paasng/paasng/platform/sourcectl/controllers/package.py +++ b/apiserver/paasng/paasng/platform/sourcectl/controllers/package.py @@ -66,7 +66,7 @@ def export(self, local_path: "PathLike", version_info: VersionInfo): if not package_storage.relative_path: return - # The source file may have a relative path (e.g. app_code/app.yaml and so on.), + # The source file may have a relative path (e.g. app_code/app_desc.yaml and so on.), # So we need to move the files in that directory into local_path local_path_obj = Path(local_path) source_path = local_path_obj / package_storage.relative_path diff --git a/apiserver/paasng/paasng/platform/sourcectl/exceptions.py b/apiserver/paasng/paasng/platform/sourcectl/exceptions.py index 1306eb9999..4ac758a2aa 100644 --- a/apiserver/paasng/paasng/platform/sourcectl/exceptions.py +++ b/apiserver/paasng/paasng/platform/sourcectl/exceptions.py @@ -101,11 +101,11 @@ class GetProcfileFormatError(GetProcfileError): class GetAppYamlError(ExceptionWithMessage): - """When no valid app.yaml can be found in application directory""" + """When no valid app_desc.yaml can be found in application directory""" class GetAppYamlFormatError(GetAppYamlError): - """The app.yaml exists but the content format is incorrect""" + """The app_desc.yaml exists but the content format is incorrect""" class GetDockerIgnoreError(ExceptionWithMessage): diff --git a/apiserver/paasng/paasng/platform/sourcectl/views.py b/apiserver/paasng/paasng/platform/sourcectl/views.py index a8ec6517f8..d88376e9bf 100644 --- a/apiserver/paasng/paasng/platform/sourcectl/views.py +++ b/apiserver/paasng/paasng/platform/sourcectl/views.py @@ -298,7 +298,7 @@ def handle_exception(self, exc): operation_description="目前仅提供给 lesscode 项目使用", ) def upload_via_url(self, request, code, module_name): - """根据 URL 方式上传源码包, 目前不校验 app.yaml""" + """根据 URL 方式上传源码包, 目前不校验 app_desc.yaml""" module = self.get_module() slz = slzs.SourcePackageUploadViaUrlSLZ(data=request.data) slz.is_valid(raise_exception=True) diff --git a/apiserver/paasng/tests/api/apigw/test_lesscode.py b/apiserver/paasng/tests/api/apigw/test_lesscode.py index d2b8217436..1ef240d94c 100644 --- a/apiserver/paasng/tests/api/apigw/test_lesscode.py +++ b/apiserver/paasng/tests/api/apigw/test_lesscode.py @@ -98,7 +98,7 @@ def contents(self): "spec_version": 2, "module": {"is_default": True, "processes": {"web": {"command": "npm run online"}}, "language": "NodeJS"}, } - return {"app.yaml": yaml.safe_dump(app_desc)} + return {"app_desc.yaml": yaml.safe_dump(app_desc)} @pytest.fixture() def tar_path(self, contents): diff --git a/apiserver/paasng/tests/paasng/platform/declarative/handlers/test_handlers.py b/apiserver/paasng/tests/paasng/platform/declarative/handlers/test_handlers.py new file mode 100644 index 0000000000..43c42463f8 --- /dev/null +++ b/apiserver/paasng/tests/paasng/platform/declarative/handlers/test_handlers.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# TencentBlueKing is pleased to support the open source community by making +# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. +# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +# Licensed under the MIT License (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific language governing permissions and +# limitations under the License. +# +# We undertake not to change the open source license (MIT license) applicable +# to the current version of the project delivered to anyone in the future. + +from textwrap import dedent + +import pytest +import yaml + +from paasng.platform.declarative.exceptions import DescriptionValidationError +from paasng.platform.declarative.handlers import get_deploy_desc_handler, get_desc_handler + +pytestmark = pytest.mark.django_db(databases=["default", "workloads"]) + + +class TestGetAppDescHandlerIncorrectVersions: + def test_ver_1(self, bk_user): + yaml_content = dedent( + """ + app_code: foo + app_name: foo + """ + ) + + with pytest.raises(DescriptionValidationError, match='version "1" is not supported'): + get_desc_handler(yaml.safe_load(yaml_content)).handle_app(bk_user) + + def test_ver_unspecified(self, bk_user): + yaml_content = dedent( + """ + bk_app_code: foo + bk_app_name: foo + """ + ) + + with pytest.raises(DescriptionValidationError, match="No spec version is specified"): + get_desc_handler(yaml.safe_load(yaml_content)).handle_app(bk_user) + + def test_ver_unknown_number(self, bk_user): + yaml_content = "spec_version: 999" + + with pytest.raises(DescriptionValidationError, match='version "999" is not supported'): + get_desc_handler(yaml.safe_load(yaml_content)).handle_app(bk_user) + + def test_ver_unknown_string(self, bk_user): + yaml_content = "spec_version: foobar" + + with pytest.raises(DescriptionValidationError, match='version "foobar" is not supported'): + get_desc_handler(yaml.safe_load(yaml_content)).handle_app(bk_user) + + +class TestGetDeployDescHandlerIncorrectVersions: + def test_ver_1(self, bk_user): + yaml_content = dedent( + """ + app_code: foo + app_name: foo + """ + ) + + with pytest.raises(ValueError, match='version "1" is not supported'): + get_deploy_desc_handler(yaml.safe_load(yaml_content)) + + def test_ver_unspecified(self, bk_user): + yaml_content = dedent( + """ + bk_app_code: foo + bk_app_name: foo + """ + ) + + with pytest.raises(ValueError, match="no spec version is specified"): + get_deploy_desc_handler(yaml.safe_load(yaml_content)) + + def test_ver_unknown_number(self, bk_user): + yaml_content = "spec_version: 999" + + with pytest.raises(ValueError, match='version "999" is not supported'): + get_deploy_desc_handler(yaml.safe_load(yaml_content)) + + def test_ver_unknown_string(self, bk_user): + yaml_content = "spec_version: foobar" + + with pytest.raises(ValueError, match='version "foobar" is not supported'): + get_deploy_desc_handler(yaml.safe_load(yaml_content)) diff --git a/apiserver/paasng/tests/paasng/platform/declarative/handlers/test_v1.py b/apiserver/paasng/tests/paasng/platform/declarative/handlers/test_v1.py deleted file mode 100644 index 498d03233c..0000000000 --- a/apiserver/paasng/tests/paasng/platform/declarative/handlers/test_v1.py +++ /dev/null @@ -1,146 +0,0 @@ -# -*- coding: utf-8 -*- -# TencentBlueKing is pleased to support the open source community by making -# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. -# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. -# Licensed under the MIT License (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://opensource.org/licenses/MIT -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific language governing permissions and -# limitations under the License. -# -# We undertake not to change the open source license (MIT license) applicable -# to the current version of the project delivered to anyone in the future. - -from typing import Dict - -import pytest - -from paasng.accessories.publish.market.models import Product -from paasng.accessories.publish.sync_market.handlers import ( - on_change_application_name, - prepare_change_application_name, -) -from paasng.platform.applications.constants import AppLanguage -from paasng.platform.applications.models import Application -from paasng.platform.declarative.application.resources import ServiceSpec -from paasng.platform.declarative.constants import AppDescPluginType -from paasng.platform.declarative.handlers import ( - DefaultDeployDescHandler, - SMartDescriptionHandler, - deploy_desc_getter_v1, - get_desc_handler, -) -from paasng.platform.declarative.models import DeploymentDescription - -pytestmark = pytest.mark.django_db(databases=["default", "workloads"]) - - -class TestSMartDescriptionHandler: - @pytest.fixture() - def app_desc(self, one_px_png) -> Dict: - return { - "author": "blueking", - "introduction": "blueking app", - "is_use_celery": False, - "version": "0.0.1", - "env": [], - "logo_b64data": one_px_png, - } - - def test_app_creation(self, random_name, bk_user, app_desc, one_px_png): - app_desc.update( - { - "app_code": random_name, - "app_name": random_name, - } - ) - SMartDescriptionHandler(app_desc).handle_app(bk_user) - application = Application.objects.get(code=random_name) - assert application is not None - # 由于 ProcessedImageField 会将 logo 扩展为 144,144, 因此这里判断对应的位置的标记位 - logo_content = application.logo.read() - assert logo_content[19] == 144 - assert logo_content[23] == 144 - - def test_app_update_existed(self, bk_app, bk_user, app_desc): - prepare_change_application_name.disconnect(on_change_application_name) - app_desc.update( - { - "app_code": bk_app.code, - "app_name": bk_app.name, - "desktop": {"width": 303, "height": 100}, - } - ) - SMartDescriptionHandler(app_desc).handle_app(bk_user) - product = Product.objects.get(code=bk_app.code) - assert product.displayoptions.width == 303 - - @pytest.mark.parametrize( - ("memory", "expected_plan_name"), - [ - (512, "default"), - (1024, "default"), - (1536, "4C2G"), - (2048, "4C2G"), - (3072, "4C4G"), - (4096, "4C4G"), - (8192, "4C4G"), - ], - ) - def test_bind_process_spec_plans(self, random_name, bk_deployment, app_desc, memory, expected_plan_name): - app_desc.update( - { - "app_code": random_name, - "app_name": random_name, - "env": [{"key": "BKAPP_FOO", "value": "1"}], - "container": {"memory": memory}, - } - ) - DefaultDeployDescHandler(app_desc, None, deploy_desc_getter_v1).handle(bk_deployment) - - desc_obj = DeploymentDescription.objects.get(deployment=bk_deployment) - assert desc_obj.spec.processes[0].res_quota_plan == expected_plan_name - - @pytest.mark.parametrize( - ("is_use_celery", "expected_services"), - [ - (True, [ServiceSpec(name="mysql"), ServiceSpec(name="rabbitmq")]), - (False, [ServiceSpec(name="mysql")]), - ], - ) - def test_app_data_to_desc(self, random_name, app_desc, is_use_celery, expected_services): - app_desc.update({"app_code": random_name, "app_name": random_name, "is_use_celery": is_use_celery}) - assert SMartDescriptionHandler(app_desc).app_desc.default_module.services == expected_services - - @pytest.mark.parametrize( - ("libraries", "expected"), [([], []), ([dict(name="foo", version="bar")], [dict(name="foo", version="bar")])] - ) - def test_libraries(self, random_name, app_desc, libraries, expected): - app_desc.update({"app_code": random_name, "app_name": random_name, "libraries": libraries}) - plugin = SMartDescriptionHandler(app_desc).app_desc.get_plugin(AppDescPluginType.APP_LIBRARIES) - assert plugin - assert plugin["data"] == expected - - -def test_app_data_to_desc(random_name): - app_data = { - "author": "blueking", - "introduction": "blueking app", - "is_use_celery": False, - "version": "0.0.1", - "env": [], - "language": "python", - "app_code": random_name, - "app_name": random_name, - } - desc = get_desc_handler(app_data).app_desc - assert desc.name_zh_cn == random_name - assert desc.code == random_name - plugin = desc.get_plugin(AppDescPluginType.APP_VERSION) - assert plugin - assert plugin["data"] == "0.0.1" - assert desc.default_module.language == AppLanguage.PYTHON diff --git a/apiserver/paasng/tests/paasng/platform/declarative/handlers/v2/test_deployment.py b/apiserver/paasng/tests/paasng/platform/declarative/handlers/v2/test_deployment.py index 82a20f7aab..0534922bae 100644 --- a/apiserver/paasng/tests/paasng/platform/declarative/handlers/v2/test_deployment.py +++ b/apiserver/paasng/tests/paasng/platform/declarative/handlers/v2/test_deployment.py @@ -42,6 +42,7 @@ def yaml_v1_normal() -> str: """A sample YAML content using v1 spec version.""" return dedent( """ + app_code: foo version: 1 module: language: python @@ -98,7 +99,6 @@ class Test__get_deploy_desc_handler: @pytest.mark.parametrize( ("yaml_fixture_name", "expected_name"), [ - ("yaml_v1_normal", "deploy_desc_getter_v1"), ("yaml_v2_normal", "deploy_desc_getter_v2"), ("yaml_v3_normal", "deploy_desc_getter_v3"), ], @@ -109,6 +109,10 @@ def test_desc_getter_name(self, yaml_fixture_name, expected_name, request): assert hasattr(handler, "desc_getter") assert handler.desc_getter.__name__ == expected_name + def test_unsupported_version(self, yaml_v1_normal): + with pytest.raises(ValueError, match='procfile data is empty.* spec version "1" is not supported'): + _ = get_deploy_desc_handler(yaml.safe_load(yaml_v1_normal)) + class TestAppDescriptionHandler: @pytest.fixture() @@ -199,7 +203,7 @@ def test_procfile_only(self, bk_module, bk_deployment): def test_invalid_desc_and_valid_procfile(self, bk_module, bk_deployment): handler = get_deploy_desc_handler( - {"not": "valid yaml"}, procfile_data={"web": "gunicorn app", "worker": "celery"} + {"spec_version": 2, "not": "valid yaml"}, procfile_data={"web": "gunicorn app", "worker": "celery"} ) with pytest.raises(DescriptionValidationError): _ = handler.handle(bk_deployment) diff --git a/apiserver/paasng/tests/paasng/platform/smart_app/conftest.py b/apiserver/paasng/tests/paasng/platform/smart_app/conftest.py index 0c320f1c86..e3a466b819 100644 --- a/apiserver/paasng/tests/paasng/platform/smart_app/conftest.py +++ b/apiserver/paasng/tests/paasng/platform/smart_app/conftest.py @@ -26,13 +26,13 @@ from paasng.platform.smart_app.services.detector import SourcePackageStatReader from paasng.platform.smart_app.services.path import ZipPath from paasng.platform.sourcectl.utils import generate_temp_dir, generate_temp_file -from tests.paasng.platform.sourcectl.packages.utils import EXAMPLE_APP_YAML, gen_tar, gen_zip +from tests.paasng.platform.sourcectl.packages.utils import V2_APP_DESC_EXAMPLE, gen_tar, gen_zip @pytest.fixture() def contents() -> Dict: """The default contents for making tar file.""" - return {"app.yaml": yaml.safe_dump(EXAMPLE_APP_YAML)} + return {"app_desc.yaml": yaml.safe_dump(V2_APP_DESC_EXAMPLE)} @pytest.fixture() diff --git a/apiserver/paasng/tests/paasng/platform/smart_app/test_detector.py b/apiserver/paasng/tests/paasng/platform/smart_app/test_detector.py index 0fd1888891..2b1c1099ba 100644 --- a/apiserver/paasng/tests/paasng/platform/smart_app/test_detector.py +++ b/apiserver/paasng/tests/paasng/platform/smart_app/test_detector.py @@ -19,124 +19,99 @@ import yaml from rest_framework.exceptions import ValidationError -from paasng.platform.declarative.constants import AppSpecVersion from paasng.platform.smart_app.services.app_desc import get_app_description from paasng.platform.smart_app.services.detector import ( - AppYamlDetector, - DetectResult, ManifestDetector, SourcePackageStatReader, + relative_path_of_app_desc, ) -from tests.paasng.platform.sourcectl.packages.utils import EXAMPLE_APP_YAML +from tests.paasng.platform.sourcectl.packages.utils import V2_APP_DESC_EXAMPLE pytestmark = pytest.mark.django_db -class TestAppYamlDetector: +class Test__relative_path_of_app_desc: @pytest.mark.parametrize( - ("spec_version", "filepath", "expected"), + ("filepath", "expected"), [ - (AppSpecVersion.VER_1, "app.yaml", DetectResult("", AppSpecVersion.VER_1)), - (AppSpecVersion.VER_1, "app.yml", DetectResult("", AppSpecVersion.VER_1)), - (AppSpecVersion.VER_1, "./app.yaml", DetectResult("./", AppSpecVersion.VER_1)), - (AppSpecVersion.VER_1, "path/to/app.yaml", DetectResult("path/to/", AppSpecVersion.VER_1)), - (AppSpecVersion.VER_1, "E:\\app.yaml", DetectResult("E:\\", AppSpecVersion.VER_1)), - (AppSpecVersion.VER_1, "app_desc.yaml", None), - (AppSpecVersion.VER_1, "app_desc.yml", None), - (AppSpecVersion.VER_1, "./app_desc.yaml", None), - (AppSpecVersion.VER_1, "path/to/app_desc.yaml", None), - (AppSpecVersion.VER_1, "E:\\app_desc.yaml", None), - (AppSpecVersion.VER_2, "app.yaml", None), - (AppSpecVersion.VER_2, "app.yml", None), - (AppSpecVersion.VER_2, "./app.yaml", None), - (AppSpecVersion.VER_2, "path/to/app.yaml", None), - (AppSpecVersion.VER_2, "E:\\app.yaml", None), - (AppSpecVersion.VER_2, "app_desc.yaml", DetectResult("", AppSpecVersion.VER_2)), - (AppSpecVersion.VER_2, "app_desc.yml", DetectResult("", AppSpecVersion.VER_2)), - (AppSpecVersion.VER_2, "./app_desc.yaml", DetectResult("./", AppSpecVersion.VER_2)), - (AppSpecVersion.VER_2, "path/to/app_desc.yaml", DetectResult("path/to/", AppSpecVersion.VER_2)), - (AppSpecVersion.VER_2, "E:\\app_desc.yaml", DetectResult("E:\\", AppSpecVersion.VER_2)), - (AppSpecVersion.VER_1, "._app.yaml", None), - (AppSpecVersion.VER_1, "./._app.yaml", None), - (AppSpecVersion.VER_1, "xxxapp.yaml", None), - (AppSpecVersion.VER_1, "./xxxapp.yaml", None), - (AppSpecVersion.VER_2, "._app_desc.yaml", None), - (AppSpecVersion.VER_2, "./._app_desc.yaml", None), - (AppSpecVersion.VER_2, "xxxapp_desc.yaml", None), - (AppSpecVersion.VER_2, "./xxxapp_desc.yaml", None), + ("app_desc.yaml", ""), + ("app_desc.yml", ""), + ("./app_desc.yaml", "./"), + ("path/to/app_desc.yaml", "path/to/"), + ("E:\\app_desc.yaml", "E:\\"), + # Not a app desc file + ("foo/bar.txt", None), + ("._app_desc.yaml", None), + ("./._app_desc.yaml", None), + ("xxxapp_desc.yaml", None), + ("./xxxapp_desc.yaml", None), + # Legacy app_desc file names are not supported + ("app.yaml", None), + ("app.yml", None), ], ) - def test_detect(self, spec_version, filepath, expected): - assert AppYamlDetector.detect(filepath, spec_version) == expected + def test_detect(self, filepath, expected): + assert relative_path_of_app_desc(filepath) == expected @pytest.mark.parametrize("package_root", ["untar_path", "zip_path"], indirect=["package_root"]) class TestManifestDetector: + @pytest.fixture + def detector(self, package_root, package_stat) -> ManifestDetector: + """The detector instance for testing.""" + return ManifestDetector( + package_root=package_root, + app_description=get_app_description(package_stat), + relative_path=package_stat.relative_path, + ) + @pytest.mark.parametrize( ("contents", "error"), [ - ({"app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, KeyError("Procfile not found.")), - ({"foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, KeyError("Procfile not found.")), + ({"app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, "Procfile not found."), + ({"foo/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, "Procfile not found."), ], ) - def test_error(self, package_root, package_stat, contents, error): - with pytest.raises(KeyError) as e: - ManifestDetector( - package_root=package_root, - app_description=get_app_description(package_stat), - relative_path=package_stat.relative_path, - ).detect() - assert str(e.value) == str(error) + def test_only_app_desc_file(self, detector, contents, error): + with pytest.raises(KeyError, match=error): + detector.detect() @pytest.mark.parametrize( ("contents", "expected"), [ - ({"app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, "./app.yaml"), - ({"foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, "./app.yaml"), + ({"app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, "./app_desc.yaml"), + ({"foo/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, "./app_desc.yaml"), ], ) - def test_detect_app_desc(self, package_root, package_stat, contents, expected): - assert ( - ManifestDetector( - package_root=package_root, - app_description=get_app_description(package_stat), - relative_path=package_stat.relative_path, - ).detect_app_desc() - == expected - ) + def test_detect_app_desc(self, detector, contents, expected): + assert detector.detect_app_desc() == expected @pytest.mark.parametrize( ("contents", "expected"), [ - ({"app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, None), + ({"app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, None), ( - {"foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML), "foo/src/requirements.txt": ""}, + {"foo/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE), "foo/src/requirements.txt": ""}, "./src/requirements.txt", ), - ({"foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML), "src/requirements.txt": ""}, None), + # Not in the same directories + ({"foo/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE), "src/requirements.txt": ""}, None), ], ) - def test_detect_dependency(self, package_root, package_stat, contents, expected): - assert ( - ManifestDetector( - package_root=package_root, - app_description=get_app_description(package_stat), - relative_path=package_stat.relative_path, - ).detect_dependency() - == expected - ) + def test_detect_dependency(self, detector, contents, expected): + assert detector.detect_dependency() == expected @pytest.mark.parametrize( ("contents", "expected"), [ - ({"app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, {}), + ({"app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, {}), ( - {"foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML), "foo/cert/bk_root_ca.cert": ""}, + {"foo/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE), "foo/cert/bk_root_ca.cert": ""}, {"root": "./cert/bk_root_ca.cert"}, ), ( { - "foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML), + "foo/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE), "foo/cert/bk_root_ca.cert": "", "foo/cert/bk_saas_sign.cert": "", }, @@ -144,62 +119,64 @@ def test_detect_dependency(self, package_root, package_stat, contents, expected) ), ], ) - def test_detect_certs(self, package_root, package_stat, contents, expected): - assert ( - ManifestDetector( - package_root=package_root, - app_description=get_app_description(package_stat), - relative_path=package_stat.relative_path, - ).detect_certs() - == expected - ) + def test_detect_certs(self, detector, contents, expected): + assert detector.detect_certs() == expected @pytest.mark.parametrize( ("contents", "expected"), [ - ({"app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, {}), + ({"app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, {}), ( - {"bar/app.yaml": yaml.dump(EXAMPLE_APP_YAML), "bar/conf/SHA256": ""}, + {"bar/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE), "bar/conf/SHA256": ""}, {"sha256": "./conf/SHA256"}, ), ( - {"bar/app.yaml": yaml.dump(EXAMPLE_APP_YAML), "bar/conf/SHA256": "", "./bar/conf/package.conf": ""}, + { + "bar/app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE), + "bar/conf/SHA256": "", + "./bar/conf/package.conf": "", + }, {"sha256": "./conf/SHA256", "package": "./conf/package.conf"}, ), ], ) - def test_detect_encryption(self, package_root, package_stat, contents, expected): - assert ( - ManifestDetector( - package_root=package_root, - app_description=get_app_description(package_stat), - relative_path=package_stat.relative_path, - ).detect_encryption() - == expected - ) + def test_detect_encryption(self, detector, contents, expected): + assert detector.detect_encryption() == expected class TestSourcePackageStatReader: + # The simple description data for validations + desc_data = {"foo": "bar"} + desc_data_str = yaml.safe_dump(desc_data) + @pytest.mark.parametrize( - ("contents", "expected_meta_info", "expected_relative_path"), + ("contents", "expected_relative_path"), [ - # 我们的打包脚本会默认打成相对路径形式 - ({"app.yaml": yaml.dump({"version": "v1"})}, {"version": "v1"}, "./"), - ({"./app.yaml": yaml.dump({"version": "v1"})}, {"version": "v1"}, "./"), - ({"app_code/app.yaml": yaml.dump({"version": "v1"})}, {"version": "v1"}, "./app_code/"), - ({"app.yml": yaml.dump({"name": "v1"})}, {"name": "v1"}, "./"), - ({"./app.yml": yaml.dump({"name": "v1"})}, {"name": "v1"}, "./"), - ({"app_code/app.yml": yaml.dump({"name": "v1"})}, {"name": "v1"}, "./app_code/"), - ({"Procfile": ""}, {}, "./"), - ({"foo": yaml.dump({"version": "v1"})}, {}, "./"), + # 打包脚本默认使用相对路径形式 + ({"app_desc.yaml": desc_data_str}, "./"), + ({"./app_desc.yaml": desc_data_str}, "./"), + ({"app_code/app_desc.yaml": desc_data_str}, "./app_code/"), + ({"app_desc.yml": desc_data_str}, "./"), ], ) - def test_get_meta_info(self, tar_path, expected_meta_info, expected_relative_path): + def test_get_meta_info_found_desc_file(self, contents, tar_path, expected_relative_path): relative_path, meta_info = SourcePackageStatReader(tar_path).get_meta_info() - assert meta_info == expected_meta_info + assert meta_info == self.desc_data assert relative_path == expected_relative_path - @pytest.mark.parametrize("contents", [{"app.yaml": "invalid-: yaml: content"}]) + @pytest.mark.parametrize( + "contents", + [ + {"Procfile": ""}, + {"foo": desc_data_str}, + ], + ) + def test_get_meta_info_no_desc_file(self, tar_path): + relative_path, meta_info = SourcePackageStatReader(tar_path).get_meta_info() + assert meta_info == {} + assert relative_path == "./" + + @pytest.mark.parametrize("contents", [{"app_desc.yaml": "invalid-: yaml: content"}]) def test_invalid_file_format(self, tar_path): with pytest.raises(ValidationError): SourcePackageStatReader(tar_path).get_meta_info() @@ -207,11 +184,15 @@ def test_invalid_file_format(self, tar_path): @pytest.mark.parametrize( ("contents", "meta_info", "version"), [ - ({"app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, EXAMPLE_APP_YAML, EXAMPLE_APP_YAML["version"]), ( - {"app.yaml": yaml.dump(EXAMPLE_APP_YAML), "logo.png": "dummy"}, - {"logo_b64data": "base64,ZHVtbXk=", "logoB64data": "base64,ZHVtbXk=", **EXAMPLE_APP_YAML}, - EXAMPLE_APP_YAML["version"], + {"app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE)}, + V2_APP_DESC_EXAMPLE, + V2_APP_DESC_EXAMPLE["app_version"], + ), + ( + {"app_desc.yaml": yaml.dump(V2_APP_DESC_EXAMPLE), "logo.png": "dummy"}, + {"logo_b64data": "base64,ZHVtbXk=", "logoB64data": "base64,ZHVtbXk="} | V2_APP_DESC_EXAMPLE, + V2_APP_DESC_EXAMPLE["app_version"], ), ], ) diff --git a/apiserver/paasng/tests/paasng/platform/smart_app/test_patcher.py b/apiserver/paasng/tests/paasng/platform/smart_app/test_patcher.py index 8bfc304e34..1bd060aef7 100644 --- a/apiserver/paasng/tests/paasng/platform/smart_app/test_patcher.py +++ b/apiserver/paasng/tests/paasng/platform/smart_app/test_patcher.py @@ -20,15 +20,15 @@ import pytest import yaml -from blue_krill.contextlib import nullcontext as does_not_raise from paasng.platform.declarative import constants +from paasng.platform.declarative.exceptions import DescriptionValidationError from paasng.platform.modules.constants import SourceOrigin from paasng.platform.smart_app.services.detector import SourcePackageStatReader from paasng.platform.smart_app.services.patcher import SourceCodePatcher from paasng.platform.smart_app.services.path import LocalFSPath from paasng.platform.sourcectl.utils import generate_temp_dir -from tests.paasng.platform.sourcectl.packages.utils import EXAMPLE_APP_YAML +from tests.paasng.platform.sourcectl.packages.utils import V1_APP_DESC_EXAMPLE pytestmark = pytest.mark.django_db EXPECTED_WEB_PROCESS = constants.WEB_PROCESS @@ -73,90 +73,25 @@ def test_module_dir(self, user_source_dir, tmp_path, tar_path, bk_module_full): assert str(patcher.module_dir.path).startswith(str(tmp_path)) @pytest.mark.parametrize( - ("contents", "target", "ctx", "expected"), + "contents", + [ + {"app.yaml": yaml.safe_dump(V1_APP_DESC_EXAMPLE)}, + ], + ) + def test_error_if_patch_unsupported_ver(self, bk_module_full, tar_path, tmp_path): + """Test if the patcher raises an error when the app description file is v1.""" + bk_module_full.name = "bar" + bk_module_full.source_origin = SourceOrigin.BK_LESS_CODE.value + stat = SourcePackageStatReader(tar_path).read() + + with pytest.raises(DescriptionValidationError): + SourceCodePatcher.patch_tarball( + module=bk_module_full, tarball_path=tar_path, working_dir=tmp_path, stat=stat + ) + + @pytest.mark.parametrize( + ("contents", "target", "expected"), [ - # 我们的打包脚本会默认打成相对路径形式 - ( - { - "app.yaml": yaml.dump( - { - **EXAMPLE_APP_YAML, - "is_use_celery": True, - } - ) - }, - "./app.yaml", - does_not_raise(), - { - **EXAMPLE_APP_YAML, - "is_use_celery": True, - }, - ), - # 测试 procfile 内容 - ( - {"app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, - "./Procfile", - does_not_raise(), - {"web": EXPECTED_WEB_PROCESS}, - ), - # 测试 ./Procfile not found - ({"foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, "./Procfile", pytest.raises(KeyError), None), - ( - {"foo/app.yaml": yaml.dump(EXAMPLE_APP_YAML)}, - "./foo/Procfile", - does_not_raise(), - {"web": EXPECTED_WEB_PROCESS}, - ), - ( - { - "foo/app.yaml": yaml.dump( - { - **EXAMPLE_APP_YAML, - "is_use_celery": True, - } - ) - }, - "./foo/Procfile", - does_not_raise(), - {"web": EXPECTED_WEB_PROCESS, "celery": constants.CELERY_PROCESS}, - ), - ( - { - "foo/app.yaml": yaml.dump( - { - **EXAMPLE_APP_YAML, - "is_use_celery_with_gevent": True, - } - ) - }, - "./foo/Procfile", - does_not_raise(), - { - "web": EXPECTED_WEB_PROCESS, - "celery": constants.CELERY_PROCESS_WITH_GEVENT, - }, - ), - ( - { - "foo/app.yaml": yaml.dump( - {**EXAMPLE_APP_YAML, "is_use_celery": True, "is_use_celery_with_gevent": True} - ) - }, - "./foo/Procfile", - does_not_raise(), - {"web": EXPECTED_WEB_PROCESS, "celery": constants.CELERY_PROCESS}, - ), - ( - {"foo/app.yaml": yaml.dump({**EXAMPLE_APP_YAML, "is_use_celery": True, "is_use_celery_beat": True})}, - "./foo/Procfile", - does_not_raise(), - { - "web": EXPECTED_WEB_PROCESS, - "celery": constants.CELERY_PROCESS, - "beat": constants.CELERY_BEAT_PROCESS, - }, - ), - # 以下测试 app_desc.yaml 规范 ( { "foo/app_desc.yaml": yaml.dump( @@ -174,7 +109,6 @@ def test_module_dir(self, user_source_dir, tmp_path, tar_path, bk_module_full): ) }, "./foo/Procfile", - does_not_raise(), # shlex 在某些情况会出现稍微偏差(这里的 ; 号位置变了) {"hello": "echo 'hello world!';"}, ), @@ -197,7 +131,6 @@ def test_module_dir(self, user_source_dir, tmp_path, tar_path, bk_module_full): "foo/Procfile": yaml.dump({"hello": "echo 'good morning!';"}), }, "./foo/Procfile", - does_not_raise(), {"hello": "echo 'good morning!';"}, ), # 测试多模块. @@ -224,7 +157,6 @@ def test_module_dir(self, user_source_dir, tmp_path, tar_path, bk_module_full): ), }, "./foo/src/bar/Procfile", - does_not_raise(), {"hello": "echo 'Hello Foo, i am Bar!'"}, ), # 测试多模块(已加密) @@ -252,63 +184,14 @@ def test_module_dir(self, user_source_dir, tmp_path, tar_path, bk_module_full): "foo/src/bar": "", }, "./foo/Procfile", - does_not_raise(), {"hello": "echo 'Hello Foo, i am Bar!'"}, ), ], ) - def test_add_procfile(self, tar_path, patched_tar, target, ctx, expected): + def test_add_procfile(self, tar_path, patched_tar, target, expected): assert tar_path.name == patched_tar.name - with tarfile.open(patched_tar) as tar, ctx: + with tarfile.open(patched_tar) as tar: fp = tar.extractfile(target) assert fp data = yaml.full_load(fp.read()) assert data == expected - - @pytest.mark.parametrize( - ("contents", "target", "ctx", "expected"), - [ - ( - { - "app.yaml": yaml.dump({**EXAMPLE_APP_YAML, "language": "python"}), - }, - "./requirements.txt", - pytest.raises(KeyError), - None, - ), - ( - { - "app.yaml": yaml.dump( - { - **EXAMPLE_APP_YAML, - "language": "python", - "libraries": [dict(name="foo", version="1.1.1"), dict(name="bar", version="2.2.2")], - } - ), - }, - "./requirements.txt", - does_not_raise(), - b"foo==1.1.1\nbar==2.2.2", - ), - ( - { - "app.yaml": yaml.dump( - { - **EXAMPLE_APP_YAML, - "language": "nodejs", - "libraries": [dict(name="foo", version="1.1.1"), dict(name="bar", version="2.2.2")], - } - ), - }, - "./requirements.txt", - pytest.raises(KeyError), - None, - ), - ], - ) - def test_add_requirements(self, tar_path, patched_tar, target, ctx, expected): - assert tar_path.name == patched_tar.name - with tarfile.open(patched_tar) as tar, ctx: - fp = tar.extractfile(target) - assert fp - assert fp.read() == expected diff --git a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_utils.py b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_utils.py index 0fd917feb7..83811d2d8f 100644 --- a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_utils.py +++ b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_utils.py @@ -33,14 +33,10 @@ ({"app_name": "阿尔法"}, False, None), ( { - "app_name": "阿尔法", - "app_name_en": "alpha", - "app_code": "foo", - "author": "blueking", - "introduction": "blueking app", - "is_use_celery": False, - "version": "0.0.1", - "env": [], + "spec_version": 2, + "app_version": "0.0.1", + "app": {"bk_app_code": "foo", "bk_app_name": "阿尔法", "bk_app_name_en": "alpha"}, + "modules": {"default": {"is_default": True, "language": "python"}}, }, True, gettext_lazy({"zh-cn": "阿尔法", "en": "alpha"}), diff --git a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/utils.py b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/utils.py index 87c27532f7..190aa6e7e4 100644 --- a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/utils.py +++ b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/utils.py @@ -86,7 +86,8 @@ def gen_zip(target_path, contents: Dict[str, Union[str, bytes]]): shutil.move(filename, target_path) -EXAMPLE_APP_YAML: dict = { +# This version of app description has been deprecated and is not supported anymore +V1_APP_DESC_EXAMPLE: dict = { "app_name": "foo", "app_code": "foo", "author": "blueking", @@ -95,3 +96,14 @@ def gen_zip(target_path, contents: Dict[str, Union[str, bytes]]): "version": "0.0.1", "env": [], } + +V2_APP_DESC_EXAMPLE: dict = { + "spec_version": 2, + "app_version": "0.0.1", + "app": { + "bk_app_code": "foo", + "bk_app_name": "foo", + "market": {"introduction": "foobar"}, + }, + "modules": {"default": {"is_default": True, "language": "python"}}, +}