diff --git a/apps/backend/plugin/tools.py b/apps/backend/plugin/tools.py index b8f2bf03b..38ec4954c 100644 --- a/apps/backend/plugin/tools.py +++ b/apps/backend/plugin/tools.py @@ -498,6 +498,9 @@ def create_pkg_record( ) ) + # 配置文件已写入DB，从插件包中移除 + os.remove(template_file_path) + proc_control, __ = models.ProcControl.objects.get_or_create( plugin_package_id=pkg_record.id, defaults=dict(module="gse_plugin", project=pkg_parse_info["project"]) ) @@ -554,7 +557,7 @@ def create_pkg_record( # 判断是否第三方插件的路径 arcname=f"{constants.PluginChildDir.EXTERNAL.value}/{project}" if is_external - else f"{constants.PluginChildDir.OFFICIAL.value}/{project}", + else f"{constants.PluginChildDir.OFFICIAL.value}/", ) logger.info( "project -> {project} version -> {version} now is pack to package_tmp_path -> {package_tmp_path}".format( diff --git a/apps/backend/tests/plugin/test_manage_commands.py b/apps/backend/tests/plugin/test_manage_commands.py new file mode 100644 index 000000000..32d537918 --- /dev/null +++ b/apps/backend/tests/plugin/test_manage_commands.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.conf import settings +from django.core.management import call_command + +from apps.backend.tests.plugin import utils +from apps.node_man import models + + +class ImportCommandTestCase(utils.PluginBaseTestCase): + @classmethod + def setUpTestData(cls): + cls.OVERWRITE_OBJ__KV_MAP[settings].update(BK_OFFICIAL_PLUGINS_INIT_PATH=cls.TMP_DIR) + super().setUpTestData() + + def test_import_command(self): + """测试导入命令""" + call_command("init_official_plugins") + self.assertTrue(models.Packages.objects.all().exists()) + self.assertTrue(models.UploadPackage.objects.all().exists()) + self.assertTrue(models.PluginConfigTemplate.objects.all().exists()) diff --git a/apps/backend/tests/plugin/test_plugin.py b/apps/backend/tests/plugin/test_plugin.py deleted file mode 100644 index acee06b80..000000000 --- a/apps/backend/tests/plugin/test_plugin.py +++ /dev/null @@ -1,792 +0,0 @@ -# -*- coding: utf-8 -*- -""" -TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. -Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at https://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" -from __future__ import absolute_import, print_function, unicode_literals - -import os -import platform -import shutil -import tarfile -import uuid - -import mock -from django.conf import settings -from django.core.management import call_command -from django.test import TestCase - -from apps.backend.plugin import tools -from apps.backend.plugin.tasks import export_plugin, package_task, run_pipeline -from apps.backend.tests.plugin import utils -from apps.backend.tests.plugin.test_plugin_status_change import TestApiBase -from apps.node_man.models import ( - DownloadRecord, - GsePluginDesc, - Packages, - PluginConfigTemplate, - ProcControl, - UploadPackage, -) - -# TODO: 后续分拆该文件 - -# 全局使用的mock -mock.patch("apps.backend.plugin.tasks.export_plugin", delay=export_plugin).start() -mock.patch("apps.backend.plugin.tasks.package_task", delay=package_task).start() -mock.patch("apps.backend.plugin.tasks.run_pipeline.delay", delay=run_pipeline).start() - -TEST_ROOT = "c:/" if platform.system() == "Windows" else "/tmp" - - -class PluginTestCase(TestCase): - @classmethod - def setUpTestData(cls): - GsePluginDesc.objects.create( - **{ - "name": "test1", - "description": "测试插件啊", - "scenario": "测试", - "category": "external", - "launch_node": "all", - "config_file": "config.yaml", - "config_format": "yaml", - "use_db": False, - } - ) - - Packages.objects.create( - pkg_name="test1.tar", - version="1.0.1", - module="gse_plugin", - project="test1", - pkg_size=10255, - pkg_path="/data/bkee/miniweb/download/windows/x86_64", - location="http://127.0.0.1/download/windows/x86_64", - md5="a95c530a7af5f492a74499e70578d150", - pkg_ctime="2019-05-05 11:54:28.070771", - pkg_mtime="2019-05-05 11:54:28.070771", - os="windows", - cpu_arch="x86_64", - is_release_version=False, - is_ready=True, - ) - - def test_one(self): - assert GsePluginDesc.objects.all().count() == 1 - - def test_two(self): - pass - - -class TestPackageFunction(TestApiBase): - temp_path = os.path.join(TEST_ROOT, uuid.uuid4().hex) - tarfile_name = "tarfile.tgz" - tarfile_path = os.path.join(temp_path, tarfile_name) - export_path = os.path.join(temp_path, "export") - upload_path = os.path.join(temp_path, "upload") - - plugin_name = "test_plugin" - - config_content = """ - name: "test_plugin" - version: "1.0.1" - description: "用于采集主机基础性能数据，包含CPU,内存，磁盘，⽹络等数据" - description_en: "用于采集主机基础性能数据，包含CPU,内存，磁盘，⽹络等数据" - scenario: "CMDB上的是实时数据，蓝鲸监控-主机监控中的基础性能数据" - scenario_en: "CMDB上的是实时数据，蓝鲸监控-主机监控中的基础性能数据" - category: official - config_file: basereport.conf - multi_config: True - config_file_path: "" - config_format: yaml - auto_launch: true - launch_node: proxy - upstream: - bkmetric - dependences: - gse_agent: "1.2.0" - bkmetric: "1.6.0" - config_templates: - - plugin_version: "*" - name: child.conf - version: 2.0.1 - file_path: etc/child - format: yaml - source_path: etc/child.conf.tpl - - plugin_version: "*" - name: test_plugin.conf - version: 1.0.2 - file_path: etc/main - format: yaml - source_path: etc/test_plugin.main.conf.tpl - is_main_config: 1 - control: - start: "./start.sh %name" - stop: "./stop.sh %name" - restart: "./restart.sh %name" - version: "./%name -v" - reload: "./%name -s reload" - health_check: "./%name -z" - node_manage_control: - package_update: false - package_remove: false - plugin_install: false - plugin_update: false - plugin_uninstall: false - plugin_upgrade: false - plugin_remove: false - plugin_restart: false - process_name: "test_process" -""" - - def setUp(self): - """测试启动初始化配置""" - # 1. 准备一个yaml配置文件 - - # 2. 创建一个打包文件，包含两层内容，一个linux，一个windows - temp_file_path = os.path.join(self.temp_path, "temp_folder") - if not os.path.exists(self.temp_path): - os.mkdir(self.temp_path) - if not os.path.exists(temp_file_path): - os.mkdir(temp_file_path) - if not os.path.exists(self.export_path): - os.mkdir(self.export_path) - if not os.path.exists(self.upload_path): - os.mkdir(self.upload_path) - for package_os, cpu_arch in (("linux", "x86_64"), ("windows", "x86")): - current_path = os.path.join(temp_file_path, "external_plugins_{}_{}".format(package_os, cpu_arch)) - os.mkdir(current_path) - - plugin_path = os.path.join(current_path, self.plugin_name) - os.mkdir(plugin_path) - - config_path = os.path.join(plugin_path, "etc") - os.mkdir(config_path) - - f1 = open(os.path.join(plugin_path, "plugin"), "w") - f1.close() - f2 = open(os.path.join(plugin_path, "project.yaml"), "w", encoding="utf-8") - f2.write(self.config_content) - f2.close() - - main_config = open(os.path.join(config_path, "test_plugin.main.conf.tpl"), "w") - main_config.close() - child_config = open(os.path.join(config_path, "child.conf.tpl"), "w") - child_config.close() - - with tarfile.open(self.tarfile_path, "w:gz") as tfile: - # self.temp_path/temp_folder下的所有文件放入temp_file_path文件下, arcname表示目标目录的目标路径 - tfile.add(temp_file_path, arcname=".", recursive=True) - - # nginx的模拟路径 - settings.DOWNLOAD_PATH = self.temp_path - settings.UPLOAD_PATH = self.upload_path - settings.EXPORT_PATH = self.export_path - - def tearDown(self): - """测试清理配置""" - - shutil.rmtree(self.temp_path) - # TODO: 文件上传时把包都临时放在了/tmp/ 注册时从tmp读取，但由于文件二级目录由uuid.uuid4().hex生成，无法清除 - # 下列只是清除了注册时的临时打包文件保证测试正常执行，但是上传时的文件并没有删除，有堆积风险 - # 不建议直接清/tmp/，考虑mock `uuid.uuid4().hex`，生成一个可见目录，采用shutil.rmtree(path)清除 - tmp_clear_files = ["/tmp/test_plugin-1.0.1-windows-x86.tgz", "/tmp/test_plugin-1.0.1-linux-x86_64.tgz"] - for file in tmp_clear_files: - if os.path.exists(file): - os.remove(file) - - def test_create_upload_record_and_register(self): - """测试创建上传包记录功能""" - - # 插件包注册后存放地址 - windows_file_path = os.path.join(settings.DOWNLOAD_PATH, "windows", "x86", "test_plugin-1.0.1.tgz") - linux_file_path = os.path.join(settings.DOWNLOAD_PATH, "linux", "x86_64", "test_plugin-1.0.1.tgz") - - # 验证创建前此时文件不存在 - self.assertFalse(os.path.exists(linux_file_path)) - self.assertFalse(os.path.exists(windows_file_path)) - - UploadPackage.create_record( - module="gse_plugin", - file_path=self.tarfile_path, - md5="abcefg", - operator="haha_test", - source_app_code="bk_nodeman", - file_name="tarfile.tgz", - ) - - # 判断路径的转移登录等是否符合预期 - self.assertEqual( - UploadPackage.objects.filter( - file_name=self.tarfile_name, - file_path=os.path.join(settings.UPLOAD_PATH, self.tarfile_name), - ).count(), - 1, - ) - self.assertTrue(os.path.exists(os.path.join(settings.UPLOAD_PATH, self.tarfile_name))) - - # 测试单独注册插件包功能 - upload_object = UploadPackage.objects.get(file_name=self.tarfile_name) - package_object_list = tools.create_package_records( - file_path=upload_object.file_path, file_name=upload_object.file_name, is_release=True - ) - - self.assertEqual( - GsePluginDesc.objects.get(name="test_plugin").node_manage_control, - { - "package_update": False, - "package_remove": False, - "plugin_install": False, - "plugin_update": False, - "plugin_uninstall": False, - "plugin_upgrade": False, - "plugin_remove": False, - "plugin_restart": False, - }, - ) - # 判断数量正确 - self.assertEqual(len(package_object_list), 2) - - # 判断写入DB数据正确 - # 1. 进程控制信息 - for package in package_object_list: - process_control = ProcControl.objects.get(plugin_package_id=package.id) - self.assertEqual(process_control.os, package.os) - self.assertEqual(process_control.start_cmd, "./start.sh %name") - self.assertEqual( - process_control.install_path, - "/usr/local/gse" if package.os != "windows" else r"C:\gse", - ) - self.assertEqual(process_control.port_range, "") - self.assertEqual(process_control.process_name, "test_process") - - # 2. 包记录信息（window， linux及版本号） - Packages.objects.get( - pkg_name="test_plugin-1.0.1.tgz", os="windows", version="1.0.1", cpu_arch="x86", creator="admin" - ) - Packages.objects.get( - pkg_name="test_plugin-1.0.1.tgz", - os="linux", - version="1.0.1", - cpu_arch="x86_64", - ) - - # 3. 验证已清理临时文件夹 - self.assertFalse(os.path.exists("/tmp/test_plugin-1.0.1-windows-x86.tgz")) - self.assertFalse(os.path.exists("/tmp/test_plugin-1.0.1-linux-x86_64.tgz")) - - # 4. 验证插件包注册并移动至nginx目录下 - self.assertTrue(os.path.exists(linux_file_path)) - self.assertTrue(os.path.exists(windows_file_path)) - - with tarfile.open(windows_file_path) as linux_tar: - linux_tar.getmember("external_plugins/%s/project.yaml" % self.plugin_name) - linux_tar.getmember("external_plugins/%s/plugin" % self.plugin_name) - - def _test_upload_api_success(self): - # 测试上传 - self.post( - path="/backend/package/upload/", - data={ - "module": "test_module", - "md5": "123", - "bk_username": "admin", - "bk_app_code": "bk_app_code", - # nginx追加的内容 - "file_local_path": self.tarfile_path, - "file_local_md5": "123", - "file_name": "tarfile.tgz", - }, - is_json=False, - ) - - # 逻辑校验 - self.assertEqual( - UploadPackage.objects.filter( - file_name=self.tarfile_name, - file_path=os.path.join(settings.UPLOAD_PATH, self.tarfile_name), - ).count(), - 1, - ) - self.assertTrue(os.path.exists(os.path.join(settings.UPLOAD_PATH, self.tarfile_name))) - - def _test_register_api_success(self): - - # 插件包注册后存放地址 - windows_file_path = os.path.join(settings.DOWNLOAD_PATH, "windows", "x86", "test_plugin-1.0.1.tgz") - linux_file_path = os.path.join(settings.DOWNLOAD_PATH, "linux", "x86_64", "test_plugin-1.0.1.tgz") - - # 验证创建前此时文件不存在 - self.assertFalse(os.path.exists(linux_file_path)) - self.assertFalse(os.path.exists(windows_file_path)) - - # 测试注册 - self.post( - path="/backend/api/plugin/create_register_task/", - data={ - "file_name": self.tarfile_name, - "is_release": True, - "bk_username": "admin", - "bk_app_code": "bk_app_code", - }, - ) - - # 逻辑校验 - # 1. 进程控制信息 - self.assertEqual(ProcControl.objects.all().count(), 2) - - # 2. 包记录信息（window， linux及版本号） - Packages.objects.get( - pkg_name="test_plugin-1.0.1.tgz", os="windows", version="1.0.1", cpu_arch="x86", creator="admin" - ) - Packages.objects.get( - pkg_name="test_plugin-1.0.1.tgz", os="linux", version="1.0.1", cpu_arch="x86_64", creator="admin" - ) - - # 3. 验证已清理临时文件夹 - self.assertFalse(os.path.exists("/tmp/test_plugin-1.0.1-windows-x86.tgz")) - self.assertFalse(os.path.exists("/tmp/test_plugin-1.0.1-linux-x86_64.tgz")) - - # 4. 验证插件包注册并移动至nginx目录下 - self.assertTrue(os.path.exists(linux_file_path)) - self.assertTrue(os.path.exists(windows_file_path)) - - with tarfile.open(linux_file_path) as linux_tar: - linux_tar.getmember("external_plugins/%s/project.yaml" % self.plugin_name) - linux_tar.getmember("external_plugins/%s/plugin" % self.plugin_name) - - # 只有一条对应的desc - self.assertEqual(GsePluginDesc.objects.filter(name=self.plugin_name).count(), 1) - - def test_create_task_register_api(self): - """测试上传文件接口""" - self._test_upload_api_success() - self._test_register_api_success() - - def test_create_task_register_optional_api(self): - """测试上传文件接口""" - self._test_upload_api_success() - - # 插件包注册后存放地址 - windows_file_path = os.path.join(settings.DOWNLOAD_PATH, "windows", "x86", "test_plugin-1.0.1.tgz") - linux_file_path = os.path.join(settings.DOWNLOAD_PATH, "linux", "x86_64", "test_plugin-1.0.1.tgz") - - # 验证创建前此时文件不存在 - self.assertFalse(os.path.exists(linux_file_path)) - self.assertFalse(os.path.exists(windows_file_path)) - - # 测试注册 - self.post( - path="/backend/api/plugin/create_register_task/", - data={ - "file_name": self.tarfile_name, - "is_release": True, - "is_template_load": True, - "select_pkg_abs_paths": ["external_plugins_windows_x86/test_plugin"], - "bk_username": "admin", - "bk_app_code": "bk_app_code", - }, - ) - - # 逻辑校验 - # 1. 进程控制信息 - self.assertEqual(ProcControl.objects.all().count(), 1) - - # 2. 包记录信息，windows已选，被导入 - Packages.objects.get( - pkg_name="test_plugin-1.0.1.tgz", os="windows", version="1.0.1", cpu_arch="x86", creator="admin" - ) - # 仅指定windows的插件包进行导入，linux插件包不应该被记录 - self.assertFalse( - Packages.objects.filter( - pkg_name="test_plugin-1.0.1.tgz", os="linux", version="1.0.1", cpu_arch="x86_64", creator="admin" - ).exists() - ) - - # 3. 验证已清理临时文件夹 - self.assertFalse(os.path.exists("/tmp/test_plugin-1.0.1-windows-x86.tgz")) - self.assertFalse(os.path.exists("/tmp/test_plugin-1.0.1-linux-x86_64.tgz")) - - # 4. 验证插件包注册并移动至nginx目录下, 仅指定windows的插件包进行导入，linux插件包不应该被记录 - self.assertFalse(os.path.exists(linux_file_path)) - self.assertTrue(os.path.exists(windows_file_path)) - - with tarfile.open(windows_file_path) as linux_tar: - linux_tar.getmember("external_plugins/%s/project.yaml" % self.plugin_name) - linux_tar.getmember("external_plugins/%s/plugin" % self.plugin_name) - - # 只有一条对应的desc - self.assertEqual(GsePluginDesc.objects.filter(name=self.plugin_name).count(), 1) - - def test_create_export_task_api(self): - self.test_create_task_register_api() - response = self.post( - path="/backend/api/plugin/create_export_task/", - data={ - "category": "gse_plugin", - "creator": "admin", - "bk_username": "admin", - "bk_app_code": "bk_app_code", - "query_params": {"project": "test_plugin", "version": "1.0.1"}, - }, - ) - record = DownloadRecord.objects.get(id=response["data"]["job_id"]) - file_path = record.file_path - self.assertTrue(os.path.exists(file_path)) - with tarfile.open(file_path) as download_tar: - download_tar.getmember( - "./external_plugins_linux_x86_64/{file_name}/plugin".format(file_name=self.plugin_name) - ) - download_tar.getmember( - "./external_plugins_windows_x86/{file_name}/plugin".format(file_name=self.plugin_name) - ) - - def test_export_with_os(self): - self.test_create_task_register_api() - response = self.post( - path="/backend/api/plugin/create_export_task/", - data={ - "category": "gse_plugin", - "creator": "admin", - "bk_username": "admin", - "bk_app_code": "bk_app_code", - "query_params": {"project": "test_plugin", "version": "1.0.1", "os": "windows"}, - }, - ) - record = DownloadRecord.objects.get(id=response["data"]["job_id"]) - file_path = record.file_path - self.assertTrue(os.path.exists(file_path)) - with tarfile.open(file_path) as download_tar: - download_tar.getmember( - "./external_plugins_windows_x86/{file_name}/plugin".format(file_name=self.plugin_name) - ) - try: - download_tar.getmember( - "./external_plugins_linux_x86_64/{file_name}/plugin".format(file_name=self.plugin_name) - ) - self.assertTrue(False) - except Exception: - pass - - def test_export_with_os_cpu_arch(self): - self.test_create_task_register_api() - utils.PluginTestObjFactory.batch_create_pkg( - [ - utils.PluginTestObjFactory.pkg_obj( - { - "pkg_name": "test_plugin-1.0.1.tgz", - "project": self.plugin_name, - "version": "1.0.1", - "cpu_arch": "x86", - } - ), - ] - ) - response = self.post( - path="/backend/api/plugin/create_export_task/", - data={ - "category": "gse_plugin", - "creator": "admin", - "bk_username": "admin", - "bk_app_code": "bk_app_code", - "query_params": {"project": "test_plugin", "version": "1.0.1", "os": "linux", "cpu_arch": "x86_64"}, - }, - ) - record = DownloadRecord.objects.get(id=response["data"]["job_id"]) - file_path = record.file_path - self.assertTrue(os.path.exists(file_path)) - with tarfile.open(file_path) as download_tar: - download_tar.getmember( - "./external_plugins_linux_x86_64/{file_name}/plugin".format(file_name=self.plugin_name) - ) - try: - download_tar.getmember( - "./external_plugins_linux_x86/{file_name}/plugin".format(file_name=self.plugin_name) - ) - download_tar.getmember( - "./external_plugins_windows_x86/{file_name}/plugin".format(file_name=self.plugin_name) - ) - self.assertTrue(False) - except Exception: - pass - - def _query_parse_api(self): - # 测试文件解析接口 - response = self.post( - path="/backend/api/plugin/parse/", - data={"file_name": self.tarfile_name, "bk_username": "admin", "bk_app_code": "bk_app_code"}, - ) - return response - - def test_parse_api_all_new_add(self): - self._test_upload_api_success() - - # 测试文件解析接口 - response = self._query_parse_api() - self.assertEqual(len(response["data"]), 2) - self.assertEqual(len([item for item in response["data"] if item["result"] and item["message"] == "新增插件"]), 2) - - first_parse_info = response["data"][0] - self.assertEqual(len(first_parse_info["config_templates"]), 2) - - def test_parse_api_yaml_file_not_find_or_unread(self): - os.remove(self.tarfile_path) - linux_yaml_file = os.path.join( - self.temp_path, "temp_folder", "external_plugins_linux_x86_64", "test_plugin", "project.yaml" - ) - os.remove(linux_yaml_file) - - windows_yaml_file = os.path.join( - self.temp_path, "temp_folder", "external_plugins_windows_x86", "test_plugin", "project.yaml" - ) - os.remove(windows_yaml_file) - - windows_yaml_file_stream = open(windows_yaml_file, "w", encoding="utf-8") - windows_yaml_file_stream.write("~6574745646") - windows_yaml_file_stream.close() - - with tarfile.open(self.tarfile_path, "w:gz") as tfile: - # self.temp_path/temp_folder下的所有文件放入temp_file_path文件下, arcname表示目标目录的目标路径 - tfile.add(os.path.join(self.temp_path, "temp_folder"), arcname=".", recursive=True) - - self._test_upload_api_success() - response = self._query_parse_api() - self.assertEqual(len(response["data"]), 2) - self.assertEqual( - len([item for item in response["data"] if not item["result"] and item["message"] == "缺少project.yaml文件"]), 1 - ) - self.assertEqual( - len( - [item for item in response["data"] if not item["result"] and item["message"] == "project.yaml文件解析读取失败"] - ), - 1, - ) - - def test_parse_api_yaml_file_lack_attr_or_category_error(self): - os.remove(self.tarfile_path) - linux_yaml_file = os.path.join( - self.temp_path, "temp_folder", "external_plugins_linux_x86_64", "test_plugin", "project.yaml" - ) - os.remove(linux_yaml_file) - lack_of_attr_content = self.config_content.replace('name: "test_plugin"', "") - linux_yaml_file_stream = open(linux_yaml_file, "w", encoding="utf-8") - linux_yaml_file_stream.write(lack_of_attr_content) - linux_yaml_file_stream.close() - - windows_yaml_file = os.path.join( - self.temp_path, "temp_folder", "external_plugins_windows_x86", "test_plugin", "project.yaml" - ) - os.remove(windows_yaml_file) - category_error_content = self.config_content.replace("category: official", "category: category_error") - windows_yaml_file_stream = open(windows_yaml_file, "w", encoding="utf-8") - windows_yaml_file_stream.write(category_error_content) - windows_yaml_file_stream.close() - - with tarfile.open(self.tarfile_path, "w:gz") as tfile: - # self.temp_path/temp_folder下的所有文件放入temp_file_path文件下, arcname表示目标目录的目标路径 - tfile.add(os.path.join(self.temp_path, "temp_folder"), arcname=".", recursive=True) - - self._test_upload_api_success() - response = self._query_parse_api() - self.assertEqual(len(response["data"]), 2) - - self.assertEqual( - len([item for item in response["data"] if not item["result"] and item["message"] == "project.yaml 文件信息缺失"]), - 1, - ) - self.assertEqual( - len( - [ - item - for item in response["data"] - if not item["result"] and item["message"] == "project.yaml 中 category 配置异常，请确认后重试" - ] - ), - 1, - ) - - def test_parse_api_not_template_and_version_update(self): - windows_template_file = os.path.join( - self.temp_path, - "temp_folder", - "external_plugins_windows_x86", - "test_plugin", - "etc", - "test_plugin.main.conf.tpl", - ) - os.remove(windows_template_file) - with tarfile.open(self.tarfile_path, "w:gz") as tfile: - # self.temp_path/temp_folder下的所有文件放入temp_file_path文件下, arcname表示目标目录的目标路径 - tfile.add(os.path.join(self.temp_path, "temp_folder"), arcname=".", recursive=True) - - Packages.objects.create( - pkg_name="test_plugin-1.0.0.tgz", - version="1.0.0", - module="gse_plugin", - project="test_plugin", - pkg_size=0, - pkg_path="", - md5="", - pkg_mtime="", - pkg_ctime="", - location="", - os="linux", - cpu_arch="x86_64", - is_release_version=True, - is_ready=True, - ) - - self._test_upload_api_success() - response = self._query_parse_api() - self.assertEqual(len(response["data"]), 2) - self.assertEqual( - len( - [ - item - for item in response["data"] - if not item["result"] and item["message"] == "找不到需要导入的配置模板文件 -> etc/test_plugin.main.conf.tpl" - ] - ), - 1, - ) - self.assertEqual(len([item for item in response["data"] if item["result"] and item["message"] == "更新插件版本"]), 1) - - def test_parse_api_low_version_or_same_version(self): - package_params = dict( - pkg_name="test_plugin-1.0.1.tgz", - version="1.0.1", - module="gse_plugin", - project="test_plugin", - pkg_size=0, - pkg_path="", - md5="", - pkg_mtime="", - pkg_ctime="", - location="", - os="linux", - cpu_arch="x86_64", - is_release_version=True, - is_ready=True, - ) - Packages.objects.create(**package_params) - package_params.update({"version": "2.0.0", "cpu_arch": "x86", "os": "windows"}) - Packages.objects.create(**package_params) - - self._test_upload_api_success() - response = self._query_parse_api() - self.assertEqual(len(response["data"]), 2) - self.assertEqual( - len([item for item in response["data"] if item["result"] and item["message"] == "已有版本插件更新"]), 1 - ) - self.assertEqual( - len([item for item in response["data"] if item["result"] and item["message"] == "低版本插件仅支持导入"]), 1 - ) - - -class TestImportCommand(TestCase): - target_path = os.path.join(TEST_ROOT, uuid.uuid4().hex) - - temp_path = os.path.join(TEST_ROOT, uuid.uuid4().hex) - - tarfile_name = "tarfile.tgz" - tarfile_path = os.path.join(temp_path, tarfile_name) - - plugin_name = "test_plugin" - - def setUp(self): - """测试启动初始化配置""" - # 1. 准备一个yaml配置文件 - config_content = """ - name: "test_plugin" - version: "1.0.1" - description: "用于采集主机基础性能数据，包含CPU,内存，磁盘，⽹络等数据" - description_en: "用于采集主机基础性能数据，包含CPU,内存，磁盘，⽹络等数据" - scenario: "CMDB上的是实时数据，蓝鲸监控-主机监控中的基础性能数据" - scenario_en: "CMDB上的是实时数据，蓝鲸监控-主机监控中的基础性能数据" - category: official - config_file: basereport.conf - multi_config: True - config_file_path: "" - config_format: yaml - auto_launch: true - launch_node: proxy - upstream: - bkmetric - dependences: - gse_agent: "1.2.0" - bkmetric: "1.6.0" - config_templates: - - plugin_version: "*" - name: child.conf - version: 2.0.1 - file_path: etc/child - format: yaml - source_path: etc/child.conf.tpl - - plugin_version: "*" - name: test_plugin.conf - version: 1.0.2 - file_path: etc/main - format: yaml - source_path: etc/test_plugin.main.conf.tpl - is_main_config: 1 - control: - start: "./start.sh %name" - stop: "./stop.sh %name" - restart: "./restart.sh %name" - version: "./%name -v" - reload: "./%name -s reload" - health_check: "./%name -z" - process_name: "test_process" -""" - - # 2. 创建一个打包文件，包含两层内容，一个linux，一个windows - temp_file_path = os.path.join(self.temp_path, "temp_folder") - if not os.path.exists(self.temp_path): - os.mkdir(self.temp_path) - os.mkdir(temp_file_path) - for package_os, cpu_arch in (("linux", "x86_64"), ("windows", "x86")): - current_path = os.path.join(temp_file_path, "external_plugins_{}_{}".format(package_os, cpu_arch)) - os.mkdir(current_path) - - plugin_path = os.path.join(current_path, self.plugin_name) - os.mkdir(plugin_path) - - config_path = os.path.join(plugin_path, "etc") - os.mkdir(config_path) - - f1 = open(os.path.join(plugin_path, "plugin"), "w") - f1.close() - f2 = open(os.path.join(plugin_path, "project.yaml"), "w", encoding="utf-8") - f2.write(config_content) - f2.close() - - main_config = open(os.path.join(config_path, "test_plugin.main.conf.tpl"), "w") - main_config.close() - child_config = open(os.path.join(config_path, "child.conf.tpl"), "w") - child_config.close() - - with tarfile.open(self.tarfile_path, "w:gz") as tfile: - tfile.add(temp_file_path, arcname=".", recursive=True) - - # nginx的模拟路径 - settings.DOWNLOAD_PATH = settings.UPLOAD_PATH = self.target_path - - settings.BK_OFFICIAL_PLUGINS_INIT_PATH = self.temp_path - - def tearDown(self): - shutil.rmtree(self.temp_path) - shutil.rmtree(self.target_path) - - def test_import_command(self): - """测试导入命令""" - if os.path.exists(settings.BK_OFFICIAL_PLUGINS_INIT_PATH): - call_command("init_official_plugins") - self.assertTrue(Packages.objects.all().exists()) - self.assertTrue(UploadPackage.objects.all().exists()) - self.assertTrue(PluginConfigTemplate.objects.all().exists()) diff --git a/apps/backend/tests/plugin/utils.py b/apps/backend/tests/plugin/utils.py index de4f21e32..ae5326d6b 100644 --- a/apps/backend/tests/plugin/utils.py +++ b/apps/backend/tests/plugin/utils.py @@ -9,46 +9,146 @@ specific language governing permissions and limitations under the License. """ import copy -import json +import os +import platform +import shutil +import tarfile +import uuid +from collections import ChainMap +from enum import Enum +from typing import Dict, List, Tuple -from django.test import Client, TestCase -from django.test.client import MULTIPART_CONTENT +import mock +from django.conf import settings -from apps.node_man import models +from apps.backend.plugin import tasks +from apps.core.files import storage +from apps.node_man import constants, models +from apps.utils import files -DEFAULT_PLUGIN_NAME = "test_plugin" +# 测试文件根路径 +from apps.utils.enum import EnhanceEnum +from apps.utils.unittest.testcase import CustomAPITestCase +TEST_ROOT = ("/tmp", "c:/")[platform.system().upper() == constants.OsType.WINDOWS] + +# 插件包yaml配置 +PROJECT_YAML_CONTENT = """ +name: "{plugin_name}" +version: "{package_version}" +description: "{description}" +description_en: "{description_en}" +scenario: "{scenario}" +scenario_en: "{scenario_en}" +category: {category} +config_format: {config_format} +config_file: {config_file} +use_db: {use_db} +is_binary: {is_binary} +launch_node: {launch_node} +auto_launch: {auto_launch} +config_templates: +- plugin_version: "{package_version}" + name: test_plugin.conf + version: {package_version} + file_path: etc + format: {config_format} + source_path: etc/{plugin_name}.conf.tpl + is_main_config: 1 +- plugin_version: "{package_version}" + name: child.conf + version: {package_version} + file_path: etc/{plugin_name} + format: {config_format} + source_path: etc/child.conf.tpl +control: + start: "./start.sh {plugin_name}" + stop: "./stop.sh {plugin_name}" + restart: "./restart.sh {plugin_name}" + version: "./{plugin_name} -v" + reload: "./{plugin_name} -s reload" + health_check: "./{plugin_name} -z" +node_manage_control: + package_update: false + package_remove: false + plugin_install: false + plugin_update: false + plugin_uninstall: false + plugin_upgrade: false + plugin_remove: false + plugin_restart: false +""" + +# 插件名称 +PLUGIN_NAME = "test_plugin" + +# 插件包版本 +PACKAGE_VERSION = "1.0.1" + +# 是否为第三方插件 +IS_EXTERNAL = False + +# models.GsePluginDesc 创建参数 +GSE_PLUGIN_DESC_PARAMS = { + "name": PLUGIN_NAME, + "description": "单元测试插件", + "description_en": "plugin for unittest", + "scenario": "单元测试插件", + "scenario_en": "plugin for unittest", + "category": constants.CategoryType.official, + "launch_node": "all", + "config_file": "test_plugin.conf", + "config_format": "yaml", + "use_db": 0, + "is_binary": 1, + "auto_launch": 0, +} + +# models.Packages 创建参数 PKG_PARAMS = { "id": 1, - "pkg_name": "test_plugin-1.0.0.tgz", - "version": "1.0.0", + "pkg_name": f"{PLUGIN_NAME}-{PACKAGE_VERSION}.tgz", + "version": PACKAGE_VERSION, "module": "gse_plugin", - "project": DEFAULT_PLUGIN_NAME, + "project": PLUGIN_NAME, "pkg_size": 0, "pkg_path": "", "md5": "", "pkg_mtime": "", "pkg_ctime": "", "location": "", - "os": "linux", - "cpu_arch": "x86_64", + "os": constants.OsType.LINUX.lower(), + "cpu_arch": constants.CpuType.x86_64, "is_release_version": True, "is_ready": True, } -GSE_PLUGIN_DESC = { - "name": DEFAULT_PLUGIN_NAME, - "description": "测试插件啊", - "scenario": "测试", - "category": "external", - "launch_node": "all", - "config_file": "config.yaml", - "config_format": "yaml", - "use_db": False, -} + +class PathSettingOverwrite(EnhanceEnum): + EXPORT_PATH = "EXPORT_PATH" + UPLOAD_PATH = "UPLOAD_PATH" + DOWNLOAD_PATH = "DOWNLOAD_PATH" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + raise NotImplementedError() + + @classmethod + def get_setting_name__path_suffix_map(cls) -> Dict[str, str]: + return {cls.EXPORT_PATH.value: "export", cls.UPLOAD_PATH.value: "upload", cls.DOWNLOAD_PATH.value: "download"} class PluginTestObjFactory: + @classmethod + def get_project_yaml_content(cls, **kwargs): + return PROJECT_YAML_CONTENT.format( + **dict( + ChainMap( + kwargs, GSE_PLUGIN_DESC_PARAMS, {"plugin_name": PLUGIN_NAME, "package_version": PACKAGE_VERSION} + ) + ) + ) + @classmethod def replace_obj_attr_values(cls, obj, obj_attr_values): # 原地修改 @@ -77,28 +177,166 @@ def pkg_obj(cls, obj_attr_values=None, is_obj=False): @classmethod def gse_plugin_desc_obj(cls, obj_attr_values=None, is_obj=False): - return cls.get_obj(GSE_PLUGIN_DESC, models.GsePluginDesc, obj_attr_values, is_obj) + return cls.get_obj(GSE_PLUGIN_DESC_PARAMS, models.GsePluginDesc, obj_attr_values, is_obj) @classmethod def batch_create_pkg(cls, packages): return cls.bulk_create(packages, models.Packages) @classmethod - def batch_create_plugin_desc(cls, plugin_descs): - return cls.bulk_create(plugin_descs, models.GsePluginDesc) + def batch_create_plugin_desc(cls, plugin_desc): + return cls.bulk_create(plugin_desc, models.GsePluginDesc) + + +class PluginBaseTestCase(CustomAPITestCase): + TMP_DIR: str = None + PLUGIN_ARCHIVE_NAME: str = None + PLUGIN_ARCHIVE_PATH: str = None + PLUGIN_ARCHIVE_MD5: str = None + OS_CPU_CHOICES = [ + (constants.OsType.LINUX.lower(), constants.CpuType.x86_64), + (constants.OsType.WINDOWS.lower(), constants.CpuType.x86), + ] + # BKAPP_PUBLIC_PATH = TMP_DIR + + API_AUTH_PARAMS: Dict[str, str] = {"bk_app_code": settings.APP_CODE, "bk_username": "admin"} + + PLUGIN_CHILD_DIR_NAME: str = (constants.PluginChildDir.OFFICIAL.value, constants.PluginChildDir.EXTERNAL.value)[ + IS_EXTERNAL + ] + + @classmethod + def setUpClass(cls): + + mock.patch("apps.backend.plugin.tasks.package_task.delay", tasks.package_task).start() + mock.patch("apps.backend.plugin.tasks.export_plugin.delay", tasks.export_plugin).start() + + cls.TMP_DIR = files.mk_and_return_tmpdir() + cls.PLUGIN_ARCHIVE_NAME = f"{PLUGIN_NAME}-{PACKAGE_VERSION}.tgz" + cls.PLUGIN_ARCHIVE_PATH = os.path.join(cls.TMP_DIR, cls.PLUGIN_ARCHIVE_NAME) + + setting_name__path_map = { + setting_name: os.path.join( + cls.TMP_DIR, PathSettingOverwrite.get_setting_name__path_suffix_map()[setting_name] + ) + for setting_name in PathSettingOverwrite.list_member_values() + } + cls.OVERWRITE_OBJ__KV_MAP = cls.OVERWRITE_OBJ__KV_MAP or {} + cls.OVERWRITE_OBJ__KV_MAP[settings] = {**cls.OVERWRITE_OBJ__KV_MAP.get(settings, {}), **setting_name__path_map} + + super().setUpClass() + + @classmethod + def setUpTestData(cls): + storage._STORAGE_OBJ_CACHE = {} + super().setUpTestData() + + def setUp(self): + for setting_name in PathSettingOverwrite.list_member_values(): + overwrite_path = os.path.join( + self.TMP_DIR, PathSettingOverwrite.get_setting_name__path_suffix_map()[setting_name] + ) + # exist_ok 目录存在直接跳过，不抛出 FileExistsError + os.makedirs(overwrite_path, exist_ok=True) + + product_tmp_dir = self.gen_test_plugin_files( + project_yaml_content=PluginTestObjFactory.get_project_yaml_content(), os_cpu_choices=self.OS_CPU_CHOICES + ) + self.pack_plugin(product_tmp_dir=product_tmp_dir) + self.PLUGIN_ARCHIVE_MD5 = files.md5sum(name=self.PLUGIN_ARCHIVE_PATH) + super().setUp() + + @classmethod + def gen_test_plugin_files(cls, project_yaml_content: str, os_cpu_choices: List[Tuple[str, str]]): + product_tmp_dir = os.path.join(cls.TMP_DIR, uuid.uuid4().hex) + for package_os, cpu_arch in os_cpu_choices: + pkg_info_dir_path = os.path.join(product_tmp_dir, f"{cls.PLUGIN_CHILD_DIR_NAME}_{package_os}_{cpu_arch}") + pkg_dir_path = os.path.join(pkg_info_dir_path, PLUGIN_NAME) + pkg_config_dir_path = os.path.join(pkg_dir_path, "etc") + pkg_executable_dir_path = os.path.join(pkg_dir_path, "bin") + + # 创建可执行文件 / 配置模板 目录 + os.makedirs(pkg_config_dir_path) + os.makedirs(pkg_executable_dir_path) + + # 创建可执行文件 + with open(os.path.join(pkg_executable_dir_path, PLUGIN_NAME), "w"): + pass + + # 写入 project.yaml + with open(os.path.join(pkg_dir_path, "project.yaml"), "w", encoding="utf-8") as project_yaml_fs: + project_yaml_fs.write(project_yaml_content) + + # 创建配置模板 + with open(os.path.join(pkg_config_dir_path, f"{PLUGIN_NAME}.conf.tpl"), "w"): + pass + with open(os.path.join(pkg_config_dir_path, f"{PLUGIN_NAME}.conf"), "w"): + pass + with open(os.path.join(pkg_config_dir_path, "child.conf.tpl"), "w"): + pass + + return product_tmp_dir + + @classmethod + def pack_plugin(cls, product_tmp_dir: str): + # 插件打包 + with tarfile.open(cls.PLUGIN_ARCHIVE_PATH, "w:gz") as tf: + tf.add(product_tmp_dir, arcname=".", recursive=True) + + def tearDown(self): + if os.path.exists(self.TMP_DIR): + shutil.rmtree(self.TMP_DIR) + super().tearDown() + + @classmethod + def tearDownClass(cls): + if os.path.exists(cls.TMP_DIR): + shutil.rmtree(cls.TMP_DIR) + + super().tearDownClass() + + +class CustomBKRepoMockStorage(storage.CustomBKRepoStorage): + mock_storage: storage.AdminFileSystemStorage = None + + def __init__( + self, + root_path=None, + username=None, + password=None, + project_id=None, + bucket=None, + endpoint_url=None, + file_overwrite=None, + ): + self.mock_storage = storage.AdminFileSystemStorage(file_overwrite=file_overwrite) + super().__init__( + root_path=root_path, + username=username, + password=password, + project_id=project_id, + bucket=bucket, + endpoint_url=endpoint_url, + file_overwrite=file_overwrite, + ) + + def path(self, name): + return self.mock_storage.path(name) + + def _open(self, name, mode="rb"): + return self.mock_storage._open(name, mode) + + def _save(self, name, content): + return self.mock_storage._save(name, content) + def exists(self, name): + return self.mock_storage.exists(name) -class UtilsClient(Client, TestCase): - def success_assert(self, response): - response_data = json.loads(response.content) - self.assertEqual(response.status_code, 200) - self.assertTrue(response_data["result"]) - return response_data + def size(self, name): + return self.mock_storage.size(name) - def get(self, path, data=None, follow=False, secure=False, **extra): - response = super().get(path, data, follow, secure, **extra) - return self.success_assert(response) + def url(self, name): + return self.mock_storage.url(name) - def post(self, path, data=None, content_type=MULTIPART_CONTENT, follow=False, secure=False, **extra): - response = super().post(path, data, content_type, follow, secure, **extra) - return self.success_assert(response) + def delete(self, name): + return self.mock_storage.delete(name) diff --git a/apps/backend/tests/plugin/views/__init__.py b/apps/backend/tests/plugin/views/__init__.py new file mode 100644 index 000000000..b402ee3b4 --- /dev/null +++ b/apps/backend/tests/plugin/views/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/apps/backend/tests/plugin/views/test_plugin_production.py b/apps/backend/tests/plugin/views/test_plugin_production.py new file mode 100644 index 000000000..9a2cedc6c --- /dev/null +++ b/apps/backend/tests/plugin/views/test_plugin_production.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import os +import random +import shutil +import tarfile +import uuid +from typing import Any, Dict, List, Optional + +import mock +from django.conf import settings + +from apps.backend.tests.plugin import utils +from apps.core.files import base, core_files_constants +from apps.node_man import constants, models +from apps.node_man.models import Packages, ProcControl +from apps.utils import files + + +class FileSystemTestCase(utils.PluginBaseTestCase): + FILE_OVERWRITE = True + + OVERWRITE_OBJ__KV_MAP = { + settings: { + "FILE_OVERWRITE": FILE_OVERWRITE, + "STORAGE_TYPE": core_files_constants.StorageType.FILE_SYSTEM.value, + }, + base.StorageFileOverwriteMixin: {"file_overwrite": FILE_OVERWRITE}, + } + + def upload_plugin(self, file_local_path: Optional[str] = None) -> Dict[str, Any]: + file_local_path = file_local_path or self.PLUGIN_ARCHIVE_PATH + md5 = files.md5sum(file_local_path) + upload_result = self.client.post( + path="/backend/package/upload/", + data={ + **self.API_AUTH_PARAMS, + "module": "gse_plugin", + "md5": md5, + # nginx 计算并回调的额外参数 + "file_name": self.PLUGIN_ARCHIVE_NAME, + "file_local_md5": md5, + "file_local_path": file_local_path, + }, + format=None, + )["data"] + file_name = upload_result["name"] + # 插件会保存到 UPLOAD_PATH + self.assertTrue( + models.UploadPackage.objects.filter( + file_name=file_name, file_path=os.path.join(settings.UPLOAD_PATH, file_name) + ).exists() + ) + return upload_result + + def register_plugin(self, file_name: str, select_pkg_relative_paths: Optional[List[str]] = None): + base_query_params = { + **self.API_AUTH_PARAMS, + "file_name": file_name, + "is_release": True, + "is_template_load": True, + } + if select_pkg_relative_paths is not None: + base_query_params["select_pkg_relative_paths"]: select_pkg_relative_paths + + self.client.post(path="/backend/api/plugin/create_register_task/", data=base_query_params) + + def parse_plugin(self, file_name: str) -> List[Dict[str, Any]]: + pkg_parse_results = self.client.post( + path="/backend/api/plugin/parse/", + data={**self.API_AUTH_PARAMS, "file_name": file_name}, + )["data"] + return pkg_parse_results + + def check_pkg_structure(self, project: str, pkg_path: str): + # 第三方插件归档的插件包路径包含 project(插件名称)层级 + + pkg_path_shims = [self.PLUGIN_CHILD_DIR_NAME] + if utils.IS_EXTERNAL: + pkg_path_shims.append(project) + + with tarfile.open(pkg_path) as tf: + # get or raise KeyError + tf.getmember(os.path.join(*pkg_path_shims, "etc", f"{project}.conf")) + tf.getmember(os.path.join(*pkg_path_shims, "project.yaml")) + tf.getmember(os.path.join(*pkg_path_shims, "bin", project)) + + # 验证写入DB的配置模板在归档插件包中被删除 + self.assertRaises(KeyError, tf.getmember, os.path.join(*pkg_path_shims, "etc", f"{project}.conf.tpl")) + self.assertRaises(KeyError, tf.getmember, os.path.join(*pkg_path_shims, "etc", "child.conf.tpl")) + + def check_and_fetch_parse_results(self, file_name: str, except_message_list: List[str]) -> List[Dict[str, Any]]: + pkg_parse_results = self.parse_plugin(file_name=file_name) + parse_message_list = [pkg_parse_result["message"] for pkg_parse_result in pkg_parse_results] + self.assertListEqual(parse_message_list, except_message_list, is_sort=True) + return pkg_parse_results + + # cases + + def test_upload__file_overwrite(self): + + save_file_name_set = set() + upload_count = random.randint(2, 10) + file_local_dir_path = os.path.join(self.TMP_DIR, uuid.uuid4().hex) + os.makedirs(file_local_dir_path, exist_ok=True) + for __ in range(upload_count): + file_local_path = os.path.join(file_local_dir_path, self.PLUGIN_ARCHIVE_NAME) + # 验证该文件上传后已被删除 + self.assertFalse(os.path.exists(file_local_path)) + + # 上传文件默认将文件移动到 UPLOAD_PATH 并删除原路径的文件，所以采取拷贝到临时目录的方式变更文件上传文件 + shutil.copy(src=self.PLUGIN_ARCHIVE_PATH, dst=file_local_path) + save_file_name_set.add(self.upload_plugin(file_local_path=file_local_path)["name"]) + + if settings.FILE_OVERWRITE: + self.assertEqual(len(save_file_name_set), 1) + else: + self.assertEqual(len(save_file_name_set), upload_count) + + def test_create_register_task(self): + upload_result = self.upload_plugin() + self.register_plugin(upload_result["name"]) + plugin_obj = models.GsePluginDesc.objects.get(name=utils.PLUGIN_NAME) + for package_os, cpu_arch in self.OS_CPU_CHOICES: + pkg_obj = Packages.objects.get( + pkg_name=self.PLUGIN_ARCHIVE_NAME, version=utils.PACKAGE_VERSION, os=package_os, cpu_arch=cpu_arch + ) + self.check_pkg_structure(project=plugin_obj.name, pkg_path=os.path.join(pkg_obj.pkg_path, pkg_obj.pkg_name)) + + self.assertEqual(ProcControl.objects.all().count(), len(self.OS_CPU_CHOICES)) + + def test_create_register_task__select_pkgs(self): + upload_result = self.upload_plugin() + cpu_arch = constants.CpuType.x86_64 + package_os = constants.OsType.LINUX.lower() + self.register_plugin( + file_name=upload_result["name"], + select_pkg_relative_paths=[ + os.path.join(f"{self.PLUGIN_CHILD_DIR_NAME}_{package_os}_{cpu_arch}", utils.PLUGIN_NAME) + ], + ) + + plugin_obj = models.GsePluginDesc.objects.get(name=utils.PLUGIN_NAME) + pkg_obj = Packages.objects.get( + pkg_name=self.PLUGIN_ARCHIVE_NAME, version=utils.PACKAGE_VERSION, os=package_os, cpu_arch=cpu_arch + ) + self.check_pkg_structure(project=plugin_obj.name, pkg_path=os.path.join(pkg_obj.pkg_path, pkg_obj.pkg_name)) + + def test_create_export_task(self): + self.test_create_register_task() + export_result = self.client.post( + path="/backend/api/plugin/create_export_task/", + data={ + **self.API_AUTH_PARAMS, + "category": "gse_plugin", + "creator": "admin", + "query_params": {"project": utils.PLUGIN_NAME, "version": utils.PACKAGE_VERSION}, + }, + )["data"] + record_obj = models.DownloadRecord.objects.get(id=export_result["job_id"]) + self.assertTrue(os.path.exists(record_obj.file_path)) + + def test_create_export_task__with_os(self): + self.test_create_register_task() + export_result = self.client.post( + path="/backend/api/plugin/create_export_task/", + data={ + **self.API_AUTH_PARAMS, + "category": "gse_plugin", + "creator": "admin", + "query_params": { + "project": utils.PLUGIN_NAME, + "version": utils.PACKAGE_VERSION, + "os": constants.OsType.LINUX.lower(), + }, + }, + )["data"] + record_obj = models.DownloadRecord.objects.get(id=export_result["job_id"]) + self.assertTrue(os.path.exists(record_obj.file_path)) + + def test_create_export_task__with_os_cpu_arch(self): + self.test_create_register_task() + export_result = self.client.post( + path="/backend/api/plugin/create_export_task/", + data={ + **self.API_AUTH_PARAMS, + "category": "gse_plugin", + "creator": "admin", + "query_params": { + "project": utils.PLUGIN_NAME, + "version": utils.PACKAGE_VERSION, + "os": constants.OsType.LINUX.lower(), + "cpu_arch": constants.CpuType.x86_64, + }, + }, + )["data"] + record_obj = models.DownloadRecord.objects.get(id=export_result["job_id"]) + self.assertTrue(os.path.exists(record_obj.file_path)) + + def test_parse(self): + self.check_and_fetch_parse_results(file_name=self.upload_plugin()["name"], except_message_list=["新增插件"] * 2) + + def test_parse__yaml_file_not_find_or_unread(self): + os.remove(self.PLUGIN_ARCHIVE_PATH) + product_tmp_dir = self.gen_test_plugin_files( + project_yaml_content="Invalid project yaml content", os_cpu_choices=self.OS_CPU_CHOICES + ) + linux_yaml_path = os.path.join(product_tmp_dir, "plugins_linux_x86_64", utils.PLUGIN_NAME, "project.yaml") + os.remove(linux_yaml_path) + self.pack_plugin(product_tmp_dir=product_tmp_dir) + + self.check_and_fetch_parse_results( + file_name=self.upload_plugin()["name"], except_message_list=["缺少project.yaml文件", "project.yaml文件解析读取失败"] + ) + + def test_parse__yaml_file_lack_attr_or_category_error(self): + os.remove(self.PLUGIN_ARCHIVE_PATH) + project_yaml_content = utils.PluginTestObjFactory.get_project_yaml_content() + product_tmp_dir = self.gen_test_plugin_files( + project_yaml_content=project_yaml_content, os_cpu_choices=self.OS_CPU_CHOICES + ) + linux_yaml_path = os.path.join(product_tmp_dir, "plugins_linux_x86_64", utils.PLUGIN_NAME, "project.yaml") + windows_yaml_path = os.path.join(product_tmp_dir, "plugins_windows_x86", utils.PLUGIN_NAME, "project.yaml") + with open(linux_yaml_path, "w+", encoding="utf-8") as fs: + lack_of_plugin_name_content = project_yaml_content.replace(f'name: "{utils.PLUGIN_NAME}"', "") + fs.write(lack_of_plugin_name_content) + with open(windows_yaml_path, "w+", encoding="utf-8") as fs: + invalid_category_content = project_yaml_content.replace( + f"category: {utils.GSE_PLUGIN_DESC_PARAMS['category']}", "category: invalid_category" + ) + fs.write(invalid_category_content) + self.pack_plugin(product_tmp_dir=product_tmp_dir) + self.check_and_fetch_parse_results( + file_name=self.upload_plugin()["name"], + except_message_list=["project.yaml 文件信息缺失", "project.yaml 中 category 配置异常，请确认后重试"], + ) + + def test_parse__not_template_and_version_update(self): + os.remove(self.PLUGIN_ARCHIVE_PATH) + project_yaml_content = utils.PluginTestObjFactory.get_project_yaml_content() + product_tmp_dir = self.gen_test_plugin_files( + project_yaml_content=project_yaml_content, os_cpu_choices=self.OS_CPU_CHOICES + ) + + # 移除windows的主配置模板 + windows_tpl_file_path = os.path.join( + product_tmp_dir, "plugins_windows_x86", utils.PLUGIN_NAME, "etc", f"{utils.PLUGIN_NAME}.conf.tpl" + ) + os.remove(windows_tpl_file_path) + + low_version_pkg_obj = utils.PluginTestObjFactory.pkg_obj( + {"version": "1.0.0", "os": constants.OsType.LINUX.lower(), "cpu_arch": constants.CpuType.x86_64}, + is_obj=True, + ) + low_version_pkg_obj.save() + + self.pack_plugin(product_tmp_dir=product_tmp_dir) + self.check_and_fetch_parse_results( + file_name=self.upload_plugin()["name"], + except_message_list=["找不到需要导入的配置模板文件 -> etc/test_plugin.conf.tpl", "更新插件版本"], + ) + + def test_parse__low_or_same_version(self): + utils.PluginTestObjFactory.batch_create_pkg( + [ + utils.PluginTestObjFactory.pkg_obj( + {"os": constants.OsType.LINUX.lower(), "cpu_arch": constants.CpuType.x86_64} + ), + utils.PluginTestObjFactory.pkg_obj( + {"version": "2.0.0", "os": constants.OsType.WINDOWS.lower(), "cpu_arch": constants.CpuType.x86} + ), + ] + ) + self.check_and_fetch_parse_results( + file_name=self.upload_plugin()["name"], except_message_list=["低版本插件仅支持导入", "已有版本插件更新"] + ) + + +class BkRepoTestCase(FileSystemTestCase): + FILE_OVERWRITE = True + OVERWRITE_OBJ__KV_MAP = { + settings: { + "FILE_OVERWRITE": FILE_OVERWRITE, + "STORAGE_TYPE": core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value, + }, + utils.CustomBKRepoMockStorage: {"file_overwrite": FILE_OVERWRITE}, + base.StorageFileOverwriteMixin: {"file_overwrite": FILE_OVERWRITE}, + } + + @classmethod + def setUpClass(cls): + mock.patch("apps.core.files.storage.CustomBKRepoStorage", utils.CustomBKRepoMockStorage).start() + super().setUpClass() + + def upload_plugin(self, file_local_path: Optional[str] = None) -> Dict[str, Any]: + file_local_path = file_local_path or self.PLUGIN_ARCHIVE_PATH + upload_result = self.client.post( + path="/backend/package/upload_cos/", + data={ + **self.API_AUTH_PARAMS, + "module": "gse_plugin", + "md5": files.md5sum(file_local_path), + # nginx 计算并回调的额外参数 + "file_name": self.PLUGIN_ARCHIVE_NAME, + "file_path": file_local_path, + }, + format=None, + )["data"] + file_name = upload_result["name"] + # 插件会保存到 UPLOAD_PATH + self.assertTrue( + models.UploadPackage.objects.filter( + file_name=file_name, file_path=os.path.join(settings.UPLOAD_PATH, file_name) + ).exists() + ) + return upload_result + + +class FileNotOverwriteTestCase(FileSystemTestCase): + FILE_OVERWRITE = False diff --git a/apps/backend/tests/plugin/test_plugin_status_change.py b/apps/backend/tests/plugin/views/test_plugin_status.py similarity index 77% rename from apps/backend/tests/plugin/test_plugin_status_change.py rename to apps/backend/tests/plugin/views/test_plugin_status.py index 8c71936d7..8f5f933d8 100644 --- a/apps/backend/tests/plugin/test_plugin_status_change.py +++ b/apps/backend/tests/plugin/views/test_plugin_status.py @@ -8,51 +8,13 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import json - -from django.test import Client, TestCase -from django.test.client import MULTIPART_CONTENT from apps.backend.tests.plugin import utils from apps.node_man import constants, models +from apps.utils.unittest.testcase import CustomAPITestCase -class TestApiBase(TestCase): - test_client = Client() - - def success_assert(self, response): - response_data = json.loads(response.content) - self.assertEqual(response.status_code, 200) - self.assertTrue(response_data["result"]) - return response_data - - def get(self, path, data=None, follow=False, secure=False, success_assert=True, **extra): - response = self.test_client.get(path, data, follow, secure, **extra) - return self.success_assert(response) - - def post( - self, - path, - data=None, - content_type=MULTIPART_CONTENT, - follow=False, - secure=False, - success_assert=True, - is_json=True, - **extra - ): - # 默认用json格式 - if is_json: - content_type = "application/json" - data = json.dumps(data) if isinstance(data, dict) else data - response = self.test_client.post(path, data, content_type, follow, secure, **extra) - if success_assert: - return self.success_assert(response) - else: - return json.loads(response.content) - - -class TestPkgStatusChange(TestApiBase): +class PluginStatusTestCase(CustomAPITestCase): def setUp(self): utils.PluginTestObjFactory.batch_create_plugin_desc([utils.PluginTestObjFactory.gse_plugin_desc_obj()]) utils.PluginTestObjFactory.batch_create_pkg( @@ -69,7 +31,7 @@ def setUp(self): def test_pkg_release(self): pkg_objs = models.Packages.objects.all() - response = self.post( + response = self.client.post( path="/backend/api/plugin/release/", data={ "id": [pkg_objs[0].id, pkg_objs[1].id], @@ -83,14 +45,14 @@ def test_pkg_release(self): models.Packages.objects.all().update(is_release_version=False) - response = self.post( + response = self.client.post( path="/backend/api/plugin/release/", data={ "md5_list": ["456"], "bk_app_code": "test", "bk_username": "test_person", "version": "1.0.1", - "name": utils.DEFAULT_PLUGIN_NAME, + "name": utils.PLUGIN_NAME, }, ) self.assertEquals(response["data"], [pkg_objs[1].id]) @@ -100,7 +62,7 @@ def test_pkg_release(self): # 测试未启用状态下不允许上下线变更 pkg_objs.update(is_ready=False, is_release_version=True) - response = self.post( + response = self.client.post( path="/backend/api/plugin/release/", data={ "id": [pkg_objs[0].id, pkg_objs[1].id], @@ -124,7 +86,7 @@ def test_pkg_status_op(self): constants.PkgStatusOpType.release, ] for op in op_order: - response = self.post( + response = self.client.post( path="/backend/api/plugin/package_status_operation/", data={ "id": [pkg_objs[0].id, pkg_objs[1].id], @@ -144,10 +106,10 @@ def test_pkg_status_op(self): ) # 下线1.0.0 - response = self.post( + response = self.client.post( path="/backend/api/plugin/package_status_operation/", data={ - "name": utils.DEFAULT_PLUGIN_NAME, + "name": utils.PLUGIN_NAME, "version": "1.0.0", "operation": constants.PkgStatusOpType.offline, "md5_list": ["123"], @@ -168,7 +130,7 @@ def test_plugin_status_op(self): constants.PluginStatusOpType.ready, ] for op in op_order: - response = self.post( + response = self.client.post( path="/backend/api/plugin/plugin_status_operation/", data={"operation": op, "id": [gse_plugin_objs[0].id], "bk_app_code": "test", "bk_username": "admin"}, ) diff --git a/apps/core/files/storage.py b/apps/core/files/storage.py index 9a0e4a0ca..f8221980a 100644 --- a/apps/core/files/storage.py +++ b/apps/core/files/storage.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ import os -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, List, Optional from bkstorages.backends import bkrepo from django.conf import settings @@ -183,7 +183,7 @@ def _handle_file_source_list( def cache_storage_obj(get_storage_func: Callable[[str, Dict], Storage]): """用于Storage 缓存读写的装饰器""" - def inner(storage_type: str = settings.STORAGE_TYPE, *args, **construct_params) -> Storage: + def inner(storage_type: Optional[str] = None, *args, **construct_params) -> Storage: # 仅默认参数情况下返回缓存 if not (construct_params or args) and storage_type in _STORAGE_OBJ_CACHE: return _STORAGE_OBJ_CACHE[storage_type] @@ -200,7 +200,7 @@ def inner(storage_type: str = settings.STORAGE_TYPE, *args, **construct_params) @cache_storage_obj -def get_storage(storage_type: str = settings.STORAGE_TYPE, safe: bool = False, **construct_params) -> BaseStorage: +def get_storage(storage_type: Optional[str] = None, safe: bool = False, **construct_params) -> BaseStorage: """ 获取 Storage :param storage_type: 文件存储类型，参考 constants.StorageType @@ -208,6 +208,7 @@ def get_storage(storage_type: str = settings.STORAGE_TYPE, safe: bool = False, * :param construct_params: storage class 构造参数，用于修改storage某些默认行为（写入仓库、base url等） :return: Storage实例 """ + storage_type = storage_type or settings.STORAGE_TYPE storage_import_path = settings.STORAGE_TYPE_IMPORT_PATH_MAP.get(storage_type) if storage_import_path is None: raise ValueError(f"please provide valid storage_type {settings.STORAGE_TYPE_IMPORT_PATH_MAP.values()}") diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 9672e4b3a..e40a2917d 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -12,6 +12,7 @@ from __future__ import unicode_literals import os +import platform import re from enum import Enum from typing import List @@ -73,7 +74,7 @@ class TimeUnit: WINDOWS_SEP = "\\" # 临时文件存放位置 -TMP_DIR = "/tmp" +TMP_DIR = ("/tmp", "c:/")[platform.system() == "Windows"] # 临时文件名格式模板 TMP_FILE_NAME_FORMAT = "nm_tf_{name}" diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 9cbe7c116..f5c25c82e 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -1096,11 +1096,17 @@ def unzip(self, local_target_dir: str) -> None: for external_type_prefix in os.listdir(package_tmp_dir): if external_type_prefix not in constants.PluginChildDir.get_optional_items(): continue + + # 官方插件的插件包目录缺少 project，为了保证导出插件包可用，解压目标目录需要补充 project 层级 + dst_shims = [local_target_dir, f"{external_type_prefix}_{self.os}_{self.cpu_arch}"] + if external_type_prefix == constants.PluginChildDir.OFFICIAL: + dst_shims.append(self.project) + # 将匹配的目录拷贝并格式化命名 # 关于拷贝目录，参考：https://stackoverflow.com/questions/1868714/ copy_tree( src=os.path.join(package_tmp_dir, external_type_prefix), - dst=os.path.join(local_target_dir, f"{external_type_prefix}_{self.os}_{self.cpu_arch}"), + dst=os.path.join(*dst_shims), ) # 移除临时解压目录 diff --git a/apps/utils/unittest/base.py b/apps/utils/unittest/base.py index e4a2406dd..926f0dd77 100644 --- a/apps/utils/unittest/base.py +++ b/apps/utils/unittest/base.py @@ -23,8 +23,11 @@ class CustomAPIClient(APIClient): @staticmethod def assert_response(response) -> Dict[str, Any]: - assert response.status_code == status.HTTP_200_OK - return json.loads(response.content) + if response.status_code != status.HTTP_200_OK: + print(json.dumps(json.loads(response.content))) + assert False + else: + return json.loads(response.content) def get(self, path, data=None, follow=False, **extra): response = super().get(path=path, data=data, follow=float, **extra) diff --git a/apps/utils/unittest/testcase.py b/apps/utils/unittest/testcase.py index 77c8edeb3..4d45472b4 100644 --- a/apps/utils/unittest/testcase.py +++ b/apps/utils/unittest/testcase.py @@ -8,7 +8,8 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from typing import Dict, List, Union +from collections import defaultdict +from typing import Any, Dict, List, Optional, Union from unittest import mock from django.contrib.auth.models import User @@ -139,7 +140,7 @@ def tearDown(self) -> None: @classmethod def setUpTestData(cls): - """Hook in testcase.__call__ , before setUpClass""" + """Hook in testcase.__call__ , after setUpClass""" super().setUpTestData() @classmethod @@ -154,7 +155,36 @@ def tearDownClass(cls): mock.patch.stopall() -class CustomBaseTestCase(AssertDataMixin, TestCaseLifeCycleMixin, TestCase): +class OverwriteSettingsMixin(TestCaseLifeCycleMixin): + OVERWRITE_OBJ__KV_MAP: Optional[Dict[Any, Dict[str, Any]]] = None + OBJ__ORIGIN_KV_MAP: Optional[Dict[Any, Dict[str, Any]]] = None + + @classmethod + def setUpClass(cls): + cls.OVERWRITE_OBJ__KV_MAP = cls.OVERWRITE_OBJ__KV_MAP or {} + cls.OBJ__ORIGIN_KV_MAP = defaultdict(lambda: defaultdict(dict)) + + super().setUpClass() + + @classmethod + def setUpTestData(cls): + for setting_obj, kv in cls.OVERWRITE_OBJ__KV_MAP.items(): + for k, v in kv.items(): + cls.OBJ__ORIGIN_KV_MAP[setting_obj][k] = getattr(setting_obj, k) + setattr(setting_obj, k, v) + super().setUpTestData() + + @classmethod + def tearDownClass(cls): + for setting_obj, origin_kv in cls.OBJ__ORIGIN_KV_MAP.items(): + for k, origin_value in origin_kv.items(): + setattr(setting_obj, k, origin_value) + + cls.OBJ__ORIGIN_KV_MAP = cls.OBJ__ORIGIN_KV_MAP = None + super().tearDownClass() + + +class CustomBaseTestCase(AssertDataMixin, OverwriteSettingsMixin, TestCase): client_class = Client @property diff --git a/requirements.txt b/requirements.txt index ccceb1231..ebecdad71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -blueapps-open==3.3.5 +blueapps-open==3.3.11 Django==2.2.6 requests==2.22.0