Skip to content

Commit

Permalink
optimization: 代码优化(#2)
Browse files Browse the repository at this point in the history
  • Loading branch information
ZhuoZhuoCrayon authored and zhangzhw8 committed Sep 22, 2021
1 parent 7d28390 commit 4b91618
Show file tree
Hide file tree
Showing 15 changed files with 254 additions and 108 deletions.
6 changes: 2 additions & 4 deletions apps/backend/components/collections/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -919,16 +919,14 @@ def execute(self, data, parent_data):
data.inputs.file_target_path = host.agent_config["temp_path"]

os_type = host.os_type.lower()
bk_os_bit = host_info.get("bk_os_bit")

# 根据节点类型、位数、系统等组装包名
arch = "x86" if bk_os_bit == "32-bit" else "x86_64"
gse_type = "proxy" if host.node_type == constants.NodeType.PROXY else "client"
package_name = f"gse_{gse_type}-{os_type}-{arch}_upgrade.tgz"
package_name = f"gse_{gse_type}-{os_type}-{host.cpu_arch}_upgrade.tgz"
files = [package_name]

# windows机器需要添加解压文件
if os_type == "windows":
if os_type == constants.OsType.WINDOWS:
files.extend(["7z.dll", "7z.exe"])
file_source = [{"files": [f"{nginx_path}/{file}" for file in files]}]

Expand Down
6 changes: 2 additions & 4 deletions apps/backend/components/collections/bulk_job_redis.py
Original file line number Diff line number Diff line change
Expand Up @@ -416,16 +416,14 @@ def execute(self, data, parent_data):
data.inputs.file_target_path = host.agent_config["temp_path"]

os_type = host.os_type.lower()
bk_os_bit = host_info.get("bk_os_bit")

# 根据节点类型、位数、系统等组装包名
arch = "x86" if bk_os_bit == "32-bit" else "x86_64"
gse_type = "proxy" if host.node_type == constants.NodeType.PROXY else "client"
package_name = f"gse_{gse_type}-{os_type}-{arch}_upgrade.tgz"
package_name = f"gse_{gse_type}-{os_type}-{host.cpu_arch}_upgrade.tgz"
files = [package_name]

# windows机器需要添加解压文件
if os_type == "windows":
if os_type == constants.OsType.WINDOWS.lower():
files.extend(["7z.dll", "7z.exe"])
file_source = [{"files": [f"{nginx_path}/{file}" for file in files]}]

Expand Down
65 changes: 31 additions & 34 deletions apps/backend/management/commands/copy_file_to_nginx.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,37 @@
from __future__ import absolute_import, unicode_literals

import os
import shutil
from typing import Iterable, List, Optional

from django.conf import settings
from django.core.management.base import BaseCommand

from apps.core.files.storage import get_storage
from apps.node_man.models import AccessPoint

IGNORED_PATH = ["__pycache__"]
from apps.utils import files


def copy_dir_files_to_storage(
source_dir_path: str, target_dir_paths: Iterable[str], ignored_dir_names: Optional[List[str]] = None
):
"""
将目录下文件保存到存储
:param source_dir_path: 源目录路径
:param target_dir_paths: 目标目录路径
:param ignored_dir_names: 忽略的目录名称
:return:
"""
storage = get_storage(file_overwrite=True)
source_file_paths = files.fetch_file_paths_from_dir(dir_path=source_dir_path, ignored_dir_names=ignored_dir_names)

for source_file_path in source_file_paths:
# 获取文件相对路径,用于拼接保存路径
file_relative_path = source_file_path.replace(source_dir_path + os.path.sep, "")
with open(source_file_path, mode="rb") as target_file_fs:
for target_dir_path in target_dir_paths:
target_file_path = os.path.join(target_dir_path, file_relative_path)
storage.save(name=target_file_path, content=target_file_fs)
target_file_fs.seek(0)


class Command(BaseCommand):
Expand All @@ -27,35 +50,9 @@ def handle(self, *args, **options):
拷贝scripts下的文件到nginx download下
"""
# 接入点配置的nginx路径
nginx_paths = [ap.nginx_path for ap in AccessPoint.objects.all() if ap.nginx_path]
nginx_paths = {ap.nginx_path for ap in AccessPoint.objects.all() if ap.nginx_path}
# 默认nginx路径
nginx_paths.append(settings.DOWNLOAD_PATH)
# 去重
nginx_paths = list(set(nginx_paths))
for _path in os.listdir(settings.BK_SCRIPTS_PATH):
if _path in IGNORED_PATH:
continue

_abspath = os.sep.join([settings.BK_SCRIPTS_PATH, _path])
if os.path.isfile(_abspath):
for dest_path in nginx_paths:
print("[Copying File]: from {} to {}".format(_abspath, dest_path))
shutil.copy2(_abspath, dest_path)

if os.path.isdir(_abspath):
for dest_path in nginx_paths:
print("[Copying Directory]: from {} to {}".format(_abspath, dest_path))
_dst_dir = os.sep.join([dest_path, _path])
shutil.rmtree(_dst_dir, ignore_errors=True)
shutil.copytree(_abspath, _dst_dir)

for filename in ["setup_agent.bat", "gsectl.bat"]:
# Windows脚本替换换行符
for dest_path in nginx_paths:
os.system(
"""
awk 'sub("$","\r")' {path}/{filename} > {dest_path}/{filename}
""".format(
path=settings.BK_SCRIPTS_PATH, dest_path=dest_path, filename=filename
)
)
nginx_paths.add(settings.DOWNLOAD_PATH)
copy_dir_files_to_storage(
source_dir_path=settings.BK_SCRIPTS_PATH, target_dir_paths=nginx_paths, ignored_dir_names=["__pycache__"]
)
11 changes: 1 addition & 10 deletions apps/backend/plugin/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,20 +305,11 @@ def validate(self, data):
)
return data

category = serializers.CharField()
category = serializers.ChoiceField(choices=DownloadRecord.CATEGORY_CHOICES)
query_params = GsePluginParamsSerializer()
creator = serializers.CharField()
bk_app_code = serializers.CharField()

def validate(self, attrs):
if attrs["category"] not in DownloadRecord.CATEGORY_TASK_DICT:
raise ValidationError(
"请求下载类型 -> {category} 暂不支持,可选项 -> {choices}".format(
category=attrs["category"], choices=DownloadRecord.CATEGORY_CHOICES
)
)
return attrs


class DeletePluginSerializer(GatewaySerializer):
name = serializers.CharField()
Expand Down
11 changes: 5 additions & 6 deletions apps/backend/plugin/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@
from apps.generic import APIViewSet
from apps.node_man import constants as const
from apps.node_man import models
from apps.utils import files
from pipeline.engine.exceptions import InvalidOperationException
from pipeline.service import task_service
from pipeline.service.pipeline_engine_adapter.adapter_api import STATE_MAP
from apps.utils import files

LOG_PREFIX_RE = re.compile(r"(\[\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}.*?\] )")
logger = logging.getLogger("app")
Expand Down Expand Up @@ -1321,11 +1321,10 @@ def upload_package_by_cos(request):
# TODO 此处的md5校验放到文件实际读取使用的地方更合理?
# file_path 不为空表示文件已在项目管理的对象存储上,此时仅需校验md5,减少文件IO
if file_path:
try:
if files.md5sum(file_obj=storage.open(name=file_path)) != md5:
raise ValidationError(_("上传文件MD5校验失败,请确认重试"))
except Exception as e:
raise ValidationError(_("文件不存在:file_path -> {file_path},error -> {err}").format(file_path=file_path, err=e))
if not storage.exists(name=file_path):
raise ValidationError(_("文件不存在:file_path -> {file_path}").format(file_path=file_path))
if files.md5sum(file_obj=storage.open(name=file_path)) != md5:
raise ValidationError(_("上传文件MD5校验失败,请确认重试"))
else:
# 创建临时存放下载插件的目录
tmp_dir = files.mk_and_return_tmpdir()
Expand Down
10 changes: 10 additions & 0 deletions apps/backend/tests/management/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
10 changes: 10 additions & 0 deletions apps/backend/tests/management/commands/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
78 changes: 78 additions & 0 deletions apps/backend/tests/management/commands/test_copy_file_to_nginx.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""

import os
import shutil
from typing import Optional

import mock
from django.conf import settings
from django.core.management import call_command

from apps.backend.tests.plugin import utils
from apps.core.files import core_files_constants
from apps.utils import files
from apps.utils.unittest.testcase import CustomBaseTestCase


class CopyFileToNginxTestCase(CustomBaseTestCase):
BK_SCRIPTS_PATH: str = os.path.join(settings.PROJECT_ROOT, "script_tools")
DOWNLOAD_PATH: Optional[str] = None
OVERWRITE_OBJ__KV_MAP = {
settings: {
"BK_SCRIPTS_PATH": BK_SCRIPTS_PATH,
"STORAGE_TYPE": core_files_constants.StorageType.FILE_SYSTEM.value,
},
}

@classmethod
def setUpClass(cls):
cls.DOWNLOAD_PATH = files.mk_and_return_tmpdir()
cls.OVERWRITE_OBJ__KV_MAP[settings]["DOWNLOAD_PATH"] = cls.DOWNLOAD_PATH
super().setUpClass()

@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.DOWNLOAD_PATH)
super().tearDownClass()

def test_case(self):
call_command("copy_file_to_nginx")
file_paths = files.fetch_file_paths_from_dir(dir_path=self.DOWNLOAD_PATH, ignored_dir_names=["__pycache__"])
for file_path in file_paths:
file_relative_path = file_path.replace(self.DOWNLOAD_PATH + os.path.sep, "")
self.assertTrue(os.path.exists(os.path.join(self.BK_SCRIPTS_PATH, file_relative_path)))

self.assertTrue(len(file_paths) != 0)
self.assertEquals(
len(file_paths),
len(files.fetch_file_paths_from_dir(dir_path=self.BK_SCRIPTS_PATH, ignored_dir_names=["__pycache__"])),
)


class BkRepoCopyFileToNginxTestCase(CopyFileToNginxTestCase):
BK_SCRIPTS_PATH: str = os.path.join(settings.PROJECT_ROOT, "script_tools")
OVERWRITE_OBJ__KV_MAP = {
settings: {
"BKREPO_USERNAME": "username",
"BKREPO_PASSWORD": "blueking",
"BKREPO_PROJECT": "project",
"BKREPO_BUCKET": "private",
"BKREPO_ENDPOINT_URL": "http://127.0.0.1",
"BK_SCRIPTS_PATH": BK_SCRIPTS_PATH,
"STORAGE_TYPE": core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value,
},
}

@classmethod
def setUpClass(cls):
mock.patch("apps.core.files.storage.CustomBKRepoStorage", utils.CustomBKRepoMockStorage).start()
super().setUpClass()
9 changes: 6 additions & 3 deletions apps/backend/tests/plugin/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,6 @@ class PluginBaseTestCase(CustomAPITestCase):
(constants.OsType.LINUX.lower(), constants.CpuType.x86_64),
(constants.OsType.WINDOWS.lower(), constants.CpuType.x86),
]
# BKAPP_PUBLIC_PATH = TMP_DIR

API_AUTH_PARAMS: Dict[str, str] = {"bk_app_code": settings.APP_CODE, "bk_username": "admin"}

PLUGIN_CHILD_DIR_NAME: str = (constants.PluginChildDir.OFFICIAL.value, constants.PluginChildDir.EXTERNAL.value)[
IS_EXTERNAL
Expand Down Expand Up @@ -232,6 +229,12 @@ def setUpTestData(cls):
super().setUpTestData()

def setUp(self):
# 设置请求附加参数
self.client.common_request_data = {
"bk_app_code": settings.APP_CODE,
"bk_username": settings.SYSTEM_USE_API_ACCOUNT,
}

for setting_name in PathSettingOverwrite.list_member_values():
overwrite_path = os.path.join(
self.TMP_DIR, PathSettingOverwrite.get_setting_name__path_suffix_map()[setting_name]
Expand Down
8 changes: 1 addition & 7 deletions apps/backend/tests/plugin/views/test_plugin_production.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ def upload_plugin(self, file_local_path: Optional[str] = None) -> Dict[str, Any]
upload_result = self.client.post(
path="/backend/package/upload/",
data={
**self.API_AUTH_PARAMS,
"module": "gse_plugin",
"md5": md5,
# nginx 计算并回调的额外参数
Expand All @@ -63,7 +62,6 @@ def upload_plugin(self, file_local_path: Optional[str] = None) -> Dict[str, Any]

def register_plugin(self, file_name: str, select_pkg_relative_paths: Optional[List[str]] = None):
base_query_params = {
**self.API_AUTH_PARAMS,
"file_name": file_name,
"is_release": True,
"is_template_load": True,
Expand All @@ -76,7 +74,7 @@ def register_plugin(self, file_name: str, select_pkg_relative_paths: Optional[Li
def parse_plugin(self, file_name: str) -> List[Dict[str, Any]]:
pkg_parse_results = self.client.post(
path="/backend/api/plugin/parse/",
data={**self.API_AUTH_PARAMS, "file_name": file_name},
data={"file_name": file_name},
)["data"]
return pkg_parse_results

Expand Down Expand Up @@ -159,7 +157,6 @@ def test_create_export_task(self):
export_result = self.client.post(
path="/backend/api/plugin/create_export_task/",
data={
**self.API_AUTH_PARAMS,
"category": "gse_plugin",
"creator": "admin",
"query_params": {"project": utils.PLUGIN_NAME, "version": utils.PACKAGE_VERSION},
Expand All @@ -173,7 +170,6 @@ def test_create_export_task__with_os(self):
export_result = self.client.post(
path="/backend/api/plugin/create_export_task/",
data={
**self.API_AUTH_PARAMS,
"category": "gse_plugin",
"creator": "admin",
"query_params": {
Expand All @@ -191,7 +187,6 @@ def test_create_export_task__with_os_cpu_arch(self):
export_result = self.client.post(
path="/backend/api/plugin/create_export_task/",
data={
**self.API_AUTH_PARAMS,
"category": "gse_plugin",
"creator": "admin",
"query_params": {
Expand Down Expand Up @@ -310,7 +305,6 @@ def upload_plugin(self, file_local_path: Optional[str] = None) -> Dict[str, Any]
upload_result = self.client.post(
path="/backend/package/upload_cos/",
data={
**self.API_AUTH_PARAMS,
"module": "gse_plugin",
"md5": files.md5sum(file_local_path),
# nginx 计算并回调的额外参数
Expand Down
Loading

0 comments on commit 4b91618

Please sign in to comment.