Skip to content

Commit

Permalink
feature: 支持 AIX 操作系统区分版本 (closed #815)
Browse files Browse the repository at this point in the history
  • Loading branch information
CohleRustW authored and ZhuoZhuoCrayon committed Jul 13, 2022
1 parent 9baf3cc commit 1bf2ba2
Show file tree
Hide file tree
Showing 13 changed files with 193 additions and 47 deletions.
21 changes: 20 additions & 1 deletion apps/backend/components/collections/agent_new/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"""
import abc
import random
import re
import time
from collections import defaultdict
from typing import (
Expand All @@ -18,6 +19,7 @@
Dict,
Iterable,
List,
Match,
Optional,
Set,
Tuple,
Expand All @@ -29,6 +31,7 @@
from django.utils.translation import ugettext_lazy as _

from apps.backend.agent.tools import InstallationTools, batch_gen_commands
from apps.backend.exceptions import OsVersionPackageValidationError
from apps.node_man import constants, models

from .. import job
Expand Down Expand Up @@ -102,7 +105,23 @@ def get_agent_upgrade_pkg_name(cls, host: models.Host) -> str:
:return:
"""
package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY]
agent_upgrade_package_name = f"gse_{package_type}-{host.os_type.lower()}-{host.cpu_arch}_upgrade.tgz"
if host.os_version:
major_version_number = None
if host.os_type == constants.OsType.AIX:
major_version_match: Optional[Match] = re.compile(r"^(?P<version>\d+).\d+.*$").search(
host.os_version or ""
)
major_version_number: Optional[str] = (
major_version_match.group("version") if major_version_match else ""
)
if not major_version_number:
raise OsVersionPackageValidationError(os_version=host.os_version, os_type=host.os_type)
agent_upgrade_package_name = (
f"gse_{package_type}-{host.os_type.lower()}{major_version_number}-{host.cpu_arch}_upgrade.tgz"
)
else:
agent_upgrade_package_name = f"gse_{package_type}-{host.os_type.lower()}-{host.cpu_arch}_upgrade.tgz"

return agent_upgrade_package_name

@staticmethod
Expand Down
16 changes: 15 additions & 1 deletion apps/backend/components/collections/agent_new/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,7 @@ def handle_report_data(self, sub_inst_id: int, success_callback_step: str) -> Di
REDIS_INST.ltrim(name, 0, -report_data_len - 1)
report_data.reverse()
cpu_arch = None
os_version = None
is_finished = False
error_log = ""
logs = []
Expand All @@ -532,6 +533,9 @@ def handle_report_data(self, sub_inst_id: int, success_callback_step: str) -> Di
logs.append(log)
if step == "report_cpu_arch":
cpu_arch = data["log"]

if step == "report_os_version":
os_version = data["log"]
# 只要匹配到成功返回步骤完成,则认为是执行完成了
if step == success_callback_step and status == "DONE":
is_finished = True
Expand All @@ -540,7 +544,7 @@ def handle_report_data(self, sub_inst_id: int, success_callback_step: str) -> Di
self.log_info(sub_inst_ids=sub_inst_id, log_content="\n".join(logs))
if error_log:
self.move_insts_to_failed([sub_inst_id], log_content=error_log)
return {"sub_inst_id": sub_inst_id, "is_finished": is_finished, "cpu_arch": cpu_arch}
return {"sub_inst_id": sub_inst_id, "is_finished": is_finished, "cpu_arch": cpu_arch, "os_version": os_version}

def _schedule(self, data, parent_data, callback_data=None):
"""通过轮询redis的方式来处理,避免使用callback的方式频繁调用schedule"""
Expand All @@ -561,18 +565,28 @@ def _schedule(self, data, parent_data, callback_data=None):
results = concurrent.batch_call(func=self.handle_report_data, params_list=params_list)
left_scheduling_sub_inst_ids = []
cpu_arch__host_id_map = defaultdict(list)
os_version__host_id_map = defaultdict(list)
for result in results:
# 对于未完成的实例,记录下来到下一次schedule中继续检查
if not result["is_finished"]:
left_scheduling_sub_inst_ids.append(result["sub_inst_id"])
# 按CPU架构对主机进行分组
bk_host_id = common_data.sub_inst_id__host_id_map.get(result["sub_inst_id"])
cpu_arch__host_id_map[result["cpu_arch"]].append(bk_host_id)
# 按操作系统版本对主机进行分组
os_version = result.get("os_version", "")
if os_version is not None:
os_version__host_id_map[os_version].append(bk_host_id)
# 批量更新CPU架构
for cpu_arch, bk_host_ids in cpu_arch__host_id_map.items():
if cpu_arch:
models.Host.objects.filter(bk_host_id__in=bk_host_ids).update(cpu_arch=cpu_arch)

# 批量更新主机操作系统版本号
for os_version, bk_host_ids in os_version__host_id_map.items():
if os_version:
models.Host.objects.filter(bk_host_id__in=bk_host_ids).update(os_version=os_version)

data.outputs.scheduling_sub_inst_ids = left_scheduling_sub_inst_ids
if not left_scheduling_sub_inst_ids:
self.finish_schedule()
Expand Down
6 changes: 6 additions & 0 deletions apps/backend/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,9 @@ class PluginParseError(BackendBaseException):
class CreatePackageRecordError(BackendBaseException):
MESSAGE = _("归档插件包信息错误")
ERROR_CODE = 10


class OsVersionPackageValidationError(BackendBaseException):
MESSAGE = _("操作系统版本安装包校验错误")
MESSAGE_TPL = _("操作系统[{os_type}]不支持 对应版本[{os_version}]的安装包")
ERROR_CODE = 11
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,13 @@ def init_redis_data(self):
"log": "aarch64",
"status": "DONE",
},
{
"timestamp": "1580870937",
"level": "INFO",
"step": "report_os_version",
"status": "DONE",
"log": "6.1.1.1",
},
{
"timestamp": "1580870937",
"level": "INFO",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,22 @@
specific language governing permissions and limitations under the License.
"""
import os.path
from abc import ABC

from django.conf import settings

from apps.backend.components.collections.agent_new import components
from apps.node_man import constants, models
from common.api import JobApi
from pipeline.component_framework.test import ComponentTestCase, ExecuteAssertion

from . import base
from . import base, utils


class PushUpgradePackageTestCase(base.JobBaseTestCase):
@classmethod
def get_default_case_name(cls) -> str:
return "下发升级包成功"
return "下发 Linux 升级包成功"

def component_cls(self):
return components.PushUpgradePackageComponent
Expand All @@ -34,3 +37,69 @@ def tearDown(self) -> None:
[os.path.join(settings.DOWNLOAD_PATH, "gse_client-linux-x86_64_upgrade.tgz")],
)
super().tearDown()


class PushAixUpgradePackageSuccessTestCase(base.JobBaseTestCase):
@classmethod
def get_default_case_name(cls) -> str:
return "下发 Aix 升级包成功"

@classmethod
def setup_obj_factory(cls):
cls.obj_factory.host_os_type_options = [constants.OsType.AIX]

def component_cls(self):
return components.PushUpgradePackageComponent

@classmethod
def setUpTestData(cls):
super().setUpTestData()
models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update(
os_version="6.1.0.0.1", cpu_arch=constants.CpuType.powerpc
)

def tearDown(self) -> None:
record = self.job_api_mock_client.call_recorder.record
fast_transfer_file_query_params = record[JobApi.fast_transfer_file][0].args[0]

self.assertEqual(
fast_transfer_file_query_params["file_source_list"][0]["file_list"],
[os.path.join(settings.DOWNLOAD_PATH, "gse_client-aix6-powerpc_upgrade.tgz")],
)
super().tearDown()


class PushAixUpgradePackageFailTestCase(utils.AgentServiceBaseTestCase, ABC):
@classmethod
def get_default_case_name(cls) -> str:
return "下发 Aix 升级包失败"

@classmethod
def setup_obj_factory(cls):
cls.obj_factory.host_os_type_options = [constants.OsType.AIX]

def component_cls(self):
return components.PushUpgradePackageComponent

@classmethod
def setUpTestData(cls):
super().setUpTestData()
models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update(
os_version=constants.CpuType.x86_64, cpu_arch=constants.CpuType.powerpc
)

def cases(self):
return [
ComponentTestCase(
name=self.get_default_case_name(),
inputs=self.common_inputs,
parent_data={},
execute_assertion=ExecuteAssertion(
success=False,
outputs={},
),
schedule_assertion=None,
execute_call_assertion=None,
patchers=[],
)
]
3 changes: 2 additions & 1 deletion apps/node_man/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -819,9 +819,10 @@ class PolicyRollBackType:
GSE_CLIENT_PACKAGES: List[str] = [
"gse_client-windows-x86.tgz",
"gse_client-windows-x86_64.tgz",
"gse_client-aix-powerpc.tgz",
"gse_client-linux-x86.tgz",
"gse_client-linux-x86_64.tgz",
"gse_client-aix6-powerpc.tgz",
"gse_client-aix7-powerpc.tgz",
]

FILES_TO_PUSH_TO_PROXY = [
Expand Down
3 changes: 1 addition & 2 deletions apps/node_man/handlers/meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -353,8 +353,7 @@ def fetch_plugin_host_condition(self):
os_dict = {"name": _("操作系统"), "id": "os_type", "children": []}

for os_type in constants.OS_TUPLE:
special_os_type = [constants.OsType.AIX, constants.OsType.SOLARIS]
if os_type in special_os_type and settings.BKAPP_RUN_ENV == constants.BkappRunEnvType.CE.value:
if os_type == constants.OsType.SOLARIS and settings.BKAPP_RUN_ENV == constants.BkappRunEnvType.CE.value:
continue
os_dict["children"].append({"id": os_type, "name": constants.OS_CHN.get(os_type, os_type)})

Expand Down
11 changes: 1 addition & 10 deletions apps/node_man/migrations/0023_init_proxy_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,9 @@


def init_proxy_package(apps, schema_editor):
packages = [
"gse_client-windows-x86.tgz",
"gse_client-windows-x86_64.tgz",
"gse_client-linux-x86.tgz",
"gse_client-linux-x86_64.tgz",
]

if settings.BKAPP_RUN_ENV != constants.BkappRunEnvType.CE.value:
packages.append("gse_client-aix-powerpc.tgz")

AccessPoint = apps.get_model("node_man", "AccessPoint")
AccessPoint.objects.update(proxy_package=packages)
AccessPoint.objects.update(proxy_package=constants.GSE_CLIENT_PACKAGES)


class Migration(migrations.Migration):
Expand Down
20 changes: 20 additions & 0 deletions apps/node_man/migrations/0060_host_os_version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Generated by Django 3.2.4 on 2022-06-09 08:29

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("node_man", "0059_auto_20220415_2150"),
]

operations = [
migrations.AddField(
model_name="host",
name="os_version",
field=models.CharField(
blank=True, default="", max_length=32, null=True, db_index=True, verbose_name="操作系统版本"
),
),
]
1 change: 1 addition & 0 deletions apps/node_man/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ class Host(models.Model):
cpu_arch = models.CharField(
_("操作系统"), max_length=16, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True
)
os_version = models.CharField(_("操作系统版本"), max_length=32, blank=True, null=True, db_index=True, default="")
node_type = models.CharField(_("节点类型"), max_length=16, choices=constants.NODE_CHOICES, db_index=True)
node_from = models.CharField(_("节点来源"), max_length=45, choices=constants.NODE_FROM_CHOICES, default="NODE_MAN")
is_manual = models.BooleanField(_("是否手动安装"), default=False)
Expand Down
3 changes: 3 additions & 0 deletions dev_log/2.2.19/xcwang_202206091632.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
---
feature:
- "支持 AIX 操作系统区分版本 (closed #815)"
Loading

0 comments on commit 1bf2ba2

Please sign in to comment.