From 1bf2ba2ebb72c82c99520c05720a11d7f9bd0cc2 Mon Sep 17 00:00:00 2001 From: xcwang <1366993017@qq.com> Date: Thu, 9 Jun 2022 16:49:23 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20=E6=94=AF=E6=8C=81=20AIX=20=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E7=B3=BB=E7=BB=9F=E5=8C=BA=E5=88=86=E7=89=88=E6=9C=AC?= =?UTF-8?q?=20(closed=20#815)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/collections/agent_new/base.py | 21 +++++- .../collections/agent_new/install.py | 16 +++- apps/backend/exceptions.py | 6 ++ .../collections/agent_new/test_install.py | 7 ++ .../agent_new/test_push_upgrade_package.py | 73 ++++++++++++++++++- apps/node_man/constants.py | 3 +- apps/node_man/handlers/meta.py | 3 +- .../migrations/0023_init_proxy_package.py | 11 +-- .../migrations/0060_host_os_version.py | 20 +++++ apps/node_man/models.py | 1 + dev_log/2.2.19/xcwang_202206091632.yaml | 3 + script_tools/setup_agent.ksh | 67 ++++++++++------- script_tools/upgrade_agent.sh.tpl | 9 ++- 13 files changed, 193 insertions(+), 47 deletions(-) create mode 100644 apps/node_man/migrations/0060_host_os_version.py create mode 100644 dev_log/2.2.19/xcwang_202206091632.yaml diff --git a/apps/backend/components/collections/agent_new/base.py b/apps/backend/components/collections/agent_new/base.py index c6b3f6265..ecd3bf158 100644 --- a/apps/backend/components/collections/agent_new/base.py +++ b/apps/backend/components/collections/agent_new/base.py @@ -10,6 +10,7 @@ """ import abc import random +import re import time from collections import defaultdict from typing import ( @@ -18,6 +19,7 @@ Dict, Iterable, List, + Match, Optional, Set, Tuple, @@ -29,6 +31,7 @@ from django.utils.translation import ugettext_lazy as _ from apps.backend.agent.tools import InstallationTools, batch_gen_commands +from apps.backend.exceptions import OsVersionPackageValidationError from apps.node_man import constants, models from .. import job @@ -102,7 +105,23 @@ def get_agent_upgrade_pkg_name(cls, host: models.Host) -> str: :return: """ package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY] - agent_upgrade_package_name = f"gse_{package_type}-{host.os_type.lower()}-{host.cpu_arch}_upgrade.tgz" + if host.os_version: + major_version_number = None + if host.os_type == constants.OsType.AIX: + major_version_match: Optional[Match] = re.compile(r"^(?P\d+).\d+.*$").search( + host.os_version or "" + ) + major_version_number: Optional[str] = ( + major_version_match.group("version") if major_version_match else "" + ) + if not major_version_number: + raise OsVersionPackageValidationError(os_version=host.os_version, os_type=host.os_type) + agent_upgrade_package_name = ( + f"gse_{package_type}-{host.os_type.lower()}{major_version_number}-{host.cpu_arch}_upgrade.tgz" + ) + else: + agent_upgrade_package_name = f"gse_{package_type}-{host.os_type.lower()}-{host.cpu_arch}_upgrade.tgz" + return agent_upgrade_package_name @staticmethod diff --git a/apps/backend/components/collections/agent_new/install.py b/apps/backend/components/collections/agent_new/install.py index 504a913d5..4078f62ee 100644 --- a/apps/backend/components/collections/agent_new/install.py +++ b/apps/backend/components/collections/agent_new/install.py @@ -515,6 +515,7 @@ def handle_report_data(self, sub_inst_id: int, success_callback_step: str) -> Di REDIS_INST.ltrim(name, 0, -report_data_len - 1) report_data.reverse() cpu_arch = None + os_version = None is_finished = False error_log = "" logs = [] @@ -532,6 +533,9 @@ def handle_report_data(self, sub_inst_id: int, success_callback_step: str) -> Di logs.append(log) if step == "report_cpu_arch": cpu_arch = data["log"] + + if step == "report_os_version": + os_version = data["log"] # 只要匹配到成功返回步骤完成,则认为是执行完成了 if step == success_callback_step and status == "DONE": is_finished = True @@ -540,7 +544,7 @@ def handle_report_data(self, sub_inst_id: int, success_callback_step: str) -> Di self.log_info(sub_inst_ids=sub_inst_id, log_content="\n".join(logs)) if error_log: self.move_insts_to_failed([sub_inst_id], log_content=error_log) - return {"sub_inst_id": sub_inst_id, "is_finished": is_finished, "cpu_arch": cpu_arch} + return {"sub_inst_id": sub_inst_id, "is_finished": is_finished, "cpu_arch": cpu_arch, "os_version": os_version} def _schedule(self, data, parent_data, callback_data=None): """通过轮询redis的方式来处理,避免使用callback的方式频繁调用schedule""" @@ -561,6 +565,7 @@ def _schedule(self, data, parent_data, callback_data=None): results = concurrent.batch_call(func=self.handle_report_data, params_list=params_list) left_scheduling_sub_inst_ids = [] cpu_arch__host_id_map = defaultdict(list) + os_version__host_id_map = defaultdict(list) for result in results: # 对于未完成的实例,记录下来到下一次schedule中继续检查 if not result["is_finished"]: @@ -568,11 +573,20 @@ def _schedule(self, data, parent_data, callback_data=None): # 按CPU架构对主机进行分组 bk_host_id = common_data.sub_inst_id__host_id_map.get(result["sub_inst_id"]) cpu_arch__host_id_map[result["cpu_arch"]].append(bk_host_id) + # 按操作系统版本对主机进行分组 + os_version = result.get("os_version", "") + if os_version is not None: + os_version__host_id_map[os_version].append(bk_host_id) # 批量更新CPU架构 for cpu_arch, bk_host_ids in cpu_arch__host_id_map.items(): if cpu_arch: models.Host.objects.filter(bk_host_id__in=bk_host_ids).update(cpu_arch=cpu_arch) + # 批量更新主机操作系统版本号 + for os_version, bk_host_ids in os_version__host_id_map.items(): + if os_version: + models.Host.objects.filter(bk_host_id__in=bk_host_ids).update(os_version=os_version) + data.outputs.scheduling_sub_inst_ids = left_scheduling_sub_inst_ids if not left_scheduling_sub_inst_ids: self.finish_schedule() diff --git a/apps/backend/exceptions.py b/apps/backend/exceptions.py index 7e7af5902..33bea4707 100644 --- a/apps/backend/exceptions.py +++ b/apps/backend/exceptions.py @@ -66,3 +66,9 @@ class PluginParseError(BackendBaseException): class CreatePackageRecordError(BackendBaseException): MESSAGE = _("归档插件包信息错误") ERROR_CODE = 10 + + +class OsVersionPackageValidationError(BackendBaseException): + MESSAGE = _("操作系统版本安装包校验错误") + MESSAGE_TPL = _("操作系统[{os_type}]不支持 对应版本[{os_version}]的安装包") + ERROR_CODE = 11 diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index 688cbbf4b..225bd2121 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -70,6 +70,13 @@ def init_redis_data(self): "log": "aarch64", "status": "DONE", }, + { + "timestamp": "1580870937", + "level": "INFO", + "step": "report_os_version", + "status": "DONE", + "log": "6.1.1.1", + }, { "timestamp": "1580870937", "level": "INFO", diff --git a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py index 896ced260..7478d6af2 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py @@ -9,19 +9,22 @@ specific language governing permissions and limitations under the License. """ import os.path +from abc import ABC from django.conf import settings from apps.backend.components.collections.agent_new import components +from apps.node_man import constants, models from common.api import JobApi +from pipeline.component_framework.test import ComponentTestCase, ExecuteAssertion -from . import base +from . import base, utils class PushUpgradePackageTestCase(base.JobBaseTestCase): @classmethod def get_default_case_name(cls) -> str: - return "下发升级包成功" + return "下发 Linux 升级包成功" def component_cls(self): return components.PushUpgradePackageComponent @@ -34,3 +37,69 @@ def tearDown(self) -> None: [os.path.join(settings.DOWNLOAD_PATH, "gse_client-linux-x86_64_upgrade.tgz")], ) super().tearDown() + + +class PushAixUpgradePackageSuccessTestCase(base.JobBaseTestCase): + @classmethod + def get_default_case_name(cls) -> str: + return "下发 Aix 升级包成功" + + @classmethod + def setup_obj_factory(cls): + cls.obj_factory.host_os_type_options = [constants.OsType.AIX] + + def component_cls(self): + return components.PushUpgradePackageComponent + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update( + os_version="6.1.0.0.1", cpu_arch=constants.CpuType.powerpc + ) + + def tearDown(self) -> None: + record = self.job_api_mock_client.call_recorder.record + fast_transfer_file_query_params = record[JobApi.fast_transfer_file][0].args[0] + + self.assertEqual( + fast_transfer_file_query_params["file_source_list"][0]["file_list"], + [os.path.join(settings.DOWNLOAD_PATH, "gse_client-aix6-powerpc_upgrade.tgz")], + ) + super().tearDown() + + +class PushAixUpgradePackageFailTestCase(utils.AgentServiceBaseTestCase, ABC): + @classmethod + def get_default_case_name(cls) -> str: + return "下发 Aix 升级包失败" + + @classmethod + def setup_obj_factory(cls): + cls.obj_factory.host_os_type_options = [constants.OsType.AIX] + + def component_cls(self): + return components.PushUpgradePackageComponent + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update( + os_version=constants.CpuType.x86_64, cpu_arch=constants.CpuType.powerpc + ) + + def cases(self): + return [ + ComponentTestCase( + name=self.get_default_case_name(), + inputs=self.common_inputs, + parent_data={}, + execute_assertion=ExecuteAssertion( + success=False, + outputs={}, + ), + schedule_assertion=None, + execute_call_assertion=None, + patchers=[], + ) + ] diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 1ee2b6b46..ca275509f 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -819,9 +819,10 @@ class PolicyRollBackType: GSE_CLIENT_PACKAGES: List[str] = [ "gse_client-windows-x86.tgz", "gse_client-windows-x86_64.tgz", - "gse_client-aix-powerpc.tgz", "gse_client-linux-x86.tgz", "gse_client-linux-x86_64.tgz", + "gse_client-aix6-powerpc.tgz", + "gse_client-aix7-powerpc.tgz", ] FILES_TO_PUSH_TO_PROXY = [ diff --git a/apps/node_man/handlers/meta.py b/apps/node_man/handlers/meta.py index b4945c8c2..93d32341b 100644 --- a/apps/node_man/handlers/meta.py +++ b/apps/node_man/handlers/meta.py @@ -353,8 +353,7 @@ def fetch_plugin_host_condition(self): os_dict = {"name": _("操作系统"), "id": "os_type", "children": []} for os_type in constants.OS_TUPLE: - special_os_type = [constants.OsType.AIX, constants.OsType.SOLARIS] - if os_type in special_os_type and settings.BKAPP_RUN_ENV == constants.BkappRunEnvType.CE.value: + if os_type == constants.OsType.SOLARIS and settings.BKAPP_RUN_ENV == constants.BkappRunEnvType.CE.value: continue os_dict["children"].append({"id": os_type, "name": constants.OS_CHN.get(os_type, os_type)}) diff --git a/apps/node_man/migrations/0023_init_proxy_package.py b/apps/node_man/migrations/0023_init_proxy_package.py index 3b3cd4929..2858b91be 100644 --- a/apps/node_man/migrations/0023_init_proxy_package.py +++ b/apps/node_man/migrations/0023_init_proxy_package.py @@ -15,18 +15,9 @@ def init_proxy_package(apps, schema_editor): - packages = [ - "gse_client-windows-x86.tgz", - "gse_client-windows-x86_64.tgz", - "gse_client-linux-x86.tgz", - "gse_client-linux-x86_64.tgz", - ] - - if settings.BKAPP_RUN_ENV != constants.BkappRunEnvType.CE.value: - packages.append("gse_client-aix-powerpc.tgz") AccessPoint = apps.get_model("node_man", "AccessPoint") - AccessPoint.objects.update(proxy_package=packages) + AccessPoint.objects.update(proxy_package=constants.GSE_CLIENT_PACKAGES) class Migration(migrations.Migration): diff --git a/apps/node_man/migrations/0060_host_os_version.py b/apps/node_man/migrations/0060_host_os_version.py new file mode 100644 index 000000000..143966f42 --- /dev/null +++ b/apps/node_man/migrations/0060_host_os_version.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.4 on 2022-06-09 08:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0059_auto_20220415_2150"), + ] + + operations = [ + migrations.AddField( + model_name="host", + name="os_version", + field=models.CharField( + blank=True, default="", max_length=32, null=True, db_index=True, verbose_name="操作系统版本" + ), + ), + ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index e08ab772c..2e6a2f681 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -322,6 +322,7 @@ class Host(models.Model): cpu_arch = models.CharField( _("操作系统"), max_length=16, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True ) + os_version = models.CharField(_("操作系统版本"), max_length=32, blank=True, null=True, db_index=True, default="") node_type = models.CharField(_("节点类型"), max_length=16, choices=constants.NODE_CHOICES, db_index=True) node_from = models.CharField(_("节点来源"), max_length=45, choices=constants.NODE_FROM_CHOICES, default="NODE_MAN") is_manual = models.BooleanField(_("是否手动安装"), default=False) diff --git a/dev_log/2.2.19/xcwang_202206091632.yaml b/dev_log/2.2.19/xcwang_202206091632.yaml new file mode 100644 index 000000000..b706560e1 --- /dev/null +++ b/dev_log/2.2.19/xcwang_202206091632.yaml @@ -0,0 +1,3 @@ +--- +feature: + - "支持 AIX 操作系统区分版本 (closed #815)" diff --git a/script_tools/setup_agent.ksh b/script_tools/setup_agent.ksh index ac65eba00..6258b8cbe 100644 --- a/script_tools/setup_agent.ksh +++ b/script_tools/setup_agent.ksh @@ -4,12 +4,14 @@ # DEFAULT DEFINITION NODE_TYPE=agent -PKG_NAME=gse_client-aix-powerpc.tgz +# PKG_NAME=gse_client-aix{version}-powerpc.tgz +PKG_NAME="" +OS_VERSION="" GSE_AGENT_RUN_DIR=/var/run/gse GSE_AGENT_DATA_DIR=/var/lib/gse GSE_AGENT_LOG_DIR=/var/log/gse -BACKUP_CONFIG_FILES=("procinfo.json") +set -A BACKUP_CONFIG_FILES "procinfo.json" # 收到如下信号或者exit退出时,执行清理逻辑 #trap quit 1 2 3 4 5 6 7 8 10 11 12 13 14 15 @@ -56,6 +58,20 @@ warn () { local L=WARN D; D="$(date +%F\ %T)"; echo "$D $L $*" | tee -a "$LOG_F err () { local L=ERROR D; D="$(date +%F\ %T)"; echo "$D $L $*" | tee -a "$LOG_FILE"; report_step_status "$D" "$L" "$@"; return 1; } fail () { local L=ERROR D; D="$(date +%F\ %T)"; echo "$D $L $*" | tee -a "$LOG_FILE"; report_step_status "$D" "$L" "$@"; exit 1; } +divide_version () { + result=$(oslevel) + log report_os_version DONE "${result}" + + if echo "${result}" | egrep -q '^6.*' ; then + OS_VERSION=6 + elif echo "${result}" | egrep -q '^7.*' ; then + OS_VERSION=7 + else + fail check_env FAILED "unsupported OS version: ${result}" + fi + PKG_NAME=gse_client-aix"${OS_VERSION}"-powerpc.tgz +} + get_cpu_arch () { local cmd=$1 CPU_ARCH=$($cmd) @@ -272,8 +288,9 @@ pre_view () { log PREVIEW - "normalized agent:" log PREVIEW - " setup path: "${AGENT_SETUP_PATH}"/bin/gse_agent" log PREVIEW - " process:" - #lsof -a -d txt -c agentWorker -c gseMaster -a -n "$AGENT_SETUP_PATH"/bin/gse_agent - log PREVIEW - " gsecmdline: $(is_gsecmdline_ok && echo OK || echo NO)" + #lsof -a -d txt -c agentWorker -c gseMaster -a -n "$AGENT_SETUP_PATH"/bin/gse_agent + # Aix 操作系统当前没有提供 gsecmdline 工具,暂不检查 + # log PREVIEW - " gsecmdline: $(is_gsecmdline_ok && echo OK || echo NO)" fi } @@ -410,7 +427,7 @@ remove_agent () { get_config () { local filename http_status - set -A config agent.conf bscp.yaml plugin_info.json + set -A config agent.conf log get_config - "request $NODE_TYPE config file(s)" @@ -452,7 +469,7 @@ setup_agent () { cd "$AGENT_SETUP_PATH/.." gunzip -dc "$TMP_DIR/$PKG_NAME" | tar xf - - agent_name=gse_agent-aix`get_aix_version` + agent_name=gse_agent if [ -f agent/bin/"$agent_name" ]; then cp -f agent/bin/"$agent_name" agent/bin/gse_agent @@ -461,14 +478,14 @@ setup_agent () { fi # update gsecmdline under /bin - cp -fp plugins/bin/gsecmdline /bin/ - chmod 775 /bin/gsecmdline + # cp -fp plugins/bin/gsecmdline /bin/ + # chmod 775 /bin/gsecmdline # setup config file get_config recovery_config_file - set -A config agent.conf bscp.yaml plugin_info.json + set -A config agent.conf for f in "${config[@]}"; do if [[ -f $TMP_DIR/$f ]]; then cp -fp "$TMP_DIR/${f}" agent/etc/${f} @@ -485,24 +502,12 @@ setup_agent () { log setup_agent DONE "agent setup succeded" } -start_basic_gse_plugin () { - log start_plugin START "start gse plugin: aixbeat, " - cd "$AGENT_SETUP_PATH/../plugins/bin" || fail start_plugin FAILED "change directory to $AGENT_SETUP_PATH/../plugins/bin failed" - version=$(get_aix_version) - if [ "$version" -eq 7 ];then - - if [[ -x ./aixbeat ]]; then - ./stop.sh aixbeat - ./start.sh aixbeat || fail start_plugin FAILED "aixbeat start failed." - fi - fi - - log start_plugin DONE "gse plugin 'aixbeat start done." -} - download_pkg () { local f http_status + # 区分下载版本 + divide_version + log download_pkg START "download gse agent package from $DOWNLOAD_URL/$PKG_NAME)." cd "$TMP_DIR" && rm -f "$PKG_NAME" "agent.conf.$LAN_ETH_IP" @@ -661,7 +666,18 @@ check_env () { [ "$CLOUD_ID" != "0" ] && node_type=pagent validate_setup_path - check_polices_${node_type}_to_upstream + # check_polices_${node_type}_to_upstream + # 目前Linux端口探测有三种方式 + + # /proc 系统发送socket + # telnet + # python -c "import socket;client_socket=socket.socket();client_socket.connect(('$ip', $_port))" + + # 但是目前这几种方式在aix系统上都有问题, 所以暂时关闭探测逻辑 + # AIX 没有 /proc文件系统 + # AIX在telnet时出现长链接无断开,无返回 + # AIX系统本身无python解释器 + check_disk_space check_dir_permission check_pkgtool @@ -740,7 +756,6 @@ for step in check_env \ remove_agent \ remove_proxy_if_exists \ setup_agent \ - start_basic_gse_plugin \ setup_startup_scripts \ setup_crontab \ check_deploy_result; do diff --git a/script_tools/upgrade_agent.sh.tpl b/script_tools/upgrade_agent.sh.tpl index 2258e782e..8b00fc47b 100644 --- a/script_tools/upgrade_agent.sh.tpl +++ b/script_tools/upgrade_agent.sh.tpl @@ -7,12 +7,13 @@ get_cpu_arch () {{ local cmd=$1 CPU_ARCH=$($cmd) CPU_ARCH=$(echo $CPU_ARCH | tr 'A-Z' 'a-z') - if [[ "$CPU_ARCH" =~ "x86_64" ]]; then + # compatible ksh regex syntax + if [[ echo "$CPU_ARCH" | egrep -q "x86_64" ]]; then return 0 - elif [[ "$CPU_ARCH" =~ "x86" || "$CPU_ARCH" =~ ^i[3456]86 ]]; then + elif [[ echo "$CPU_ARCH" | egrep -q "x86" || echo "$CPU_ARCH" | egrep -q '^i[3456]86' ]]; then CPU_ARCH="x86" return 0 - elif [[ "$CPU_ARCH" =~ "aarch" ]]; then + elif [[ echo "$CPU_ARCH" | egrep -q "aarch" ]]; then return 0 else return 1 @@ -112,7 +113,7 @@ remove_crontab () {{ fi }} -if grep "$CPU_ARCH" <<<{package_name} > /dev/null; then +if echo {package_name} | grep "$CPU_ARCH" > /dev/null; then echo "pkg:{package_name} and arch:$CPU_ARCH is ok" else echo "pkg:{package_name} and arch:$CPU_ARCH is bad"