From 8e4aad641f7e5b9a9017afc2023d82a132285c64 Mon Sep 17 00:00:00 2001 From: crayon <873217631@qq.com> Date: Tue, 19 Oct 2021 17:39:55 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20PaaS=E5=AE=B9=E5=99=A8=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E9=80=82=E9=85=8D=20(closed=20#10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +- app_desc.yaml | 131 ++++++++++++ .../management/commands/copy_file_to_nginx.py | 42 +++- .../commands/test_copy_file_to_nginx.py | 2 + apps/node_man/handlers/debug.py | 4 +- apps/node_man/handlers/iam.py | 4 +- apps/node_man/iam_provider.py | 2 +- apps/node_man/periodic_tasks/__init__.py | 4 + .../periodic_tasks/gse_svr_discovery.py | 109 ++++++++++ apps/node_man/tests/test_debug.py | 8 +- apps/node_man/urls.py | 4 +- bin/post_compile | 1 + blueking/component/conf.py | 3 +- common/api/domains.py | 14 +- common/api/modules/bk_node.py | 8 + config/__init__.py | 13 +- config/default.py | 194 ++++++++++++------ config/prod.py | 4 - config/stag.py | 4 - dev_log/2.1.354/crayon_202110191739.yaml | 3 + env/__init__.py | 12 ++ env/constants.py | 25 +++ env/paas_version_diff/__init__.py | 46 +++++ env/paas_version_diff/paas_v2.py | 47 +++++ env/paas_version_diff/paas_v3.py | 15 ++ env/paas_version_diff/paas_v3_container.py | 70 +++++++ pack.sh | 1 + requirements.txt | 2 + .../workflows/pre-commit/generate_dev_log.py | 2 +- .../templates/nodeman#bin#environ.sh | 36 ++-- 30 files changed, 700 insertions(+), 114 deletions(-) create mode 100644 app_desc.yaml create mode 100644 apps/node_man/periodic_tasks/gse_svr_discovery.py create mode 100644 dev_log/2.1.354/crayon_202110191739.yaml create mode 100644 env/__init__.py create mode 100644 env/constants.py create mode 100644 env/paas_version_diff/__init__.py create mode 100644 env/paas_version_diff/paas_v2.py create mode 100644 env/paas_version_diff/paas_v3.py create mode 100644 env/paas_version_diff/paas_v3_container.py diff --git a/.gitignore b/.gitignore index 65092b264..714f743e7 100644 --- a/.gitignore +++ b/.gitignore @@ -91,7 +91,6 @@ celerybeat-schedule # Environments .env .venv -env/ venv/ ENV/ @@ -177,3 +176,6 @@ tests/postman/bkman # Thumbnails ._* + +.helm +!env/ diff --git a/app_desc.yaml b/app_desc.yaml new file mode 100644 index 000000000..7b0aa1f8f --- /dev/null +++ b/app_desc.yaml @@ -0,0 +1,131 @@ +spec_version: 2 +app_version: "{{APP_VERSION}}" +app: + region: default + bk_app_code: "bk_nodeman" + bk_app_name: "节点管理" + market: + category: 运维工具 + introduction: 通过节点管理，可以对蓝鲸体系中的gse agent进行管理，包括状态查询、版本更新、配置管理、健康检查、进程管理等。 + display_options: + width: 1300 + height: 720 + is_win_maximize: True + open_mode: "new_tab" +modules: + default: + is_default: True + source_dir: src + language: Python + services: + - name: mysql + - name: bkrepo + env_variables: + - key: BKAPP_IS_V3_CONTAINER + value: "True" + description: 是否运行在V3容器版本 + - key: PIP_VERSION + value: "20.2.3" + description: 固化pip版本 + - key: STORAGE_TYPE + value: "BLUEKING_ARTIFACTORY" + description: 存储类型 + - key: BKAPP_RUN_ENV + value: "ce" + description: 运行环境，app_desc 暂时只在社区版使用，为了简化部署在此声明配置 + - key: GSE_ENABLE_SVR_DISCOVERY + value: "True" + description: 是否启用 gse svr 服务发现，启用后，默认接入点会通过zk的方式，自动更新gse svr信息 + + svc_discovery: + bk_saas: + - bk_app_code: "bk_iam" + - bk_app_code: "bk_nodeman" + module_name: "backend" + - bk_app_code: "bk_nodeman" + module_name: "default" + processes: + web: + command: gunicorn wsgi -w 4 -b :$PORT --access-logfile - --error-logfile - --access-logformat '[%(h)s] %({request_id}i)s %(u)s %(t)s "%(r)s" %(s)s %(D)s %(b)s "%(f)s" "%(a)s"' + plan: 4C2G5R + replicas: 5 + + backend: + is_default: False + source_dir: src + language: Python + services: + - name: rabbitmq + - name: redis + - name: bkrepo + shared_from: default + - name: mysql + shared_from: default + env_variables: + - key: BKAPP_IS_V3_CONTAINER + value: "True" + description: 是否运行在V3容器版本 + - key: REDIS_MODE + value: "standalone" + description: 后台配置的Redis模式 + - key: BACKEND_CONFIG + value: "True" + description: 是否启用后台配置，用于同一份代码区分SaaS和后台的差异化配置 + - key: PIP_VERSION + value: "20.2.3" + description: 固化pip版本 + - key: BKAPP_IS_PAAS_DEPLOY + value: "True" + description: 是否基于PaaS部署 + - key: STORAGE_TYPE + value: "BLUEKING_ARTIFACTORY" + description: 存储类型 + - key: BKAPP_RUN_ENV + value: "ce" + description: 运行环境，app_desc 暂时只在社区版使用，为了简化部署在此声明配置 + - key: GSE_ENABLE_SVR_DISCOVERY + value: "True" + description: 是否启用 gse svr 服务发现，启用后，默认接入点会通过zk的方式，自动更新gse svr信息 + + svc_discovery: + bk_saas: + - bk_app_code: "bk_iam" + - bk_app_code: "bk_nodeman" + module_name: "backend" + - bk_app_code: "bk_nodeman" + module_name: "default" + + + processes: + backend-web: + command: gunicorn --timeout 300 -w 8 -b :$PORT -k gevent wsgi:application --access-logfile - --error-logfile - --access-logformat '[%(h)s] %({request_id}i)s %(u)s %(t)s "%(r)s" %(s)s %(D)s %(b)s "%(f)s" "%(a)s"' + plan: 4C2G5R + replicas: 5 + celery-beat: + command: celery -A apps.backend beat -l info + plan: 4C2G5R + replicas: 1 + dworker: + command: celery -A apps.backend worker -Q default --autoscale=8,2 --maxtasksperchild=50 -O fair --time-limit=1800 + plan: 4C2G5R + replicas: 5 + bworker: + command: celery -A apps.backend worker -Q backend --autoscale=16,2 --maxtasksperchild=50 -O fair --time-limit=1800 + plan: 4C2G5R + replicas: 5 + baworker: + command: celery -A apps.backend worker -Q backend_additional_task -c 10 -O fair --time-limit=1800 --maxtasksperchild=50 + plan: 4C2G5R + replicas: 5 + pworker: + command: celery -A apps.backend worker -Q pipeline,pipeline_priority -n pipeline_worker@%h --maxtasksperchild=50 --autoscale=16,2 -O fair --time-limit=1800 + plan: 4C2G5R + replicas: 5 + psworker: + command: celery -A apps.backend worker -Q service_schedule,service_schedule_priority -n schedule_worker@%h --maxtasksperchild=50 -c 50 -P eventlet -O fair --time-limit=1800 + plan: 4C2G5R + replicas: 5 + paworker: + command: celery -A apps.backend worker -Q pipeline_additional_task,pipeline_additional_task_priority -n common_worker@%h -l info --autoscale=16,2 --maxtasksperchild=50 -O fair --time-limit=1800 + plan: 4C2G5R + replicas: 5 diff --git a/apps/backend/management/commands/copy_file_to_nginx.py b/apps/backend/management/commands/copy_file_to_nginx.py index c82844d41..735ed3c85 100644 --- a/apps/backend/management/commands/copy_file_to_nginx.py +++ b/apps/backend/management/commands/copy_file_to_nginx.py @@ -16,10 +16,15 @@ from django.conf import settings from django.core.management.base import BaseCommand +from apps.core.files.storage import constants as core_files_constants from apps.core.files.storage import get_storage from apps.node_man.models import AccessPoint from apps.utils import files +from . import utils + +log_and_print = utils.get_log_and_print("copy_file_to_nginx(storage)") + def copy_dir_files_to_storage( source_dir_path: str, target_dir_paths: Iterable[str], ignored_dir_names: Optional[List[str]] = None @@ -31,6 +36,12 @@ def copy_dir_files_to_storage( :param ignored_dir_names: 忽略的目录名称 :return: """ + + log_and_print( + f"source_dir_path -> {source_dir_path} \n target_dir_path -> {target_dir_paths} \n " + f"ignored_dir_names -> {ignored_dir_names}" + ) + storage = get_storage(file_overwrite=True) source_file_paths = files.fetch_file_paths_from_dir(dir_path=source_dir_path, ignored_dir_names=ignored_dir_names) @@ -43,16 +54,41 @@ def copy_dir_files_to_storage( storage.save(name=target_file_path, content=target_file_fs) target_file_fs.seek(0) + # 如果使用了蓝鲸制品库，更新默认接入点的下载地址 + if storage.storage_type == core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value: + default_ap = AccessPoint.objects.all().first() + default_save_path = default_ap.nginx_path or settings.DOWNLOAD_PATH + if default_save_path.startswith("/"): + default_save_path = default_save_path[1:] + package_download_url = os.path.join( + storage.endpoint_url, "generic", storage.project_id, storage.bucket, default_save_path + ) + log_and_print(f"storage_type -> {storage.storage_type}, init package_download_url -> {package_download_url}") + + default_ap.package_inner_url = package_download_url + default_ap.package_outer_url = package_download_url + default_ap.save() + class Command(BaseCommand): def handle(self, *args, **options): """ 拷贝scripts下的文件到nginx download下 """ + + if not settings.BK_BACKEND_CONFIG: + log_and_print("command only work on settings.BK_BACKEND_CONFIG == True") + return + # 接入点配置的nginx路径 - nginx_paths = {ap.nginx_path for ap in AccessPoint.objects.all() if ap.nginx_path} + target_dir_paths = {ap.nginx_path for ap in AccessPoint.objects.all() if ap.nginx_path} # 默认nginx路径 - nginx_paths.add(settings.DOWNLOAD_PATH) + target_dir_paths.add(settings.DOWNLOAD_PATH) + copy_dir_files_to_storage( - source_dir_path=settings.BK_SCRIPTS_PATH, target_dir_paths=nginx_paths, ignored_dir_names=["__pycache__"] + source_dir_path=settings.BK_SCRIPTS_PATH, + target_dir_paths=target_dir_paths, + ignored_dir_names=["__pycache__"], ) + + log_and_print("success.") diff --git a/apps/backend/tests/management/commands/test_copy_file_to_nginx.py b/apps/backend/tests/management/commands/test_copy_file_to_nginx.py index 3dbc6b6c3..e9c872088 100644 --- a/apps/backend/tests/management/commands/test_copy_file_to_nginx.py +++ b/apps/backend/tests/management/commands/test_copy_file_to_nginx.py @@ -30,6 +30,7 @@ class CopyFileToNginxTestCase(CustomBaseTestCase): settings: { "BK_SCRIPTS_PATH": BK_SCRIPTS_PATH, "STORAGE_TYPE": core_files_constants.StorageType.FILE_SYSTEM.value, + "BK_BACKEND_CONFIG": True, }, } @@ -69,6 +70,7 @@ class BkRepoCopyFileToNginxTestCase(CopyFileToNginxTestCase): "BKREPO_ENDPOINT_URL": "http://127.0.0.1", "BK_SCRIPTS_PATH": BK_SCRIPTS_PATH, "STORAGE_TYPE": core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value, + "BK_BACKEND_CONFIG": True, }, } diff --git a/apps/node_man/handlers/debug.py b/apps/node_man/handlers/debug.py index bc7d1d3b0..84a7d98c7 100644 --- a/apps/node_man/handlers/debug.py +++ b/apps/node_man/handlers/debug.py @@ -93,7 +93,7 @@ def subscription_details(self, subscription_id): "task_actions": task.actions, "is_auto_trigger": task.is_auto_trigger, "create_time": task.create_time, - "details": f"{settings.BK_NODEMAN_URL}/api/debug/fetch_task_details?" + "details": f"{settings.BK_NODEMAN_HOST}/api/debug/fetch_task_details?" f"subscription_id={subscription_id}&task_id={task.id}", } @@ -171,7 +171,7 @@ def fetch_subscriptions_by_host(self, bk_host_id): result.append( { "subscription_id": record.subscription_id, - "subscription_detail": f"{settings.BK_NODEMAN_URL}/api/debug/fetch_subscription_details?" + "subscription_detail": f"{settings.BK_NODEMAN_HOST}/api/debug/fetch_subscription_details?" f"subscription_id={record.subscription_id}", } ) diff --git a/apps/node_man/handlers/iam.py b/apps/node_man/handlers/iam.py index e4373d6bd..5fa2127af 100644 --- a/apps/node_man/handlers/iam.py +++ b/apps/node_man/handlers/iam.py @@ -40,7 +40,7 @@ class IamHandler(APIModel): ] if settings.USE_IAM: - _iam = IAM(settings.APP_CODE, settings.SECRET_KEY, settings.BK_IAM_HOST, settings.BK_IAM_ESB_PAAS_HOST) + _iam = IAM(settings.APP_CODE, settings.SECRET_KEY, settings.BK_IAM_INNER_HOST, settings.BK_PAAS_INNER_HOST) else: _iam = object @@ -322,7 +322,7 @@ def fetch_redirect_url(self, params, username): apply_info["related_resource_types"][0]["instances"] = instances data["actions"].append(apply_info) ok, message, result = IamHandler._iam._client.get_apply_url(bk_token="", bk_username=username, data=data) - return result or settings.BK_IAM_URL + return result or settings.BK_IAM_SAAS_HOST @staticmethod def return_resource_instance_creator(resource_id, instance_id, instance_name, creator): diff --git a/apps/node_man/iam_provider.py b/apps/node_man/iam_provider.py index 250ca8bb7..c9ef5d888 100644 --- a/apps/node_man/iam_provider.py +++ b/apps/node_man/iam_provider.py @@ -575,7 +575,7 @@ class IamRegister(object): ] def __init__(self): - self._iam = IAM(settings.APP_CODE, settings.SECRET_KEY, settings.BK_IAM_HOST, settings.BK_PAAS_INNER_HOST) + self._iam = IAM(settings.APP_CODE, settings.SECRET_KEY, settings.BK_IAM_INNER_HOST, settings.BK_PAAS_INNER_HOST) def register_system(self): # ***需要将placeholder改为内网访问地址*** diff --git a/apps/node_man/periodic_tasks/__init__.py b/apps/node_man/periodic_tasks/__init__.py index 6f1ca143f..d46e8f07d 100644 --- a/apps/node_man/periodic_tasks/__init__.py +++ b/apps/node_man/periodic_tasks/__init__.py @@ -19,5 +19,9 @@ from .sync_proc_status_task import sync_proc_status_task # noqa from .update_proxy_file import update_proxy_file # noqa +# 是否启用 gse svr 服务发现，启用后，默认接入点会通过zk的方式，自动更新gse svr信息 +if getattr(settings, "GSE_ENABLE_SVR_DISCOVERY", False): + from .gse_svr_discovery import gse_svr_discovery # noqa + if getattr(settings, "CONFIG_POLICY_BY_TENCENT_VPC", False): from .configuration_policy import configuration_policy # noqa diff --git a/apps/node_man/periodic_tasks/gse_svr_discovery.py b/apps/node_man/periodic_tasks/gse_svr_discovery.py new file mode 100644 index 000000000..9c57d229c --- /dev/null +++ b/apps/node_man/periodic_tasks/gse_svr_discovery.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from telnetlib import Telnet +from typing import List, Optional, Tuple + +from celery.schedules import crontab +from celery.task import periodic_task +from django.conf import settings +from kazoo.client import KazooClient +from kazoo.exceptions import NoAuthError, NoNodeError + +from apps.node_man import models +from common.log import logger + + +def check_ip_ports_reachable(host: str, ports: List[str]) -> bool: + + for port in ports: + try: + with Telnet(host=host, port=port, timeout=2): + pass + except (ConnectionRefusedError, TimeoutError): + logger.error(f"host -> {host}, port -> {port} not reachable.") + return False + return True + + +class ZkSafeClient: + + zk_client: Optional[KazooClient] + + def __init__(self, hosts: str, auth_data: List[Tuple[str, str]], **kwargs): + self.zk_client = KazooClient(hosts=hosts, auth_data=auth_data, **kwargs) + + def __enter__(self) -> KazooClient: + self.zk_client.start() + return self.zk_client + + def __exit__(self, exc_type, exc_val, exc_tb): + self.zk_client.stop() + + +@periodic_task( + queue="default", + options={"queue": "default"}, + run_every=crontab(hour="*", minute="*/1", day_of_week="*", day_of_month="*", month_of_year="*"), +) +def gse_svr_discovery(): + if not settings.GSE_ENABLE_SVR_DISCOVERY: + logger.info(f"GSE_ENABLE_SVR_DISCOVERY == {settings.GSE_ENABLE_SVR_DISCOVERY}, skip") + return + ap = models.AccessPoint.objects.all().first() + if not ap: + logger.error("not access point, skip") + return + + logger.info(f"gse_svr_discovery: access_point -> {ap.name}") + + zk_node_path__ap_field_map = { + "/gse/config/server/dataserver/all": "dataserver", + "/gse/config/server/taskserver/all": "taskserver", + "/gse/config/server/btfiles/all": "btfileserver", + } + zk_node_path__check_ports_map = { + "/gse/config/server/dataserver/all": [ap.port_config["data_port"]], + "/gse/config/server/taskserver/all": [ap.port_config["io_port"]], + "/gse/config/server/btfiles/all": [ap.port_config["file_svr_port"]], + } + + is_change = False + auth_data = None + zk_hosts_str = ",".join(f"{zk_host['zk_ip']}:{zk_host['zk_port']}" for zk_host in ap.zk_hosts) + if ap.zk_account and ap.zk_password: + auth_data = [("digest", f"{ap.zk_account}:{ap.zk_password}")] + + with ZkSafeClient(hosts=zk_hosts_str, auth_data=auth_data) as zk_client: + for zk_node_path, ap_field in zk_node_path__ap_field_map.items(): + try: + svr_ips = zk_client.get_children(path=zk_node_path) + # 过滤不可达的ip + svr_ips = [ + svr_ip + for svr_ip in svr_ips + if check_ip_ports_reachable(svr_ip, zk_node_path__check_ports_map[zk_node_path]) + ] + + except NoNodeError: + logger.error(f"zk_node_path -> {zk_node_path} not exist") + except NoAuthError: + logger.error(f"zk_node_path -> {zk_node_path} no auth, please check zk account.") + except Exception as e: + logger.exception(f"failed to get zk_node_path -> {zk_node_path}, err: {e}") + else: + if not svr_ips: + logger.info(f"zk_node_path -> {zk_node_path} get empty ip list, skip.") + continue + logger.info(f"zk_node_path -> {zk_node_path}, svr_ips -> {svr_ips}") + setattr(ap, ap_field, [{"inner_ip": svr_ip, "outer_ip": svr_ip} for svr_ip in svr_ips]) + is_change = True + if is_change: + ap.save() diff --git a/apps/node_man/tests/test_debug.py b/apps/node_man/tests/test_debug.py index 0906d2068..7d37796b1 100644 --- a/apps/node_man/tests/test_debug.py +++ b/apps/node_man/tests/test_debug.py @@ -84,8 +84,8 @@ def test_subscription_details(self): self.assertEqual(result[0]["task_id"], self.subscription_task_obj.id) self.assertEqual( result[0]["details"], - f"{settings.BK_NODEMAN_URL}/api/debug/fetch_task_details?" - f"subscription_id={self.subscription_obj.id}&task_id={self.subscription_task_obj.id}" + f"{settings.BK_NODEMAN_HOST}/api/debug/fetch_task_details?" + f"subscription_id={self.subscription_obj.id}&task_id={self.subscription_task_obj.id}", ) def test_fetch_hosts_by_subscription(self): @@ -99,8 +99,8 @@ def test_fetch_subscriptions_by_host(self): [ { "subscription_id": self.subscription_obj.id, - "subscription_detail": f"{settings.BK_NODEMAN_URL}/api/debug/fetch_subscription_details?" - f"subscription_id={self.subscription_obj.id}", + "subscription_detail": f"{settings.BK_NODEMAN_HOST}/api/debug/fetch_subscription_details?" + f"subscription_id={self.subscription_obj.id}", } ], ) diff --git a/apps/node_man/urls.py b/apps/node_man/urls.py index 4572a83f2..e8bc9b54a 100644 --- a/apps/node_man/urls.py +++ b/apps/node_man/urls.py @@ -30,20 +30,20 @@ cmdb, debug, host, + install_channel, job, meta, permission, plugin, policy, tjj, - install_channel, ) from apps.node_man.views.healthz import HealthzViewSet from apps.node_man.views.host_v2 import HostV2ViewSet from apps.node_man.views.plugin import GsePluginViewSet from apps.node_man.views.plugin_v2 import PluginV2ViewSet -iam = IAM(settings.APP_CODE, settings.SECRET_KEY, settings.BK_IAM_HOST, settings.BK_PAAS_INNER_HOST) +iam = IAM(settings.APP_CODE, settings.SECRET_KEY, settings.BK_IAM_INNER_HOST, settings.BK_PAAS_INNER_HOST) router = routers.DefaultRouter(trailing_slash=True) diff --git a/bin/post_compile b/bin/post_compile index 10a58f3aa..2b2894fdf 100644 --- a/bin/post_compile +++ b/bin/post_compile @@ -1,3 +1,4 @@ #!/bin/bash python manage.py createcachetable django_cache python manage.py migrate +python manage.py copy_file_to_nginx diff --git a/blueking/component/conf.py b/blueking/component/conf.py index ff211b505..070eaea17 100644 --- a/blueking/component/conf.py +++ b/blueking/component/conf.py @@ -17,8 +17,9 @@ APP_CODE = settings.APP_ID SECRET_KEY = settings.APP_TOKEN - COMPONENT_SYSTEM_HOST = getattr(settings, "BK_PAAS_INNER_HOST", settings.BK_PAAS_HOST) + COMPONENT_SYSTEM_HOST = settings.BK_COMPONENT_API_URL DEFAULT_BK_API_VER = getattr(settings, "DEFAULT_BK_API_VER", "v2") + except Exception: APP_CODE = "" SECRET_KEY = "" diff --git a/common/api/domains.py b/common/api/domains.py index 6d6f92a93..488823029 100644 --- a/common/api/domains.py +++ b/common/api/domains.py @@ -12,13 +12,23 @@ from django.conf import settings -BK_PAAS_HOST = settings.BK_PAAS_INNER_HOST or settings.BK_PAAS_HOST or "" ESB_PREFIX_V2 = os.getenv("ESB_PREFIX_V2") or "/api/c/compapi/v2/" def gen_api_root(api_gw_env_key: str, suffix: str) -> str: """生成API根路径，首先从环境变量获取，若环境变量没有，则按默认规则拼接""" - return os.getenv(api_gw_env_key) or f"{BK_PAAS_HOST}/{ESB_PREFIX_V2}/{suffix}/" + api_gw_env_val = os.getenv(api_gw_env_key) + if api_gw_env_val: + return api_gw_env_val + + api_root = f"{settings.BK_COMPONENT_API_URL}/{ESB_PREFIX_V2}/{suffix}/" + if api_root.startswith("http://"): + api_root = api_root[:7] + api_root[7:].replace("//", "/") + elif api_root.startswith("https://"): + api_root = api_root[:8] + api_root[8:].replace("//", "/") + else: + api_root = api_root.replace("//", "/") + return api_root # 蓝鲸平台模块域名 diff --git a/common/api/modules/bk_node.py b/common/api/modules/bk_node.py index ac47a8b85..533c9bc09 100644 --- a/common/api/modules/bk_node.py +++ b/common/api/modules/bk_node.py @@ -20,6 +20,14 @@ class _BKNodeApi(object): def __init__(self): + self.metric_list = DataAPI( + method="POST", + url=BK_NODE_APIGATEWAY_ROOT + "backend/package/upload_cos/", + module=self.MODULE, + description=u"上传插件", + before_request=add_esb_info_before_request, + ) + self.create_subscription = DataAPI( method="POST", url=BK_NODE_APIGATEWAY_ROOT + "backend/api/subscription/create/", diff --git a/config/__init__.py b/config/__init__.py index cb70df7d4..ef5ae7674 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -11,7 +11,7 @@ import os -__all__ = ["celery_app", "ENVIRONMENT", "RUN_VER", "APP_CODE", "SECRET_KEY", "BK_URL", "BASE_DIR"] +__all__ = ["celery_app", "RUN_VER", "APP_CODE", "SECRET_KEY", "BK_URL", "BASE_DIR"] # This will make sure the app is always imported when # Django starts so that shared_task will use this app. @@ -28,16 +28,11 @@ def get_env_or_raise(key): return value -# V3判断环境的环境变量为BKPAAS_ENVIRONMENT -if "BKPAAS_ENVIRONMENT" in os.environ: - ENVIRONMENT = os.getenv("BKPAAS_ENVIRONMENT", "dev") -# V2判断环境的环境变量为BK_ENV -else: - PAAS_V2_ENVIRONMENT = os.environ.get("BK_ENV", "development") - ENVIRONMENT = {"development": "dev", "testing": "stag", "production": "prod"}.get(PAAS_V2_ENVIRONMENT) - # SaaS运行版本，如非必要请勿修改 RUN_VER = os.environ.get("BKPAAS_ENGINE_REGION", "open") +# 兼容 V3 取值差异 +if RUN_VER == "default": + RUN_VER = "open" APP_ID = APP_CODE = os.environ.get("APP_ID", "bk_nodeman") APP_TOKEN = SECRET_KEY = os.environ.get("APP_TOKEN", "") diff --git a/config/default.py b/config/default.py index eeb832fd5..7810dac7e 100644 --- a/config/default.py +++ b/config/default.py @@ -9,13 +9,15 @@ specific language governing permissions and limitations under the License. """ import sys -from distutils.util import strtobool from enum import Enum +from typing import Dict, Optional from blueapps.conf.default_settings import * # noqa +from blueapps.conf.log import get_logging_config_dict +import env +from apps.utils.enum import EnhanceEnum from apps.utils.env import get_type_env -from config import ENVIRONMENT # pipeline 配置 from pipeline.celery.settings import CELERY_QUEUES as PIPELINE_CELERY_QUEUES @@ -167,35 +169,44 @@ # =============================================================================== # 项目配置 # =============================================================================== -BK_PAAS_HOST = os.getenv("BK_PAAS_HOST", "") or BK_URL + +CONF_PATH = os.path.abspath(__file__) +PROJECT_ROOT = os.path.dirname(os.path.dirname(CONF_PATH)) +PYTHON_BIN = os.path.dirname(sys.executable) + + +BK_PAAS_HOST = os.getenv("BK_PAAS_HOST", "") BK_PAAS_INNER_HOST = os.getenv("BK_PAAS_INNER_HOST") or BK_PAAS_HOST +# ESB、APIGW 的域名，新增于PaaSV3，如果取不到该值，则使用 BK_PAAS_INNER_HOST +BK_COMPONENT_API_URL = env.BK_COMPONENT_API_URL -BK_NODEMAN_URL = os.getenv("BK_NODEMAN_URL", f"{BK_PAAS_INNER_HOST}/o/bk_nodeman") -BK_CMDB_HOST = os.environ.get("BK_CMDB_HOST", BK_PAAS_HOST.replace("paas", "cmdb")) +BK_NODEMAN_HOST = env.BK_NODEMAN_HOST +# 节点管理后台外网域名，用于构造文件导入导出的API URL +BK_NODEMAN_BACKEND_HOST = env.BK_NODEMAN_BACKEND_HOST BK_JOB_HOST = os.environ.get("BK_JOB_HOST", BK_PAAS_HOST.replace("paas", "job")) -BK_IAM_ESB_PAAS_HOST = os.getenv("BK_IAM_ESB_PAAS_HOST") or BK_PAAS_INNER_HOST -BK_IAM_HOST = os.getenv("BK_IAM_V3_INNER_HOST", "http://bkiam.service.consul:9081") -BK_IAM_SYSTEM_ID = os.getenv("BK_IAM_SYSTEM_ID", "bk_nodeman") -BK_IAM_CMDB_SYSTEM_ID = os.getenv("BK_IAM_CMDB_SYSTEM_ID", "bk_cmdb") -BK_IAM_URL = os.getenv("BK_IAM_URL", f"{BK_PAAS_HOST}/o/bk_iam/apply-custom-perm") +# 使用权限中心 +USE_IAM = bool(os.getenv("BKAPP_USE_IAM", False)) + +BK_IAM_APP_CODE = os.getenv("BK_IAM_V3_APP_CODE", "bk_iam") +BK_IAM_INNER_HOST = os.getenv("BK_IAM_V3_INNER_HOST", "http://bkiam.service.consul:9081") +BK_IAM_SAAS_HOST = env.BK_IAM_SAAS_HOST + +BK_IAM_SYSTEM_ID = os.getenv("BKAPP_IAM_SYSTEM_ID", "bk_nodeman") +BK_IAM_CMDB_SYSTEM_ID = os.getenv("BKAPP_IAM_CMDB_SYSTEM_ID", "bk_cmdb") +BK_IAM_MIGRATION_JSON_PATH = os.path.join(PROJECT_ROOT, "support-files/bkiam") +BK_IAM_RESOURCE_API_HOST = env.BK_IAM_RESOURCE_API_HOST + BK_IAM_MIGRATION_APP_NAME = "iam_migrations" BK_IAM_SKIP = False -CONF_PATH = os.path.abspath(__file__) -PROJECT_ROOT = os.path.dirname(os.path.dirname(CONF_PATH)) -PYTHON_BIN = os.path.dirname(sys.executable) - BK_DOCS_CENTER_HOST = os.getenv("BK_DOCS_CENTER_HOST") INIT_SUPERUSER = ["admin"] DEBUG = False SHOW_EXCEPTION_DETAIL = False -# 使用权限中心 -USE_IAM = bool(os.getenv("BKAPP_USE_IAM", False)) - # 并发数 CONCURRENT_NUMBER = int(os.getenv("CONCURRENT_NUMBER", 50) or 50) @@ -255,7 +266,7 @@ MAKO_TEMPLATE_DIR = [os.path.join(PROJECT_ROOT, directory) for directory in ["static/dist", "templates"]] -VUE_INDEX = f"{ENVIRONMENT}/index.html" +VUE_INDEX = f"{env.ENVIRONMENT}/index.html" TEMPLATES = [ { @@ -314,14 +325,11 @@ class StorageType(Enum): # 是否覆盖同名文件 FILE_OVERWRITE = get_type_env("FILE_OVERWRITE", _type=bool, default=False) -# 节点管理后台外网域名，用于构造文件导入导出的API URL -BACKEND_HOST = os.getenv("BKAPP_BACKEND_HOST", "") - BKREPO_USERNAME = os.getenv("BKREPO_USERNAME") BKREPO_PASSWORD = os.getenv("BKREPO_PASSWORD") BKREPO_PROJECT = os.getenv("BKREPO_PROJECT") # 默认文件存放仓库 -BKREPO_BUCKET = os.getenv("BKREPO_BUCKET") +BKREPO_BUCKET = os.getenv("BKREPO_PUBLIC_BUCKET") # 对象存储平台域名 BKREPO_ENDPOINT_URL = os.getenv("BKREPO_ENDPOINT_URL") @@ -335,10 +343,10 @@ class StorageType(Enum): DEFAULT_FILE_STORAGE = STORAGE_TYPE_IMPORT_PATH_MAP[STORAGE_TYPE] # 本地文件系统上传文件后台API -FILE_SYSTEM_UPLOAD_API = f"{BACKEND_HOST}/backend/package/upload/" +FILE_SYSTEM_UPLOAD_API = f"{BK_NODEMAN_BACKEND_HOST}/backend/package/upload/" # 对象存储上传文件后台API -COS_UPLOAD_API = f"{BACKEND_HOST}/backend/package/upload_cos/" +COS_UPLOAD_API = f"{BK_NODEMAN_BACKEND_HOST}/backend/package/upload_cos/" # 暂时存在多个上传API的原因：原有文件上传接口被Nginx转发 STORAGE_TYPE_UPLOAD_API_MAP = { @@ -348,7 +356,7 @@ class StorageType(Enum): DEFAULT_FILE_UPLOAD_API = STORAGE_TYPE_UPLOAD_API_MAP[STORAGE_TYPE] -BKAPP_NODEMAN_DOWNLOAD_API = f"{BACKEND_HOST}/backend/export/download/" +BKAPP_NODEMAN_DOWNLOAD_API = f"{BK_NODEMAN_BACKEND_HOST}/backend/export/download/" PUBLIC_PATH = os.getenv("BKAPP_PUBLIC_PATH") or "/data/bkee/public/bknodeman/" @@ -413,7 +421,47 @@ class StorageType(Enum): CELERY_TASK_RESULT_EXPIRES = 60 * 30 # 30分钟丢弃结果 # 后台配置 -BK_BACKEND_CONFIG = bool(os.getenv("BK_BACKEND_CONFIG", None)) +BK_BACKEND_CONFIG = env.BK_BACKEND_CONFIG +BKAPP_IS_PAAS_DEPLOY = env.BKAPP_IS_PAAS_DEPLOY + + +class ConfigRedisMode(EnhanceEnum): + """ + 后台配置的Redis模式 + 背景：脱离PaaS的后台部署，运维提供的Redis模式切换控制变量 + """ + + STANDALONE = "standalone" + SENTINEL = "sentinel" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return {cls.STANDALONE: "单例模式", cls.SENTINEL: "哨兵模式"} + + +class RedisMode(EnhanceEnum): + """标准Redis模式""" + + SINGLE = "single" + REPLICATION = "replication" + CLUSTER = "cluster" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return {cls.SINGLE: "单例模式", cls.REPLICATION: "主从模式", cls.CLUSTER: "集群模式"} + + @classmethod + def get_config_mode__redis_mode_map(cls): + """获取配置Redis模式 - 标准Redis模式映射""" + return { + ConfigRedisMode.SENTINEL.value: cls.REPLICATION.value, + ConfigRedisMode.STANDALONE.value: cls.SINGLE.value, + } + + @classmethod + def get_standard_redis_mode(cls, config_redis_mode: str, default: Optional[str] = None) -> Optional[str]: + return cls.get_config_mode__redis_mode_map().get(config_redis_mode, default) + if BK_BACKEND_CONFIG: IS_LOCAL = False @@ -440,11 +488,11 @@ class StorageType(Enum): DATABASES = { "default": { "ENGINE": "django.db.backends.mysql", # 默认用mysql - "NAME": os.getenv("BK_NODEMAN_MYSQL_NAME", "bk_nodeman"), # 数据库名 - "USER": os.getenv("BK_NODEMAN_MYSQL_USER"), - "PASSWORD": os.getenv("BK_NODEMAN_MYSQL_PASSWORD"), - "HOST": os.getenv("BK_NODEMAN_MYSQL_HOST"), - "PORT": os.getenv("BK_NODEMAN_MYSQL_PORT"), + "NAME": os.getenv("MYSQL_NAME", "bk_nodeman"), # 数据库名 + "USER": os.getenv("MYSQL_USER"), + "PASSWORD": os.getenv("MYSQL_PASSWORD"), + "HOST": os.getenv("MYSQL_HOST"), + "PORT": os.getenv("MYSQL_PORT"), "OPTIONS": {"isolation_level": "repeatable read"}, } } @@ -453,39 +501,54 @@ class StorageType(Enum): REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"] = [ "apps.utils.drf.CsrfExemptSessionAuthentication", ] - # redis 集群sentinel模式 - REDIS_HOST = os.getenv("BK_NODEMAN_REDIS_SENTINEL_HOST") - REDIS_PORT = os.getenv("BK_NODEMAN_REDIS_SENTINEL_PORT") - REDIS_PASSWD = os.getenv("BK_NODEMAN_REDIS_PASSWORD") - REDIS_MASTER_NAME = os.getenv("BK_NODEMAN_REDIS_MASTER_NAME") - REDIS = { - "host": REDIS_HOST, - "port": REDIS_PORT, - "password": REDIS_PASSWD, - "service_name": REDIS_MASTER_NAME, - "mode": "replication", # 哨兵模式，可选 single, cluster, replication - } # BROKER_URL BROKER_URL = "amqp://{user}:{passwd}@{host}:{port}/{vhost}".format( - user=os.getenv("BK_NODEMAN_RABBITMQ_USERNAME"), - passwd=os.getenv("BK_NODEMAN_RABBITMQ_PASSWORD"), - host=os.getenv("BK_NODEMAN_RABBITMQ_HOST"), - port=os.getenv("BK_NODEMAN_RABBITMQ_PORT"), - vhost=os.getenv("BK_NODEMAN_RABBITMQ_VHOST") or "bk_bknodeman", + user=os.getenv("RABBITMQ_USER"), + passwd=os.getenv("RABBITMQ_PASSWORD"), + host=os.getenv("RABBITMQ_HOST"), + port=os.getenv("RABBITMQ_PORT"), + vhost=os.getenv("RABBITMQ_VHOST") or "bk_bknodeman", ) - # celery redbeat config - if BKAPP_RUN_ENV == "ce": + CONFIG_REDIS_MODE = os.getenv("REDIS_MODE", ConfigRedisMode.SENTINEL.value) + REDIS_MODE = RedisMode.get_standard_redis_mode(CONFIG_REDIS_MODE, default=RedisMode.REPLICATION.value) + + REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") + REDIS_MASTER_NAME = os.getenv("REDIS_MASTER_NAME") + + if REDIS_MODE == "replication": + # redis 集群sentinel模式 + REDIS_HOST = os.getenv("REDIS_SENTINEL_HOST") + REDIS_PORT = os.getenv("REDIS_SENTINEL_PORT") + # # celery redbeat config + REDBEAT_REDIS_URL = "redis-sentinel://redis-sentinel:{port}/0".format(port=REDIS_PORT or 26379) + + # 临时兼容社区版单例配置 + if BKAPP_RUN_ENV == "ce": + REDBEAT_REDIS_URL = "redis://:{passwd}@{host}:{port}/0".format( + passwd=REDIS_PASSWORD, host=REDIS_HOST, port=REDIS_PORT or 6379 + ) + REDIS_MODE = "single" + else: + REDIS_HOST = os.getenv("REDIS_HOST") + REDIS_PORT = os.getenv("REDIS_PORT") + # # celery redbeat config REDBEAT_REDIS_URL = "redis://:{passwd}@{host}:{port}/0".format( - passwd=REDIS_PASSWD, host=REDIS_HOST, port=REDIS_PORT or 6379 + passwd=REDIS_PASSWORD, host=REDIS_HOST, port=REDIS_PORT or 6379 ) - REDIS["mode"] = "single" - else: - REDBEAT_REDIS_URL = "redis-sentinel://redis-sentinel:{port}/0".format(port=REDIS_PORT or 26379) + + REDIS = { + "host": REDIS_HOST, + "port": REDIS_PORT, + "password": REDIS_PASSWORD, + "service_name": REDIS_MASTER_NAME, + "mode": REDIS_MODE, # 哨兵模式，可选 single, cluster, replication + } + REDBEAT_REDIS_OPTIONS = { "sentinels": [(REDIS_HOST, REDIS_PORT)], - "password": REDIS_PASSWD, + "password": REDIS_PASSWORD, "service_name": REDIS_MASTER_NAME or "mymaster", "socket_timeout": 0.1, "retry_period": 60, @@ -500,16 +563,18 @@ class StorageType(Enum): BKAPP_EE_SOPS_TEMPLATE_ID = os.getenv("BKAPP_EE_SOPS_TEMPLATE_ID") BKAPP_REQUEST_EE_SOPS_BK_BIZ_ID = os.getenv("BKAPP_REQUEST_EE_SOPS_BK_BIZ_ID") - from blueapps.patch.log import get_paas_v2_logging_config_dict + # 后台不基于PaaS部署的情况下，需要重定向日志 + if not BKAPP_IS_PAAS_DEPLOY: + from blueapps.patch.log import get_paas_v2_logging_config_dict - # 日志 - BK_LOG_DIR = os.getenv("BK_LOG_DIR", "./../bk_nodeman/logs") - LOGGING = get_paas_v2_logging_config_dict( - is_local=IS_LOCAL, bk_log_dir=BK_LOG_DIR, log_level=locals().get("LOG_LEVEL", "INFO") - ) -else: - from blueapps.conf.log import get_logging_config_dict + # 日志 + BK_LOG_DIR = os.getenv("BK_LOG_DIR", "./../bk_nodeman/logs") + LOGGING = get_paas_v2_logging_config_dict( + is_local=IS_LOCAL, bk_log_dir=BK_LOG_DIR, log_level=locals().get("LOG_LEVEL", "INFO") + ) +# 基于PaaS部署的后台，以及SaaS需要修改日志编码 +if not BK_BACKEND_CONFIG or BKAPP_IS_PAAS_DEPLOY: LOGGING = get_logging_config_dict(locals()) LOGGING["handlers"]["root"]["encoding"] = "utf-8" LOGGING["handlers"]["component"]["encoding"] = "utf-8" @@ -559,6 +624,9 @@ class StorageType(Enum): GSE_PROCESS_STATUS_DATAID = os.getenv("GSE_PROCESS_STATUS_DATAID") or 1200000 GSE_PROCESS_EVENT_DATAID = os.getenv("GSE_PROCESS_EVENT_DATAID") or 1100008 +# 是否启用 gse svr 服务发现，启用后，默认接入点会通过zk的方式，自动更新gse svr信息 +GSE_ENABLE_SVR_DISCOVERY = get_type_env(key="GSE_ENABLE_SVR_DISCOVERY", default=False, _type=bool) + # 是否使用CMDB订阅机制去主动触发插件下发 USE_CMDB_SUBSCRIPTION_TRIGGER = get_type_env(key="BKAPP_USE_CMDB_SUBSCRIPTION_TRIGGER", default=True, _type=bool) diff --git a/config/prod.py b/config/prod.py index cca83adab..878801c97 100644 --- a/config/prod.py +++ b/config/prod.py @@ -22,7 +22,3 @@ # 只对正式环境日志级别进行配置，可以在这里修改 LOG_LEVEL = os.environ.get("LOG_LEVEL", "ERROR") - -BK_IAM_INNER_HOST = os.getenv("BK_IAM_V3_INNER_HOST") -BK_IAM_MIGRATION_JSON_PATH = os.path.join(PROJECT_ROOT, "support-files/bkiam") -BK_IAM_RESOURCE_API_HOST = os.getenv("BKAPP_IAM_RESOURCE_API_HOST", "{}{}".format(BK_PAAS_INNER_HOST, SITE_URL)) diff --git a/config/stag.py b/config/stag.py index dfb58a683..771c6b3b7 100644 --- a/config/stag.py +++ b/config/stag.py @@ -16,10 +16,6 @@ else: from blueapps.patch.settings_paas_services import * # noqa -BK_IAM_INNER_HOST = os.getenv("BK_IAM_V3_INNER_HOST") -BK_IAM_MIGRATION_JSON_PATH = os.path.join(PROJECT_ROOT, "support-files/bkiam") -BK_IAM_RESOURCE_API_HOST = os.getenv("BKAPP_IAM_RESOURCE_API_HOST", "{}{}".format(BK_PAAS_INNER_HOST, SITE_URL)) - # 预发布环境 RUN_MODE = "STAGING" diff --git a/dev_log/2.1.354/crayon_202110191739.yaml b/dev_log/2.1.354/crayon_202110191739.yaml new file mode 100644 index 000000000..196975573 --- /dev/null +++ b/dev_log/2.1.354/crayon_202110191739.yaml @@ -0,0 +1,3 @@ +--- +feature: + - "PaaS容器部署适配 (closed #10)" diff --git a/env/__init__.py b/env/__init__.py new file mode 100644 index 000000000..6c302cee7 --- /dev/null +++ b/env/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from .paas_version_diff import * # noqa diff --git a/env/constants.py b/env/constants.py new file mode 100644 index 000000000..8b0d76ed1 --- /dev/null +++ b/env/constants.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +from enum import Enum +from typing import Dict + +from apps.utils.enum import EnhanceEnum + + +class BkPaaSVersion(EnhanceEnum): + V2 = 2 + V3 = 3 + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return {cls.V2: "V2", cls.V3: "V3具备容器及二进制配置差异"} diff --git a/env/paas_version_diff/__init__.py b/env/paas_version_diff/__init__.py new file mode 100644 index 000000000..c7d85a57e --- /dev/null +++ b/env/paas_version_diff/__init__.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from apps.utils.env import get_type_env + +from .. import constants + +# 差异化赋值后的统一命名 settings +__all__ = [ + # PaaS 部署环境，标准化为 stag / dev + "ENVIRONMENT", + # 是否为后台配置 + "BK_BACKEND_CONFIG", + # 后台是否为PaaS部署 + "BKAPP_IS_PAAS_DEPLOY", + # esb 访问地址 + "BK_COMPONENT_API_URL", + # 节点管理SaaS访问地址 + "BK_NODEMAN_HOST", + # 节点管理后台访问地址 + "BK_NODEMAN_BACKEND_HOST", + # 权限中心 SaaS 地址 + "BK_IAM_SAAS_HOST", + # 提供给权限中心的资源回调地址 + "BK_IAM_RESOURCE_API_HOST", +] + +# PaaS 版本 +BKPAAS_MAJOR_VERSION = get_type_env(key="BKPAAS_MAJOR_VERSION", default=constants.BkPaaSVersion.V2.value, _type=int) +# 是否为 PaaS V3 容器化部署版本 +BKAPP_IS_V3_CONTAINER = get_type_env(key="BKAPP_IS_V3_CONTAINER", default=False, _type=bool) + +if BKPAAS_MAJOR_VERSION == constants.BkPaaSVersion.V3.value: + if BKAPP_IS_V3_CONTAINER: + from .paas_v3_container import * # noqa + else: + from .paas_v3 import * # noqa +else: + from .paas_v2 import * # noqa diff --git a/env/paas_version_diff/paas_v2.py b/env/paas_version_diff/paas_v2.py new file mode 100644 index 000000000..9bb4a35c3 --- /dev/null +++ b/env/paas_version_diff/paas_v2.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from apps.utils.env import get_type_env + +APP_CODE = get_type_env(key="APP_ID", default="", _type=str) + +BK_PAAS_HOST = get_type_env(key="BK_PAAS_HOST", default="", _type=str) + +BK_PAAS_INNER_HOST = get_type_env(key="BK_PAAS_INNER_HOST", default=BK_PAAS_HOST, _type=str) + +PAAS_V2_ENVIRONMENT = get_type_env(key="BK_ENV", default="development", _type=str) + +# define envs + +# PaaS 部署环境 +ENVIRONMENT = {"development": "dev", "testing": "stag", "production": "prod"}.get(PAAS_V2_ENVIRONMENT) + +# 是否启用后台配置 +BK_BACKEND_CONFIG = get_type_env(key="BK_BACKEND_CONFIG", default=False, _type=bool) + +BKAPP_IS_PAAS_DEPLOY = False + +# esb 访问地址 +BK_COMPONENT_API_URL = BK_PAAS_HOST or BK_PAAS_INNER_HOST or "" + +# 节点管理 default 模块地址 +BK_NODEMAN_HOST = get_type_env(key="BK_NODEMAN_URL", default=f"{BK_PAAS_INNER_HOST}/o/{APP_CODE}", _type=str) + +# 节点管理后台地址 +BK_NODEMAN_BACKEND_HOST = get_type_env(key="BKAPP_BACKEND_HOST", default="", _type=str) + +BK_IAM_APP_CODE = get_type_env(key="BK_IAM_V3_APP_CODE", default="bk_iam", _type=str) + +BK_IAM_SAAS_HOST = get_type_env(key="BK_IAM_V3_SAAS_HOST", default=f"{BK_PAAS_HOST}/o/{BK_IAM_APP_CODE}/", _type=str) + +BK_IAM_RESOURCE_API_HOST = get_type_env( + key="BKAPP_IAM_RESOURCE_API_HOST", default=f"{BK_PAAS_INNER_HOST}/o/{APP_CODE}/", _type=str +) diff --git a/env/paas_version_diff/paas_v3.py b/env/paas_version_diff/paas_v3.py new file mode 100644 index 000000000..85b5cf63c --- /dev/null +++ b/env/paas_version_diff/paas_v3.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from .paas_v2 import * # noqa + +# PaaS 二进制版本 v2 v3 主要的差异配置 +ENVIRONMENT = get_type_env(key="BKPAAS_ENVIRONMENT", default="dev", _type=str) diff --git a/env/paas_version_diff/paas_v3_container.py b/env/paas_version_diff/paas_v3_container.py new file mode 100644 index 000000000..489e44a6e --- /dev/null +++ b/env/paas_version_diff/paas_v3_container.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import base64 +import json +import os +from collections import defaultdict +from typing import Dict, List + +from apps.utils.env import get_type_env + +DEFAULT_MODULE_NAME = "default" + +APP_CODE = get_type_env(key="BKPAAS_APP_ID", default="", _type=str) + +ENVIRONMENT = get_type_env(key="BKPAAS_ENVIRONMENT", default="dev", _type=str) + +BKPAAS_SERVICE_ADDRESSES_BKSAAS = os.getenv("BKPAAS_SERVICE_ADDRESSES_BKSAAS") +BKPAAS_SERVICE_ADDRESSES_BKSAAS_LIST: List[Dict[str, Dict[str, str]]] = ( + json.loads(base64.b64decode(BKPAAS_SERVICE_ADDRESSES_BKSAAS).decode("utf-8")) + if BKPAAS_SERVICE_ADDRESSES_BKSAAS + else {} +) + +APP_CODE__SAAS_MODULE_HOST_MAP: Dict[str, Dict[str, str]] = defaultdict(lambda: defaultdict(str)) + +for item in BKPAAS_SERVICE_ADDRESSES_BKSAAS_LIST: + module_info = item["key"] + bk_app_code = module_info.get("bk_app_code") + module_name = module_info.get("module_name") + + if not bk_app_code: + continue + if not module_name or module_name == "None": + module_name = DEFAULT_MODULE_NAME + + APP_CODE__SAAS_MODULE_HOST_MAP[bk_app_code][module_name] = item["value"].get(ENVIRONMENT) + +# define envs + +# 是否启用后台配置 +BK_BACKEND_CONFIG = get_type_env(key="BACKEND_CONFIG", default=False, _type=bool) + +BKAPP_IS_PAAS_DEPLOY = get_type_env(key="BKAPP_IS_PAAS_DEPLOY", default=False, _type=bool) + +# esb 访问地址 +BK_COMPONENT_API_URL = get_type_env(key="BK_COMPONENT_API_URL", _type=str) + +# 节点管理 default 模块地址 +BK_NODEMAN_HOST = APP_CODE__SAAS_MODULE_HOST_MAP[APP_CODE][DEFAULT_MODULE_NAME] + +# 节点管理后台地址 +BK_NODEMAN_BACKEND_HOST = get_type_env( + key="BKAPP_BACKEND_HOST", default=APP_CODE__SAAS_MODULE_HOST_MAP[APP_CODE]["backend"], _type=str +) + +BK_IAM_APP_CODE = get_type_env(key="BK_IAM_V3_APP_CODE", default="bk_iam", _type=str) + +BK_IAM_SAAS_HOST = get_type_env( + key="BK_IAM_V3_SAAS_HOST", default=APP_CODE__SAAS_MODULE_HOST_MAP[BK_IAM_APP_CODE][DEFAULT_MODULE_NAME], _type=str +) + +BK_IAM_RESOURCE_API_HOST = get_type_env(key="BKAPP_IAM_RESOURCE_API_HOST", default=BK_NODEMAN_HOST, _type=str) diff --git a/pack.sh b/pack.sh index 1f363f98f..563a1b25d 100755 --- a/pack.sh +++ b/pack.sh @@ -49,6 +49,7 @@ cp -R bk_nodeman/script_tools ${NODEMAN_DIST_DIR} cp -R bk_nodeman/config ${NODEMAN_DIST_DIR} cp -R bk_nodeman/iam ${NODEMAN_DIST_DIR} cp -R bk_nodeman/upgrade ${NODEMAN_DIST_DIR} +cp -R bk_nodeman/env ${NODEMAN_DIST_DIR} cp bk_nodeman/manage.py ${NODEMAN_DIST_DIR} cp bk_nodeman/settings.py ${NODEMAN_DIST_DIR} diff --git a/requirements.txt b/requirements.txt index 5415ccdee..ec8f5ca1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,3 +65,5 @@ ddtrace==0.14.1 bkstorages==1.0.1 ruamel.yaml==0.17.16 + +kazoo==2.8.0 diff --git a/scripts/workflows/pre-commit/generate_dev_log.py b/scripts/workflows/pre-commit/generate_dev_log.py index 020f820c8..66a92e9d7 100644 --- a/scripts/workflows/pre-commit/generate_dev_log.py +++ b/scripts/workflows/pre-commit/generate_dev_log.py @@ -142,7 +142,7 @@ def main() -> int: commit_message_file_path = get_commit_message_result["commit_message_file_path"] if WF_CMD_PATTERN not in commit_message_untreated: print("workflow command not found, skip") - return 1 + return 0 commit_message = commit_message_untreated.replace(WF_CMD_PATTERN, "") if commit_message.endswith("\n"): diff --git a/support-files/templates/nodeman#bin#environ.sh b/support-files/templates/nodeman#bin#environ.sh index b630f64e3..6ee034aea 100755 --- a/support-files/templates/nodeman#bin#environ.sh +++ b/support-files/templates/nodeman#bin#environ.sh @@ -51,25 +51,31 @@ export BKAPP_USE_IAM="__BK_NODEMAN_USE_IAM__" export BK_IAM_V3_INNER_HOST="__BK_IAM_PRIVATE_URL__" # 数据库 -export BK_NODEMAN_MYSQL_NAME="__BK_NODEMAN_MYSQL_NAME__" -export BK_NODEMAN_MYSQL_USER="__BK_NODEMAN_MYSQL_USER__" -export BK_NODEMAN_MYSQL_PASSWORD="__BK_NODEMAN_MYSQL_PASSWORD__" -export BK_NODEMAN_MYSQL_HOST="__BK_NODEMAN_MYSQL_HOST__" -export BK_NODEMAN_MYSQL_PORT="__BK_NODEMAN_MYSQL_PORT__" +export MYSQL_NAME="__BK_NODEMAN_MYSQL_NAME__" +export MYSQL_USER="__BK_NODEMAN_MYSQL_USER__" +export MYSQL_PASSWORD="__BK_NODEMAN_MYSQL_PASSWORD__" +export MYSQL_HOST="__BK_NODEMAN_MYSQL_HOST__" +export MYSQL_PORT="__BK_NODEMAN_MYSQL_PORT__" # Redis -export BK_NODEMAN_REDIS_SENTINEL_HOST="__BK_NODEMAN_REDIS_SENTINEL_HOST__" -export BK_NODEMAN_REDIS_SENTINEL_PORT=__BK_NODEMAN_REDIS_SENTINEL_PORT__ -export BK_NODEMAN_REDIS_SENTINEL_PASSWORD="BK_NODEMAN_REDIS_SENTINEL_PASSWORD" -export BK_NODEMAN_REDIS_PASSWORD="__BK_NODEMAN_REDIS_PASSWORD__" -export BK_NODEMAN_REDIS_MASTER_NAME="__BK_NODEMAN_REDIS_MASTER_NAME__" +# standalone: 单实例 +# sentinel： 哨兵 +export REDIS_MODE="__BK_NODEMAN_REDIS_MODE__" +export REDIS_SENTINEL_HOST="__BK_NODEMAN_REDIS_SENTINEL_HOST__" +export REDIS_SENTINEL_PORT=__BK_NODEMAN_REDIS_SENTINEL_PORT__ +export REDIS_SENTINEL_PASSWORD="__BK_NODEMAN_REDIS_SENTINEL_PASSWORD__" +export REDIS_PASSWORD="__BK_NODEMAN_REDIS_PASSWORD__" +export REDIS_MASTER_NAME="__BK_NODEMAN_REDIS_MASTER_NAME__" +# 单实例需要配置下面的变量 +export REDIS_HOST="__BK_NODEMAN_REDIS_HOST__" +export REDIS_PORT=__BK_NODEMAN_REDIS_PORT__ # RabbitMQ -export BK_NODEMAN_RABBITMQ_USERNAME="__BK_NODEMAN_RABBITMQ_USERNAME__" -export BK_NODEMAN_RABBITMQ_PASSWORD="__BK_NODEMAN_RABBITMQ_PASSWORD__" -export BK_NODEMAN_RABBITMQ_HOST="__BK_NODEMAN_RABBITMQ_HOST__" -export BK_NODEMAN_RABBITMQ_PORT=__BK_NODEMAN_RABBITMQ_PORT__ -export BK_NODEMAN_RABBITMQ_VHOST="__BK_NODEMAN_RABBITMQ_VHOST__" +export RABBITMQ_USER="__BK_NODEMAN_RABBITMQ_USERNAME__" +export RABBITMQ_PASSWORD="__BK_NODEMAN_RABBITMQ_PASSWORD__" +export RABBITMQ_HOST="__BK_NODEMAN_RABBITMQ_HOST__" +export RABBITMQ_PORT=__BK_NODEMAN_RABBITMQ_PORT__ +export RABBITMQ_VHOST="__BK_NODEMAN_RABBITMQ_VHOST__" # GSE export BKAPP_GSE_AGENT_HOME="__BK_GSE_AGENT_HOME__"