From bf1c2dab0ac84ca29c2f63bec520cd2bde4cdace Mon Sep 17 00:00:00 2001 From: wklken Date: Thu, 19 May 2022 11:16:28 +0800 Subject: [PATCH] feat(common/request_id): add request_id support (#444) close #360 --- .../bkuser_core/config/common/django_basic.py | 1 + .../enhanced_account/middlewares.py | 5 +- src/api/poetry.lock | 25 +++++- src/api/pyproject.toml | 1 + src/bkuser_global/local.py | 84 +++++++++++++++++++ src/bkuser_global/logging.py | 30 +++++++ src/bkuser_global/middlewares.py | 32 +++++++ src/login/bklogin/components/esb.py | 2 + .../bklogin/config/common/django_basic.py | 1 + src/login/poetry.lock | 25 +++++- src/login/pyproject.toml | 1 + src/saas/bkuser_shell/apis/viewset.py | 4 +- .../config/common/django_basic.py | 1 + src/saas/poetry.lock | 25 +++++- src/saas/pyproject.toml | 1 + 15 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 src/bkuser_global/local.py diff --git a/src/api/bkuser_core/config/common/django_basic.py b/src/api/bkuser_core/config/common/django_basic.py index 9a40f151a..aef1f8e2d 100644 --- a/src/api/bkuser_core/config/common/django_basic.py +++ b/src/api/bkuser_core/config/common/django_basic.py @@ -39,6 +39,7 @@ MIDDLEWARE = [ "django_prometheus.middleware.PrometheusBeforeMiddleware", + "bkuser_global.middlewares.RequestProvider", "django.middleware.common.CommonMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.csrf.CsrfViewMiddleware", diff --git a/src/api/bkuser_core/enhanced_account/middlewares.py b/src/api/bkuser_core/enhanced_account/middlewares.py index 31ca4b7ca..3afc5a1b1 100644 --- a/src/api/bkuser_core/enhanced_account/middlewares.py +++ b/src/api/bkuser_core/enhanced_account/middlewares.py @@ -87,7 +87,10 @@ def from_request(cls, request: HttpRequest): return cls( remote_addr=request.META.get("REMOTE_ADDR", cls.__missing_term), path_info=request.META.get("PATH_INFO", "None"), - request_id=request.META.get("HTTP_X_BKAPI_REQUEST_ID", cls.__missing_term), + request_id=( + request.META.get("HTTP_X_REQUEST_ID") + or request.META.get("HTTP_X_BKAPI_REQUEST_ID", cls.__missing_term) + ), query_string=request.META.get("QUERY_STRING", "None"), method=request.META.get("REQUEST_METHOD", cls.__missing_term), agent_header=request.META.get("HTTP_USER_AGENT", cls.__missing_term), diff --git a/src/api/poetry.lock b/src/api/poetry.lock index a3b753a8c..04dad6fec 100644 --- a/src/api/poetry.lock +++ b/src/api/poetry.lock @@ -2041,6 +2041,25 @@ type = "legacy" url = "https://mirrors.tencent.com/pypi/simple" reference = "tencent-mirrors" +[[package]] +name = "werkzeug" +version = "2.0.3" +description = "The comprehensive WSGI web application library." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +dataclasses = {version = "*", markers = "python_version < \"3.7\""} + +[package.extras] +watchdog = ["watchdog"] + +[package.source] +type = "legacy" +url = "https://mirrors.tencent.com/pypi/simple" +reference = "tencent-mirrors" + [[package]] name = "whitenoise" version = "5.2.0" @@ -2112,7 +2131,7 @@ reference = "tencent-mirrors" [metadata] lock-version = "1.1" python-versions = "3.6.14" -content-hash = "d3225b34e5cfb8c8b733a4f3e17fa42ac55ce5ed9dae8fe6e64df6c8dc335a53" +content-hash = "82957c9fdf07bf52e977df16ec98df904056ed069afb54c74abce24ff6b8918b" [metadata.files] aenum = [ @@ -3003,6 +3022,10 @@ wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] +werkzeug = [ + {file = "Werkzeug-2.0.3-py3-none-any.whl", hash = "sha256:1421ebfc7648a39a5c58c601b154165d05cf47a3cd0ccb70857cbdacf6c8f2b8"}, + {file = "Werkzeug-2.0.3.tar.gz", hash = "sha256:b863f8ff057c522164b6067c9e28b041161b4be5ba4d0daceeaa50a163822d3c"}, +] whitenoise = [ {file = "whitenoise-5.2.0-py2.py3-none-any.whl", hash = "sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"}, {file = "whitenoise-5.2.0.tar.gz", hash = "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7"}, diff --git a/src/api/pyproject.toml b/src/api/pyproject.toml index afa4b5d44..01286be43 100644 --- a/src/api/pyproject.toml +++ b/src/api/pyproject.toml @@ -42,6 +42,7 @@ sentry-sdk = "1.5.6" pyjwt = "^2.3.0" apigw-manager = "^1.0.3" django-celery-results = "2.0.1" +werkzeug = "2.0.3" [tool.poetry.dev-dependencies] ipython = "^7.15.0" diff --git a/src/bkuser_global/local.py b/src/bkuser_global/local.py new file mode 100644 index 000000000..7be66e2e3 --- /dev/null +++ b/src/bkuser_global/local.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available. +Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import uuid + +from werkzeug.local import Local as _Local +from werkzeug.local import release_local + +_local = _Local() + + +def new_request_id(): + return uuid.uuid4().hex + + +class Singleton(object): + _instance = None + + def __new__(cls, *args, **kwargs): + if not isinstance(cls._instance, cls): + cls._instance = object.__new__(cls, *args, **kwargs) + return cls._instance + + +class Local(Singleton): + """local对象 + 必须配合中间件RequestProvider使用 + """ + + @property + def request(self): + """获取全局request对象""" + request = getattr(_local, "request", None) + # if not request: + # raise RuntimeError("request object not in local") + return request + + @request.setter + def request(self, value): + """设置全局request对象""" + _local.request = value + + @property + def request_id(self): + # celery后台没有request对象 + if self.request: + return self.request.request_id + + return new_request_id() + + def get_http_request_id(self): + """从接入层获取request_id，或者生成一个新的request_id""" + # 在从header中获取 + request_id = self.request.META.get("HTTP_X_REQUEST_ID") or self.request.META.get("HTTP_X_BKAPI_REQUEST_ID", "") + + if request_id: + return request_id + + # 最后主动生成一个 + return new_request_id() + + @property + def request_username(self) -> str: + try: + # celery后台，openAPI都可能没有user，需要判断 + if self.request and hasattr(self.request, "user"): + return self.request.user.username + except Exception: # pylint: disable=broad-except + return "" + + return "" + + def release(self): + release_local(_local) + + +local = Local() diff --git a/src/bkuser_global/logging.py b/src/bkuser_global/logging.py index 70ebdc16f..ff45cb6d9 100644 --- a/src/bkuser_global/logging.py +++ b/src/bkuser_global/logging.py @@ -8,8 +8,22 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import logging import os +from .local import local + + +class RequestIDFilter(logging.Filter): + """ + request id log filter + 日志记录中增加request id + """ + + def filter(self, record): + record.request_id = local.request_id + return True + class LoggingType: """日志输出类型""" @@ -105,20 +119,28 @@ def get_stdout_logging(log_level: str, package_name: str, formatter: str = "json return { "version": 1, "disable_existing_loggers": False, + "filters": { + "request_id_filter": { + "()": RequestIDFilter, + } + }, "formatters": formatters, "handlers": { "null": {"level": "DEBUG", "class": "logging.NullHandler"}, "root": { "class": log_class, "formatter": formatter, + "filters": ["request_id_filter"], }, "component": { "class": log_class, "formatter": formatter, + "filters": ["request_id_filter"], }, "iam": { "class": log_class, "formatter": "iam", + "filters": ["request_id_filter"], }, }, "loggers": get_loggers(package_name, log_level), @@ -135,6 +157,11 @@ def get_file_logging(log_level: str, logging_dir: str, file_name: str, package_n return { "version": 1, "disable_existing_loggers": False, + "filters": { + "request_id_filter": { + "()": RequestIDFilter, + } + }, "formatters": formatters, "handlers": { "null": {"level": "DEBUG", "class": "logging.NullHandler"}, @@ -144,6 +171,7 @@ def get_file_logging(log_level: str, logging_dir: str, file_name: str, package_n "filename": os.path.join(logging_dir, f"{file_name}.log"), "maxBytes": 1024 * 1024 * 10, "backupCount": 5, + "filters": ["request_id_filter"], }, "component": { "class": log_class, @@ -151,6 +179,7 @@ def get_file_logging(log_level: str, logging_dir: str, file_name: str, package_n "filename": os.path.join(logging_dir, "component.log"), "maxBytes": 1024 * 1024 * 10, "backupCount": 5, + "filters": ["request_id_filter"], }, "iam": { "class": log_class, @@ -158,6 +187,7 @@ def get_file_logging(log_level: str, logging_dir: str, file_name: str, package_n "filename": os.path.join(logging_dir, "iam.log"), "maxBytes": 1024 * 1024 * 10, "backupCount": 5, + "filters": ["request_id_filter"], }, }, "loggers": get_loggers(package_name, log_level), diff --git a/src/bkuser_global/middlewares.py b/src/bkuser_global/middlewares.py index 6864db78e..2ab1f9410 100644 --- a/src/bkuser_global/middlewares.py +++ b/src/bkuser_global/middlewares.py @@ -14,6 +14,8 @@ from django.utils import timezone from django.utils.deprecation import MiddlewareMixin +from .local import local + logger = logging.getLogger(__name__) @@ -24,3 +26,33 @@ def process_request(self, request): timezone.activate(pytz.timezone(tzname)) else: timezone.deactivate() + + +class RequestProvider(object): + """request_id中间件 + 调用链使用 + """ + + def __init__(self, get_response=None): + self.get_response = get_response + + def __call__(self, request): + local.request = request + request.request_id = local.get_http_request_id() + + response = self.get_response(request) + response["X-Request-Id"] = request.request_id + + local.release() + + return response + + # Compatibility methods for Django <1.10 + def process_request(self, request): + local.request = request + request.request_id = local.get_http_request_id() + + def process_response(self, request, response): + response["X-Request-Id"] = request.request_id + local.release() + return response diff --git a/src/login/bklogin/components/esb.py b/src/login/bklogin/components/esb.py index 7a0ce3a7b..1888ee4d5 100644 --- a/src/login/bklogin/components/esb.py +++ b/src/login/bklogin/components/esb.py @@ -17,12 +17,14 @@ from .util import _remove_sensitive_info from bklogin.common.log import logger +from bkuser_global.local import local def _call_esb_api(http_func, url_path, data, timeout=30): # 默认请求头 headers = { "Content-Type": "application/json", + "X-Request-Id": local.request_id, } # Note: 目前企业版ESB调用的鉴权信息都是与接口的参数一起的，并非在header头里 diff --git a/src/login/bklogin/config/common/django_basic.py b/src/login/bklogin/config/common/django_basic.py index f7d3c1ef6..f9c2b40d0 100644 --- a/src/login/bklogin/config/common/django_basic.py +++ b/src/login/bklogin/config/common/django_basic.py @@ -48,6 +48,7 @@ MIDDLEWARE = ( "django_prometheus.middleware.PrometheusBeforeMiddleware", + "bkuser_global.middlewares.RequestProvider", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.locale.LocaleMiddleware", "django.middleware.common.CommonMiddleware", diff --git a/src/login/poetry.lock b/src/login/poetry.lock index 6789dda37..1239dac19 100644 --- a/src/login/poetry.lock +++ b/src/login/poetry.lock @@ -1344,6 +1344,25 @@ type = "legacy" url = "https://mirrors.tencent.com/pypi/simple" reference = "tencent-mirrors" +[[package]] +name = "werkzeug" +version = "2.0.3" +description = "The comprehensive WSGI web application library." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +dataclasses = {version = "*", markers = "python_version < \"3.7\""} + +[package.extras] +watchdog = ["watchdog"] + +[package.source] +type = "legacy" +url = "https://mirrors.tencent.com/pypi/simple" +reference = "tencent-mirrors" + [[package]] name = "xlrd" version = "1.0.0" @@ -1425,7 +1444,7 @@ reference = "tencent-mirrors" [metadata] lock-version = "1.1" python-versions = "3.6.14" -content-hash = "b172e229f3b0eb9cfb32c8648c1f5f6b2f9203383d5ef6debd204133691fdd0e" +content-hash = "bee2dfd1cd594de6e23e029362b86d1c05c7edbd6a64bc1af608092e456e9de5" [metadata.files] appnope = [ @@ -2033,6 +2052,10 @@ wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] +werkzeug = [ + {file = "Werkzeug-2.0.3-py3-none-any.whl", hash = "sha256:1421ebfc7648a39a5c58c601b154165d05cf47a3cd0ccb70857cbdacf6c8f2b8"}, + {file = "Werkzeug-2.0.3.tar.gz", hash = "sha256:b863f8ff057c522164b6067c9e28b041161b4be5ba4d0daceeaa50a163822d3c"}, +] xlrd = [ {file = "xlrd-1.0.0-py3-none-any.whl", hash = "sha256:05f55eb39a68f1c3d336de186aeb90c98ea5add722813738d8ae8b97819c5924"}, {file = "xlrd-1.0.0.tar.gz", hash = "sha256:0ff87dd5d50425084f7219cb6f86bb3eb5aa29063f53d50bf270ed007e941069"}, diff --git a/src/login/pyproject.toml b/src/login/pyproject.toml index 8299be55f..5c8d6bb81 100644 --- a/src/login/pyproject.toml +++ b/src/login/pyproject.toml @@ -25,6 +25,7 @@ blue-krill = "^1.0.7" python-json-logger = "^2.0.2" sentry-sdk = "1.5.6" django-decorator-include = "^3.0" +werkzeug = "2.0.3" [tool.poetry.dev-dependencies] ipython = "^7.15.0" diff --git a/src/saas/bkuser_shell/apis/viewset.py b/src/saas/bkuser_shell/apis/viewset.py index eb3950a14..c2bc1a288 100644 --- a/src/saas/bkuser_shell/apis/viewset.py +++ b/src/saas/bkuser_shell/apis/viewset.py @@ -11,7 +11,6 @@ import json import logging import math -import uuid from collections import OrderedDict from typing import Callable, Optional @@ -21,6 +20,7 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.viewsets import GenericViewSet +from bkuser_global.local import local from bkuser_global.utils import force_str_2_bool from bkuser_shell.common.core_client import get_api_client from bkuser_shell.common.response import Response @@ -177,5 +177,5 @@ def make_default_headers(operator: str) -> dict: settings.API_APP_CODE_HEADER_NAME: settings.APP_ID, settings.API_APP_SECRET_HEADER_NAME: settings.APP_TOKEN, "Accept-Language": get_language(), - "X-Request-ID": uuid.uuid4().hex, + "X-Request-ID": local.request_id, } diff --git a/src/saas/bkuser_shell/config/common/django_basic.py b/src/saas/bkuser_shell/config/common/django_basic.py index 5bf38bcbf..eadddf860 100644 --- a/src/saas/bkuser_shell/config/common/django_basic.py +++ b/src/saas/bkuser_shell/config/common/django_basic.py @@ -35,6 +35,7 @@ # ============================================================================== MIDDLEWARE = [ "django_prometheus.middleware.PrometheusBeforeMiddleware", + "bkuser_global.middlewares.RequestProvider", "django.middleware.common.CommonMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.csrf.CsrfViewMiddleware", diff --git a/src/saas/poetry.lock b/src/saas/poetry.lock index 9fe0985cb..9867c5a1e 100644 --- a/src/saas/poetry.lock +++ b/src/saas/poetry.lock @@ -1446,6 +1446,25 @@ type = "legacy" url = "https://mirrors.tencent.com/pypi/simple" reference = "tencent-mirrors" +[[package]] +name = "werkzeug" +version = "2.0.3" +description = "The comprehensive WSGI web application library." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +dataclasses = {version = "*", markers = "python_version < \"3.7\""} + +[package.extras] +watchdog = ["watchdog"] + +[package.source] +type = "legacy" +url = "https://mirrors.tencent.com/pypi/simple" +reference = "tencent-mirrors" + [[package]] name = "whitenoise" version = "5.2.0" @@ -1482,7 +1501,7 @@ reference = "tencent-mirrors" [metadata] lock-version = "1.1" python-versions = "3.6.14" -content-hash = "5c0be13d56627a0b3d41328d9242a0e3e9b67dc32dfd105207b29f31df896dbe" +content-hash = "23fddded1879ec5bd032316179edb7ebfa4f91ec07da75b9599134c4f7a3324c" [metadata.files] aenum = [ @@ -2150,6 +2169,10 @@ wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] +werkzeug = [ + {file = "Werkzeug-2.0.3-py3-none-any.whl", hash = "sha256:1421ebfc7648a39a5c58c601b154165d05cf47a3cd0ccb70857cbdacf6c8f2b8"}, + {file = "Werkzeug-2.0.3.tar.gz", hash = "sha256:b863f8ff057c522164b6067c9e28b041161b4be5ba4d0daceeaa50a163822d3c"}, +] whitenoise = [ {file = "whitenoise-5.2.0-py2.py3-none-any.whl", hash = "sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"}, {file = "whitenoise-5.2.0.tar.gz", hash = "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7"}, diff --git a/src/saas/pyproject.toml b/src/saas/pyproject.toml index 4ba4156a4..934b837ec 100644 --- a/src/saas/pyproject.toml +++ b/src/saas/pyproject.toml @@ -28,6 +28,7 @@ pyyaml = "^5.3.1" django-environ = "^0.4.5" django-prometheus = "^2.1.0" sentry-sdk = "1.5.6" +werkzeug = "2.0.3" [tool.poetry.dev-dependencies] ipython = "^7.15.0"