feat(common/request_id): add request_id support (#444)

close #360

src/api/bkuser_core/config/common/django_basic.py

@@ -39,6 +39,7 @@
 
 MIDDLEWARE = [
     "django_prometheus.middleware.PrometheusBeforeMiddleware",
+    "bkuser_global.middlewares.RequestProvider",
     "django.middleware.common.CommonMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",

src/api/bkuser_core/enhanced_account/middlewares.py

@@ -87,7 +87,10 @@ def from_request(cls, request: HttpRequest):
         return cls(
             remote_addr=request.META.get("REMOTE_ADDR", cls.__missing_term),
             path_info=request.META.get("PATH_INFO", "None"),
-            request_id=request.META.get("HTTP_X_BKAPI_REQUEST_ID", cls.__missing_term),
+            request_id=(
+                request.META.get("HTTP_X_REQUEST_ID")
+                or request.META.get("HTTP_X_BKAPI_REQUEST_ID", cls.__missing_term)
+            ),
             query_string=request.META.get("QUERY_STRING", "None"),
             method=request.META.get("REQUEST_METHOD", cls.__missing_term),
             agent_header=request.META.get("HTTP_USER_AGENT", cls.__missing_term),

src/api/pyproject.toml

@@ -42,6 +42,7 @@ sentry-sdk = "1.5.6"
 pyjwt = "^2.3.0"
 apigw-manager = "^1.0.3"
 django-celery-results = "2.0.1"
+werkzeug = "2.0.3"
 
 [tool.poetry.dev-dependencies]
 ipython = "^7.15.0"

src/bkuser_global/local.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import uuid
+
+from werkzeug.local import Local as _Local
+from werkzeug.local import release_local
+
+_local = _Local()
+
+
+def new_request_id():
+    return uuid.uuid4().hex
+
+
+class Singleton(object):
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if not isinstance(cls._instance, cls):
+            cls._instance = object.__new__(cls, *args, **kwargs)
+        return cls._instance
+
+
+class Local(Singleton):
+    """local对象
+    必须配合中间件RequestProvider使用
+    """
+
+    @property
+    def request(self):
+        """获取全局request对象"""
+        request = getattr(_local, "request", None)
+        # if not request:
+        #     raise RuntimeError("request object not in local")
+        return request
+
+    @request.setter
+    def request(self, value):
+        """设置全局request对象"""
+        _local.request = value
+
+    @property
+    def request_id(self):
+        # celery后台没有request对象
+        if self.request:
+            return self.request.request_id
+
+        return new_request_id()
+
+    def get_http_request_id(self):
+        """从接入层获取request_id，或者生成一个新的request_id"""
+        # 在从header中获取
+        request_id = self.request.META.get("HTTP_X_REQUEST_ID") or self.request.META.get("HTTP_X_BKAPI_REQUEST_ID", "")
+
+        if request_id:
+            return request_id
+
+        # 最后主动生成一个
+        return new_request_id()
+
+    @property
+    def request_username(self) -> str:
+        try:
+            # celery后台，openAPI都可能没有user，需要判断
+            if self.request and hasattr(self.request, "user"):
+                return self.request.user.username
+        except Exception:  # pylint: disable=broad-except
+            return ""
+
+        return ""
+
+    def release(self):
+        release_local(_local)
+
+
+local = Local()

src/bkuser_global/logging.py

@@ -8,8 +8,22 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+import logging
 import os
 
+from .local import local
+
+
+class RequestIDFilter(logging.Filter):
+    """
+    request id log filter
+    日志记录中增加request id
+    """
+
+    def filter(self, record):
+        record.request_id = local.request_id
+        return True
+
 
 class LoggingType:
     """日志输出类型"""
@@ -105,20 +119,28 @@ def get_stdout_logging(log_level: str, package_name: str, formatter: str = "json
     return {
         "version": 1,
         "disable_existing_loggers": False,
+        "filters": {
+            "request_id_filter": {
+                "()": RequestIDFilter,
+            }
+        },
         "formatters": formatters,
         "handlers": {
             "null": {"level": "DEBUG", "class": "logging.NullHandler"},
             "root": {
                 "class": log_class,
                 "formatter": formatter,
+                "filters": ["request_id_filter"],
             },
             "component": {
                 "class": log_class,
                 "formatter": formatter,
+                "filters": ["request_id_filter"],
             },
             "iam": {
                 "class": log_class,
                 "formatter": "iam",
+                "filters": ["request_id_filter"],
             },
         },
         "loggers": get_loggers(package_name, log_level),
@@ -135,6 +157,11 @@ def get_file_logging(log_level: str, logging_dir: str, file_name: str, package_n
     return {
         "version": 1,
         "disable_existing_loggers": False,
+        "filters": {
+            "request_id_filter": {
+                "()": RequestIDFilter,
+            }
+        },
         "formatters": formatters,
         "handlers": {
             "null": {"level": "DEBUG", "class": "logging.NullHandler"},
@@ -144,20 +171,23 @@ def get_file_logging(log_level: str, logging_dir: str, file_name: str, package_n
                 "filename": os.path.join(logging_dir, f"{file_name}.log"),
                 "maxBytes": 1024 * 1024 * 10,
                 "backupCount": 5,
+                "filters": ["request_id_filter"],
             },
             "component": {
                 "class": log_class,
                 "formatter": formatter,
                 "filename": os.path.join(logging_dir, "component.log"),
                 "maxBytes": 1024 * 1024 * 10,
                 "backupCount": 5,
+                "filters": ["request_id_filter"],
             },
             "iam": {
                 "class": log_class,
                 "formatter": "iam",
                 "filename": os.path.join(logging_dir, "iam.log"),
                 "maxBytes": 1024 * 1024 * 10,
                 "backupCount": 5,
+                "filters": ["request_id_filter"],
             },
         },
         "loggers": get_loggers(package_name, log_level),

src/bkuser_global/middlewares.py

@@ -14,6 +14,8 @@
 from django.utils import timezone
 from django.utils.deprecation import MiddlewareMixin
 
+from .local import local
+
 logger = logging.getLogger(__name__)
 
 
@@ -24,3 +26,33 @@ def process_request(self, request):
             timezone.activate(pytz.timezone(tzname))
         else:
             timezone.deactivate()
+
+
+class RequestProvider(object):
+    """request_id中间件
+    调用链使用
+    """
+
+    def __init__(self, get_response=None):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        local.request = request
+        request.request_id = local.get_http_request_id()
+
+        response = self.get_response(request)
+        response["X-Request-Id"] = request.request_id
+
+        local.release()
+
+        return response
+
+    # Compatibility methods for Django <1.10
+    def process_request(self, request):
+        local.request = request
+        request.request_id = local.get_http_request_id()
+
+    def process_response(self, request, response):
+        response["X-Request-Id"] = request.request_id
+        local.release()
+        return response

src/login/bklogin/components/esb.py

@@ -17,12 +17,14 @@
 
 from .util import _remove_sensitive_info
 from bklogin.common.log import logger
+from bkuser_global.local import local
 
 
 def _call_esb_api(http_func, url_path, data, timeout=30):
     # 默认请求头
     headers = {
         "Content-Type": "application/json",
+        "X-Request-Id": local.request_id,
     }
 
     # Note: 目前企业版ESB调用的鉴权信息都是与接口的参数一起的，并非在header头里

src/login/bklogin/config/common/django_basic.py

@@ -48,6 +48,7 @@
 
 MIDDLEWARE = (
     "django_prometheus.middleware.PrometheusBeforeMiddleware",
+    "bkuser_global.middlewares.RequestProvider",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.locale.LocaleMiddleware",
     "django.middleware.common.CommonMiddleware",

src/login/pyproject.toml

@@ -25,6 +25,7 @@ blue-krill = "^1.0.7"
 python-json-logger = "^2.0.2"
 sentry-sdk = "1.5.6"
 django-decorator-include = "^3.0"
+werkzeug = "2.0.3"
 
 [tool.poetry.dev-dependencies]
 ipython = "^7.15.0"

src/saas/bkuser_shell/apis/viewset.py

@@ -11,7 +11,6 @@
 import json
 import logging
 import math
-import uuid
 from collections import OrderedDict
 from typing import Callable, Optional
 
@@ -21,6 +20,7 @@
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.viewsets import GenericViewSet
 
+from bkuser_global.local import local
 from bkuser_global.utils import force_str_2_bool
 from bkuser_shell.common.core_client import get_api_client
 from bkuser_shell.common.response import Response
@@ -177,5 +177,5 @@ def make_default_headers(operator: str) -> dict:
         settings.API_APP_CODE_HEADER_NAME: settings.APP_ID,
         settings.API_APP_SECRET_HEADER_NAME: settings.APP_TOKEN,
         "Accept-Language": get_language(),
-        "X-Request-ID": uuid.uuid4().hex,
+        "X-Request-ID": local.request_id,
     }

src/saas/bkuser_shell/config/common/django_basic.py

@@ -35,6 +35,7 @@
 # ==============================================================================
 MIDDLEWARE = [
     "django_prometheus.middleware.PrometheusBeforeMiddleware",
+    "bkuser_global.middlewares.RequestProvider",
     "django.middleware.common.CommonMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",