Merge branch 'master' into acme_clinet_sign_alg

docs/administration.rst

@@ -35,6 +35,37 @@ Basic Configuration
 
         LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log"
 
+.. data:: LOG_REQUEST_HEADERS
+    :noindex:
+
+    ::
+        Defaults to False (off).  This adds logging similar to a webserver, where each request made to the API is logged.
+        Useful for tracing where requests are being made from, or for auditing purposes.
+
+        LOG_REQUEST_HEADERS = True
+
+.. data:: LOG_SANITIZE_REQUEST_HEADERS
+    :noindex:
+
+    ::
+        Defaults to True (on).  This sanitizes the requests logging to remove the query parameters,
+        as those parameters often contain sensitivity information.
+
+        LOG_SANITIZE_REQUEST_HEADERS = True
+
+    .. warning::
+        This should never be used in a production environment as it exposes sensitivite information.
+
+.. data:: LOG_REQUEST_HEADERS_SKIP_ENDPOINT
+    :noindex:
+
+    ::
+        Defaults to the metrics and healthcheck endpoints.  Some endpoints are not useful to log and can generate a lot of noise.
+        If an endpoint is listed here, it will be skipped and not logged.  It is only recommended to add endpoints that are purely
+        informational or only used internally.
+
+        LOG_REQUEST_HEADERS_SKIP_ENDPOINT = ["/metrics", "/healthcheck"]
+
 .. data:: DEBUG
     :noindex:
 

lemur/__init__.py

@@ -9,7 +9,9 @@
 .. moduleauthor:: Hossein Shafagh <hshafagh@netflix.com>
 
 """
+import socket
 import time
+import urllib.parse
 from flask import g, request
 
 from lemur import factory
@@ -109,6 +111,59 @@ def before_request():
 
     @app.after_request
     def after_request(response):
+        def sanitize_path(*, path: str) -> str:
+            """
+            Sanitizes the given path, if it starts with "/api/1/" and LOG_SANITIZE_REQUEST_HEADERS is enabled.
+            It replaces query parameters with '<sanitized_query_parameters>'.
+
+            :param path: The URL path to be sanitized.
+            :return: The sanitized URL path.
+            """
+            # Handle empty paths
+            if not path:
+                return path
+
+            # Don't mess with paths that aren't part of the API
+            if not path.startswith("/api/1/"):
+                return path
+
+            # Skip sanitizing the path if we're told to
+            if not app.config.get("LOG_SANITIZE_REQUEST_HEADERS", True):
+                return path
+
+            parsed_path = urllib.parse.urlparse(path)
+            if parsed_path.query:
+                return urllib.parse.urlunparse(
+                    (
+                        parsed_path.scheme,
+                        parsed_path.netloc,
+                        parsed_path.path,
+                        parsed_path.params,
+                        "<sanitized_query_parameters>",
+                        parsed_path.fragment
+                    )
+                )
+            return path
+
+        # Log request headers
+        skip_endpoints = any(
+            endpoint in request.full_path for endpoint in app.config.get("LOG_REQUEST_HEADERS_SKIP_ENDPOINT", [])
+        )
+        if app.config.get("LOG_REQUEST_HEADERS", False) and not skip_endpoints:
+            app.logger.info({
+                "lemur": socket.gethostname(),
+                "ingress-ip": request.remote_addr,
+                "request-id": request.headers.get("X-Request-Id"),
+                "ip": request.headers.get("X-Real-Ip", request.remote_addr),
+                "method": request.method,
+                "scheme": request.headers.get("X-Scheme", request.scheme),
+                "path": sanitize_path(path=request.full_path),
+                "status": response.status_code,
+                "user-agent": request.headers.get("User-Agent"),
+                "referer": sanitize_path(path=request.headers.get("Referer")),
+                "host": request.headers.get("Host")
+            })
+
         # Update custom response headers
         response.headers.update(custom_response_headers)
 

lemur/default.conf.py

@@ -16,6 +16,9 @@
 
 LOG_LEVEL = "DEBUG"
 LOG_FILE = "lemur.log"
+LOG_REQUEST_HEADERS = False
+LOG_SANITIZE_REQUEST_HEADERS = True
+LOG_REQUEST_HEADERS_SKIP_ENDPOINT = ["/metrics", "/healthcheck"]
 
 # Set of controls to use around ingesting user group information from the IDP
 # Allows mapping user groups to Lemur roles and automatically creating them

lemur/manage.py

@@ -126,7 +126,9 @@ def cli(script_info, config):
 LOG_LEVEL = "DEBUG"
 LOG_FILE = "lemur.log"
 LOG_UPGRADE_FILE = "db_upgrade.log"
-
+LOG_REQUEST_HEADERS = False
+LOG_SANITIZE_REQUEST_HEADERS = True
+LOG_REQUEST_HEADERS_SKIP_ENDPOINT = ["/metrics", "/healthcheck"]  # These endpoints are noisy so skip them by default
 
 # Database