feat: implement a request id to trace requests #586

lapis2-docs/astro.config.mjs

@@ -97,6 +97,10 @@ export default defineConfig({
                             link: '/concepts/variant-query/',
                             onlyIfFeature: 'sarsCoV2VariantQuery',
                         },
+                        {
+                            label: 'Request Id',
+                            link: '/concepts/request-id/',
+                        },
                     ]),
                 },
                 {

lapis2-docs/src/content/docs/concepts/request-id.mdx

@@ -0,0 +1,23 @@
+---
+title: Request Id
+description: Request Id
+---
+
+LAPIS uses a request id to identify requests.
+This is useful for debugging and logging purposes.
+
+:::note
+If you report an error that occurred in LAPIS, please include the request id.
+This will greatly simplify identifying the problem in our log files.
+:::
+
+You can provide a request id in the request as an HTTP header `X-Request-Id`.
+This is useful if you want to correlate requests in your own log files with the LAPIS log files
+to track problems across systems.
+
+:::caution
+If you use the `X-Request-Id` header, make sure that the value is unique.
+:::
+
+If you don't specify a request id, LAPIS will generate one for you.
+It is contained in the response header `X-Request-Id` and in `info` in the response body.

lapis2-docs/tests/docs.spec.ts

@@ -21,6 +21,7 @@ const conceptsPages = [
     'Request methods: GET and POST',
     'Response format',
     'Variant query',
+    'Request Id',
 ];
 
 const userTutorialPages = ['Plot the global distribution of all sequences in R'];

lapis2/src/main/kotlin/org/genspectrum/lapis/LapisSpringConfig.kt

@@ -1,6 +1,10 @@
 package org.genspectrum.lapis
 
 import com.fasterxml.jackson.module.kotlin.readValue
+import io.swagger.v3.oas.models.media.Content
+import io.swagger.v3.oas.models.media.MediaType
+import io.swagger.v3.oas.models.media.Schema
+import io.swagger.v3.oas.models.parameters.HeaderParameter
 import mu.KotlinLogging
 import org.genspectrum.lapis.auth.DataOpennessAuthorizationFilterFactory
 import org.genspectrum.lapis.config.DatabaseConfig
@@ -16,9 +20,12 @@ import org.genspectrum.lapis.config.SequenceFilterFields
 import org.genspectrum.lapis.logging.RequestContext
 import org.genspectrum.lapis.logging.RequestContextLogger
 import org.genspectrum.lapis.logging.StatisticsLogObjectMapper
+import org.genspectrum.lapis.openApi.REQUEST_ID_HEADER
+import org.genspectrum.lapis.openApi.REQUEST_ID_HEADER_DESCRIPTION
 import org.genspectrum.lapis.openApi.buildOpenApiSchema
 import org.genspectrum.lapis.util.TimeFactory
 import org.genspectrum.lapis.util.YamlObjectMapper
+import org.springdoc.core.customizers.OperationCustomizer
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -34,6 +41,22 @@ class LapisSpringConfig {
         referenceGenomeSchema: ReferenceGenomeSchema,
     ) = buildOpenApiSchema(sequenceFilterFields, databaseConfig, referenceGenomeSchema)
 
+    @Bean
+    fun headerCustomizer() = OperationCustomizer { operation, _ ->
+        val foundRequestIdHeaderParameter = operation.parameters?.any { it.name == REQUEST_ID_HEADER }
+        if (foundRequestIdHeaderParameter == false || foundRequestIdHeaderParameter == null) {
+            operation.addParametersItem(
+                HeaderParameter().apply {
+                    name = REQUEST_ID_HEADER
+                    required = false
+                    description = REQUEST_ID_HEADER_DESCRIPTION
+                    content = Content().addMediaType("text/plain", MediaType().schema(Schema<String>()))
+                },
+            )
+        }
+        operation
+    }
+
     @Bean
     fun databaseConfig(
         @Value("\${lapis.databaseConfig.path}") configPath: String,

lapis2/src/main/kotlin/org/genspectrum/lapis/logging/RequestId.kt

@@ -0,0 +1,44 @@
+package org.genspectrum.lapis.logging
+
+import jakarta.servlet.FilterChain
+import jakarta.servlet.http.HttpServletRequest
+import jakarta.servlet.http.HttpServletResponse
+import org.genspectrum.lapis.openApi.REQUEST_ID_HEADER
+import org.slf4j.MDC
+import org.springframework.core.annotation.Order
+import org.springframework.stereotype.Component
+import org.springframework.web.context.annotation.RequestScope
+import org.springframework.web.filter.OncePerRequestFilter
+import java.util.UUID
+
+@Component
+@RequestScope
+class RequestIdContext {
+    var requestId: String? = null
+}
+
+private const val REQUEST_ID_MDC_KEY = "RequestId"
+private const val HIGH_PRECEDENCE_BUT_LOW_ENOUGH_TO_HAVE_REQUEST_SCOPE_AVAILABLE = -100
+
+@Component
+@Order(HIGH_PRECEDENCE_BUT_LOW_ENOUGH_TO_HAVE_REQUEST_SCOPE_AVAILABLE)
+class RequestIdFilter(private val requestIdContext: RequestIdContext) : OncePerRequestFilter() {
+
+    override fun doFilterInternal(
+        request: HttpServletRequest,
+        response: HttpServletResponse,
+        filterChain: FilterChain,
+    ) {
+        val requestId = request.getHeader(REQUEST_ID_HEADER) ?: UUID.randomUUID().toString()
+
+        MDC.put(REQUEST_ID_MDC_KEY, requestId)
+        requestIdContext.requestId = requestId
+        response.addHeader(REQUEST_ID_HEADER, requestId)
+
+        try {
+            filterChain.doFilter(request, response)
+        } finally {
+            MDC.remove(REQUEST_ID_MDC_KEY)
+        }
+    }
+}

lapis2/src/main/kotlin/org/genspectrum/lapis/openApi/OpenApiDocs.kt

@@ -255,12 +255,14 @@ private fun computePrimitiveFieldFilters(
     }
 
 private fun lapisResponseSchema(dataSchema: Schema<Any>) =
-    Schema<Any>().type("object").properties(
-        mapOf(
-            "data" to Schema<Any>().type("array").items(dataSchema),
-            "info" to infoResponseSchema(),
-        ),
-    ).required(listOf("data", "info"))
+    Schema<Any>().type("object")
+        .properties(
+            mapOf(
+                "data" to Schema<Any>().type("array").items(dataSchema),
+                "info" to infoResponseSchema(),
+            ),
+        )
+        .required(listOf("data", "info"))
 
 private fun infoResponseSchema() =
     Schema<LapisInfo>().type("object")
@@ -270,8 +272,10 @@ private fun infoResponseSchema() =
                 "dataVersion" to Schema<String>().type("string")
                     .description(LAPIS_DATA_VERSION_RESPONSE_DESCRIPTION)
                     .example(LAPIS_DATA_VERSION_EXAMPLE),
+                "requestId" to Schema<String>().type("string").description(REQUEST_ID_HEADER_DESCRIPTION),
             ),
-        ).required(listOf("dataVersion"))
+        )
+        .required(listOf("dataVersion"))
 
 private fun metadataFieldSchemas(databaseConfig: DatabaseConfig) =
     databaseConfig.schema.metadata.associate { it.name to Schema<String>().type(mapToOpenApiType(it.type)) }

lapis2/src/main/kotlin/org/genspectrum/lapis/openApi/Schemas.kt

@@ -69,6 +69,12 @@ const val LAPIS_DATA_VERSION_HEADER_DESCRIPTION = "$LAPIS_DATA_VERSION_DESCRIPTI
 const val LAPIS_DATA_VERSION_RESPONSE_DESCRIPTION = "$LAPIS_DATA_VERSION_DESCRIPTION " +
     "Same as the value returned in the info object in the header '$LAPIS_DATA_VERSION_HEADER'."
 
+const val REQUEST_ID_HEADER = "X-Request-ID"
+const val REQUEST_ID_HEADER_DESCRIPTION = """
+A UUID that uniquely identifies the request for tracing purposes.
+If none if provided in the request, LAPIS will generate one.
+"""
+
 @Target(AnnotationTarget.FUNCTION, AnnotationTarget.CLASS)
 @Retention(AnnotationRetention.RUNTIME)
 @Operation
@@ -81,6 +87,11 @@ const val LAPIS_DATA_VERSION_RESPONSE_DESCRIPTION = "$LAPIS_DATA_VERSION_DESCRIP
             description = LAPIS_DATA_VERSION_HEADER_DESCRIPTION,
             schema = Schema(type = "string"),
         ),
+        Header(
+            name = REQUEST_ID_HEADER,
+            description = REQUEST_ID_HEADER_DESCRIPTION,
+            schema = Schema(type = "string"),
+        ),
     ],
 )
 annotation class LapisResponseAnnotation(
@@ -161,7 +172,7 @@ annotation class LapisAlignedMultiSegmentedNucleotideSequenceResponse
 @Retention(AnnotationRetention.RUNTIME)
 @Parameter(
     description =
-        "Valid filters for sequence data. This may be empty. Only provide the fields that should be filtered by.",
+    "Valid filters for sequence data. This may be empty. Only provide the fields that should be filtered by.",
     schema = Schema(ref = "#/components/schemas/$PRIMITIVE_FIELD_FILTERS_SCHEMA"),
     explode = Explode.TRUE,
     style = ParameterStyle.FORM,

lapis2/src/main/kotlin/org/genspectrum/lapis/request/LapisInfo.kt

@@ -3,9 +3,11 @@ package org.genspectrum.lapis.request
 import io.swagger.v3.oas.annotations.media.Schema
 import org.genspectrum.lapis.controller.LapisErrorResponse
 import org.genspectrum.lapis.controller.LapisResponse
+import org.genspectrum.lapis.logging.RequestIdContext
 import org.genspectrum.lapis.openApi.LAPIS_DATA_VERSION_EXAMPLE
 import org.genspectrum.lapis.openApi.LAPIS_DATA_VERSION_RESPONSE_DESCRIPTION
 import org.genspectrum.lapis.openApi.LAPIS_INFO_DESCRIPTION
+import org.genspectrum.lapis.openApi.REQUEST_ID_HEADER_DESCRIPTION
 import org.genspectrum.lapis.silo.DataVersion
 import org.springframework.core.MethodParameter
 import org.springframework.http.MediaType
@@ -21,12 +23,18 @@ data class LapisInfo(
         description = LAPIS_DATA_VERSION_RESPONSE_DESCRIPTION,
         example = LAPIS_DATA_VERSION_EXAMPLE,
     ) var dataVersion: String? = null,
+    @Schema(description = REQUEST_ID_HEADER_DESCRIPTION)
+    var requestId: String? = null,
 )
 
 const val LAPIS_DATA_VERSION_HEADER = "Lapis-Data-Version"
 
 @ControllerAdvice
-class ResponseBodyAdviceDataVersion(private val dataVersion: DataVersion) : ResponseBodyAdvice<Any> {
+class ResponseBodyAdviceDataVersion(
+    private val dataVersion: DataVersion,
+    private val requestIdContext: RequestIdContext,
+) : ResponseBodyAdvice<Any> {
+
     override fun beforeBodyWrite(
         body: Any?,
         returnType: MethodParameter,
@@ -38,13 +46,15 @@ class ResponseBodyAdviceDataVersion(private val dataVersion: DataVersion) : Resp
         response.headers.add(LAPIS_DATA_VERSION_HEADER, dataVersion.dataVersion)
 
         when (body) {
-            is LapisResponse<*> -> return LapisResponse(body.data, LapisInfo(dataVersion.dataVersion))
-            is LapisErrorResponse -> return LapisErrorResponse(body.error, LapisInfo(dataVersion.dataVersion))
+            is LapisResponse<*> -> return LapisResponse(body.data, getLapisInfo())
+            is LapisErrorResponse -> return LapisErrorResponse(body.error, getLapisInfo())
         }
 
         return body
     }
 
+    private fun getLapisInfo() = LapisInfo(dataVersion.dataVersion, requestIdContext.requestId)
+
     override fun supports(
         returnType: MethodParameter,
         converterType: Class<out HttpMessageConverter<*>>,

lapis2/src/main/resources/logback.xml

@@ -1,6 +1,6 @@
 <configuration>
 
-    <property name="PATTERN" value="%date %level [%thread] %class: %message%n"/>
+    <property name="PATTERN" value="%date %level [%thread] [%X{RequestId}] %class: %message%n"/>
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <layout class="ch.qos.logback.classic.PatternLayout">

lapis2/src/test/resources/logback-test.xml

@@ -1,7 +1,7 @@
 <configuration>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <layout class="ch.qos.logback.classic.PatternLayout">
-            <Pattern>%date %level [%thread] %class: %message%n</Pattern>
+            <Pattern>%date %level [%thread] [%X{RequestId}] %class: %message%n</Pattern>
         </layout>
     </appender>
 

siloLapisTests/test/requestId.spec.ts

@@ -0,0 +1,23 @@
+import { expect } from 'chai';
+import { lapisClient } from './common';
+
+describe('The request id', () => {
+  it('should be returned when explicitly specified', async () => {
+    const requestID = 'hardcodedRequestIdInTheTest';
+
+    const result = await lapisClient.postAggregated1({
+      aggregatedPostRequest: {},
+      xRequestID: requestID,
+    });
+
+    expect(result.info.requestId).equals(requestID);
+  });
+
+  it('should be generated when none is specified', async () => {
+    const result = await lapisClient.postAggregated1({
+      aggregatedPostRequest: {},
+    });
+
+    expect(result.info.requestId).length.is.at.least(1);
+  });
+});