Skip to content

Commit

Permalink
feat: make it possible to return data from /details as CSV #284
Browse files Browse the repository at this point in the history
  • Loading branch information
fengelniederhammer committed Aug 3, 2023
1 parent a5cd5a4 commit d3107d6
Show file tree
Hide file tree
Showing 17 changed files with 593 additions and 214 deletions.
1 change: 1 addition & 0 deletions lapis2/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ dependencies {
implementation 'io.github.microutils:kotlin-logging-jvm:3.0.5'
antlr 'org.antlr:antlr4:4.13.0'
implementation 'org.antlr:antlr4-runtime:4.13.0'
implementation 'org.apache.commons:commons-csv:1.10.0'

testImplementation('org.springframework.boot:spring-boot-starter-test') {
exclude group: "org.mockito"
Expand Down
27 changes: 19 additions & 8 deletions lapis2/src/main/kotlin/org/genspectrum/lapis/OpenApiDocs.kt
Original file line number Diff line number Diff line change
Expand Up @@ -11,27 +11,28 @@ import org.genspectrum.lapis.config.SequenceFilterFields
import org.genspectrum.lapis.controller.AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION
import org.genspectrum.lapis.controller.AGGREGATED_REQUEST_SCHEMA
import org.genspectrum.lapis.controller.AGGREGATED_RESPONSE_SCHEMA
import org.genspectrum.lapis.controller.AMINO_ACID_MUTATIONS_PROPERTY
import org.genspectrum.lapis.controller.AMINO_ACID_MUTATIONS_SCHEMA
import org.genspectrum.lapis.controller.DETAILS_FIELDS_DESCRIPTION
import org.genspectrum.lapis.controller.DETAILS_REQUEST_SCHEMA
import org.genspectrum.lapis.controller.DETAILS_RESPONSE_SCHEMA
import org.genspectrum.lapis.controller.FIELDS_PROPERTY
import org.genspectrum.lapis.controller.FORMAT_PROPERTY
import org.genspectrum.lapis.controller.LIMIT_DESCRIPTION
import org.genspectrum.lapis.controller.LIMIT_PROPERTY
import org.genspectrum.lapis.controller.LIMIT_SCHEMA
import org.genspectrum.lapis.controller.MIN_PROPORTION_PROPERTY
import org.genspectrum.lapis.controller.NUCLEOTIDE_MUTATIONS_PROPERTY
import org.genspectrum.lapis.controller.NUCLEOTIDE_MUTATIONS_SCHEMA
import org.genspectrum.lapis.controller.OFFSET_DESCRIPTION
import org.genspectrum.lapis.controller.OFFSET_PROPERTY
import org.genspectrum.lapis.controller.OFFSET_SCHEMA
import org.genspectrum.lapis.controller.ORDER_BY_FIELDS_SCHEMA
import org.genspectrum.lapis.controller.ORDER_BY_PROPERTY
import org.genspectrum.lapis.controller.REQUEST_SCHEMA_WITH_MIN_PROPORTION
import org.genspectrum.lapis.controller.SEQUENCE_FILTERS_SCHEMA
import org.genspectrum.lapis.request.AMINO_ACID_MUTATIONS_PROPERTY
import org.genspectrum.lapis.request.AminoAcidMutation
import org.genspectrum.lapis.request.FIELDS_PROPERTY
import org.genspectrum.lapis.request.LIMIT_PROPERTY
import org.genspectrum.lapis.request.MIN_PROPORTION_PROPERTY
import org.genspectrum.lapis.request.NUCLEOTIDE_MUTATIONS_PROPERTY
import org.genspectrum.lapis.request.NucleotideMutation
import org.genspectrum.lapis.request.OFFSET_PROPERTY
import org.genspectrum.lapis.request.ORDER_BY_PROPERTY
import org.genspectrum.lapis.request.OrderByField
import org.genspectrum.lapis.response.COUNT_PROPERTY

Expand All @@ -48,7 +49,8 @@ fun buildOpenApiSchema(sequenceFilterFields: SequenceFilterFields, databaseConfi
Pair(AMINO_ACID_MUTATIONS_PROPERTY, aminoAcidMutations()) +
Pair(ORDER_BY_PROPERTY, orderByPostSchema()) +
Pair(LIMIT_PROPERTY, limitSchema()) +
Pair(OFFSET_PROPERTY, offsetSchema())
Pair(OFFSET_PROPERTY, offsetSchema()) +
Pair(FORMAT_PROPERTY, formatSchema())

return OpenAPI()
.components(
Expand Down Expand Up @@ -225,3 +227,12 @@ private fun offsetSchema() = Schema<Int>()
private fun fieldsSchema() = Schema<String>()
.type("array")
.items(Schema<String>().type("string"))

private fun formatSchema() = Schema<String>()
.type("string")
.description(
"The data format of the response. It is preferred to set the Accept header instead," +
" but having this as a parameters enables users to share links to download CSV files.",
)
._enum(listOf("csv", "tsv", "json"))
._default("json")
Original file line number Diff line number Diff line change
@@ -1,26 +1,21 @@
package org.genspectrum.lapis.auth

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import jakarta.servlet.FilterChain
import jakarta.servlet.http.HttpServletRequest
import jakarta.servlet.http.HttpServletResponse
import mu.KotlinLogging
import org.genspectrum.lapis.config.AccessKeys
import org.genspectrum.lapis.config.AccessKeysReader
import org.genspectrum.lapis.config.DatabaseConfig
import org.genspectrum.lapis.config.OpennessLevel
import org.genspectrum.lapis.controller.ACCESS_KEY_PROPERTY
import org.genspectrum.lapis.controller.LapisHttpErrorResponse
import org.genspectrum.lapis.util.CachedBodyHttpServletRequest
import org.springframework.http.HttpStatus
import org.springframework.http.MediaType
import org.springframework.stereotype.Component
import org.springframework.web.filter.OncePerRequestFilter

const val ACCESS_KEY_PROPERTY = "accessKey"

private val log = KotlinLogging.logger {}

@Component
class DataOpennessAuthorizationFilterFactory(
private val databaseConfig: DatabaseConfig,
Expand Down Expand Up @@ -63,7 +58,7 @@ abstract class DataOpennessAuthorizationFilter(protected val objectMapper: Objec
}

private fun makeRequestBodyReadableMoreThanOnce(request: HttpServletRequest) =
CachedBodyHttpServletRequest(request)
CachedBodyHttpServletRequest(request, objectMapper)

abstract fun isAuthorizedForEndpoint(request: CachedBodyHttpServletRequest): AuthorizationResult
}
Expand All @@ -75,7 +70,7 @@ sealed interface AuthorizationResult {
fun failure(message: String): AuthorizationResult = Failure(message)
}

object Success : AuthorizationResult
data object Success : AuthorizationResult

class Failure(val message: String) : AuthorizationResult
}
Expand Down Expand Up @@ -103,9 +98,9 @@ private class ProtectedDataAuthorizationFilter(
return AuthorizationResult.success()
}

val requestFields = getRequestFields(request)
val requestFields = request.getRequestFields()

val accessKey = requestFields[ACCESS_KEY_PROPERTY]
val accessKey = requestFields[ACCESS_KEY_PROPERTY]?.textValue()
?: return AuthorizationResult.failure("An access key is required to access ${request.requestURI}.")

if (accessKeys.fullAccessKey == accessKey) {
Expand All @@ -121,23 +116,4 @@ private class ProtectedDataAuthorizationFilter(

return AuthorizationResult.failure("You are not authorized to access ${request.requestURI}.")
}

private fun getRequestFields(request: CachedBodyHttpServletRequest): Map<String, String> {
if (request.parameterNames.hasMoreElements()) {
return request.parameterMap.mapValues { (_, value) -> value.joinToString() }
}

if (request.contentLength == 0) {
log.warn { "Could not read access key from body, because content length is 0." }
return emptyMap()
}

return try {
objectMapper.readValue(request.inputStream)
} catch (exception: Exception) {
log.error { "Failed to read access key from request body: ${exception.message}" }
log.debug { exception.stackTraceToString() }
emptyMap()
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package org.genspectrum.lapis.controller

import com.fasterxml.jackson.databind.JsonNode
import org.apache.commons.csv.CSVFormat
import org.apache.commons.csv.CSVPrinter
import org.genspectrum.lapis.silo.DetailsData
import org.springframework.stereotype.Component
import java.io.StringWriter

interface CsvRecord {
fun asArray(): Array<String>
}

@Component
class CsvWriter {
fun write(headers: Array<String>, data: List<CsvRecord>): String {
val stringWriter = StringWriter()
CSVPrinter(
stringWriter,
CSVFormat.DEFAULT.builder().setRecordSeparator("\n").setHeader(*headers).build(),
).use {
for (datum in data) {
it.printRecord(*datum.asArray())
}
}
return stringWriter.toString().trim()
}
}

fun DetailsData.asCsvRecord() = JsonValuesCsvRecord(this.values)

data class JsonValuesCsvRecord(val values: Collection<JsonNode>) : CsvRecord {
override fun asArray() = values.map { it.asText() }.toTypedArray()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package org.genspectrum.lapis.controller

import com.fasterxml.jackson.databind.ObjectMapper
import jakarta.servlet.FilterChain
import jakarta.servlet.http.HttpServletRequest
import jakarta.servlet.http.HttpServletRequestWrapper
import jakarta.servlet.http.HttpServletResponse
import mu.KotlinLogging
import org.genspectrum.lapis.util.CachedBodyHttpServletRequest
import org.springframework.stereotype.Component
import org.springframework.web.filter.OncePerRequestFilter
import java.util.Collections
import java.util.Enumeration

private val log = KotlinLogging.logger {}

@Component
class DataFormatParameterFilter(val objectMapper: ObjectMapper) : OncePerRequestFilter() {

override fun doFilterInternal(
request: HttpServletRequest,
response: HttpServletResponse,
filterChain: FilterChain,
) {
val reReadableRequest = CachedBodyHttpServletRequest(request, objectMapper)

filterChain.doFilter(AcceptHeaderModifyingRequestWrapper(reReadableRequest), response)
}
}

class AcceptHeaderModifyingRequestWrapper(
private val reReadableRequest: CachedBodyHttpServletRequest,
) : HttpServletRequestWrapper(reReadableRequest) {

override fun getHeader(name: String): String? {
if (name.equals("Accept", ignoreCase = true)) {
when (reReadableRequest.getRequestFields()[FORMAT_PROPERTY]?.textValue()?.uppercase()) {
"CSV" -> {
log.debug { "Overwriting Accept header to text/csv due to format property" }
return "text/csv"
}

else -> {}
}
}

return super.getHeader(name)
}

override fun getHeaders(name: String): Enumeration<String>? {
if (name.equals("Accept", ignoreCase = true)) {
when (reReadableRequest.getRequestFields()[FORMAT_PROPERTY]?.textValue()?.uppercase()) {
"CSV" -> {
log.debug { "Overwriting Accept header to text/csv due to format property" }
return Collections.enumeration(listOf("text/csv"))
}

else -> {}
}
}

return super.getHeaders(name)
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.genspectrum.lapis.controller

import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import mu.KotlinLogging
import org.genspectrum.lapis.model.SiloNotImplementedError
import org.genspectrum.lapis.silo.SiloException
Expand All @@ -9,78 +8,92 @@ import org.springframework.http.MediaType
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.ControllerAdvice
import org.springframework.web.bind.annotation.ExceptionHandler
import org.springframework.web.bind.annotation.ResponseStatus
import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler

private val log = KotlinLogging.logger {}

@ControllerAdvice
class ExceptionHandler : ResponseEntityExceptionHandler() {
@ExceptionHandler(Throwable::class)
fun handleUnexpectedException(e: Throwable): ResponseEntity<String> {
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
fun handleUnexpectedException(e: Throwable): ResponseEntity<LapisHttpErrorResponse> {
log.error(e) { "Caught unexpected exception: ${e.message}" }

return ResponseEntity
.status(HttpStatus.INTERNAL_SERVER_ERROR)
.contentType(MediaType.APPLICATION_JSON)
.body(
jacksonObjectMapper().writeValueAsString(
LapisHttpErrorResponse(
"Unexpected error",
"${e.message}",
),
LapisHttpErrorResponse(
"Unexpected error",
"${e.message}",
),
)
}

@ExceptionHandler(IllegalArgumentException::class)
fun handleIllegalArgumentException(e: IllegalArgumentException): ResponseEntity<String> {
@ResponseStatus(HttpStatus.BAD_REQUEST)
fun handleIllegalArgumentException(e: IllegalArgumentException): ResponseEntity<LapisHttpErrorResponse> {
log.error(e) { "Caught IllegalArgumentException: ${e.message}" }

return ResponseEntity
.status(HttpStatus.BAD_REQUEST)
.contentType(MediaType.APPLICATION_JSON)
.body(
jacksonObjectMapper().writeValueAsString(
LapisHttpErrorResponse(
"Bad request",
"${e.message}",
),
LapisHttpErrorResponse(
"Bad request",
"${e.message}",
),
)
}

@ExceptionHandler(AddForbiddenToOpenApiDocsHelper::class)
@ResponseStatus(HttpStatus.FORBIDDEN)
fun handleForbiddenException(e: AddForbiddenToOpenApiDocsHelper): ResponseEntity<LapisHttpErrorResponse> {
return ResponseEntity
.status(HttpStatus.FORBIDDEN)
.contentType(MediaType.APPLICATION_JSON)
.body(
LapisHttpErrorResponse(
"Forbidden",
"${e.message}",
),
)
}

@ExceptionHandler(SiloException::class)
fun handleSiloException(e: SiloException): ResponseEntity<String> {
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
fun handleSiloException(e: SiloException): ResponseEntity<LapisHttpErrorResponse> {
log.error(e) { "Caught SiloException: ${e.message}" }

return ResponseEntity
.status(HttpStatus.INTERNAL_SERVER_ERROR)
.contentType(MediaType.APPLICATION_JSON)
.body(
jacksonObjectMapper().writeValueAsString(
LapisHttpErrorResponse(
"Silo error",
"${e.message}",
),
LapisHttpErrorResponse(
"Silo error",
"${e.message}",
),
)
}

@ExceptionHandler(SiloNotImplementedError::class)
fun handleNotImplementedError(e: SiloNotImplementedError): ResponseEntity<String> {
@ResponseStatus(HttpStatus.NOT_IMPLEMENTED)
fun handleNotImplementedError(e: SiloNotImplementedError): ResponseEntity<LapisHttpErrorResponse> {
log.error(e) { "Caught SiloNotImplementedError: ${e.message}" }
return ResponseEntity
.status(HttpStatus.NOT_IMPLEMENTED)
.contentType(MediaType.APPLICATION_JSON)
.body(
jacksonObjectMapper().writeValueAsString(
LapisHttpErrorResponse(
"Not implemented",
"${e.message}",
),
LapisHttpErrorResponse(
"Not implemented",
"${e.message}",
),
)
}
}

/** This is not yet actually thrown, but makes "403 Forbidden" appear in OpenAPI docs. */
class AddForbiddenToOpenApiDocsHelper(message: String) : Exception(message)

data class LapisHttpErrorResponse(val title: String, val message: String)
Loading

0 comments on commit d3107d6

Please sign in to comment.