Skip to content

Commit

Permalink
feat: implement orderBy, limit and offset
Browse files Browse the repository at this point in the history 
issue: #290
  • Loading branch information
@fengelniederhammer
fengelniederhammer committed Jul 24, 2023
1 parent 0c6e7ad commit ae85534
Show file tree
Hide file tree
Showing 16 changed files with 777 additions and 46 deletions.
60 changes: 58 additions & 2 deletions lapis2/src/main/kotlin/org/genspectrum/lapis/OpenApiDocs.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,24 @@ import org.genspectrum.lapis.controller.AMINO_ACID_MUTATIONS_SCHEMA
import org.genspectrum.lapis.controller.DETAILS_FIELDS_DESCRIPTION
import org.genspectrum.lapis.controller.DETAILS_REQUEST_SCHEMA
import org.genspectrum.lapis.controller.DETAILS_RESPONSE_SCHEMA
import org.genspectrum.lapis.controller.LIMIT_DESCRIPTION
import org.genspectrum.lapis.controller.LIMIT_SCHEMA
import org.genspectrum.lapis.controller.NUCLEOTIDE_MUTATIONS_SCHEMA
import org.genspectrum.lapis.controller.OFFSET_DESCRIPTION
import org.genspectrum.lapis.controller.OFFSET_SCHEMA
import org.genspectrum.lapis.controller.ORDER_BY_FIELDS_SCHEMA
import org.genspectrum.lapis.controller.REQUEST_SCHEMA_WITH_MIN_PROPORTION
import org.genspectrum.lapis.controller.SEQUENCE_FILTERS_SCHEMA
import org.genspectrum.lapis.request.AMINO_ACID_MUTATIONS_PROPERTY
import org.genspectrum.lapis.request.AminoAcidMutation
import org.genspectrum.lapis.request.FIELDS_PROPERTY
import org.genspectrum.lapis.request.LIMIT_PROPERTY
import org.genspectrum.lapis.request.MIN_PROPORTION_PROPERTY
import org.genspectrum.lapis.request.NUCLEOTIDE_MUTATIONS_PROPERTY
import org.genspectrum.lapis.request.NucleotideMutation
import org.genspectrum.lapis.request.OFFSET_PROPERTY
import org.genspectrum.lapis.request.ORDER_BY_PROPERTY
import org.genspectrum.lapis.request.OrderByField
import org.genspectrum.lapis.response.COUNT_PROPERTY

fun buildOpenApiSchema(sequenceFilterFields: SequenceFilterFields, databaseConfig: DatabaseConfig): OpenAPI {
Expand All @@ -36,7 +45,10 @@ fun buildOpenApiSchema(sequenceFilterFields: SequenceFilterFields, databaseConfi

val sequenceFilters = requestProperties +
Pair(NUCLEOTIDE_MUTATIONS_PROPERTY, nucleotideMutations()) +
Pair(AMINO_ACID_MUTATIONS_PROPERTY, aminoAcidMutations())
Pair(AMINO_ACID_MUTATIONS_PROPERTY, aminoAcidMutations()) +
Pair(ORDER_BY_PROPERTY, orderByPostSchema()) +
Pair(LIMIT_PROPERTY, limitSchema()) +
Pair(OFFSET_PROPERTY, offsetSchema())

return OpenAPI()
.components(
Expand Down Expand Up @@ -85,7 +97,10 @@ fun buildOpenApiSchema(sequenceFilterFields: SequenceFilterFields, databaseConfi
.properties(metadataFieldSchemas(databaseConfig)),
)
.addSchemas(NUCLEOTIDE_MUTATIONS_SCHEMA, nucleotideMutations())
.addSchemas(AMINO_ACID_MUTATIONS_SCHEMA, aminoAcidMutations()),
.addSchemas(AMINO_ACID_MUTATIONS_SCHEMA, aminoAcidMutations())
.addSchemas(ORDER_BY_FIELDS_SCHEMA, orderByGetSchema())
.addSchemas(LIMIT_SCHEMA, limitSchema())
.addSchemas(OFFSET_SCHEMA, offsetSchema()),
)
}

Expand Down Expand Up @@ -164,6 +179,47 @@ private fun aminoAcidMutations() =
),
)

private fun orderByGetSchema() = Schema<List<String>>()
.type("array")
.items(orderByFieldStringSchema())
.description("The fields by which the result is ordered in ascending order.")

private fun orderByPostSchema() = Schema<List<String>>()
.type("array")
.items(
Schema<String>().anyOf(
listOf(
orderByFieldStringSchema(),
Schema<OrderByField>()
.type("object")
.description("The fields by which the result is ordered with ascending or descending order.")
.required(listOf("field"))
.properties(
mapOf(
"field" to orderByFieldStringSchema(),
"type" to Schema<String>()
.type("string")
._enum(listOf("ascending", "descending"))
._default("ascending"),
),
),
),
),
)

private fun orderByFieldStringSchema() = Schema<String>()
.type("string")
.example("country")
.description("The field by which the result is ordered.")

private fun limitSchema() = Schema<Int>()
.type("integer")
.description(LIMIT_DESCRIPTION)

private fun offsetSchema() = Schema<Int>()
.type("integer")
.description(OFFSET_DESCRIPTION)

// This is a function so that the resulting schema can be reused in multiple places. The setters mutate the instance.
private fun fieldsSchema() = Schema<String>()
.type("array")
Expand Down
80 changes: 79 additions & 1 deletion lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,14 @@ import org.genspectrum.lapis.request.AMINO_ACID_MUTATIONS_PROPERTY
import org.genspectrum.lapis.request.AminoAcidMutation
import org.genspectrum.lapis.request.DEFAULT_MIN_PROPORTION
import org.genspectrum.lapis.request.FIELDS_PROPERTY
import org.genspectrum.lapis.request.LIMIT_PROPERTY
import org.genspectrum.lapis.request.MIN_PROPORTION_PROPERTY
import org.genspectrum.lapis.request.MutationProportionsRequest
import org.genspectrum.lapis.request.NUCLEOTIDE_MUTATIONS_PROPERTY
import org.genspectrum.lapis.request.NucleotideMutation
import org.genspectrum.lapis.request.OFFSET_PROPERTY
import org.genspectrum.lapis.request.ORDER_BY_PROPERTY
import org.genspectrum.lapis.request.OrderByField
import org.genspectrum.lapis.request.SequenceFiltersRequestWithFields
import org.genspectrum.lapis.response.AggregationData
import org.genspectrum.lapis.response.MutationData
Expand All @@ -38,11 +42,17 @@ const val DETAILS_RESPONSE_SCHEMA = "DetailsResponse"

const val NUCLEOTIDE_MUTATIONS_SCHEMA = "NucleotideMutations"
const val AMINO_ACID_MUTATIONS_SCHEMA = "AminoAcidMutations"
const val ORDER_BY_FIELDS_SCHEMA = "OrderByFields"
const val LIMIT_SCHEMA = "Limit"
const val OFFSET_SCHEMA = "Offset"

const val AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION =
"The fields to stratify by. If empty, only the overall count is returned"
const val DETAILS_FIELDS_DESCRIPTION =
"The fields that the response items should contain. If empty, all fields are returned"
const val LIMIT_DESCRIPTION = "The maximum number of entries to return in the response"
const val OFFSET_DESCRIPTION = "The offset of the first entry to return in the response. " +
"This is useful for pagination in combination with \"limit\"."

@RestController
class LapisController(private val siloQueryModel: SiloQueryModel, private val requestContext: RequestContext) {
Expand All @@ -54,6 +64,9 @@ class LapisController(private val siloQueryModel: SiloQueryModel, private val re
FIELDS_PROPERTY,
NUCLEOTIDE_MUTATIONS_PROPERTY,
AMINO_ACID_MUTATIONS_PROPERTY,
ORDER_BY_PROPERTY,
LIMIT_PROPERTY,
OFFSET_PROPERTY,
)
}

Expand All @@ -66,6 +79,13 @@ class LapisController(private val siloQueryModel: SiloQueryModel, private val re
@Parameter(description = AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION)
@RequestParam
fields: List<String>?,
@Parameter(
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
description = "The fields to order by." +
" Fields specified here must either be \"count\" or also be present in \"fields\".",
)
@RequestParam
orderBy: List<OrderByField>?,
@Parameter(
schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_MUTATIONS_SCHEMA"),
explode = Explode.TRUE,
Expand All @@ -75,12 +95,27 @@ class LapisController(private val siloQueryModel: SiloQueryModel, private val re
@Parameter(schema = Schema(ref = "#/components/schemas/$AMINO_ACID_MUTATIONS_SCHEMA"))
@RequestParam
aminoAcidMutations: List<AminoAcidMutation>?,
@Parameter(
schema = Schema(ref = "#/components/schemas/$LIMIT_SCHEMA"),
description = LIMIT_DESCRIPTION,
)
@RequestParam
limit: Int? = null,
@Parameter(
schema = Schema(ref = "#/components/schemas/$OFFSET_SCHEMA"),
description = OFFSET_DESCRIPTION,
)
@RequestParam
offset: Int? = null,
): List<AggregationData> {
val request = SequenceFiltersRequestWithFields(
sequenceFilters?.filter { !nonSequenceFilterFields.contains(it.key) } ?: emptyMap(),
nucleotideMutations ?: emptyList(),
aminoAcidMutations ?: emptyList(),
fields ?: emptyList(),
orderBy ?: emptyList(),
limit,
offset,
)

requestContext.filter = request
Expand Down Expand Up @@ -118,12 +153,33 @@ class LapisController(private val siloQueryModel: SiloQueryModel, private val re
aminoAcidMutations: List<AminoAcidMutation>?,
@RequestParam(defaultValue = DEFAULT_MIN_PROPORTION.toString())
minProportion: Double,
@Parameter(
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
description = "The fields of the response to order by.",
)
@RequestParam
orderBy: List<OrderByField>?,
@Parameter(
schema = Schema(ref = "#/components/schemas/$LIMIT_SCHEMA"),
description = LIMIT_DESCRIPTION,
)
@RequestParam
limit: Int? = null,
@Parameter(
schema = Schema(ref = "#/components/schemas/$OFFSET_SCHEMA"),
description = OFFSET_DESCRIPTION,
)
@RequestParam
offset: Int? = null,
): List<MutationData> {
val request = MutationProportionsRequest(
sequenceFilters?.filter { !nonSequenceFilterFields.contains(it.key) } ?: emptyMap(),
nucleotideMutations ?: emptyList(),
aminoAcidMutations ?: emptyList(),
minProportion,
orderBy ?: emptyList(),
limit,
offset,
)

requestContext.filter = request
Expand All @@ -149,21 +205,43 @@ class LapisController(private val siloQueryModel: SiloQueryModel, private val re
@SequenceFilters
@RequestParam
sequenceFilters: Map<String, String>?,
@Parameter(description = AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION)
@Parameter(description = DETAILS_FIELDS_DESCRIPTION)
@RequestParam
fields: List<String>?,
@Parameter(
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
description = "The fields of the response to order by." +
" Fields specified here must also be present in \"fields\".",
)
@RequestParam
orderBy: List<OrderByField>?,
@Parameter(schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_MUTATIONS_SCHEMA"))
@RequestParam
nucleotideMutations: List<NucleotideMutation>?,
@Parameter(schema = Schema(ref = "#/components/schemas/$AMINO_ACID_MUTATIONS_SCHEMA"))
@RequestParam
aminoAcidMutations: List<AminoAcidMutation>?,
@Parameter(
schema = Schema(ref = "#/components/schemas/$LIMIT_SCHEMA"),
description = LIMIT_DESCRIPTION,
)
@RequestParam
limit: Int? = null,
@Parameter(
schema = Schema(ref = "#/components/schemas/$OFFSET_SCHEMA"),
description = OFFSET_DESCRIPTION,
)
@RequestParam
offset: Int? = null,
): List<DetailsData> {
val request = SequenceFiltersRequestWithFields(
sequenceFilters?.filter { !nonSequenceFilterFields.contains(it.key) } ?: emptyMap(),
nucleotideMutations ?: emptyList(),
aminoAcidMutations ?: emptyList(),
fields ?: emptyList(),
orderBy ?: emptyList(),
limit,
offset,
)

requestContext.filter = request
Expand Down
21 changes: 18 additions & 3 deletions lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,37 @@ class SiloQueryModel(

fun aggregate(sequenceFilters: SequenceFiltersRequestWithFields) = siloClient.sendQuery(
SiloQuery(
SiloAction.aggregated(sequenceFilters.fields),
SiloAction.aggregated(
sequenceFilters.fields,
sequenceFilters.orderByFields,
sequenceFilters.limit,
sequenceFilters.offset,
),
siloFilterExpressionMapper.map(sequenceFilters),
),
)

fun computeMutationProportions(sequenceFilters: MutationProportionsRequest) =
siloClient.sendQuery(
SiloQuery(
SiloAction.mutations(sequenceFilters.minProportion),
SiloAction.mutations(
sequenceFilters.minProportion,
sequenceFilters.orderByFields,
sequenceFilters.limit,
sequenceFilters.offset,
),
siloFilterExpressionMapper.map(sequenceFilters),
),
)

fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields) = siloClient.sendQuery(
SiloQuery(
SiloAction.details(sequenceFilters.fields),
SiloAction.details(
sequenceFilters.fields,
sequenceFilters.orderByFields,
sequenceFilters.limit,
sequenceFilters.offset,
),
siloFilterExpressionMapper.map(sequenceFilters),
),
)
Expand Down
52 changes: 46 additions & 6 deletions lapis2/src/main/kotlin/org/genspectrum/lapis/request/CommonSequenceFilters.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,28 @@ import org.genspectrum.lapis.auth.ACCESS_KEY_PROPERTY

const val NUCLEOTIDE_MUTATIONS_PROPERTY = "nucleotideMutations"
const val AMINO_ACID_MUTATIONS_PROPERTY = "aminoAcidMutations"
const val ORDER_BY_PROPERTY = "orderBy"
const val LIMIT_PROPERTY = "limit"
const val OFFSET_PROPERTY = "offset"

private val nonSequenceFilterPrimitiveFields = listOf(
LIMIT_PROPERTY,
OFFSET_PROPERTY,
ACCESS_KEY_PROPERTY,
)

interface CommonSequenceFilters {
val sequenceFilters: Map<String, String>
val nucleotideMutations: List<NucleotideMutation>
val aaMutations: List<AminoAcidMutation>
val orderByFields: List<OrderByField>
val limit: Int?
val offset: Int?

fun isEmpty() = sequenceFilters.isEmpty() && nucleotideMutations.isEmpty() && aaMutations.isEmpty()
}

fun parseCommonFields(
node: JsonNode,
codec: ObjectCodec,
): Triple<List<NucleotideMutation>, List<AminoAcidMutation>, Map<String, String>> {
fun parseCommonFields(node: JsonNode, codec: ObjectCodec): ParsedCommonFields {
val nucleotideMutations = when (val nucleotideMutationsNode = node.get(NUCLEOTIDE_MUTATIONS_PROPERTY)) {
null -> emptyList()
is ArrayNode -> nucleotideMutationsNode.map { codec.treeToValue(it, NucleotideMutation::class.java) }
Expand All @@ -37,14 +46,45 @@ fun parseCommonFields(
)
}

val orderByFields = when (val orderByNode = node.get(ORDER_BY_PROPERTY)) {
null -> emptyList()
is ArrayNode -> orderByNode.map { codec.treeToValue(it, OrderByField::class.java) }
else -> throw IllegalArgumentException(
"orderBy must be an array or null",
)
}

val limitNode = node.get(LIMIT_PROPERTY)
val limit = when (limitNode?.nodeType) {
null -> null
JsonNodeType.NULL, JsonNodeType.NUMBER -> limitNode.asInt()
else -> throw IllegalArgumentException("limit must be a number or null")
}

val offsetNode = node.get(OFFSET_PROPERTY)
val offset = when (offsetNode?.nodeType) {
null -> null
JsonNodeType.NULL, JsonNodeType.NUMBER -> offsetNode.asInt()
else -> throw IllegalArgumentException("offset must be a number or null")
}

val sequenceFilters = node.fields()
.asSequence()
.filter { isStringOrNumber(it.value) }
.filter { it.key != ACCESS_KEY_PROPERTY }
.filter { !nonSequenceFilterPrimitiveFields.contains(it.key) }
.associate { it.key to it.value.asText() }
return Triple(nucleotideMutations, aminoAcidMutations, sequenceFilters)
return ParsedCommonFields(nucleotideMutations, aminoAcidMutations, sequenceFilters, orderByFields, limit, offset)
}

data class ParsedCommonFields(
val nucleotideMutations: List<NucleotideMutation>,
val aminoAcidMutations: List<AminoAcidMutation>,
val sequenceFilters: Map<String, String>,
val orderByFields: List<OrderByField>,
val limit: Int?,
val offset: Int?,
)

private fun isStringOrNumber(jsonNode: JsonNode) =
when (jsonNode.nodeType) {
JsonNodeType.STRING,
Expand Down
Loading

0 comments on commit ae85534

Please sign in to comment.