Skip to content

Commit

Permalink
feat: make fields case-insensitive #502
Browse files Browse the repository at this point in the history
  • Loading branch information
fengelniederhammer committed Dec 18, 2023
1 parent 7ad9746 commit 22a6467
Show file tree
Hide file tree
Showing 16 changed files with 408 additions and 168 deletions.
Original file line number Diff line number Diff line change
@@ -1,23 +1,25 @@
package org.genspectrum.lapis.config

import java.util.Locale

typealias SequenceFilterFieldName = String
typealias LowercaseName = String

const val SARS_COV2_VARIANT_QUERY_FEATURE = "sarsCoV2VariantQuery"

val FEATURES_FOR_SEQUENCE_FILTERS = listOf(
SARS_COV2_VARIANT_QUERY_FEATURE,
)

data class SequenceFilterFields(val fields: Map<SequenceFilterFieldName, SequenceFilterFieldType>) {
data class SequenceFilterFields(val fields: Map<LowercaseName, SequenceFilterField>) {
companion object {
fun fromDatabaseConfig(databaseConfig: DatabaseConfig): SequenceFilterFields {
val metadataFields = databaseConfig.schema.metadata
.map(::mapToSequenceFilterFields)
.flatten()
.toMap()
.flatMap(::mapToSequenceFilterField)
.associateBy { it.name.lowercase(Locale.US) }

val featuresFields = if (databaseConfig.schema.features.isEmpty()) {
emptyMap<SequenceFilterFieldName, SequenceFilterFieldType>()
emptyMap()
} else {
databaseConfig.schema.features
.filter { it.name in FEATURES_FOR_SEQUENCE_FILTERS }
Expand All @@ -29,35 +31,76 @@ data class SequenceFilterFields(val fields: Map<SequenceFilterFieldName, Sequenc
}
}

private fun mapToSequenceFilterFields(databaseMetadata: DatabaseMetadata) =
data class SequenceFilterField(
val name: SequenceFilterFieldName,
val type: SequenceFilterFieldType,
)

private fun mapToSequenceFilterField(databaseMetadata: DatabaseMetadata) =
when (databaseMetadata.type) {
MetadataType.STRING -> listOf(databaseMetadata.name to SequenceFilterFieldType.String)
MetadataType.PANGO_LINEAGE -> listOf(databaseMetadata.name to SequenceFilterFieldType.PangoLineage)
MetadataType.STRING -> listOf(
SequenceFilterField(
name = databaseMetadata.name,
type = SequenceFilterFieldType.String,
),
)

MetadataType.PANGO_LINEAGE -> listOf(
SequenceFilterField(
name = databaseMetadata.name,
type = SequenceFilterFieldType.PangoLineage,
),
)

MetadataType.DATE -> listOf(
databaseMetadata.name to SequenceFilterFieldType.Date,
"${databaseMetadata.name}From" to SequenceFilterFieldType.DateFrom(databaseMetadata.name),
"${databaseMetadata.name}To" to SequenceFilterFieldType.DateTo(databaseMetadata.name),
SequenceFilterField(name = databaseMetadata.name, type = SequenceFilterFieldType.Date),
SequenceFilterField(
name = "${databaseMetadata.name}From",
type = SequenceFilterFieldType.DateFrom(databaseMetadata.name),
),
SequenceFilterField(
name = "${databaseMetadata.name}To",
type = SequenceFilterFieldType.DateTo(databaseMetadata.name),
),
)

MetadataType.INT -> listOf(
databaseMetadata.name to SequenceFilterFieldType.Int,
"${databaseMetadata.name}From" to SequenceFilterFieldType.IntFrom(databaseMetadata.name),
"${databaseMetadata.name}To" to SequenceFilterFieldType.IntTo(databaseMetadata.name),
SequenceFilterField(name = databaseMetadata.name, type = SequenceFilterFieldType.Int),
SequenceFilterField(
name = "${databaseMetadata.name}From",
type = SequenceFilterFieldType.IntFrom(databaseMetadata.name),
),
SequenceFilterField(
name = "${databaseMetadata.name}To",
type = SequenceFilterFieldType.IntTo(databaseMetadata.name),
),
)

MetadataType.FLOAT -> listOf(
databaseMetadata.name to SequenceFilterFieldType.Float,
"${databaseMetadata.name}From" to SequenceFilterFieldType.FloatFrom(databaseMetadata.name),
"${databaseMetadata.name}To" to SequenceFilterFieldType.FloatTo(databaseMetadata.name),
SequenceFilterField(name = databaseMetadata.name, type = SequenceFilterFieldType.Float),
SequenceFilterField(
name = "${databaseMetadata.name}From",
type = SequenceFilterFieldType.FloatFrom(databaseMetadata.name),
),
SequenceFilterField(
name = "${databaseMetadata.name}To",
type = SequenceFilterFieldType.FloatTo(databaseMetadata.name),
),
)

MetadataType.NUCLEOTIDE_INSERTION -> emptyList()
MetadataType.AMINO_ACID_INSERTION -> emptyList()
}

private const val VARIANT_QUERY_FIELD = "variantQuery"

private fun mapToSequenceFilterFieldsFromFeatures(databaseFeature: DatabaseFeature) =
when (databaseFeature.name) {
SARS_COV2_VARIANT_QUERY_FEATURE -> "variantQuery" to SequenceFilterFieldType.VariantQuery
SARS_COV2_VARIANT_QUERY_FEATURE -> VARIANT_QUERY_FIELD.lowercase(Locale.US) to SequenceFilterField(
name = VARIANT_QUERY_FIELD,
type = SequenceFilterFieldType.VariantQuery,
)

else -> throw IllegalArgumentException(
"Unknown feature '${databaseFeature.name}'",
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import org.genspectrum.lapis.openApi.REQUEST_SCHEMA_WITH_MIN_PROPORTION
import org.genspectrum.lapis.request.AminoAcidInsertion
import org.genspectrum.lapis.request.AminoAcidMutation
import org.genspectrum.lapis.request.CommonSequenceFilters
import org.genspectrum.lapis.request.Field
import org.genspectrum.lapis.request.MutationProportionsRequest
import org.genspectrum.lapis.request.NucleotideInsertion
import org.genspectrum.lapis.request.NucleotideMutation
Expand Down Expand Up @@ -85,7 +86,7 @@ class LapisController(
sequenceFilters: Map<String, String>?,
@FieldsToAggregateBy
@RequestParam
fields: List<String>?,
fields: List<Field>?,
@AggregatedOrderByFields
@RequestParam
orderBy: List<OrderByField>?,
Expand Down Expand Up @@ -140,7 +141,7 @@ class LapisController(
sequenceFilters: Map<String, String>?,
@FieldsToAggregateBy
@RequestParam
fields: List<String>?,
fields: List<Field>?,
@AggregatedOrderByFields
@RequestParam
orderBy: List<OrderByField>?,
Expand Down Expand Up @@ -193,7 +194,7 @@ class LapisController(
sequenceFilters: Map<String, String>?,
@FieldsToAggregateBy
@RequestParam
fields: List<String>?,
fields: List<Field>?,
@AggregatedOrderByFields
@RequestParam
orderBy: List<OrderByField>?,
Expand Down Expand Up @@ -683,7 +684,7 @@ class LapisController(
sequenceFilters: Map<String, String>?,
@DetailsFields
@RequestParam
fields: List<String>?,
fields: List<Field>?,
@DetailsOrderByFields
@RequestParam
orderBy: List<OrderByField>?,
Expand Down Expand Up @@ -736,7 +737,7 @@ class LapisController(
sequenceFilters: Map<String, String>?,
@DetailsFields
@RequestParam
fields: List<String>?,
fields: List<Field>?,
@DetailsOrderByFields
@RequestParam
orderBy: List<OrderByField>?,
Expand Down Expand Up @@ -786,7 +787,7 @@ class LapisController(
sequenceFilters: Map<String, String>?,
@DetailsFields
@RequestParam
fields: List<String>?,
fields: List<Field>?,
@DetailsOrderByFields
@RequestParam
orderBy: List<OrderByField>?,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.genspectrum.lapis.model

import org.genspectrum.lapis.config.SequenceFilterField
import org.genspectrum.lapis.config.SequenceFilterFieldType
import org.genspectrum.lapis.config.SequenceFilterFields
import org.genspectrum.lapis.controller.BadRequestException
Expand Down Expand Up @@ -27,6 +28,7 @@ import org.genspectrum.lapis.silo.True
import org.springframework.stereotype.Component
import java.time.LocalDate
import java.time.format.DateTimeParseException
import java.util.Locale

data class SequenceFilterValue(val type: SequenceFilterFieldType, val value: String, val originalKey: String)

Expand All @@ -45,8 +47,8 @@ class SiloFilterExpressionMapper(
val allowedSequenceFiltersWithType = sequenceFilters
.sequenceFilters
.map { (key, value) ->
val nullableType = allowedSequenceFilterFields.fields[key]
val (filterExpressionId, type) = mapToFilterExpressionIdentifier(nullableType, key)
val nullableField = allowedSequenceFilterFields.fields[key.lowercase(Locale.US)]
val (filterExpressionId, type) = mapToFilterExpressionIdentifier(nullableField, key)
filterExpressionId to SequenceFilterValue(type, value, key)
}
.groupBy({ it.first }, { it.second })
Expand Down Expand Up @@ -87,26 +89,27 @@ class SiloFilterExpressionMapper(
}

private fun mapToFilterExpressionIdentifier(
type: SequenceFilterFieldType?,
field: SequenceFilterField?,
key: SequenceFilterFieldName,
): Pair<Pair<SequenceFilterFieldName, Filter>, SequenceFilterFieldType> {
val type = field?.type
val filterExpressionId = when (type) {
is SequenceFilterFieldType.DateFrom -> Pair(type.associatedField, Filter.DateBetween)
is SequenceFilterFieldType.DateTo -> Pair(type.associatedField, Filter.DateBetween)
SequenceFilterFieldType.Date -> Pair(key, Filter.DateBetween)
SequenceFilterFieldType.PangoLineage -> Pair(key, Filter.PangoLineage)
SequenceFilterFieldType.String -> Pair(key, Filter.StringEquals)
SequenceFilterFieldType.VariantQuery -> Pair(key, Filter.VariantQuery)
SequenceFilterFieldType.Int -> Pair(key, Filter.IntEquals)
SequenceFilterFieldType.Date -> Pair(field.name, Filter.DateBetween)
SequenceFilterFieldType.PangoLineage -> Pair(field.name, Filter.PangoLineage)
SequenceFilterFieldType.String -> Pair(field.name, Filter.StringEquals)
SequenceFilterFieldType.VariantQuery -> Pair(field.name, Filter.VariantQuery)
SequenceFilterFieldType.Int -> Pair(field.name, Filter.IntEquals)
is SequenceFilterFieldType.IntFrom -> Pair(type.associatedField, Filter.IntBetween)
is SequenceFilterFieldType.IntTo -> Pair(type.associatedField, Filter.IntBetween)
SequenceFilterFieldType.Float -> Pair(key, Filter.FloatEquals)
SequenceFilterFieldType.Float -> Pair(field.name, Filter.FloatEquals)
is SequenceFilterFieldType.FloatFrom -> Pair(type.associatedField, Filter.FloatBetween)
is SequenceFilterFieldType.FloatTo -> Pair(type.associatedField, Filter.FloatBetween)

null -> throw BadRequestException(
"'$key' is not a valid sequence filter key. Valid keys are: " +
allowedSequenceFilterFields.fields.keys.joinToString(),
allowedSequenceFilterFields.fields.values.joinToString { it.name },
)
}
return Pair(filterExpressionId, type)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class SiloQueryModel(
siloClient.sendQuery(
SiloQuery(
SiloAction.aggregated(
sequenceFilters.fields,
sequenceFilters.fields.map { it.fieldName },
sequenceFilters.orderByFields,
sequenceFilters.limit,
sequenceFilters.offset,
Expand Down Expand Up @@ -88,7 +88,7 @@ class SiloQueryModel(
siloClient.sendQuery(
SiloQuery(
SiloAction.details(
sequenceFilters.fields,
sequenceFilters.fields.map { it.fieldName },
sequenceFilters.orderByFields,
sequenceFilters.limit,
sequenceFilters.offset,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,8 @@ private fun mapToOpenApiType(type: MetadataType): String =

private fun primitiveSequenceFilterFieldSchemas(sequenceFilterFields: SequenceFilterFields) =
sequenceFilterFields.fields
.map { (fieldName, fieldType) -> fieldName to Schema<String>().type(fieldType.openApiType) }
.toMap()
.values
.associate { (fieldName, field) -> fieldName to Schema<String>().type(field.openApiType) }

private fun requestSchemaForCommonSequenceFilters(
requestProperties: Map<SequenceFilterFieldName, Schema<out Any>>,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.genspectrum.lapis.request

import org.genspectrum.lapis.config.DatabaseConfig
import org.springframework.stereotype.Component

@Component
class CaseInsensitiveFieldsCleaner(databaseConfig: DatabaseConfig) {
private val fieldsMap = databaseConfig.schema.metadata.map { it.name }.associateBy { it.lowercase() }

fun clean(fieldName: String) = fieldsMap[fieldName.lowercase()]

fun getKnownFields() = fieldsMap.values
}
20 changes: 20 additions & 0 deletions lapis2/src/main/kotlin/org/genspectrum/lapis/request/Field.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.genspectrum.lapis.request

import org.genspectrum.lapis.controller.BadRequestException
import org.springframework.core.convert.converter.Converter
import org.springframework.stereotype.Component

data class Field(val fieldName: String)

@Component
class FieldConverter(private val caseInsensitiveFieldsCleaner: CaseInsensitiveFieldsCleaner) :
Converter<String, Field> {
override fun convert(source: String): Field {
val cleaned = caseInsensitiveFieldsCleaner.clean(source)
?: throw BadRequestException(
"Unknown field: $source, known values are ${caseInsensitiveFieldsCleaner.getKnownFields()}",
)

return Field(cleaned)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,14 @@ enum class Order {
}

@JsonComponent
class OrderByFieldDeserializer : JsonDeserializer<OrderByField>() {
class OrderByFieldDeserializer(private val orderByFieldsCleaner: OrderByFieldsCleaner) :
JsonDeserializer<OrderByField>() {
override fun deserialize(
jsonParser: JsonParser,
ctxt: DeserializationContext,
): OrderByField {
return when (val value = jsonParser.readValueAsTree<JsonNode>()) {
is TextNode -> OrderByField(value.asText(), Order.ASCENDING)
is TextNode -> OrderByField(orderByFieldsCleaner.clean(value.asText()), Order.ASCENDING)
is ObjectNode -> deserializeOrderByField(value)
else -> throw BadRequestException("orderByField must be a string or an object")
}
Expand All @@ -52,11 +53,16 @@ class OrderByFieldDeserializer : JsonDeserializer<OrderByField>() {
else -> throw BadRequestException("orderByField type must be \"ascending\" or \"descending\"")
}

return OrderByField(fieldNode.asText(), ascending)
return OrderByField(orderByFieldsCleaner.clean(fieldNode.asText()), ascending)
}
}

@Component
class OrderByFieldConverter : Converter<String, OrderByField> {
override fun convert(source: String) = OrderByField(source, Order.ASCENDING)
class OrderByFieldConverter(private val orderByFieldsCleaner: OrderByFieldsCleaner) : Converter<String, OrderByField> {
override fun convert(source: String) = OrderByField(orderByFieldsCleaner.clean(source), Order.ASCENDING)
}

@Component
class OrderByFieldsCleaner(private val caseInsensitiveFieldsCleaner: CaseInsensitiveFieldsCleaner) {
fun clean(fieldName: String): String = caseInsensitiveFieldsCleaner.clean(fieldName) ?: fieldName
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,15 @@ data class SequenceFiltersRequestWithFields(
override val aaMutations: List<AminoAcidMutation>,
override val nucleotideInsertions: List<NucleotideInsertion>,
override val aminoAcidInsertions: List<AminoAcidInsertion>,
val fields: List<String>,
val fields: List<Field>,
override val orderByFields: List<OrderByField> = emptyList(),
override val limit: Int? = null,
override val offset: Int? = null,
) : CommonSequenceFilters

@JsonComponent
class SequenceFiltersRequestWithFieldsDeserializer : JsonDeserializer<SequenceFiltersRequestWithFields>() {
class SequenceFiltersRequestWithFieldsDeserializer(private val fieldConverter: FieldConverter) :
JsonDeserializer<SequenceFiltersRequestWithFields>() {
override fun deserialize(
jsonParser: JsonParser,
ctxt: DeserializationContext,
Expand All @@ -32,7 +33,7 @@ class SequenceFiltersRequestWithFieldsDeserializer : JsonDeserializer<SequenceFi

val fields = when (val fields = node.get(FIELDS_PROPERTY)) {
null -> emptyList()
is ArrayNode -> fields.asSequence().map { it.asText() }.toList()
is ArrayNode -> fields.asSequence().map { fieldConverter.convert(it.asText()) }.toList()
else -> throw BadRequestException(
"fields must be an array or null",
)
Expand Down
Loading

0 comments on commit 22a6467

Please sign in to comment.