Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add duration logical type #233

Merged
merged 1 commit into from
Jul 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 122 additions & 20 deletions api/avro4k-core.api

Large diffs are not rendered by default.

72 changes: 29 additions & 43 deletions src/main/kotlin/com/github/avrokotlin/avro4k/Annotations.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,18 @@

package com.github.avrokotlin.avro4k

import com.github.avrokotlin.avro4k.internal.asAvroLogicalType
import com.github.avrokotlin.avro4k.internal.nonNullSerialName
import com.github.avrokotlin.avro4k.serializer.BigDecimalSerializer
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.SerialInfo
import kotlinx.serialization.descriptors.SerialDescriptor
import org.apache.avro.LogicalType
import org.intellij.lang.annotations.Language

/**
* Adds a property to the Avro schema or field. Its value could be any valid JSON or just a string.
*
* When annotated on a value class or its underlying field, the props are applied to the underlying type.
*
* Only works with classes (data, enum & object types) and class properties (not enum values).
* Fails at runtime when used in value classes wrapping a named schema (fixed, enum or record).
*/
@SerialInfo
@Repeatable
Expand All @@ -24,26 +23,13 @@ public annotation class AvroProp(
@Language("JSON") val value: String,
)

/**
* To be used with [BigDecimalSerializer] to specify the scale, precision, type and rounding mode of the decimal value.
*
* Can be used with [AvroFixed] to serialize value as a fixed type.
*/
@SerialInfo
@ExperimentalSerializationApi
@Target(AnnotationTarget.PROPERTY)
public annotation class AvroDecimal(
val scale: Int = 2,
val precision: Int = 8,
)

/**
* Adds documentation to:
* - a record's field
* - a record
* - an enum
* - a record's field when annotated on a data class property
* - a record when annotated on a data class or object
* - an enum type when annotated on an enum class
*
* Ignored in inline classes.
* Only works with classes (data, enum & object types) and class properties (not enum values). Ignored in value classes.
*/
@SerialInfo
@Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS)
Expand All @@ -52,16 +38,34 @@ public annotation class AvroDoc(val value: String)
/**
* Adds aliases to a field of a record. It helps to allow having different names for the same field for better compatibility when changing a schema.
*
* Ignored in inline classes.
* Only works with classes (data, enum & object types) and class properties (not enum values). Ignored in value classes.
*
* @param value The aliases for the annotated property. Note that the given aliases won't be changed by the configured [AvroConfiguration.fieldNamingStrategy].
*/
@SerialInfo
@Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS)
public annotation class AvroAlias(vararg val value: String)

/**
* To be used with [BigDecimalSerializer] to specify the scale, precision, type and rounding mode of the decimal value.
*
* Can be used with [AvroFixed] to serialize value as a fixed type.
*
* Only works with [java.math.BigDecimal] property type.
*/
@SerialInfo
@ExperimentalSerializationApi
@Target(AnnotationTarget.PROPERTY)
public annotation class AvroDecimal(
val scale: Int = 2,
val precision: Int = 8,
)

/**
* Indicates that the annotated property should be encoded as an Avro fixed type.
*
* Only works with [ByteArray], [String] and [java.math.BigDecimal] property types.
*
* @param size The number of bytes of the fixed type. Note that smaller values will be padded with 0s during encoding, but not unpadded when decoding.
*/
@SerialInfo
Expand All @@ -76,7 +80,7 @@ public annotation class AvroFixed(val size: Int)
* - Nulls have to be represented as a json `null`. To set the string `"null"`, don't forget to quote the string, example: `""""null""""` or `"\"null\""`.
* - Any non json content will be treated as a string
*
* Ignored in inline classes.
* Only works with data class properties (not enum values). Ignored in value classes.
*/
@SerialInfo
@Target(AnnotationTarget.PROPERTY)
Expand All @@ -87,27 +91,9 @@ public annotation class AvroDefault(
/**
* Sets the enum default value when decoded an unknown enum value.
*
* It must be annotated on an enum value. Otherwise, it will be ignored.
* Only works with enum classes.
*/
@SerialInfo
@ExperimentalSerializationApi
@Target(AnnotationTarget.PROPERTY)
public annotation class AvroEnumDefault

/**
* Adds a logical type to the given serializer, where the logical type name is the descriptor's name.
*
* To use it:
* ```kotlin
* object YourCustomLogicalTypeSerializer : KSerializer<YourType> {
* override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("YourType", PrimitiveKind.STRING)
* .asAvroLogicalType()
* }
* ```
*
* For more complex needs, please file an feature request [here](https://github.com/avro-kotlin/avro4k/issues).
*/
@ExperimentalSerializationApi
public fun SerialDescriptor.asAvroLogicalType(): SerialDescriptor {
return asAvroLogicalType { LogicalType(nonNullSerialName) }
}
public annotation class AvroEnumDefault
24 changes: 5 additions & 19 deletions src/main/kotlin/com/github/avrokotlin/avro4k/Avro.kt
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,8 @@ import com.github.avrokotlin.avro4k.internal.EnumResolver
import com.github.avrokotlin.avro4k.internal.PolymorphicResolver
import com.github.avrokotlin.avro4k.internal.RecordResolver
import com.github.avrokotlin.avro4k.internal.schema.ValueVisitor
import com.github.avrokotlin.avro4k.serializer.BigDecimalSerializer
import com.github.avrokotlin.avro4k.serializer.BigIntegerSerializer
import com.github.avrokotlin.avro4k.serializer.InstantSerializer
import com.github.avrokotlin.avro4k.serializer.LocalDateSerializer
import com.github.avrokotlin.avro4k.serializer.LocalDateTimeSerializer
import com.github.avrokotlin.avro4k.serializer.LocalTimeSerializer
import com.github.avrokotlin.avro4k.serializer.URLSerializer
import com.github.avrokotlin.avro4k.serializer.UUIDSerializer
import com.github.avrokotlin.avro4k.serializer.JavaStdLibSerializersModule
import com.github.avrokotlin.avro4k.serializer.JavaTimeSerializersModule
import kotlinx.serialization.BinaryFormat
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.ExperimentalSerializationApi
Expand All @@ -21,8 +15,8 @@ import kotlinx.serialization.SerializationStrategy
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.modules.EmptySerializersModule
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.modules.contextual
import kotlinx.serialization.modules.overwriteWith
import kotlinx.serialization.modules.plus
import kotlinx.serialization.serializer
import okio.Buffer
import org.apache.avro.Schema
Expand All @@ -47,16 +41,8 @@ public sealed class Avro(

public companion object Default : Avro(
AvroConfiguration(),
SerializersModule {
contextual(UUIDSerializer)
contextual(URLSerializer)
contextual(BigIntegerSerializer)
contextual(BigDecimalSerializer)
contextual(InstantSerializer)
contextual(LocalDateSerializer)
contextual(LocalTimeSerializer)
contextual(LocalDateTimeSerializer)
}
JavaStdLibSerializersModule +
JavaTimeSerializersModule
)

public fun schema(descriptor: SerialDescriptor): Schema {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
package com.github.avrokotlin.avro4k.internal

import com.github.avrokotlin.avro4k.AvroDecoder
import com.github.avrokotlin.avro4k.AvroEncoder
import com.github.avrokotlin.avro4k.internal.decoder.direct.AbstractAvroDirectDecoder
import com.github.avrokotlin.avro4k.serializer.AvroDuration
import com.github.avrokotlin.avro4k.serializer.AvroDurationSerializer
import com.github.avrokotlin.avro4k.serializer.AvroSerializer
import com.github.avrokotlin.avro4k.serializer.SchemaSupplierContext
import com.github.avrokotlin.avro4k.serializer.SerialDescriptorWithAvroSchemaDelegate
import com.github.avrokotlin.avro4k.serializer.createSchema
import com.github.avrokotlin.avro4k.serializer.fixed
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.InternalSerializationApi
import kotlinx.serialization.KSerializer
import kotlinx.serialization.SerializationException
import kotlinx.serialization.SerializationStrategy
import kotlinx.serialization.builtins.ByteArraySerializer
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.descriptors.PrimitiveKind
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.descriptors.StructureKind
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.internal.AbstractCollectionSerializer
import org.apache.avro.Schema
import kotlin.io.encoding.Base64
import kotlin.io.encoding.ExperimentalEncodingApi
import kotlin.time.Duration
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.milliseconds

/**
* This middleware is here to intercept some native types like kotlin Duration or ByteArray as we want to apply some
* specific rules on them for generating custom schemas or having specific serialization strategies.
*/
@Suppress("UNCHECKED_CAST")
internal object SerializerLocatorMiddleware {
fun <T> apply(serializer: SerializationStrategy<T>): SerializationStrategy<T> {
return when {
serializer === ByteArraySerializer() -> AvroByteArraySerializer
serializer === Duration.serializer() -> KotlinDurationSerializer
else -> serializer
} as SerializationStrategy<T>
}

@OptIn(InternalSerializationApi::class)
fun <T> apply(deserializer: DeserializationStrategy<T>): DeserializationStrategy<T> {
return when {
deserializer === ByteArraySerializer() -> AvroByteArraySerializer
deserializer === Duration.serializer() -> KotlinDurationSerializer
deserializer is AbstractCollectionSerializer<*, T, *> -> AvroCollectionSerializer(deserializer)
else -> deserializer
} as DeserializationStrategy<T>
}

fun apply(descriptor: SerialDescriptor): SerialDescriptor {
return when {
descriptor.isCollectionOfBytes() -> SerialDescriptorWithAvroSchemaDelegate(descriptor, AvroByteArraySerializer)
descriptor == String.serializer().descriptor -> AvroStringSerialDescriptor
descriptor == Duration.serializer().descriptor -> KotlinDurationSerializer.descriptor
else -> descriptor
}
}

private fun SerialDescriptor.isCollectionOfBytes() = kind === StructureKind.LIST && elementsCount == 1 && getElementDescriptor(0).kind === PrimitiveKind.BYTE
}

private val AvroStringSerialDescriptor: SerialDescriptor =
SerialDescriptorWithAvroSchemaDelegate(String.serializer().descriptor) { context ->
context.fixed?.createSchema() ?: Schema.create(Schema.Type.STRING)
}

private object KotlinDurationSerializer : AvroSerializer<Duration>(Duration::class.qualifiedName!!) {
private const val MILLIS_PER_DAY = 1000 * 60 * 60 * 24

override fun getSchema(context: SchemaSupplierContext): Schema {
return AvroDurationSerializer.DURATION_SCHEMA
}

override fun serializeAvro(
encoder: AvroEncoder,
value: Duration,
) {
AvroDurationSerializer.serializeAvro(encoder, value.toAvroDuration())
}

override fun deserializeAvro(decoder: AvroDecoder): Duration {
return AvroDurationSerializer.deserializeAvro(decoder).toKotlinDuration()
}

override fun serializeGeneric(
encoder: Encoder,
value: Duration,
) {
encoder.encodeString(value.toString())
}

override fun deserializeGeneric(decoder: Decoder): Duration {
return Duration.parse(decoder.decodeString())
}

private fun AvroDuration.toKotlinDuration(): Duration {
if (months == UInt.MAX_VALUE && days == UInt.MAX_VALUE && millis == UInt.MAX_VALUE) {
return Duration.INFINITE
}
if (months != 0u) {
throw SerializationException("java.time.Duration cannot contains months")
}
return days.toLong().days + millis.toLong().milliseconds
}

private fun Duration.toAvroDuration(): AvroDuration {
if (isNegative()) {
throw SerializationException("${Duration::class.qualifiedName} cannot be converted to ${AvroDuration::class.qualifiedName} as it cannot be negative")
}
if (isInfinite()) {
return AvroDuration(
months = UInt.MAX_VALUE,
days = UInt.MAX_VALUE,
millis = UInt.MAX_VALUE
)
}
val millis = inWholeMilliseconds
return AvroDuration(
months = 0u,
days = (millis / MILLIS_PER_DAY).toUInt(),
millis = (millis % MILLIS_PER_DAY).toUInt()
)
}
}

private object AvroByteArraySerializer : AvroSerializer<ByteArray>(ByteArray::class.qualifiedName!!) {
override fun getSchema(context: SchemaSupplierContext): Schema {
return context.fixed?.createSchema() ?: Schema.create(Schema.Type.BYTES)
}

override fun serializeAvro(
encoder: AvroEncoder,
value: ByteArray,
) {
// encoding related to the type (fixed or bytes) is handled in AvroEncoder
encoder.encodeBytes(value)
}

override fun deserializeAvro(decoder: AvroDecoder): ByteArray {
// decoding related to the type (fixed or bytes) is handled in AvroDecoder
return decoder.decodeBytes()
}

@OptIn(ExperimentalEncodingApi::class)
override fun serializeGeneric(
encoder: Encoder,
value: ByteArray,
) {
encoder.encodeString(Base64.Mime.encode(value))
}

@OptIn(ExperimentalEncodingApi::class)
override fun deserializeGeneric(decoder: Decoder): ByteArray {
return Base64.Mime.decode(decoder.decodeString())
}
}

@OptIn(InternalSerializationApi::class)
internal class AvroCollectionSerializer<T>(private val original: AbstractCollectionSerializer<*, T, *>) : KSerializer<T> {
override val descriptor: SerialDescriptor
get() = original.descriptor

override fun deserialize(decoder: Decoder): T {
if (decoder is AbstractAvroDirectDecoder) {
var result: T? = null
decoder.decodedCollectionSize = -1
do {
result = original.merge(decoder, result)
} while (decoder.decodedCollectionSize > 0)
return result!!
}
return original.deserialize(decoder)
}

override fun serialize(
encoder: Encoder,
value: T,
) {
original.serialize(encoder, value)
}
}
Loading