Added ability to buffered read huge strings in custom KSerializers

core/api/kotlinx-serialization-core.api

@@ -413,6 +413,10 @@ public abstract class kotlinx/serialization/encoding/AbstractEncoder : kotlinx/s
 	public fun shouldEncodeElementDefault (Lkotlinx/serialization/descriptors/SerialDescriptor;I)Z
 }
 
+public abstract interface class kotlinx/serialization/encoding/ChunkedDecoder {
+	public abstract fun decodeStringChunked (Lkotlin/jvm/functions/Function1;)V
+}
+
 public abstract interface class kotlinx/serialization/encoding/CompositeDecoder {
 	public static final field Companion Lkotlinx/serialization/encoding/CompositeDecoder$Companion;
 	public static final field DECODE_DONE I

core/commonMain/src/kotlinx/serialization/encoding/ChunkedDecoder.kt

@@ -0,0 +1,13 @@
+package kotlinx.serialization.encoding
+
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.descriptors.PrimitiveKind
+
+public interface ChunkedDecoder {
+    /**
+     * Decodes a string value by chunks (16k by default), outputs string them to consumer.
+     * Corresponding kind is [PrimitiveKind.STRING].
+     */
+    @ExperimentalSerializationApi
+    public fun decodeStringChunked(consumeChunk:(chunk:String) -> Unit)
+}

formats/json-tests/jvmTest/src/kotlinx/serialization/json/JsonChunkedDecoderTest.kt

@@ -0,0 +1,82 @@
+package kotlinx.serialization.json
+
+import kotlinx.serialization.*
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.encoding.*
+import org.junit.Test
+import java.io.*
+import java.util.*
+import kotlin.random.Random
+import kotlin.test.*
+
+
+@Serializable(with = LargeStringSerializer::class)
+data class LargeBinaryData(val binaryData: ByteArray) {
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as LargeBinaryData
+
+        if (!binaryData.contentEquals(other.binaryData)) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return binaryData.contentHashCode()
+    }
+}
+
+@Serializable
+data class ClassWithBinaryDataField(val binaryField: LargeBinaryData)
+
+object LargeStringSerializer : KSerializer<LargeBinaryData> {
+    private val b64Decoder: Base64.Decoder = Base64.getDecoder()
+    override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("LargeStringContent", PrimitiveKind.STRING)
+
+    override fun deserialize(decoder: Decoder): LargeBinaryData {
+        require(decoder is ChunkedDecoder) { "Only chunked decoder supported" }
+
+        var reminder = ""
+        val decodedBytes = ByteArrayOutputStream().use { bos ->
+            decoder.decodeStringChunked {
+                val actualChunk = reminder + it
+                val reminderLength = actualChunk.length % 4
+                val alignedLength = actualChunk.length - reminderLength
+                val alignedChunk = actualChunk.take(alignedLength)
+                reminder = actualChunk.takeLast(reminderLength)
+                bos.write(b64Decoder.decode(alignedChunk))
+            }
+            bos.toByteArray()
+        }
+
+        return LargeBinaryData(decodedBytes)
+    }
+
+    override fun serialize(encoder: Encoder, value: LargeBinaryData) {
+        encoder.encodeString(Base64.getEncoder().encodeToString(value.binaryData))
+    }
+}
+
+
+class JsonChunkedDecoderTest:JsonTestBase() {
+
+    @Test
+    fun decodeBase64String() {
+        val sourceObject = ClassWithBinaryDataField(LargeBinaryData(Random.nextBytes(16 * 1024))) // After encoding will be more than BATCH_SIZE (16k)
+        val serializedObject = Json.encodeToString(sourceObject)
+
+        JsonTestingMode.values().forEach { mode ->
+            if (mode == JsonTestingMode.TREE) {
+                assertFails("Only chunked decoder supported") {
+                    Json.decodeFromString<ClassWithBinaryDataField>(serializedObject, mode)
+                }
+            } else {
+                val deserializedObject = Json.decodeFromString<ClassWithBinaryDataField>(serializedObject, mode)
+                assertEquals(sourceObject.binaryField, deserializedObject.binaryField)
+            }
+        }
+    }
+}

formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt

@@ -14,6 +14,7 @@ import kotlinx.serialization.json.*
 import kotlinx.serialization.modules.*
 import kotlin.jvm.*
 
+
 /**
  * [JsonDecoder] which reads given JSON from [AbstractJsonLexer] field by field.
  */
@@ -24,7 +25,7 @@ internal open class StreamingJsonDecoder(
     @JvmField internal val lexer: AbstractJsonLexer,
     descriptor: SerialDescriptor,
     discriminatorHolder: DiscriminatorHolder?
-) : JsonDecoder, AbstractDecoder() {
+) : JsonDecoder, ChunkedDecoder, AbstractDecoder() {
 
     // A mutable reference to the discriminator that have to be skipped when in optimistic phase
     // of polymorphic serialization, see `decodeSerializableValue`
@@ -343,6 +344,10 @@ internal open class StreamingJsonDecoder(
         }
     }
 
+    override fun decodeStringChunked(consumeChunk: (chunk: String) -> Unit) {
+        lexer.consumeStringChunked(consumeChunk)
+    }
+
     override fun decodeInline(descriptor: SerialDescriptor): Decoder =
         if (descriptor.isUnsignedNumber) JsonDecoderForUnsignedTypes(lexer, json)
         else super.decodeInline(descriptor)

formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/AbstractJsonLexer.kt

@@ -4,7 +4,6 @@
 
 package kotlinx.serialization.json.internal
 
-import kotlinx.serialization.json.internal.*
 import kotlinx.serialization.json.internal.CharMappings.CHAR_TO_TOKEN
 import kotlinx.serialization.json.internal.CharMappings.ESCAPE_2_CHAR
 import kotlin.js.*
@@ -307,6 +306,8 @@ internal abstract class AbstractJsonLexer {
      */
     abstract fun consumeKeyString(): String
 
+    abstract fun consumeStringChunked(consumeChunk: (stringChunk: String) -> Unit)
+
     fun consumeString(): String {
         if (peekedString != null) {
             return takePeeked()

formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/JsonLexer.kt

@@ -131,6 +131,32 @@ internal class ReaderJsonLexer(
         preload(spaceLeft)
     }
 
+    override fun consumeStringChunked(consumeChunk: (stringChunk: String) -> Unit) {
+        consumeNextToken(STRING)
+        var currentPosition = this.currentPosition
+        var lastPosition = currentPosition
+        var char = source[currentPosition] // Avoid two range checks visible in the profiler
+        while (char != STRING) {
+            if (++currentPosition >= source.length) {
+                // end of chunk
+                writeRange(lastPosition, currentPosition, consumeChunk)
+                currentPosition = prefetchOrEof(currentPosition)
+                if (currentPosition == -1)
+                    fail("EOF", currentPosition)
+                lastPosition = currentPosition
+            }
+            char = source[currentPosition]
+        }
+        writeRange(lastPosition, currentPosition, consumeChunk)
+        this.currentPosition = currentPosition + 1 // Consume closing '"' (STRING)
+    }
+
+    private fun writeRange(fromIndex: Int, toIndex: Int, consumeChunk: (stringChunk: String) -> Unit) {
+        val tmp = StringBuilder()
+        tmp.appendRange(source, fromIndex, toIndex)
+        consumeChunk(tmp.toString())
+    }
+
     override fun consumeKeyString(): String {
         /*
          * For strings we assume that escaped symbols are rather an exception, so firstly

formats/json/commonMain/src/kotlinx/serialization/json/internal/lexer/StringJsonLexer.kt

@@ -97,6 +97,10 @@ internal class StringJsonLexer(override val source: String) : AbstractJsonLexer(
         return source.substring(current, closingQuote)
     }
 
+    override fun consumeStringChunked(consumeChunk: (stringChunk: String) -> Unit) {
+        consumeKeyString().chunked(BATCH_SIZE).forEach(consumeChunk)
+    }
+
     override fun consumeLeadingMatchingValue(keyToMatch: String, isLenient: Boolean): String? {
         val positionSnapshot = currentPosition
         try {