Rewrite and restructure JSON parser

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/CoerceInputValuesBenchmark.kt

@@ -16,8 +16,6 @@ import java.util.concurrent.*
 @Fork(2)
 open class CoerceInputValuesBenchmark {
 
-    // Specific benchmark to isolate effect on #1156. Remove after release of 1.0.1
-
     @Serializable
     class Holder(
         val i1: Int,

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/PrimitiveValuesBenchmark.kt

@@ -59,4 +59,3 @@ open class PrimitiveValuesBenchmark {
     @Benchmark
     fun encodeLong(): LongHolder = Json.decodeFromString(LongHolder.serializer(), longValue)
 }
-

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterBenchmark.kt

@@ -23,5 +23,5 @@ open class TwitterBenchmark {
 
     // Order of magnitude: 4-7 op/ms
     @Benchmark
-    fun parseTwitter() = Json.decodeFromString(MacroTwitterFeed.serializer(), input)
+    fun parseTwitter() = Json.decodeFromString(Twitter.serializer(), input)
 }

benchmark/src/jmh/kotlin/kotlinx/benchmarks/model/MacroTwitter.kt

@@ -19,9 +19,9 @@ data class TwitterStatus(
     val source: String,
     val truncated: Boolean,
     val in_reply_to_status_id: Long?,
-    val in_reply_to_status_id_str: Long?,
+    val in_reply_to_status_id_str: String?,
     val in_reply_to_user_id: Long?,
-    val in_reply_to_user_id_str: Long?,
+    val in_reply_to_user_id_str: String?,
     val in_reply_to_screen_name: String?,
     val user: TwitterUser,
     val geo: String?,

benchmark/src/jmh/kotlin/kotlinx/benchmarks/model/Twitter.kt

@@ -4,7 +4,7 @@ import kotlinx.serialization.*
 import kotlinx.serialization.json.*
 
 fun main() {
-    val s = MacroTwitterFeed::class.java.getResource("/twitter.json").readBytes().decodeToString()
+    val s = MacroTwitterFeed::class.java.getResource("/twitter_macro.json").readBytes().decodeToString()
     println(Json.decodeFromString<MacroTwitterFeed>(s))
 }
 

core/commonMain/src/kotlinx/serialization/internal/PluginGeneratedSerialDescriptor.kt

@@ -31,10 +31,9 @@ internal open class PluginGeneratedSerialDescriptor(
     private val elementsOptionality = BooleanArray(elementsCount)
     public override val serialNames: Set<String> get() = indices.keys
 
-    // don't change lazy mode: KT-32871, KT-32872
-    private val indices: Map<String, Int> by lazy { buildIndices() }
+    private var indices: Map<String, Int> = emptyMap()
     // Cache child serializers, they are not cached by the implementation for nullable types
-    private val childSerializers by lazy { generatedSerializer?.childSerializers() ?: emptyArray() }
+    private val childSerializers: Array<KSerializer<*>> by lazy { generatedSerializer?.childSerializers() ?: emptyArray() }
 
     // Lazy because of JS specific initialization order (#789)
     internal val typeParameterDescriptors: Array<SerialDescriptor> by lazy {
@@ -48,6 +47,9 @@ internal open class PluginGeneratedSerialDescriptor(
         names[++added] = name
         elementsOptionality[added] = isOptional
         propertiesAnnotations[added] = null
+        if (added == elementsCount - 1) {
+            indices = buildIndices()
+        }
     }
 
     public fun pushAnnotation(annotation: Annotation) {

docs/basic-serialization.md

@@ -411,7 +411,7 @@ Attempts to explicitly specify its value in the serial format, even if the speci
 value is equal to the default one, produces the following exception.
 
 ```text
-Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 60: Encountered an unknown key 'language'.
+Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 42: Encountered an unknown key 'language'.
 Use 'ignoreUnknownKeys = true' in 'Json {}' builder to ignore unknown keys.
 ```   
 

formats/json/commonMain/src/kotlinx/serialization/json/Json.kt

@@ -83,7 +83,7 @@ public sealed class Json(internal val configuration: JsonConf) : StringFormat {
         val reader = JsonReader(string)
         val input = StreamingJsonDecoder(this, WriteMode.OBJ, reader)
         val result = input.decodeSerializableValue(deserializer)
-        if (!reader.isDone) { error("Reader has not consumed the whole input: $reader") }
+        reader.expectEof()
         return result
     }
     /**

formats/json/commonMain/src/kotlinx/serialization/json/JsonElementSerializers.kt

@@ -96,6 +96,9 @@ internal object JsonNullSerializer : KSerializer<JsonNull> {
 
     override fun deserialize(decoder: Decoder): JsonNull {
         verify(decoder)
+        if (decoder.decodeNotNullMark()) {
+            throw JsonDecodingException("Expected 'null' literal")
+        }
         decoder.decodeNull()
         return JsonNull
     }

formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonParser.kt

@@ -13,70 +13,64 @@ internal class JsonParser(
     private val isLenient = configuration.isLenient
 
     private fun readObject(): JsonElement {
-        reader.requireTokenClass(TC_BEGIN_OBJ) { "Expected start of the object" }
-        reader.nextToken()
-        // Prohibit leading comma
-        reader.require(reader.tokenClass != TC_COMMA, reader.currentPosition) { "Unexpected leading comma" }
+        var lastToken = reader.consumeNextToken(TC_BEGIN_OBJ)
+        if (reader.peekNextToken() == TC_COMMA) reader.fail("Unexpected leading comma")
         val result = linkedMapOf<String, JsonElement>()
-        var valueExpected = false
-        while (reader.canBeginValue) {
-            valueExpected = false
-            val key = if (isLenient) reader.takeString() else reader.takeStringQuoted()
-            reader.requireTokenClass(TC_COLON) { "Expected ':'" }
-            reader.nextToken()
+        while (reader.canConsumeValue()) {
+            // Read key and value
+            val key = if (isLenient) reader.consumeStringLenient() else reader.consumeString()
+            reader.consumeNextToken(TC_COLON)
             val element = read()
             result[key] = element
-            if (reader.tokenClass != TC_COMMA) {
-                // Prohibit whitespaces instead of commas {a:b c:d}
-                reader.requireTokenClass(TC_END_OBJ) { "Expected end of the object or comma" }
-            } else {
-                valueExpected = true
-                reader.nextToken()
+            // Verify the next token
+            lastToken = reader.consumeNextToken()
+            if (lastToken != TC_COMMA && lastToken != TC_END_OBJ) {
+                reader.fail("Expected end of the object or comma")
             }
         }
-        reader.require(!valueExpected && reader.tokenClass == TC_END_OBJ, reader.currentPosition) { "Expected end of the object" }
-        reader.nextToken()
+        // Check for the correct ending
+        if (lastToken == TC_BEGIN_OBJ) { // Case of empty object
+            reader.consumeNextToken(TC_END_OBJ)
+        } else if (lastToken == TC_COMMA) { // Trailing comma
+            reader.fail("Unexpected trailing comma")
+        }
         return JsonObject(result)
     }
 
     private fun readArray(): JsonElement {
-        reader.requireTokenClass(TC_BEGIN_LIST) { "Expected start of the array" }
-        reader.nextToken()
+        var lastToken = reader.consumeNextToken()
         // Prohibit leading comma
-        reader.require(reader.tokenClass != TC_COMMA, reader.currentPosition) { "Unexpected leading comma" }
+        if (reader.peekNextToken() == TC_COMMA) reader.fail("Unexpected leading comma")
         val result = arrayListOf<JsonElement>()
-        var valueExpected = false
-        while (reader.canBeginValue) {
-            valueExpected = false
+        while (reader.canConsumeValue()) {
             val element = read()
             result.add(element)
-            if (reader.tokenClass != TC_COMMA) {
-                // Prohibit whitespaces instead of commas [a b]
-                reader.requireTokenClass(TC_END_LIST) { "Expected end of the array or comma" }
-            } else {
-                valueExpected = true
-                reader.nextToken()
+            lastToken = reader.consumeNextToken()
+            if (lastToken != TC_COMMA) {
+                reader.require(lastToken == TC_END_LIST) { "Expected end of the array or comma" }
             }
         }
-        // Prohibit trailing commas
-        reader.require(!valueExpected, reader.currentPosition) { "Unexpected trailing comma" }
-        reader.nextToken()
+        // Check for the correct ending
+        if (lastToken == TC_BEGIN_LIST) { // Case of empty object
+            reader.consumeNextToken(TC_END_LIST)
+        } else if (lastToken == TC_COMMA) { // Trailing comma
+            reader.fail("Unexpected trailing comma")
+        }
         return JsonArray(result)
     }
 
-    private fun readValue(isString: Boolean): JsonElement {
-        val str = if (isLenient) {
-            reader.takeString()
+    private fun readValue(isString: Boolean): JsonPrimitive {
+        val string = if (isLenient || !isString) {
+            reader.consumeStringLenient()
         } else {
-            if (isString) reader.takeStringQuoted() else reader.takeString()
+            reader.consumeString()
         }
-        return JsonLiteral(str, isString)
+        if (string == NULL) return JsonNull
+        return JsonLiteral(string, isString)
     }
 
     fun read(): JsonElement {
-        if (!reader.canBeginValue) reader.fail("Can't begin reading value from here")
-        return when (reader.tokenClass) {
-            TC_NULL -> JsonNull.also { reader.nextToken() }
+        return when (reader.peekNextToken()) {
             TC_STRING -> readValue(isString = true)
             TC_OTHER -> readValue(isString = false)
             TC_BEGIN_OBJ -> readObject()