Fix incorrect skipElement() when string literals contain ] or }

Kotlin · Apr 14, 2021 · f7b7f19 · f7b7f19
1 parent 6087755
commit f7b7f19
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 11 deletions.
diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonLexer.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonLexer.kt
@@ -422,15 +422,19 @@ internal class JsonLexer(private val source: String) {
         }
     }
 
-    fun skipElement() {
+    fun skipElement(allowLenientStrings: Boolean) {
         val tokenStack = mutableListOf<Byte>()
         var lastToken = peekNextToken()
         if (lastToken != TC_BEGIN_LIST && lastToken != TC_BEGIN_OBJ) {
             consumeStringLenient()
             return
         }
         while (true) {
-            lastToken = consumeNextToken()
+            lastToken = peekNextToken()
+            if (lastToken == TC_STRING) {
+                if (allowLenientStrings) consumeStringLenient() else consumeKeyString()
+                continue
+            }
             when (lastToken) {
                 TC_BEGIN_LIST, TC_BEGIN_OBJ -> {
                     tokenStack.add(lastToken)
@@ -442,7 +446,6 @@ internal class JsonLexer(private val source: String) {
                         source
                     )
                     tokenStack.removeAt(tokenStack.size - 1)
-                    if (tokenStack.size == 0) return
                 }
                 TC_END_OBJ -> {
                     if (tokenStack.last() != TC_BEGIN_OBJ) throw JsonDecodingException(
@@ -451,9 +454,10 @@ internal class JsonLexer(private val source: String) {
                         source
                     )
                     tokenStack.removeAt(tokenStack.size - 1)
-                    if (tokenStack.size == 0) return
                 }
             }
+            consumeNextToken()
+            if (tokenStack.size == 0) return
         }
     }
 

diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt
@@ -162,7 +162,7 @@ internal open class StreamingJsonDecoder(
 
     private fun handleUnknown(key: String): Boolean {
         if (configuration.ignoreUnknownKeys) {
-            lexer.skipElement()
+            lexer.skipElement(configuration.isLenient)
         } else {
             lexer.failOnUnknownKey(key)
         }

diff --git a/formats/json/commonTest/src/kotlinx/serialization/json/JsonModesTest.kt b/formats/json/commonTest/src/kotlinx/serialization/json/JsonModesTest.kt
@@ -4,9 +4,8 @@
 
 package kotlinx.serialization.json
 
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.SerializationException
-import kotlinx.serialization.test.assertStringFormAndRestored
+import kotlinx.serialization.*
+import kotlinx.serialization.test.*
 import kotlin.test.*
 
 class JsonModesTest : JsonTestBase() {
@@ -48,22 +47,44 @@ class JsonModesTest : JsonTestBase() {
                 "{a: 0, strangeField: {a: b, c: {d: e}, f: [g,h,j] }}",
                 useStreaming
             ),
-            data)
+            data
+        )
         assertEquals(
             lenient.decodeFromString(
                 JsonOptionalTests.Data.serializer(),
                 "{strangeField: {a: b, c: {d: e}, f: [g,h,j] }, a: 0}",
                 useStreaming
             ),
-            data)
+            data
+        )
+    }
+
+    @Test
+    fun ignoreKeysCanIgnoreWeirdStringValues() {
+        val data = JsonOptionalTests.Data()
+        fun doTest(input: String) {
+            assertEquals(data, lenient.decodeFromString(input))
+        }
+        doTest("{a: 0, strangeField: [\"imma string with } bracket\", \"sss\"]}")
+        doTest("{a: 0, strangeField: [\"imma string with ] bracket\", \"sss\"]}")
+        doTest("{a: 0, strangeField: \"imma string with } bracket\"}")
+        doTest("{a: 0, strangeField: \"imma string with ] bracket\"}")
+        doTest("{a: 0, strangeField: {key: \"imma string with ] bracket\"}}")
+        doTest("{a: 0, strangeField: {key: \"imma string with } bracket\"}}")
+        doTest("""{"a": 0, "strangeField": {"key": "imma string with } bracket"}}""")
+        doTest("""{"a": 0, "strangeField": {"key": "imma string with ] bracket"}}""")
+        doTest("""{"a": 0, "strangeField": ["imma string with ] bracket"]}""")
+        doTest("""{"a": 0, "strangeField": ["imma string with } bracket"]}""")
     }
 
     @Test
     fun testSerializeQuotedJson() = parametrizedTest { useStreaming ->
         assertEquals(
             """{"a":10,"e":false,"c":"Hello"}""", default.encodeToString(
                 JsonTransientTest.Data.serializer(),
-                JsonTransientTest.Data(10, 100), useStreaming))
+                JsonTransientTest.Data(10, 100), useStreaming
+            )
+        )
     }
 
     @Test