From b5ff6d1dc5b1adb0b767645efa6eb7a3af874c21 Mon Sep 17 00:00:00 2001 From: Valentin Rocher Date: Mon, 7 Mar 2022 08:25:18 +0100 Subject: [PATCH 1/3] Support unicode symbols in strings Fixes #72 --- .../com/akuleshov7/ktoml/tree/TomlValue.kt | 56 +++++++++++++------ .../com/akuleshov7/ktoml/utils/Utils.kt | 2 + .../ktoml/parsers/ValueParserTest.kt | 24 ++++++++ .../com/akuleshov7/ktoml/utils/UtilsJs.kt | 17 ++++++ .../com/akuleshov7/ktoml/utils/UtilsJvm.kt | 5 ++ .../com/akuleshov7/ktoml/utils/UtilsLinux.kt | 12 ++++ .../com/akuleshov7/ktoml/utils/UtilsMac.kt | 12 ++++ .../com/akuleshov7/ktoml/utils/UtilsWin.kt | 12 ++++ 8 files changed, 123 insertions(+), 17 deletions(-) create mode 100644 ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt create mode 100644 ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt create mode 100644 ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt create mode 100644 ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt create mode 100644 ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt diff --git a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt index 0a10467c..857eafe0 100644 --- a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt +++ b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt @@ -9,6 +9,7 @@ import com.akuleshov7.ktoml.exceptions.ParseException import com.akuleshov7.ktoml.parsers.trimBrackets import com.akuleshov7.ktoml.parsers.trimQuotes import com.akuleshov7.ktoml.parsers.trimSingleQuotes +import com.akuleshov7.ktoml.utils.appendCodePointCompat import kotlinx.datetime.* /** @@ -104,13 +105,14 @@ internal constructor( private fun String.convertSpecialCharacters(lineNo: Int): String { val resultString = StringBuilder() - var updatedOnPreviousStep = false var i = 0 - while (i < this.length) { - val newCharacter = if (this[i] == '\\' && i != this.length - 1) { - updatedOnPreviousStep = true - when (this[i + 1]) { - // table that is used to convert escaped string literals to proper char symbols + while (i < length) { + val c = get(i) + var offset = 1 + if (c == '\\' && i != lastIndex) { + // Escaped + val next = get(i + 1) + val simpleEscape = when (next) { 't' -> '\t' 'b' -> '\b' 'r' -> '\r' @@ -118,24 +120,44 @@ internal constructor( '\\' -> '\\' '\'' -> '\'' '"' -> '"' + 'u', 'U' -> null else -> throw ParseException( "According to TOML documentation unknown" + - " escape symbols are not allowed. Please check: [\\${this[i + 1]}]", + " escape symbols are not allowed. Please check: [\\$next]", lineNo ) } + offset++ + if (simpleEscape != null) { + resultString.append(simpleEscape) + } else { + // Unicode + val nbUnicodeChars = if (next == 'u') 4 else 8 + offset += nbUnicodeChars + if (i + 1 + nbUnicodeChars >= length) { + val invalid = substring(i + 1) + throw ParseException( + "According to TOML documentation unknown" + + " escape symbols are not allowed. Please check: [\\$invalid]", + lineNo + ) + } + val hexCode = substring(i + 2, i + 2 + nbUnicodeChars) + val codePoint = hexCode.toInt(16) + try { + resultString.appendCodePointCompat(codePoint) + } catch (e: IllegalArgumentException) { + throw ParseException( + "According to TOML documentation unknown" + + " escape symbols are not allowed. Please check: [\\$next$hexCode]", + lineNo + ) + } + } } else { - this[i] - } - // need to skip the next character if we have processed special escaped symbol - if (updatedOnPreviousStep) { - updatedOnPreviousStep = false - i += 2 - } else { - i += 1 + resultString.append(c) } - - resultString.append(newCharacter) + i += offset } return resultString.toString() } diff --git a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt index 96f78fe2..2792ea59 100644 --- a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt +++ b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt @@ -25,3 +25,5 @@ public fun findPrimitiveTableInAstByName(children: List, fullTableName return findPrimitiveTableInAstByName(children.map { it.children }.flatten(), fullTableName) } + +internal expect fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder \ No newline at end of file diff --git a/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt b/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt index 615296c0..f5de541a 100644 --- a/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt +++ b/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt @@ -85,6 +85,18 @@ class ValueParserTest { test = TomlKeyValuePrimitive(Pair("a", "\"hello\t\\\\\\\\world\""), 0) assertEquals("hello\t\\\\world", test.value.content) + test = TomlKeyValuePrimitive("a" to "\"Ɣ is greek\"", 0) + assertEquals("Ɣ is greek", test.value.content) + + test = TomlKeyValuePrimitive("a" to "\"\\u0194 is greek\"", 0) + assertEquals("Ɣ is greek", test.value.content) + + test = TomlKeyValuePrimitive("a" to "\"\\U0001F615 is emoji\"", 0) + assertEquals("\uD83D\uDE15 is emoji", test.value.content) + + test = TomlKeyValuePrimitive("a" to "\"\uD83D\uDE15 is emoji\"", 0) + assertEquals("\uD83D\uDE15 is emoji", test.value.content) + // regression test related to comments with an equals symbol after it var pairTest = "lineCaptureGroup = 1 # index `warningTextHasLine = false`\n".splitKeyValue(0) @@ -105,6 +117,18 @@ class ValueParserTest { 0 ) } + assertFailsWith { + TomlKeyValuePrimitive("a" to "val\\ue", 0) + } + assertFailsWith { + TomlKeyValuePrimitive("a" to "\\x33", 0) + } + assertFailsWith { + TomlKeyValuePrimitive("a" to "\\UFFFFFFFF", 0) + } + assertFailsWith { + TomlKeyValuePrimitive("a" to "\\U00D80000", 0) + } } } diff --git a/ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt b/ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt new file mode 100644 index 00000000..5cd6fd30 --- /dev/null +++ b/ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt @@ -0,0 +1,17 @@ +package com.akuleshov7.ktoml.utils + +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { + return when (codePoint) { + in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) + } + else -> throw IllegalArgumentException() + } +} + +private const val MAX_CODE_POINT = 0X10FFFF +private const val MIN_SUPPLEMENTARY_CODE_POINT: Int = 0x10000 +private const val MIN_LOW_SURROGATE: Int = '\uDC00'.code +private const val MIN_HIGH_SURROGATE: Int = '\uD800'.code \ No newline at end of file diff --git a/ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt b/ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt new file mode 100644 index 00000000..41d56ea2 --- /dev/null +++ b/ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt @@ -0,0 +1,5 @@ +package com.akuleshov7.ktoml.utils + +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { + return appendCodePoint(codePoint) +} \ No newline at end of file diff --git a/ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt b/ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt new file mode 100644 index 00000000..8cbed267 --- /dev/null +++ b/ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt @@ -0,0 +1,12 @@ +package com.akuleshov7.ktoml.utils + +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { + return when (codePoint) { + in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) + } + else -> throw IllegalArgumentException() + } +} \ No newline at end of file diff --git a/ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt b/ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt new file mode 100644 index 00000000..8cbed267 --- /dev/null +++ b/ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt @@ -0,0 +1,12 @@ +package com.akuleshov7.ktoml.utils + +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { + return when (codePoint) { + in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) + } + else -> throw IllegalArgumentException() + } +} \ No newline at end of file diff --git a/ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt b/ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt new file mode 100644 index 00000000..8cbed267 --- /dev/null +++ b/ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt @@ -0,0 +1,12 @@ +package com.akuleshov7.ktoml.utils + +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { + return when (codePoint) { + in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) + } + else -> throw IllegalArgumentException() + } +} \ No newline at end of file From 45d6cfc61309a1d58e073088608d6c07ee85d013 Mon Sep 17 00:00:00 2001 From: Valentin Rocher Date: Mon, 7 Mar 2022 08:58:44 +0100 Subject: [PATCH 2/3] fix diktat issues --- .../com/akuleshov7/ktoml/tree/TomlValue.kt | 92 +++++++++++-------- .../com/akuleshov7/ktoml/utils/Utils.kt | 11 ++- .../com/akuleshov7/ktoml/utils/UtilsJs.kt | 30 +++--- .../com/akuleshov7/ktoml/utils/UtilsJvm.kt | 8 +- .../com/akuleshov7/ktoml/utils/UtilsLinux.kt | 21 +++-- .../com/akuleshov7/ktoml/utils/UtilsMac.kt | 21 +++-- .../com/akuleshov7/ktoml/utils/UtilsWin.kt | 21 +++-- 7 files changed, 120 insertions(+), 84 deletions(-) diff --git a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt index 857eafe0..593fee74 100644 --- a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt +++ b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/tree/TomlValue.kt @@ -76,6 +76,12 @@ internal constructor( ) : this(content.verifyAndTrimQuotes(lineNo), lineNo) public companion object { + private const val COMPLEX_UNICODE_LENGTH = 8 + private const val COMPLEX_UNICODE_PREFIX = 'U' + private const val HEX_RADIX = 16 + private const val SIMPLE_UNICODE_LENGTH = 4 + private const val SIMPLE_UNICODE_PREFIX = 'u' + private fun String.verifyAndTrimQuotes(lineNo: Int): Any = if (startsWith("\"") && endsWith("\"")) { trimQuotes() @@ -107,60 +113,68 @@ internal constructor( val resultString = StringBuilder() var i = 0 while (i < length) { - val c = get(i) + val currentChar = get(i) var offset = 1 - if (c == '\\' && i != lastIndex) { + if (currentChar == '\\' && i != lastIndex) { // Escaped val next = get(i + 1) - val simpleEscape = when (next) { - 't' -> '\t' - 'b' -> '\b' - 'r' -> '\r' - 'n' -> '\n' - '\\' -> '\\' - '\'' -> '\'' - '"' -> '"' - 'u', 'U' -> null + offset++ + when (next) { + 't' -> resultString.append('\t') + 'b' -> resultString.append('\b') + 'r' -> resultString.append('\r') + 'n' -> resultString.append('\n') + '\\' -> resultString.append('\\') + '\'' -> resultString.append('\'') + '"' -> resultString.append('"') + SIMPLE_UNICODE_PREFIX, COMPLEX_UNICODE_PREFIX -> + offset += resultString.appendEscapedUnicode(this, next, i + 2, lineNo) else -> throw ParseException( "According to TOML documentation unknown" + " escape symbols are not allowed. Please check: [\\$next]", lineNo ) } - offset++ - if (simpleEscape != null) { - resultString.append(simpleEscape) - } else { - // Unicode - val nbUnicodeChars = if (next == 'u') 4 else 8 - offset += nbUnicodeChars - if (i + 1 + nbUnicodeChars >= length) { - val invalid = substring(i + 1) - throw ParseException( - "According to TOML documentation unknown" + - " escape symbols are not allowed. Please check: [\\$invalid]", - lineNo - ) - } - val hexCode = substring(i + 2, i + 2 + nbUnicodeChars) - val codePoint = hexCode.toInt(16) - try { - resultString.appendCodePointCompat(codePoint) - } catch (e: IllegalArgumentException) { - throw ParseException( - "According to TOML documentation unknown" + - " escape symbols are not allowed. Please check: [\\$next$hexCode]", - lineNo - ) - } - } } else { - resultString.append(c) + resultString.append(currentChar) } i += offset } return resultString.toString() } + + private fun StringBuilder.appendEscapedUnicode( + fullString: String, + marker: Char, + codeStartIndex: Int, + lineNo: Int + ): Int { + val nbUnicodeChars = if (marker == SIMPLE_UNICODE_PREFIX) { + SIMPLE_UNICODE_LENGTH + } else { + COMPLEX_UNICODE_LENGTH + } + if (codeStartIndex + nbUnicodeChars > fullString.length) { + val invalid = fullString.substring(codeStartIndex - 1) + throw ParseException( + "According to TOML documentation unknown" + + " escape symbols are not allowed. Please check: [\\$invalid]", + lineNo + ) + } + val hexCode = fullString.substring(codeStartIndex, codeStartIndex + nbUnicodeChars) + val codePoint = hexCode.toInt(HEX_RADIX) + try { + appendCodePointCompat(codePoint) + } catch (e: IllegalArgumentException) { + throw ParseException( + "According to TOML documentation unknown" + + " escape symbols are not allowed. Please check: [\\$marker$hexCode]", + lineNo + ) + } + return nbUnicodeChars + } } } diff --git a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt index 2792ea59..7e68f1fa 100644 --- a/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt +++ b/ktoml-core/src/commonMain/kotlin/com/akuleshov7/ktoml/utils/Utils.kt @@ -7,6 +7,15 @@ package com.akuleshov7.ktoml.utils import com.akuleshov7.ktoml.tree.TomlNode import com.akuleshov7.ktoml.tree.TomlTablePrimitive +/** + * Append a code point to a [StringBuilder] + * + * @param codePoint code point + * @return [StringBuilder] with appended code point + */ +@Throws(IllegalArgumentException::class) +internal expect fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder + /** * searching (BFS) the table with the [fullTableName] * @@ -25,5 +34,3 @@ public fun findPrimitiveTableInAstByName(children: List, fullTableName return findPrimitiveTableInAstByName(children.map { it.children }.flatten(), fullTableName) } - -internal expect fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder \ No newline at end of file diff --git a/ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt b/ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt index 5cd6fd30..54c9d788 100644 --- a/ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt +++ b/ktoml-core/src/jsMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJs.kt @@ -1,17 +1,21 @@ -package com.akuleshov7.ktoml.utils +/** + * Specific implementation for utilities + */ -internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { - return when (codePoint) { - in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) - in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> { - append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) - append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) - } - else -> throw IllegalArgumentException() - } -} +package com.akuleshov7.ktoml.utils -private const val MAX_CODE_POINT = 0X10FFFF +@Suppress("LONG_NUMERICAL_VALUES_SEPARATED") +private const val MAX_CODE_POINT = 0x10FFFFf private const val MIN_SUPPLEMENTARY_CODE_POINT: Int = 0x10000 private const val MIN_LOW_SURROGATE: Int = '\uDC00'.code -private const val MIN_HIGH_SURROGATE: Int = '\uD800'.code \ No newline at end of file +private const val MIN_HIGH_SURROGATE: Int = '\uD800'.code + +@Suppress("MAGIC_NUMBER") +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder = when (codePoint) { + in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) + } + else -> throw IllegalArgumentException() +} diff --git a/ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt b/ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt index 41d56ea2..36e20bd8 100644 --- a/ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt +++ b/ktoml-core/src/jvmMain/kotlin/com/akuleshov7/ktoml/utils/UtilsJvm.kt @@ -1,5 +1,7 @@ +/** + * Specific implementation for utilities + */ + package com.akuleshov7.ktoml.utils -internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { - return appendCodePoint(codePoint) -} \ No newline at end of file +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder = appendCodePoint(codePoint) diff --git a/ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt b/ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt index 8cbed267..458f07f2 100644 --- a/ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt +++ b/ktoml-core/src/linuxX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsLinux.kt @@ -1,12 +1,15 @@ +/** + * Specific implementation for utilities + */ + package com.akuleshov7.ktoml.utils -internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { - return when (codePoint) { - in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) - in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { - append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) - append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) - } - else -> throw IllegalArgumentException() +@Suppress("MAGIC_NUMBER") +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder = when (codePoint) { + in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) } -} \ No newline at end of file + else -> throw IllegalArgumentException() +} diff --git a/ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt b/ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt index 8cbed267..458f07f2 100644 --- a/ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt +++ b/ktoml-core/src/macosX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsMac.kt @@ -1,12 +1,15 @@ +/** + * Specific implementation for utilities + */ + package com.akuleshov7.ktoml.utils -internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { - return when (codePoint) { - in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) - in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { - append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) - append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) - } - else -> throw IllegalArgumentException() +@Suppress("MAGIC_NUMBER") +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder = when (codePoint) { + in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) } -} \ No newline at end of file + else -> throw IllegalArgumentException() +} diff --git a/ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt b/ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt index 8cbed267..458f07f2 100644 --- a/ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt +++ b/ktoml-core/src/mingwX64Main/kotlin/com/akuleshov7/ktoml/utils/UtilsWin.kt @@ -1,12 +1,15 @@ +/** + * Specific implementation for utilities + */ + package com.akuleshov7.ktoml.utils -internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder { - return when (codePoint) { - in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) - in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { - append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) - append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) - } - else -> throw IllegalArgumentException() +@Suppress("MAGIC_NUMBER") +internal actual fun StringBuilder.appendCodePointCompat(codePoint: Int): StringBuilder = when (codePoint) { + in 0 until Char.MIN_SUPPLEMENTARY_CODE_POINT -> append(codePoint.toChar()) + in Char.MIN_SUPPLEMENTARY_CODE_POINT..Char.MAX_CODE_POINT -> { + append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10)) + append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff)) } -} \ No newline at end of file + else -> throw IllegalArgumentException() +} From b28aeab8c48cdb1b292336d2d2e175e44d360a07 Mon Sep 17 00:00:00 2001 From: Valentin Rocher Date: Mon, 7 Mar 2022 09:02:13 +0100 Subject: [PATCH 3/3] add test from TOML spec --- .../kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt b/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt index f5de541a..48c8da7a 100644 --- a/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt +++ b/ktoml-core/src/commonTest/kotlin/com/akuleshov7/ktoml/parsers/ValueParserTest.kt @@ -97,6 +97,9 @@ class ValueParserTest { test = TomlKeyValuePrimitive("a" to "\"\uD83D\uDE15 is emoji\"", 0) assertEquals("\uD83D\uDE15 is emoji", test.value.content) + test = TomlKeyValuePrimitive("a" to "\"I'm a string. \\\"You can quote me\\\". Name\\tJos\\u00E9\\nLocation\\tSF.\"", 0) + assertEquals("I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF.", test.value.content) + // regression test related to comments with an equals symbol after it var pairTest = "lineCaptureGroup = 1 # index `warningTextHasLine = false`\n".splitKeyValue(0)