ktor3 upgrade to 3.0.0-rc-1

README.md

@@ -32,7 +32,7 @@ Ksoup is adept at handling all varieties of HTML found in the wild.
 Include the dependency in `commonMain`. Latest version [![Maven Central](https://img.shields.io/maven-central/v/com.fleeksoft.ksoup/ksoup.svg)](https://central.sonatype.com/artifact/com.fleeksoft.ksoup/ksoup)
 
 Ksoup published in four variants. Pick the one that suits your needs and start building!
-1. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 3.0.0-beta-2](https://github.com/ktorio/ktor)**
+1. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 3.0.0-rc-1](https://github.com/ktorio/ktor)**
    ```kotlin
    implementation("com.fleeksoft.ksoup:ksoup:<version>")
 

gradle.properties

@@ -8,9 +8,9 @@ android.nonTransitiveRClass=true
 kotlin.native.ignoreIncorrectDependencies=true
 kotlin.mpp.enableCInteropCommonization=true
 kotlin.mpp.applyDefaultHierarchyTemplate=false
-# common, kotlinx, korlibs, okio, ktor2
+# dev, common, kotlinx, korlibs, okio, ktor2
 # dev will include all modules in settings.gradle.kts but use kotlinx dep for engine
-libBuildType=common
+libBuildType=okio
 
 
 SONATYPE_HOST=CENTRAL_PORTAL

gradle/libs.versions.toml

@@ -3,8 +3,8 @@ agp = "8.5.2"
 kotlin = "2.0.20"
 compileSdk = "34"
 minSdk = "21"
-libraryVersion = "0.1.6"
-ktor = "3.0.0-beta-2"
+libraryVersion = "0.1.7"
+ktor = "3.0.0-rc-1"
 ktor2 = "2.3.12"
 coroutines = "1.8.1"
 kotlinxDatetime = "0.6.1"
@@ -17,7 +17,7 @@ dokka = "1.9.20"
 korlibs = "6.0.1"
 mavenPublish = "0.29.0"
 
-stately-concurrent = "2.0.7"
+stately = "2.1.0"
 jsoup = "1.18.1"
 
 [libraries]
@@ -26,6 +26,12 @@ kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-c
 kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" }
 ktor2-io = { module = "io.ktor:ktor-io", version.ref = "ktor2" }
 ktor2-http = { module = "io.ktor:ktor-http", version.ref = "ktor2" }
+ktor2-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor2" }
+ktor2-client-darwin = { module = "io.ktor:ktor-client-darwin", version.ref = "ktor2" }
+ktor2-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor2" }
+ktor2-client-okhttp = { module = "io.ktor:ktor-client-okhttp", version.ref = "ktor2" }
+ktor2-client-js = { module = "io.ktor:ktor-client-js", version.ref = "ktor2" }
+ktor2-client-win = { module = "io.ktor:ktor-client-winhttp", version.ref = "ktor2" }
 ktor-io = { module = "io.ktor:ktor-io", version.ref = "ktor" }
 ktor-http = { module = "io.ktor:ktor-http", version.ref = "ktor" }
 ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
@@ -40,7 +46,7 @@ kotlinx-io = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "
 codepoints = { module = "de.cketti.unicode:kotlin-codepoints-deluxe", version.ref = "codepoints" }
 korlibs-io = { module = "com.soywiz:korlibs-io", version.ref = "korlibs" }
 korlibs-io-network-core = { module = "com.soywiz:korlibs-io-network-core", version.ref = "korlibs" }
-stately-concurrent = { module = "co.touchlab:stately-concurrent-collections", version.ref = "stately-concurrent" }
+stately-concurrency = { module = "co.touchlab:stately-concurrency", version.ref = "stately" }
 jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" }
 okio = { module = "com.squareup.okio:okio", version.ref = "okio" }
 okio-nodefilesystem = { module = "com.squareup.okio:okio-nodefilesystem", version.ref = "okio" }

ksoup-network-ktor2/module.yaml

@@ -14,19 +14,19 @@ repositories:
 dependencies:
   - $libs.kotlinx.coroutines.core
   - ../ksoup: compile-only
-  - $libs.ktor.client.core: exported
+  - $libs.ktor2.client.core: exported
 
 dependencies@jvmAndAndroid:
-  - $libs.ktor.client.okhttp
+  - $libs.ktor2.client.okhttp
 
 dependencies@apple:
-  - $libs.ktor.client.darwin
+  - $libs.ktor2.client.darwin
 
 dependencies@js:
-  - $libs.ktor.client.js
+  - $libs.ktor2.client.js
 
 dependencies@mingw:
-  - $libs.ktor.client.win
+  - $libs.ktor2.client.win
 
 dependencies@linux:
-  - $libs.ktor.client.cio
+  - $libs.ktor2.client.cio

ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/SourceExtNetwork.kt

@@ -3,6 +3,6 @@ package com.fleeksoft.ksoup.network
 import com.fleeksoft.ksoup.io.SourceReader
 import com.fleeksoft.ksoup.io.from
 import io.ktor.client.statement.*
-import io.ktor.utils.io.*
+import io.ktor.util.*
 
 suspend fun HttpResponse.asSourceReader() = SourceReader.from(this.bodyAsChannel().toByteArray())

ksoup-test/module.yaml

@@ -17,7 +17,7 @@ test-dependencies:
   - $libs.codepoints
   - $libs.kotlinx.coroutines.test
   - $libs.kotlinx.datetime
-  - $libs.stately.concurrent
+  - $libs.stately.concurrency
 
 test-dependencies@jvmAndAndroid:
   - $libs.jsoup

ksoup-test/test/com/fleeksoft/ksoup/parser/HtmlParserTest.kt

@@ -2251,6 +2251,26 @@ class HtmlParserTest {
         assertEquals("<img multi=\"&#x1f4af;\" single=\"&#x1f4af;\" hexsingle=\"&#x1f4af;\">", img.outerHtml())
     }
 
+    @Test
+    fun tableInPInQuirksMode() {
+        var html = "<p><span><table><tbody><tr><td><span>Hello table data</span></td></tr></tbody></table></span></p>"
+        var doc: Document = Ksoup.parse(html)
+        assertEquals(Document.QuirksMode.quirks, doc.quirksMode())
+        assertEquals(
+            "<p><span><table><tbody><tr><td><span>Hello table data</span></td></tr></tbody></table></span></p>",  // quirks, allows table in p
+            TextUtil.normalizeSpaces(doc.body().html())
+        )
+
+        // doctype set, no quirks
+        html = "<!DOCTYPE html><p><span><table><tbody><tr><td><span>Hello table data</span></td></tr></tbody></table></span></p>"
+        doc = Ksoup.parse(html)
+        assertEquals(Document.QuirksMode.noQuirks, doc.quirksMode())
+        assertEquals(
+            "<p><span></span></p><table><tbody><tr><td><span>Hello table data</span></td></tr></tbody></table><p></p>",  // no quirks, p gets closed
+            TextUtil.normalizeSpaces(doc.body().html())
+        )
+    }
+
     companion object {
         private fun dupeAttributeData(): List<Pair<String, String>> {
             return listOf(

ksoup-test/test/com/fleeksoft/ksoup/select/SelectorTest.kt

@@ -1,7 +1,6 @@
 package com.fleeksoft.ksoup.select
 
 import com.fleeksoft.ksoup.Ksoup
-import com.fleeksoft.ksoup.TestHelper
 import com.fleeksoft.ksoup.nodes.Document
 import com.fleeksoft.ksoup.nodes.Element
 import com.fleeksoft.ksoup.parser.Parser
@@ -85,7 +84,7 @@ class SelectorTest {
 //        Locale.setDefault(locale)
         val h =
             "<div Title=Foo /><div Title=Bar /><div Style=Qux /><div title=Balim /><div title=SLIM />" +
-                "<div data-name='with spaces'/>"
+                    "<div data-name='with spaces'/>"
         val doc = Ksoup.parse(h)
         val withTitle = doc.select("[title]")
         assertEquals(4, withTitle.size)
@@ -1226,6 +1225,55 @@ class SelectorTest {
         assertSelectedOwnText(emptyAttr, "Three")
     }
 
+    @Test
+    fun divHasSpanPreceding() {
+        val html = "<div><span>abc</span><a>def</a></div>"
+        val q = "div:has(span + a)"
+
+        val doc: Document = Ksoup.parse(html)
+        val els: Elements = doc.select(q)
+        assertEquals(1, els.size)
+        assertEquals("div", els.first()?.normalName())
+    }
+
+    @Test
+    fun divHasDivPreceding() {
+        val html = """
+            <div id=1>
+            <div 1><span>hello</span></div>
+            <div 2><span>there</span></div>
+
+            </div>
+            """.trimIndent()
+
+        val q = "div:has(>div + div)"
+
+        val doc: Document = Ksoup.parse(html)
+        val els: Elements = doc.select(q)
+        assertEquals(1, els.size)
+        assertEquals("div", els.first()?.normalName())
+        assertEquals("1", els.first()?.id())
+    }
+
+    @Test
+    fun nestedMultiHas() {
+        val html =
+            "<html>" +
+                    "<head></head>" +
+                    "<body>" +
+                    "<div id=o>" +
+                    "<div id=i1><span id=s1>hello</span></div>" +
+                    "<div id=i2><span id=s2>world</span></div>" +
+                    "</div>" +
+                    "</body></html>"
+        val document: Document = Ksoup.parse(html)
+
+        val q = "div:has(> div:has(> span) + div:has(> span))"
+        val els: Elements = document.select(q)
+        assertEquals(1, els.size)
+        assertEquals("o", els[0].id())
+    }
+
     companion object {
         /** Test that the selected elements match exactly the specified IDs.  */
         fun assertSelectedIds(

ksoup/module.yaml

@@ -13,7 +13,7 @@ repositories:
 dependencies:
   - ../ksoup-engine-common: exported
   - $libs.codepoints
-  - $libs.stately.concurrent
+  - $libs.stately.concurrency
 
 test-dependencies:
   - $kotlin-test

ksoup/src/com/fleeksoft/ksoup/parser/HtmlTreeBuilderState.kt

@@ -22,7 +22,6 @@ public enum class HtmlTreeBuilderState {
                 tb.insertCommentNode(t.asComment())
             } else if (t.isDoctype()) {
                 // todo: parse error check on expected doctypes
-                // todo: quirk state check on doctype ids
                 val d: Token.Doctype = t.asDoctype()
                 val doctype =
                     DocumentType(
@@ -33,10 +32,16 @@ public enum class HtmlTreeBuilderState {
                 doctype.setPubSysKey(d.pubSysKey)
                 tb.document.appendChild(doctype)
                 tb.onNodeInserted(doctype)
-                if (d.isForceQuirks) tb.document.quirksMode(Document.QuirksMode.quirks)
+                // todo: quirk state check on more doctype ids, if deemed useful (most are ancient legacy and presumably irrelevant)
+                if (d.isForceQuirks || doctype.name() != "html" || doctype.publicId().equals("HTML", ignoreCase = true)) tb.document.quirksMode(
+                    Document.QuirksMode.quirks
+                )
                 tb.transition(BeforeHtml)
             } else {
                 // todo: check not iframe srcdoc
+
+                // todo: check not iframe srcdoc
+                tb.document.quirksMode(Document.QuirksMode.quirks) // missing doctype
                 tb.transition(BeforeHtml)
                 return tb.process(t) // re-process token
             }

ksoup/src/com/fleeksoft/ksoup/select/StructuralEvaluator.kt

@@ -1,5 +1,6 @@
 package com.fleeksoft.ksoup.select
 
+import com.fleeksoft.ksoup.internal.SoftPool
 import com.fleeksoft.ksoup.internal.StringUtil
 import com.fleeksoft.ksoup.nodes.Element
 import com.fleeksoft.ksoup.nodes.NodeIterator
@@ -48,8 +49,7 @@ public abstract class StructuralEvaluator(public val evaluator: Evaluator) : Eva
 
     internal class Has(evaluator: Evaluator) : StructuralEvaluator(evaluator) {
         companion object {
-            private val nodeIterator: ThreadLocal<NodeIterator<Element>> =
-                ThreadLocal { NodeIterator(Element("html"), Element::class) }
+            private val ElementIterPool: SoftPool<NodeIterator<Element>> = SoftPool { NodeIterator(Element("html"), Element::class) }
         }
 
         private val checkSiblings = evalWantsSiblings(evaluator) // evaluating against siblings (or children)
@@ -63,16 +63,18 @@ public abstract class StructuralEvaluator(public val evaluator: Evaluator) : Eva
                     }
                     sib = sib.nextElementSibling()
                 }
-            } else {
-                // otherwise we only want to match children (or below), and not the input element. And we want to minimize GCs so reusing the Iterator obj
-                val it = nodeIterator.get()
-                it.restart(element)
+            }
+            // otherwise we only want to match children (or below), and not the input element. And we want to minimize GCs so reusing the Iterator obj
+            val it = ElementIterPool.borrow()
+            it.restart(element)
+            try {
                 while (it.hasNext()) {
                     val el = it.next()
                     if (el === element) continue  // don't match self, only descendants
-
                     if (evaluator.matches(element, el)) return true
                 }
+            } finally {
+                ElementIterPool.release(it)
             }
             return false
         }

publishToMaven.sh

@@ -16,7 +16,7 @@ if [ "$1" == "--remote" ]; then
 fi
 
 # Default build types if none are passed
-default_build_types=("kotlinx" "korlibs" "ktor2" "okio")
+default_build_types=("common" "kotlinx" "korlibs" "ktor2" "okio")
 
 # If build types are passed, use them; otherwise, use the default list
 if [ "$#" -ge 1 ]; then

settings.gradle.kts

@@ -24,24 +24,26 @@ dependencyResolutionManagement {
 val libBuildType = settings.providers.gradleProperty("libBuildType").get()
 
 include("ksoup-engine-common")
-if (libBuildType == "korlibs" || libBuildType == "common") {
+if (libBuildType == "korlibs" || libBuildType == "dev") {
     include("ksoup-engine-korlibs", "ksoup-network-korlibs")
 }
 
-if (libBuildType == "kotlinx" || libBuildType == "common") {
+if (libBuildType == "kotlinx" || libBuildType == "dev") {
     include("ksoup-engine-kotlinx", "ksoup-network")
 }
 
-if (libBuildType == "okio" || libBuildType == "common") {
+if (libBuildType == "okio" || libBuildType == "dev") {
     include("ksoup-engine-okio", "ksoup-network-ktor2")
 }
 
-if (libBuildType == "ktor2" || libBuildType == "common") {
+if (libBuildType == "ktor2" || libBuildType == "dev") {
     include("ksoup-engine-ktor2", "ksoup-network-ktor2")
 }
 
-include("ksoup")
-include("ksoup-test")
+if (libBuildType != "common") {
+    include("ksoup")
+    include("ksoup-test")
+}
 
 //include("sample:shared", "sample:desktop")
 //include("sample:android", "sample:ios")