Merge remote-tracking branch 'upstream/main' into linuxArm64

# Conflicts:
#	gradle.properties
#	gradle/libs.versions.toml
#	ktor-test-dispatcher/linux/src/TestLinux.kt
#	ktor-test-dispatcher/linuxX64/src/TestLinuxX64.kt
#	ktor-test-dispatcher/posix/src/TestPosix.kt

CHANGELOG.md

@@ -1,3 +1,34 @@
+# 2.3.1
+> Published 31 May 2023
+
+### Bugfixes
+* AndroidClientEngine: the engine double-parses query parameters before sending a request ([KTOR-5814](https://youtrack.jetbrains.com/issue/KTOR-5814))
+* Flaky tests in WinHttp engine ([KTOR-5946](https://youtrack.jetbrains.com/issue/KTOR-5946))
+* Electron/Node.js detection doesn't work correctly ([KTOR-5906](https://youtrack.jetbrains.com/issue/KTOR-5906))
+* Curl sometimes fails with `API function called from within callback` ([KTOR-5918](https://youtrack.jetbrains.com/issue/KTOR-5918))
+* Bearer auth token refresh hangs after prior refresh threw an exception ([KTOR-5879](https://youtrack.jetbrains.com/issue/KTOR-5879))
+* HOCON: "No configuration setting found for key" error after merging ([KTOR-5895](https://youtrack.jetbrains.com/issue/KTOR-5895))
+* Ktor Client Unable to Stream Responses in Javascript ([KTOR-5867](https://youtrack.jetbrains.com/issue/KTOR-5867))
+* Darwin engine does not support streaming of request body ([KTOR-5899](https://youtrack.jetbrains.com/issue/KTOR-5899))
+* The Logging plugin doesn't log full kotlinx deserialization errors ([KTOR-5421](https://youtrack.jetbrains.com/issue/KTOR-5421))
+* XForwardedHeaders should set `remoteAddress` in addition to `remoteHost` ([KTOR-5786](https://youtrack.jetbrains.com/issue/KTOR-5786))
+* Sessions: Set-Cookie is added on every api request ([KTOR-912](https://youtrack.jetbrains.com/issue/KTOR-912))
+* RateLimitters for every request key live in memory forever ([KTOR-5872](https://youtrack.jetbrains.com/issue/KTOR-5872))
+* Significant delay between getting a part and starting reading from its provider for multipart/form-data requests ([KTOR-5248](https://youtrack.jetbrains.com/issue/KTOR-5248))
+* getTimeMillis has seconds precision on native ([KTOR-5878](https://youtrack.jetbrains.com/issue/KTOR-5878))
+* A coroutine closed due to cancellation is considered by the JsWebSocketSession to be closed on failure ([KTOR-2932](https://youtrack.jetbrains.com/issue/KTOR-2932))
+* WebSockets: requests to a non-existing route cause server to lock up after responding with 404 (potential DOS) ([KTOR-5829](https://youtrack.jetbrains.com/issue/KTOR-5829))
+* testApplication: NPE when test server doesn't reply with an HTTP upgrade ([KTOR-5815](https://youtrack.jetbrains.com/issue/KTOR-5815))
+* GMTDate timestamp doesn't reflect timezone when created using `Calendar.toDate` method ([KTOR-5813](https://youtrack.jetbrains.com/issue/KTOR-5813))
+
+### Improvements
+* Warn when the RateLimit plugin installed after the routing ([KTOR-5915](https://youtrack.jetbrains.com/issue/KTOR-5915))
+* Allow access to RateLimiters related to call ([KTOR-5876](https://youtrack.jetbrains.com/issue/KTOR-5876))
+* Multipart: Support not writing a temporary file for binary data ([KTOR-5864](https://youtrack.jetbrains.com/issue/KTOR-5864))
+* Make System Property to Set outgoingToBeProcessed Size for WebSockets ([KTOR-5855](https://youtrack.jetbrains.com/issue/KTOR-5855))
+* Support optional properties in YAML ([KTOR-5796](https://youtrack.jetbrains.com/issue/KTOR-5796))
+* YAML config does not support reading variables from itself ([KTOR-5797](https://youtrack.jetbrains.com/issue/KTOR-5797))
+
 # 2.3.0
 > Published 19 April 2023
 

README.md

@@ -39,6 +39,10 @@ fun main(args: Array<String>) {
 * Runs embedded web server on `localhost:8080`
 * Installs routing and responds with `Hello, world!` when receiving a GET http request for the root path
 
+## Start using Ktor
+
+Build your first Kotlin HTTP or RESTful application using Ktor: [start.ktor.io](https://start.ktor.io)
+
 ## Principles
 
 #### Unopinionated

buildSrc/src/main/kotlin/test/server/tests/Content.kt

@@ -19,6 +19,10 @@ import test.server.*
 internal fun Application.contentTestServer() {
     routing {
         route("/content") {
+            get("/uri") {
+                call.respondText { call.request.local.uri }
+            }
+
             get("/empty") {
                 call.respond("")
             }

gradle.properties

@@ -4,13 +4,13 @@
 
 # ktor
 # Makes Ktor import and resolve
-ktor.ide.jvmAndCommonOnly=true
+ktor.ide.jvmAndCommonOnly=false
 
 # sytleguide
 kotlin.code.style=official
 
 # config
-version=2.3.1-SNAPSHOT
+version=2.3.1
 
 # gradle
 org.gradle.daemon=true
@@ -32,7 +32,7 @@ kotlin.native.binary.memoryModel=experimental
 #kotlinx.atomicfu.enableJsIrTransformation=true
 
 kotlin_version=1.8.10
-coroutines_version=1.7.0-RC
+coroutines_version=1.7.1
 atomicfu_version=0.20.2
 slf4j_version=1.7.36
 junit_version=4.13.2

gradle/libs.versions.toml

@@ -1,14 +1,14 @@
 [versions]
 kotlin-version = "1.8.10"
 kotlinx-html-version = "0.8.1-SNAPSHOT"
-coroutines-version = "1.7.0-RC"
+coroutines-version = "1.7.1"
 atomicfu-version = "0.20.2"
 serialization-version = "1.5.1"
 validator-version = "0.8.0"
 ktlint-version = "3.10.0"
 
 netty-version = "4.1.92.Final"
-netty-tcnative-version = "2.0.60.Final"
+netty-tcnative-version = "2.0.61.Final"
 
 jetty-version = "9.4.51.v20230217"
 jetty-jakarta-version = "11.0.15"
@@ -26,7 +26,7 @@ okhttp-version = "4.11.0"
 
 json-simple-version = "1.1.1"
 gson-version = "2.10.1"
-jackson-version = "2.15.0"
+jackson-version = "2.15.1"
 
 junit-version = "4.13.2"
 slf4j-version = "2.0.7"

ktor-client/ktor-client-android/jvm/src/io/ktor/client/engine/android/AndroidClientEngine.kt

@@ -44,7 +44,7 @@ public class AndroidClientEngine(override val config: AndroidEngineConfig) : Htt
 
         val requestTime = GMTDate()
 
-        val url: String = URLBuilder().takeFrom(data.url).buildString()
+        val url: String = data.url.toString()
         val outgoingContent: OutgoingContent = data.body
         val contentLength: Long? = data.headers[HttpHeaders.ContentLength]?.toLong()
             ?: outgoingContent.contentLength

ktor-client/ktor-client-cio/jvm/src/io/ktor/client/plugins/websocket/cio/buildersCio.kt

@@ -14,7 +14,6 @@ import kotlinx.coroutines.*
 /**
  * Creates a raw [ClientWebSocketSession]: no ping-pong and other service messages are used.
  */
-@OptIn(ExperimentalCoroutinesApi::class)
 public suspend fun HttpClient.webSocketRawSession(
     method: HttpMethod = HttpMethod.Get,
     host: String? = null,

ktor-client/ktor-client-core/common/src/io/ktor/client/plugins/websocket/builders.kt

@@ -23,7 +23,6 @@ public fun HttpClientConfig<*>.WebSockets(config: WebSockets.Config.() -> Unit)
 /**
  * Opens a [DefaultClientWebSocketSession].
  */
-@OptIn(ExperimentalCoroutinesApi::class)
 public suspend fun HttpClient.webSocketSession(
     block: HttpRequestBuilder.() -> Unit
 ): DefaultClientWebSocketSession {

ktor-client/ktor-client-curl/desktop/src/io/ktor/client/engine/curl/CurlProcessor.kt

@@ -36,9 +36,7 @@ internal class CurlProcessor(coroutineContext: CoroutineContext) {
             init.join()
         }
 
-        curlScope.launch {
-            runEventLoop()
-        }
+        runEventLoop()
     }
 
     suspend fun executeRequest(request: CurlRequestData): CurlSuccess {
@@ -48,7 +46,7 @@ internal class CurlProcessor(coroutineContext: CoroutineContext) {
         return result.await()
     }
 
-    @OptIn(ExperimentalCoroutinesApi::class)
+    @OptIn(DelicateCoroutinesApi::class)
     private fun runEventLoop() {
         curlScope.launch {
             val api = curlApi!!

ktor-client/ktor-client-curl/desktop/src/io/ktor/client/engine/curl/internal/CurlCallbacks.kt

@@ -42,9 +42,13 @@ internal fun onBodyChunkReceived(
     }
 
     val chunkSize = (size * count).toInt()
-    val written = body.writeAvailable(1) { dst: Buffer ->
-        val toWrite = minOf(chunkSize - wrapper.bytesWritten.value, dst.writeRemaining)
-        dst.writeFully(buffer, wrapper.bytesWritten.value, toWrite)
+    val written = try {
+        body.writeAvailable(1) { dst: Buffer ->
+            val toWrite = minOf(chunkSize - wrapper.bytesWritten.value, dst.writeRemaining)
+            dst.writeFully(buffer, wrapper.bytesWritten.value, toWrite)
+        }
+    } catch (cause: Throwable) {
+        return -1
     }
     if (written > 0) {
         wrapper.bytesWritten += written
@@ -79,8 +83,12 @@ internal fun onBodyChunkRequested(
     if (body.isClosedForRead) {
         return if (body.closedCause != null) -1 else 0
     }
-    val readCount = body.readAvailable(1) { source: Buffer ->
-        source.readAvailable(buffer, 0, requested)
+    val readCount = try {
+        body.readAvailable(1) { source: Buffer ->
+            source.readAvailable(buffer, 0, requested)
+        }
+    } catch (cause: Throwable) {
+        return -1
     }
     if (readCount > 0) {
         return readCount

ktor-client/ktor-client-java/jvm/src/io/ktor/client/engine/java/JavaHttpWebSocket.kt

@@ -87,7 +87,6 @@ internal class JavaHttpWebSocket(
 
     init {
         launch {
-            @OptIn(ExperimentalCoroutinesApi::class)
             _outgoing.consumeEach { frame ->
                 when (frame.frameType) {
                     FrameType.TEXT -> {

ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/QueryTest.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package io.ktor.client.tests
+
+import io.ktor.client.request.*
+import io.ktor.client.statement.*
+import io.ktor.client.tests.utils.*
+import io.ktor.http.*
+import kotlin.test.*
+
+class QueryTest : ClientLoader() {
+    @Test
+    fun queryParametersArentModified() = clientTests {
+        test { client ->
+            val result = client.get {
+                url {
+                    encodedParameters.apply {
+                        url.encodedParameters.appendAll("", emptyList())
+                        url.takeFrom("$TEST_SERVER/content/uri?Expires")
+                    }
+                }
+            }.bodyAsText()
+            assertEquals("/content/uri?&Expires", result)
+        }
+    }
+}

ktor-client/ktor-client-winhttp/windows/src/io/ktor/client/engine/winhttp/internal/WinHttpConnect.kt

@@ -31,8 +31,7 @@ internal class WinHttpConnect(private val hConnect: COpaquePointer) : Closeable
     fun openRequest(
         method: HttpMethod,
         url: Url,
-        httpVersion: String?,
-        chunkedMode: WinHttpChunkedMode
+        httpVersion: String?
     ): COpaquePointer? {
         var openFlags = WINHTTP_FLAG_ESCAPE_DISABLE or
             WINHTTP_FLAG_ESCAPE_DISABLE_QUERY or
@@ -42,10 +41,6 @@ internal class WinHttpConnect(private val hConnect: COpaquePointer) : Closeable
             openFlags = openFlags or WINHTTP_FLAG_SECURE
         }
 
-        if (chunkedMode == WinHttpChunkedMode.Automatic) {
-            openFlags = openFlags or WINHTTP_FLAG_AUTOMATIC_CHUNKING
-        }
-
         return WinHttpOpenRequest(
             hConnect,
             method.value,

ktor-client/ktor-client-winhttp/windows/src/io/ktor/client/engine/winhttp/internal/WinHttpCoroutine.kt

@@ -25,18 +25,27 @@ internal suspend inline fun <T> Closeable.closeableCoroutine(
     }
 
     state.handlers[WinHttpCallbackStatus.RequestError.value] = { statusInfo, _ ->
-        val result = statusInfo!!.reinterpret<WINHTTP_ASYNC_RESULT>().pointed
-        continuation.resumeWithException(getWinHttpException(errorMessage, result.dwError))
+        if (continuation.isActive) {
+            val result = statusInfo!!.reinterpret<WINHTTP_ASYNC_RESULT>().pointed
+            continuation.resumeWithException(getWinHttpException(errorMessage, result.dwError))
+        } else {
+            close()
+        }
     }
 
     state.handlers[WinHttpCallbackStatus.SecureFailure.value] = { statusInfo, _ ->
-        val securityCode = statusInfo!!.reinterpret<UIntVar>().pointed.value
-        continuation.resumeWithException(getWinHttpException(errorMessage, securityCode))
+        if (continuation.isActive) {
+            val securityCode = statusInfo!!.reinterpret<UIntVar>().pointed.value
+            continuation.resumeWithException(getWinHttpException(errorMessage, securityCode))
+        } else {
+            close()
+        }
     }
 
     try {
         block(continuation)
     } catch (cause: Throwable) {
-        continuation.resumeWithException(cause)
+        if (continuation.isActive) continuation.resumeWithException(cause)
+        else close()
     }
 }

ktor-client/ktor-client-winhttp/windows/src/io/ktor/client/engine/winhttp/internal/WinHttpRequest.kt

@@ -7,6 +7,7 @@ package io.ktor.client.engine.winhttp.internal
 import io.ktor.client.engine.winhttp.*
 import io.ktor.client.request.*
 import io.ktor.http.*
+import io.ktor.http.content.*
 import io.ktor.utils.io.core.*
 import kotlinx.atomicfu.*
 import kotlinx.cinterop.*
@@ -20,16 +21,14 @@ import kotlin.coroutines.*
 internal class WinHttpRequest(
     hSession: COpaquePointer,
     data: HttpRequestData,
-    private val config: WinHttpClientEngineConfig
+    config: WinHttpClientEngineConfig
 ) : Closeable {
     private val connect: WinHttpConnect
     private val hRequest: COpaquePointer
     private val closed = atomic(false)
     private val requestClosed = atomic(false)
     private val connectReference: StableRef<WinHttpConnect>
 
-    val chunkedMode: WinHttpChunkedMode
-
     init {
         val hConnect = WinHttpConnect(hSession, data.url.host, data.url.port.convert(), 0)
             ?: throw getWinHttpException("Unable to create connection")
@@ -44,24 +43,9 @@ internal class WinHttpRequest(
             else -> null
         }
 
-        // Try to open request with proposed chunking encoding.
-        // If it fails fall back to programmatic encoding.
-        var detectedChunkedMode = getChunkedMode(data)
-        val request = connect.openRequest(data.method, data.url, httpVersion, detectedChunkedMode)
-        if (request == null) {
-            val errorCode = GetLastError().toInt()
-            if (errorCode != ERROR_INVALID_PARAMETER || detectedChunkedMode != WinHttpChunkedMode.Automatic) {
-                throw getWinHttpException("Unable to open request")
-            }
-
-            detectedChunkedMode = WinHttpChunkedMode.Enabled
-        }
-
-        hRequest = request ?: connect.openRequest(data.method, data.url, httpVersion, detectedChunkedMode)
+        hRequest = connect.openRequest(data.method, data.url, httpVersion)
             ?: throw getWinHttpException("Unable to open request")
 
-        chunkedMode = detectedChunkedMode
-
         configureFeatures()
 
         enableHttpProtocols(protocolVersion)
@@ -152,8 +136,6 @@ internal class WinHttpRequest(
 
     /**
      * Construct a body after receiving response headers.
-     *
-     * @param produceBody is response body producer.
      */
     private fun getResponseData() = memScoped {
         val dwStatusCode = alloc<UIntVar>()
@@ -296,20 +278,12 @@ internal class WinHttpRequest(
         }
     }
 
-    /**
-     * Computes possible chunking mode for request body processing.
-     *
-     * In HTTP 1 request must have one of the `Content-Length` or `Transfer-Encoding` headers.
-     * In HTTP 2 supported chunked encoding out of the box.
-     */
-    private fun getChunkedMode(data: HttpRequestData): WinHttpChunkedMode {
+    internal fun isChunked(data: HttpRequestData): Boolean {
+        if (data.body is OutgoingContent.NoContent) return false
         val contentLength = data.body.contentLength ?: data.body.headers[HttpHeaders.ContentLength]?.toLong()
-        val transferEncoding = data.body.headers[HttpHeaders.TransferEncoding]
-        return when {
-            contentLength != null || transferEncoding != null -> WinHttpChunkedMode.Disabled
-            config.protocolVersion.major >= 2 -> WinHttpChunkedMode.Automatic
-            else -> WinHttpChunkedMode.Enabled
-        }
+        return contentLength == null ||
+            data.headers[HttpHeaders.TransferEncoding] == "chunked" ||
+            data.body.headers[HttpHeaders.TransferEncoding] == "chunked"
     }
 
     /**

ktor-client/ktor-client-winhttp/windows/src/io/ktor/client/engine/winhttp/internal/WinHttpRequestProducer.kt

@@ -27,7 +27,7 @@ internal class WinHttpRequestProducer(
     private val data: HttpRequestData
 ) {
     private val closed = atomic(false)
-    private val chunked: Boolean = request.chunkedMode == WinHttpChunkedMode.Enabled && !data.isUpgradeRequest()
+    private val chunked: Boolean = request.isChunked(data)
 
     fun getHeaders(): Map<String, String> {
         val headers = data.headersToMap()
@@ -102,6 +102,7 @@ internal class WinHttpRequestProducer(
         return result
     }
 
+    @OptIn(DelicateCoroutinesApi::class)
     private suspend fun OutgoingContent.toByteChannel(): ByteReadChannel? = when (this) {
         is OutgoingContent.ByteArrayContent -> ByteReadChannel(bytes())
         is OutgoingContent.WriteChannelContent -> GlobalScope.writer(coroutineContext) {

ktor-client/ktor-client-winhttp/windows/src/io/ktor/client/engine/winhttp/internal/WinHttpResponseConsumer.kt

@@ -10,6 +10,7 @@ import kotlinx.cinterop.*
 import kotlinx.coroutines.*
 import kotlin.coroutines.*
 
+@OptIn(DelicateCoroutinesApi::class)
 internal fun WinHttpRequest.readBody(callContext: CoroutineContext): ByteReadChannel {
     return GlobalScope.writer(callContext) {
         val readBuffer = ByteArrayPool.borrow()