Clients improvement

skrapers/src/main/kotlin/ru/sokomishalov/skraper/SkraperClient.kt

@@ -15,13 +15,20 @@
  */
 package ru.sokomishalov.skraper
 
+import ru.sokomishalov.skraper.client.HttpMethodType
+import ru.sokomishalov.skraper.client.HttpMethodType.GET
 import ru.sokomishalov.skraper.model.URLString
 
 /**
  * @author sokomishalov
  */
 interface SkraperClient {
 
-    suspend fun fetch(url: URLString, headers: Map<String, String> = emptyMap()): ByteArray?
+    suspend fun fetch(
+            url: URLString,
+            method: HttpMethodType = GET,
+            headers: Map<String, String> = emptyMap(),
+            body: ByteArray? = null
+    ): ByteArray?
 
 }

skrapers/src/main/kotlin/ru/sokomishalov/skraper/SkraperClientExtensions.kt

@@ -18,6 +18,8 @@ package ru.sokomishalov.skraper
 import com.fasterxml.jackson.databind.JsonNode
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
+import ru.sokomishalov.skraper.client.HttpMethodType
+import ru.sokomishalov.skraper.client.HttpMethodType.GET
 import ru.sokomishalov.skraper.internal.serialization.readJsonNodes
 import ru.sokomishalov.skraper.model.URLString
 import java.nio.charset.Charset
@@ -28,24 +30,36 @@ import kotlin.text.Charsets.UTF_8
  * @author sokomishalov
  */
 
-suspend fun SkraperClient.fetchBytes(url: URLString, headers: Map<String, String> = emptyMap()): ByteArray? {
+suspend fun SkraperClient.fetchBytes(
+        url: URLString,
+        method: HttpMethodType = GET,
+        headers: Map<String, String> = emptyMap(),
+        body: ByteArray? = null
+): ByteArray? {
     return runCatching {
-        fetch(url = url, headers = headers)
+        fetch(url, method, headers, body)
     }.getOrNull()
 }
 
-suspend fun SkraperClient.fetchJson(url: URLString, headers: Map<String, String> = emptyMap()): JsonNode? {
+suspend fun SkraperClient.fetchJson(
+        url: URLString,
+        method: HttpMethodType = GET,
+        headers: Map<String, String> = emptyMap(),
+        body: ByteArray? = null
+): JsonNode? {
     return runCatching {
-        fetch(url = url, headers = headers)?.run {
-            readJsonNodes()
-        }
+        fetch(url, method, headers, body)?.run { readJsonNodes() }
     }.getOrNull()
 }
 
-suspend fun SkraperClient.fetchDocument(url: URLString, headers: Map<String, String> = emptyMap(), charset: Charset = UTF_8): Document? {
+suspend fun SkraperClient.fetchDocument(
+        url: URLString,
+        method: HttpMethodType = GET,
+        headers: Map<String, String> = emptyMap(),
+        body: ByteArray? = null,
+        charset: Charset = UTF_8
+): Document? {
     return runCatching {
-        fetch(url = url, headers = headers)?.run {
-            Jsoup.parse(toString(charset))
-        }
+        fetch(url, method, headers, body)?.run { Jsoup.parse(toString(charset)) }
     }.getOrNull()
 }

skrapers/src/main/kotlin/ru/sokomishalov/skraper/client/HttpMethodType.kt

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2019-present Mikhael Sokolov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@file:Suppress("unused")
+
+package ru.sokomishalov.skraper.client
+
+/**
+ * @author sokomishalov
+ */
+enum class HttpMethodType {
+    GET,
+    HEAD,
+    POST,
+    PUT,
+    PATCH,
+    DELETE,
+    OPTIONS,
+    TRACE
+}

skrapers/src/main/kotlin/ru/sokomishalov/skraper/client/jdk/DefaultBlockingSkraperClient.kt

@@ -15,9 +15,10 @@
  */
 package ru.sokomishalov.skraper.client.jdk
 
-import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Dispatchers.IO
 import kotlinx.coroutines.withContext
 import ru.sokomishalov.skraper.SkraperClient
+import ru.sokomishalov.skraper.client.HttpMethodType
 import ru.sokomishalov.skraper.internal.net.openStreamForRedirectable
 import ru.sokomishalov.skraper.model.URLString
 import java.net.URL
@@ -29,9 +30,14 @@ import java.net.URL
  */
 object DefaultBlockingSkraperClient : SkraperClient {
 
-    override suspend fun fetch(url: URLString, headers: Map<String, String>): ByteArray? {
-        return withContext(Dispatchers.IO) {
-            URL(url).openStreamForRedirectable(headers = headers)
+    override suspend fun fetch(
+            url: URLString,
+            method: HttpMethodType,
+            headers: Map<String, String>,
+            body: ByteArray?
+    ): ByteArray? {
+        return withContext(IO) {
+            URL(url).openStreamForRedirectable(method, headers, body)
         }.use {
             it.readBytes()
         }

skrapers/src/main/kotlin/ru/sokomishalov/skraper/client/ktor/KtorSkraperClient.kt

@@ -16,23 +16,47 @@
 package ru.sokomishalov.skraper.client.ktor
 
 import io.ktor.client.HttpClient
-import io.ktor.client.request.get
 import io.ktor.client.request.header
+import io.ktor.client.request.request
+import io.ktor.content.ByteArrayContent
+import io.ktor.http.ContentType
+import io.ktor.http.HttpHeaders.UnsafeHeadersList
+import io.ktor.http.HttpMethod
+import io.ktor.http.takeFrom
 import ru.sokomishalov.skraper.SkraperClient
+import ru.sokomishalov.skraper.client.HttpMethodType
 import ru.sokomishalov.skraper.model.URLString
 
 class KtorSkraperClient(
         private val client: HttpClient = DEFAULT_CLIENT
 ) : SkraperClient {
 
-    override suspend fun fetch(url: URLString, headers: Map<String, String>): ByteArray? {
-        return client.get(url) {
-            headers.forEach { (k, v) -> header(k, v) }
+    override suspend fun fetch(
+            url: URLString,
+            method: HttpMethodType,
+            headers: Map<String, String>,
+            body: ByteArray?
+    ): ByteArray? {
+        return client.request {
+            this.url.takeFrom(url)
+            this.method = HttpMethod.parse(method.name)
+            headers
+                    .filterKeys { it !in UnsafeHeadersList }
+                    .forEach { (k, v) ->
+                        header(k, v)
+                    }
+            body?.let {
+                this.body = ByteArrayContent(
+                        bytes = it,
+                        contentType = headers["Content-Type"]?.let { t -> ContentType.parse(t) }
+                )
+            }
         }
     }
 
     companion object {
-        val DEFAULT_CLIENT = HttpClient {
+        @JvmStatic
+        val DEFAULT_CLIENT: HttpClient = HttpClient {
             followRedirects = true
         }
     }

skrapers/src/main/kotlin/ru/sokomishalov/skraper/client/okhttp3/OkHttp3SkraperClient.kt

@@ -18,11 +18,15 @@ package ru.sokomishalov.skraper.client.okhttp3
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.suspendCancellableCoroutine
 import okhttp3.*
+import okhttp3.MediaType.Companion.toMediaType
+import okio.BufferedSink
 import ru.sokomishalov.skraper.SkraperClient
+import ru.sokomishalov.skraper.client.HttpMethodType
 import ru.sokomishalov.skraper.model.URLString
 import java.io.IOException
 import kotlin.coroutines.resumeWithException
 
+
 /**
  * Huge appreciation to my russian colleague
  * @see <a href="https://github.com/gildor/kotlin-coroutines-okhttp/blob/master/src/main/kotlin/ru/gildor/coroutines/okhttp/CallAwait.kt">link</a>
@@ -34,11 +38,17 @@ class OkHttp3SkraperClient(
 ) : SkraperClient {
 
     @Suppress("BlockingMethodInNonBlockingContext")
-    override suspend fun fetch(url: URLString, headers: Map<String, String>): ByteArray? {
+    override suspend fun fetch(
+            url: URLString,
+            method: HttpMethodType,
+            headers: Map<String, String>,
+            body: ByteArray?
+    ): ByteArray? {
         val request = Request
                 .Builder()
                 .url(url)
                 .headers(Headers.headersOf(*(headers.flatMap { listOf(it.key, it.value) }.toTypedArray())))
+                .method(method = method.name, body = body?.createRequest(contentType = headers["Content-Type"]))
                 .build()
 
         return client
@@ -48,31 +58,34 @@ class OkHttp3SkraperClient(
                 ?.bytes()
     }
 
-    companion object {
-        val DEFAULT_CLIENT: OkHttpClient = OkHttpClient
-                .Builder()
-                .followRedirects(true)
-                .followSslRedirects(true)
-                .build()
-    }
-
     @OptIn(ExperimentalCoroutinesApi::class)
     private suspend fun Call.await(): Response {
         return suspendCancellableCoroutine { continuation ->
             enqueue(object : Callback {
-                override fun onResponse(call: Call, response: Response) {
-                    continuation.resume(response) {}
-                }
-
-                override fun onFailure(call: Call, e: IOException) {
-                    if (continuation.isCancelled) return
-                    continuation.resumeWithException(e)
-                }
+                override fun onResponse(call: Call, response: Response) = continuation.resume(response) { Unit }
+                override fun onFailure(call: Call, e: IOException) = if (continuation.isCancelled.not()) continuation.resumeWithException(e) else Unit
             })
 
             continuation.invokeOnCancellation {
-                runCatching { cancel() }
+                runCatching { cancel() }.getOrNull()
             }
         }
     }
+
+    private fun ByteArray.createRequest(contentType: String?): RequestBody? {
+        return object : RequestBody() {
+            override fun contentType(): MediaType? = contentType?.toMediaType()
+            override fun contentLength(): Long = this@createRequest.size.toLong()
+            override fun writeTo(sink: BufferedSink) = sink.write(this@createRequest).run { Unit }
+        }
+    }
+
+    companion object {
+        @JvmStatic
+        val DEFAULT_CLIENT: OkHttpClient = OkHttpClient
+                .Builder()
+                .followRedirects(true)
+                .followSslRedirects(true)
+                .build()
+    }
 }

skrapers/src/main/kotlin/ru/sokomishalov/skraper/client/reactornetty/ReactorNettySkraperClient.kt

@@ -13,14 +13,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+@file:Suppress("ReactorUnusedPublisher")
+
 package ru.sokomishalov.skraper.client.reactornetty
 
+import io.netty.handler.codec.http.HttpMethod
 import io.netty.handler.ssl.SslContextBuilder
 import io.netty.handler.ssl.util.InsecureTrustManagerFactory
 import kotlinx.coroutines.reactive.awaitFirstOrNull
+import reactor.core.publisher.Mono
+import reactor.netty.ByteBufFlux
+import reactor.netty.ByteBufMono
 import reactor.netty.http.client.HttpClient
 import ru.sokomishalov.skraper.SkraperClient
+import ru.sokomishalov.skraper.client.HttpMethodType
 import ru.sokomishalov.skraper.model.URLString
+import kotlin.text.Charsets.UTF_8
 
 /**
  * @author sokomishalov
@@ -29,16 +37,26 @@ class ReactorNettySkraperClient(
         private val client: HttpClient = DEFAULT_CLIENT
 ) : SkraperClient {
 
-    override suspend fun fetch(url: URLString, headers: Map<String, String>): ByteArray? {
+    override suspend fun fetch(
+            url: URLString,
+            method: HttpMethodType,
+            headers: Map<String, String>,
+            body: ByteArray?
+    ): ByteArray? {
         return client
                 .headers { headers.forEach { (k, v) -> it[k] = v } }
-                .get()
+                .request(HttpMethod.valueOf(method.name))
                 .uri(url)
+                .send(when (body) {
+                    null -> ByteBufMono.empty()
+                    else -> ByteBufFlux.fromString(Mono.just(body.toString(UTF_8)))
+                })
                 .responseSingle { _, u -> u.asByteArray() }
                 .awaitFirstOrNull()
     }
 
     companion object {
+        @JvmStatic
         val DEFAULT_CLIENT: HttpClient = HttpClient
                 .create()
                 .followRedirect(true)

skrapers/src/main/kotlin/ru/sokomishalov/skraper/client/spring/SpringReactiveSkraperClient.kt

@@ -17,13 +17,16 @@ package ru.sokomishalov.skraper.client.spring
 
 import io.netty.handler.ssl.SslContextBuilder
 import io.netty.handler.ssl.util.InsecureTrustManagerFactory
+import org.springframework.http.HttpMethod
+import org.springframework.http.HttpMethod.GET
 import org.springframework.http.client.reactive.ReactorClientHttpConnector
 import org.springframework.web.reactive.function.client.ExchangeStrategies
 import org.springframework.web.reactive.function.client.WebClient
 import org.springframework.web.reactive.function.client.awaitBodyOrNull
 import org.springframework.web.reactive.function.client.awaitExchange
 import reactor.netty.http.client.HttpClient
 import ru.sokomishalov.skraper.SkraperClient
+import ru.sokomishalov.skraper.client.HttpMethodType
 import ru.sokomishalov.skraper.model.URLString
 
 /**
@@ -33,16 +36,23 @@ class SpringReactiveSkraperClient(
         private val webClient: WebClient = DEFAULT_CLIENT
 ) : SkraperClient {
 
-    override suspend fun fetch(url: URLString, headers: Map<String, String>): ByteArray? {
+    override suspend fun fetch(
+            url: URLString,
+            method: HttpMethodType,
+            headers: Map<String, String>,
+            body: ByteArray?
+    ): ByteArray? {
         return webClient
-                .get()
+                .method(HttpMethod.resolve(method.name) ?: GET)
                 .uri(url)
                 .headers { headers.forEach { (k, v) -> it[k] = v } }
+                .apply { body?.let { bodyValue(it) } }
                 .awaitExchange()
                 .awaitBodyOrNull()
     }
 
     companion object {
+        @JvmStatic
         val DEFAULT_CLIENT: WebClient = WebClient
                 .builder()
                 .clientConnector(