Implement a HttpUrlConnection and Apache HC4 based channels (#335)

changelog/@unreleased/pr-335.v2.yml

@@ -0,0 +1,5 @@
+type: feature
+feature:
+  description: Implement a HttpUrlConnection and Apache HC4 based channels
+  links:
+  - https://github.com/palantir/dialogue/pull/335

dialogue-apache-hc4-client/build.gradle

@@ -0,0 +1,13 @@
+apply from: "$rootDir/gradle/publish-jar.gradle"
+
+dependencies {
+    api project(':dialogue-core')
+    api project(':dialogue-target')
+    api 'com.palantir.conjure.java.runtime:client-config'
+    api 'org.apache.httpcomponents:httpclient'
+    implementation project(':dialogue-blocking-channels')
+    implementation 'com.palantir.safe-logging:preconditions'
+
+    testCompile project(':dialogue-client-test-lib')
+    testCompile project(':dialogue-serde')
+}

dialogue-apache-hc4-client/src/main/java/com/palantir/dialogue/hc4/ApacheHttpClientBlockingChannel.java

@@ -0,0 +1,199 @@
+/*
+ * (c) Copyright 2020 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.palantir.dialogue.hc4;
+
+import com.palantir.dialogue.Endpoint;
+import com.palantir.dialogue.Headers;
+import com.palantir.dialogue.HttpMethod;
+import com.palantir.dialogue.Request;
+import com.palantir.dialogue.RequestBody;
+import com.palantir.dialogue.Response;
+import com.palantir.dialogue.UrlBuilder;
+import com.palantir.dialogue.blocking.BlockingChannel;
+import com.palantir.logsafe.Preconditions;
+import com.palantir.logsafe.exceptions.SafeRuntimeException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+final class ApacheHttpClientBlockingChannel implements BlockingChannel {
+    private static final Logger log = LoggerFactory.getLogger(ApacheHttpClientBlockingChannel.class);
+
+    private final CloseableHttpClient client;
+    private final UrlBuilder baseUrl;
+
+    ApacheHttpClientBlockingChannel(CloseableHttpClient client, URL baseUrl) {
+        this.client = client;
+        this.baseUrl = UrlBuilder.from(baseUrl);
+    }
+
+    @Override
+    public Response execute(Endpoint endpoint, Request request) throws IOException {
+        // Create base request given the URL
+        UrlBuilder url = baseUrl.newBuilder();
+        endpoint.renderPath(request.pathParams(), url);
+        request.queryParams().forEach(url::queryParam);
+        URL target = url.build();
+        RequestBuilder builder =
+                RequestBuilder.create(endpoint.httpMethod().name()).setUri(target.toString());
+
+        // Fill headers
+        request.headerParams().forEach(builder::addHeader);
+
+        if (request.body().isPresent()) {
+            Preconditions.checkArgument(
+                    endpoint.httpMethod() != HttpMethod.GET, "GET endpoints must not have a request body");
+            RequestBody body = request.body().get();
+            builder.setEntity(new RequestBodyEntity(body));
+        }
+        return new HttpClientResponse(client.execute(builder.build()));
+    }
+
+    private static final class HttpClientResponse implements Response {
+
+        private final CloseableHttpResponse response;
+        private Map<String, List<String>> headers;
+
+        HttpClientResponse(CloseableHttpResponse response) {
+            this.response = response;
+        }
+
+        @Override
+        public InputStream body() {
+            try {
+                return response.getEntity().getContent();
+            } catch (IOException e) {
+                throw new SafeRuntimeException("Failed to get response stream", e);
+            }
+        }
+
+        @Override
+        public int code() {
+            return response.getStatusLine().getStatusCode();
+        }
+
+        @Override
+        public Map<String, List<String>> headers() {
+            if (headers == null) {
+                Map<String, List<String>> tmpHeaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+                for (Header header : response.getAllHeaders()) {
+                    String value = header.getValue();
+                    if (value != null) {
+                        tmpHeaders
+                                .computeIfAbsent(header.getName(), _name -> new ArrayList<>(1))
+                                .add(header.getValue());
+                    }
+                }
+                headers = tmpHeaders;
+            }
+            return headers;
+        }
+
+        @Override
+        public Optional<String> getFirstHeader(String header) {
+            return Optional.ofNullable(response.getFirstHeader(header)).map(Header::getValue);
+        }
+
+        @Override
+        public void close() {
+            try {
+                response.close();
+            } catch (IOException | RuntimeException e) {
+                log.warn("Failed to close response", e);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return "HttpClientResponse{response=" + response + '}';
+        }
+    }
+
+    private static final class RequestBodyEntity implements HttpEntity {
+
+        private final RequestBody requestBody;
+        private final Header contentType;
+
+        RequestBodyEntity(RequestBody requestBody) {
+            this.requestBody = requestBody;
+            this.contentType = new BasicHeader(Headers.CONTENT_TYPE, requestBody.contentType());
+        }
+
+        @Override
+        public boolean isRepeatable() {
+            return false;
+        }
+
+        @Override
+        public boolean isChunked() {
+            return true;
+        }
+
+        @Override
+        public long getContentLength() {
+            // unknown
+            return -1;
+        }
+
+        @Override
+        public Header getContentType() {
+            return contentType;
+        }
+
+        @Override
+        public Header getContentEncoding() {
+            return null;
+        }
+
+        @Override
+        public InputStream getContent() throws UnsupportedOperationException {
+            throw new UnsupportedOperationException("getContent is not supported, writeTo should be used");
+        }
+
+        @Override
+        public void writeTo(OutputStream outStream) throws IOException {
+            requestBody.writeTo(outStream);
+        }
+
+        @Override
+        public boolean isStreaming() {
+            // Applies to responses.
+            return false;
+        }
+
+        @Override
+        public void consumeContent() {}
+
+        @Override
+        public String toString() {
+            return "RequestBodyEntity{requestBody=" + requestBody + '}';
+        }
+    }
+}

dialogue-apache-hc4-client/src/main/java/com/palantir/dialogue/hc4/ApacheHttpClientChannels.java

@@ -0,0 +1,117 @@
+/*
+ * (c) Copyright 2020 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.palantir.dialogue.hc4;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.primitives.Ints;
+import com.palantir.conjure.java.api.config.service.UserAgent;
+import com.palantir.conjure.java.client.config.CipherSuites;
+import com.palantir.conjure.java.client.config.ClientConfiguration;
+import com.palantir.conjure.java.client.config.NodeSelectionStrategy;
+import com.palantir.dialogue.Channel;
+import com.palantir.dialogue.blocking.BlockingChannelAdapter;
+import com.palantir.dialogue.core.Channels;
+import com.palantir.logsafe.Preconditions;
+import com.palantir.logsafe.SafeArg;
+import com.palantir.logsafe.exceptions.SafeIllegalArgumentException;
+import com.palantir.tritium.metrics.registry.TaggedMetricRegistry;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.concurrent.TimeUnit;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.ProxyAuthenticationStrategy;
+import org.apache.http.impl.conn.SystemDefaultRoutePlanner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class ApacheHttpClientChannels {
+    private static final Logger log = LoggerFactory.getLogger(ApacheHttpClientChannels.class);
+
+    private ApacheHttpClientChannels() {}
+
+    public static Channel create(ClientConfiguration conf, UserAgent baseAgent, TaggedMetricRegistry metrics) {
+        Preconditions.checkArgument(
+                !conf.fallbackToCommonNameVerification(), "fallback-to-common-name-verification is not supported");
+        Preconditions.checkArgument(!conf.meshProxy().isPresent(), "Mesh proxy is not supported");
+        Preconditions.checkArgument(
+                conf.clientQoS() == ClientConfiguration.ClientQoS.ENABLED, "Disabling client QOS is not supported");
+        Preconditions.checkArgument(
+                conf.serverQoS() == ClientConfiguration.ServerQoS.AUTOMATIC_RETRY,
+                "Propagating QoS exceptions is not supported");
+        Preconditions.checkArgument(!conf.proxyCredentials().isPresent(), "Proxy credentials are not supported");
+        if (conf.nodeSelectionStrategy() != NodeSelectionStrategy.ROUND_ROBIN) {
+            log.warn(
+                    "Dialogue currently only supports ROUND_ROBIN node selection strategy. {} will be ignored",
+                    SafeArg.of("requestedStrategy", conf.nodeSelectionStrategy()));
+        }
+        long socketTimeoutMillis =
+                Math.max(conf.readTimeout().toMillis(), conf.writeTimeout().toMillis());
+        int connectTimeout = Ints.checkedCast(conf.connectTimeout().toMillis());
+        // TODO(ckozak): close resources?
+        CloseableHttpClient client = HttpClients.custom()
+                .setDefaultRequestConfig(RequestConfig.custom()
+                        .setSocketTimeout(Ints.checkedCast(socketTimeoutMillis))
+                        .setConnectTimeout(connectTimeout)
+                        // Don't allow clients to block forever waiting on a connection to become available
+                        .setConnectionRequestTimeout(connectTimeout)
+                        // Match okhttp, disallow redirects
+                        .setRedirectsEnabled(false)
+                        .setRelativeRedirectsAllowed(false)
+                        .build())
+                .setDefaultSocketConfig(
+                        SocketConfig.custom().setSoKeepAlive(true).build())
+                .evictIdleConnections(55, TimeUnit.SECONDS)
+                .setMaxConnPerRoute(1000)
+                .setMaxConnTotal(Integer.MAX_VALUE)
+                // TODO(ckozak): proxy credentials
+                .setRoutePlanner(new SystemDefaultRoutePlanner(null, conf.proxy()))
+                .setProxyAuthenticationStrategy(ProxyAuthenticationStrategy.INSTANCE)
+                .disableAutomaticRetries()
+                // Must be disabled otherwise connections are not reused when client certificates are provided
+                .disableConnectionState()
+                // Match okhttp behavior disabling cookies
+                .disableCookieManagement()
+                // Dialogue handles content-compression with ContentDecodingChannel
+                .disableContentCompression()
+                .setSSLSocketFactory(
+                        new SSLConnectionSocketFactory(
+                                conf.sslSocketFactory(),
+                                new String[] {"TLSv1.2"},
+                                conf.enableGcmCipherSuites()
+                                        ? CipherSuites.allCipherSuites()
+                                        : CipherSuites.fastCipherSuites(),
+                                new DefaultHostnameVerifier()))
+                .build();
+        ImmutableList<Channel> channels = conf.uris().stream()
+                .map(uri -> BlockingChannelAdapter.of(new ApacheHttpClientBlockingChannel(client, url(uri))))
+                .collect(ImmutableList.toImmutableList());
+
+        return Channels.create(channels, baseAgent, metrics);
+    }
+
+    private static URL url(String uri) {
+        try {
+            return new URL(uri);
+        } catch (MalformedURLException e) {
+            throw new SafeIllegalArgumentException("Failed to parse URL", e);
+        }
+    }
+}

dialogue-apache-hc4-client/src/test/java/com/palantir/dialogue/hc4/ApacheApacheHttpClientChannelsTest.java

@@ -0,0 +1,44 @@
+/*
+ * (c) Copyright 2020 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.palantir.dialogue.hc4;
+
+import com.palantir.conjure.java.api.config.service.ServiceConfiguration;
+import com.palantir.conjure.java.api.config.service.UserAgent;
+import com.palantir.conjure.java.api.config.ssl.SslConfiguration;
+import com.palantir.conjure.java.client.config.ClientConfigurations;
+import com.palantir.dialogue.AbstractChannelTest;
+import com.palantir.dialogue.Channel;
+import com.palantir.tritium.metrics.registry.DefaultTaggedMetricRegistry;
+import java.net.URL;
+import java.nio.file.Paths;
+
+public final class ApacheApacheHttpClientChannelsTest extends AbstractChannelTest {
+
+    private static final SslConfiguration SSL_CONFIG = SslConfiguration.of(
+            Paths.get("src/test/resources/trustStore.jks"), Paths.get("src/test/resources/keyStore.jks"), "keystore");
+
+    @Override
+    protected Channel createChannel(URL baseUrl) {
+        ServiceConfiguration serviceConf = ServiceConfiguration.builder()
+                .addUris(baseUrl.toString())
+                .security(SSL_CONFIG)
+                .build();
+        return ApacheHttpClientChannels.create(
+                ClientConfigurations.of(serviceConf),
+                UserAgent.of(UserAgent.Agent.of("test-service", "1.0.0")),
+                new DefaultTaggedMetricRegistry());
+    }
+}

dialogue-blocking-channels/build.gradle

@@ -0,0 +1,12 @@
+apply from: "$rootDir/gradle/publish-jar.gradle"
+
+dependencies {
+    api project(':dialogue-target')
+    api 'com.google.guava:guava'
+    implementation 'com.palantir.tracing:tracing'
+
+    testCompile 'junit:junit'
+    testCompile 'org.assertj:assertj-core'
+    testCompile 'org.mockito:mockito-core'
+    testCompile 'org.awaitility:awaitility'
+}