Track host resolution failure metrics (#1582)

Track host resolution failure metrics

changelog/@unreleased/pr-1582.v2.yml

@@ -0,0 +1,5 @@
+type: improvement
+improvement:
+  description: Track host resolution failure metrics
+  links:
+  - https://github.com/palantir/dialogue/pull/1582

dialogue-apache-hc5-client/src/main/java/com/palantir/dialogue/hc5/ApacheHttpClientChannels.java

@@ -517,7 +517,7 @@ public Socket createSocket(final HttpContext context) throws IOException {
                     // No maximum time to live
                     TimeValue.NEG_ONE_MILLISECOND,
                     null,
-                    new InstrumentedDnsResolver(SystemDefaultDnsResolver.INSTANCE, name),
+                    new InstrumentedDnsResolver(SystemDefaultDnsResolver.INSTANCE, name, conf.taggedMetricRegistry()),
                     new InstrumentedManagedHttpConnectionFactory(
                             ManagedHttpClientConnectionFactory.INSTANCE, conf.taggedMetricRegistry(), name));
             connectionManager.setDefaultSocketConfig(SocketConfig.custom()

dialogue-apache-hc5-client/src/main/java/com/palantir/dialogue/hc5/InstrumentedDnsResolver.java

@@ -16,11 +16,13 @@
 
 package com.palantir.dialogue.hc5;
 
+import com.codahale.metrics.Meter;
 import com.palantir.logsafe.SafeArg;
 import com.palantir.logsafe.UnsafeArg;
 import com.palantir.logsafe.logger.SafeLogger;
 import com.palantir.logsafe.logger.SafeLoggerFactory;
 import com.palantir.tracing.CloseableTracer;
+import com.palantir.tritium.metrics.registry.TaggedMetricRegistry;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import org.apache.hc.client5.http.DnsResolver;
@@ -30,11 +32,13 @@ final class InstrumentedDnsResolver implements DnsResolver {
 
     private static final SafeLogger log = SafeLoggerFactory.get(InstrumentedDnsResolver.class);
     private final DnsResolver delegate;
+    private final Meter errorMeter;
     private final String clientName;
 
-    InstrumentedDnsResolver(DnsResolver delegate, String clientName) {
+    InstrumentedDnsResolver(DnsResolver delegate, String clientName, TaggedMetricRegistry metricRegistry) {
         this.delegate = delegate;
         this.clientName = clientName;
+        this.errorMeter = DialogueClientMetrics.of(metricRegistry).connectionResolutionError(clientName);
     }
 
     @Override
@@ -57,6 +61,8 @@ public InetAddress[] resolve(String host) throws UnknownHostException {
             }
             return resolved;
         } catch (Throwable t) {
+            recordFailure();
+
             if (debugLoggingEnabled) {
                 long durationNanos = System.nanoTime() - startNanos;
                 log.debug(
@@ -89,6 +95,8 @@ public String resolveCanonicalHostname(String host) throws UnknownHostException
             }
             return resolved;
         } catch (Throwable t) {
+            recordFailure();
+
             if (debugLoggingEnabled) {
                 long durationNanos = System.nanoTime() - startNanos;
                 log.debug(
@@ -102,6 +110,10 @@ public String resolveCanonicalHostname(String host) throws UnknownHostException
         }
     }
 
+    private void recordFailure() {
+        errorMeter.mark();
+    }
+
     @Override
     public String toString() {
         return "InstrumentedDnsResolver{" + delegate + '}';

dialogue-apache-hc5-client/src/main/metrics/dialogue-apache-client-metrics.yml

@@ -39,3 +39,8 @@ namespaces:
         type: timer
         tags: [client-name, client-type]
         docs: Reports the time spent creating a new connection. This includes both connecting the socket and the full TLS handshake.
+
+      connection.resolution.error:
+        type: meter
+        tags: [client-name]
+        docs: Rate that connections have failed due to host resolution.

dialogue-apache-hc5-client/src/test/java/com/palantir/dialogue/hc5/ApacheHttpClientChannelsTest.java

@@ -17,8 +17,10 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.fail;
 
 import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
 import com.codahale.metrics.Metric;
 import com.google.common.collect.Lists;
 import com.google.common.collect.MoreCollectors;
@@ -124,6 +126,33 @@ public void metrics() throws Exception {
         }
     }
 
+    @Test
+    public void countsUnknownHostExceptions() throws Exception {
+        ClientConfiguration conf = TestConfigurations.create("http://unused");
+
+        try (ApacheHttpClientChannels.CloseableClient client =
+                ApacheHttpClientChannels.createCloseableHttpClient(conf, "testClient")) {
+
+            Meter connectionResolutionError =
+                    DialogueClientMetrics.of(conf.taggedMetricRegistry()).connectionResolutionError("testClient");
+
+            assertThat(connectionResolutionError.getCount()).isZero();
+
+            Channel channel =
+                    ApacheHttpClientChannels.createSingleUri("http://unknown-host-for-testing.unused", client);
+            ListenableFuture<Response> future =
+                    channel.execute(TestEndpoint.GET, Request.builder().build());
+
+            try (Response response = Futures.getChecked(future, UnknownHostException.class)) {
+                fail("This request should have failed with an unknown host exception! (code: %d)", response.code());
+            } catch (UnknownHostException _exception) {
+                // No-op, this is expected
+            }
+
+            assertThat(connectionResolutionError.getCount()).isEqualTo(1L);
+        }
+    }
+
     @Test
     public void supportsContentLengthHeader() {
         testContentLength(

dialogue-clients/metrics.md

@@ -11,6 +11,7 @@ Dialogue client response metrics provided by the Apache client channel.
 - `dialogue.client.create` tagged `client-name`, `client-type` (meter): Marked every time a new client is created.
 - `dialogue.client.close` tagged `client-name`, `client-type` (meter): Marked every time an Apache client is successfully closed and any underlying resources released (e.g. connections and background threads).
 - `dialogue.client.connection.create` tagged `client-name`, `client-type` (timer): Reports the time spent creating a new connection. This includes both connecting the socket and the full TLS handshake.
+- `dialogue.client.connection.resolution.error` tagged `client-name` (meter): Rate that connections have failed due to host resolution.
 
 ### dialogue.client.pool
 Connection pool metrics from the dialogue Apache client.