diff --git a/dialogue-clients/src/main/java/com/palantir/dialogue/clients/AugmentClientConfig.java b/dialogue-clients/src/main/java/com/palantir/dialogue/clients/AugmentClientConfig.java
index de1a54418d..2eb3831e20 100644
--- a/dialogue-clients/src/main/java/com/palantir/dialogue/clients/AugmentClientConfig.java
+++ b/dialogue-clients/src/main/java/com/palantir/dialogue/clients/AugmentClientConfig.java
@@ -27,6 +27,7 @@
 import com.palantir.tritium.metrics.registry.TaggedMetricRegistry;
 import java.security.Provider;
 import java.util.Optional;
+import javax.net.ssl.SSLContext;
 import org.immutables.value.Value;
 
 /**
@@ -64,6 +65,14 @@ static ClientConfiguration getClientConf(ServiceConfiguration serviceConfig, Aug
         ClientConfiguration.Builder builder =
                 ClientConfiguration.builder().from(ClientConfigurations.of(serviceConfig));
 
+        SSLContext context = augment.securityProvider()
+                .map(provider -> SslSocketFactories.createSslContext(serviceConfig.security(), provider))
+                .orElseGet(() -> SslSocketFactories.createSslContext(serviceConfig.security()));
+        // Reduce the session cache size for clients. We expect TLS connections to be reused, thus the cache isn't
+        // terribly important.
+        context.getClientSessionContext().setSessionCacheSize(100);
+        builder.sslSocketFactory(context.getSocketFactory());
+
         if (!serviceConfig.maxNumRetries().isPresent()) {
             augment.maxNumRetries().ifPresent(builder::maxNumRetries);
         }