GitHub#454 Sanitise HTTP message logs

components/api/src/main/java/com/hotels/styx/api/HttpHeader.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -98,6 +98,6 @@ public boolean equals(Object o) {
 
     @Override
     public String toString() {
-        return name + "=" + HEADER_VALUES_JOINER.join(values);
+        return name + ":" + HEADER_VALUES_JOINER.join(values);
     }
 }

components/api/src/main/java/com/hotels/styx/api/HttpRequest.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -349,8 +349,7 @@ public String toString() {
         return toStringHelper(this)
                 .add("version", version)
                 .add("method", method)
-                .add("uri", url)
-                .add("headers", headers)
+                .add("url", url)
                 .add("id", id)
                 .toString();
     }

components/api/src/main/java/com/hotels/styx/api/HttpResponse.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -214,7 +214,6 @@ public String toString() {
         return toStringHelper(this)
                 .add("version", version)
                 .add("status", status)
-                .add("headers", headers)
                 .toString();
     }
 

components/api/src/main/java/com/hotels/styx/api/LiveHttpRequest.java

@@ -388,8 +388,7 @@ public String toString() {
         return toStringHelper(this)
                 .add("version", version)
                 .add("method", method)
-                .add("uri", url)
-                .add("headers", headers)
+                .add("url", url)
                 .add("id", id)
                 .toString();
     }

components/api/src/main/java/com/hotels/styx/api/LiveHttpResponse.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -235,7 +235,6 @@ public String toString() {
         return toStringHelper(this)
                 .add("version", version)
                 .add("status", status)
-                .add("headers", headers)
                 .toString();
     }
 

components/api/src/test/java/com/hotels/styx/api/HttpHeaderTest.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -31,15 +31,15 @@ public void rejectsEmptyValuesList() {
     @Test
     public void createsSingleValueHeader() {
         HttpHeader header = header("name", "value");
-        assertThat(header.toString(), is("name=value"));
+        assertThat(header.toString(), is("name:value"));
         assertThat(header.value(), is("value"));
         assertThat(header.values(), contains("value"));
     }
 
     @Test
     public void createsMultipleValueHeader() {
         HttpHeader header = header("name", "value1", "value2");
-        assertThat(header.toString(), is("name=value1, value2"));
+        assertThat(header.toString(), is("name:value1, value2"));
         assertThat(header.value(), is("value1"));
         assertThat(header.values(), contains("value1", "value2"));
     }

components/api/src/test/java/com/hotels/styx/api/HttpRequestTest.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -131,10 +131,8 @@ public void canUseBuilderToSetRequestProperties() {
                 .cookies(requestCookie("cfoo", "bar"))
                 .build();
 
-        assertThat(request.toString(), is("HttpRequest{version=HTTP/1.1, method=PATCH, uri=https://hotels.com, " +
-                "headers=[headerName=a, Cookie=cfoo=bar, Host=hotels.com], id=id}"));
-
-        assertThat(request.headers("headerName"), is(singletonList("a")));
+        assertThat(request.toString(), is("HttpRequest{version=HTTP/1.1, method=PATCH, url=https://hotels.com, id=id}"));
+        assertThat(request.headers().toString(), is("[headerName=a, Cookie=cfoo=bar, Host=hotels.com]"));
     }
 
     @Test

components/api/src/test/java/com/hotels/styx/api/LiveHttpRequestTest.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -137,9 +137,8 @@ public void canUseBuilderToSetRequestProperties() {
                 .cookies(requestCookie("cfoo", "bar"))
                 .build();
 
-        assertThat(request.toString(), is("LiveHttpRequest{version=HTTP/1.0, method=PATCH, uri=https://hotels.com, headers=[headerName=a, Cookie=cfoo=bar, Host=hotels.com], id=id}"));
-
-        assertThat(request.headers("headerName"), is(singletonList("a")));
+        assertThat(request.toString(), is("LiveHttpRequest{version=HTTP/1.0, method=PATCH, url=https://hotels.com, id=id}"));
+        assertThat(request.headers().toString(), is("[headerName=a, Cookie=cfoo=bar, Host=hotels.com]"));
     }
 
     @Test

components/client/src/main/java/com/hotels/styx/client/StyxBackendServiceClient.java

@@ -52,6 +52,7 @@
 import static com.hotels.styx.api.extension.service.StickySessionConfig.stickySessionDisabled;
 import static com.hotels.styx.client.StyxHeaderConfig.ORIGIN_ID_DEFAULT;
 import static com.hotels.styx.client.stickysession.StickySessionCookie.newStickySessionCookie;
+import static com.hotels.styx.common.format.HttpMessageFormatter.formatRequest;
 import static io.netty.handler.codec.http.HttpMethod.HEAD;
 import static java.util.Collections.emptyList;
 import static java.util.Objects.nonNull;
@@ -255,9 +256,9 @@ private static String hosts(Iterable<RemoteHost> origins) {
         }
     }
 
-    private static void logError(LiveHttpRequest rewrittenRequest, Throwable throwable) {
+    private static void logError(LiveHttpRequest request, Throwable throwable) {
         LOGGER.error("Error Handling request={} exceptionClass={} exceptionMessage=\"{}\"",
-                new Object[]{rewrittenRequest, throwable.getClass().getName(), throwable.getMessage()});
+                new Object[]{formatRequest(request), throwable.getClass().getName(), throwable.getMessage()});
     }
 
     private LiveHttpResponse removeUnexpectedResponseBody(LiveHttpRequest request, LiveHttpResponse response) {

components/client/src/main/java/com/hotels/styx/client/StyxHttpClient.java

@@ -38,6 +38,7 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.hotels.styx.api.HttpHeaderNames.HOST;
 import static com.hotels.styx.api.HttpHeaderNames.USER_AGENT;
+import static com.hotels.styx.common.format.HttpMessageFormatter.formatRequest;
 import static com.hotels.styx.api.extension.Origin.newOriginBuilder;
 import static com.hotels.styx.client.HttpConfig.newHttpConfigBuilder;
 import static java.util.Objects.requireNonNull;
@@ -158,7 +159,7 @@ private static Origin originFromRequest(LiveHttpRequest request, Boolean isHttps
                 .orElseGet(() -> {
                     checkArgument(request.url().isAbsolute(), "host header is not set for request=%s", request);
                     return request.url().authority().map(Url.Authority::hostAndPort)
-                            .orElseThrow(() -> new IllegalArgumentException("Cannot send request " + request + " as URL is not absolute and no HOST header is present"));
+                            .orElseThrow(() -> new IllegalArgumentException("Cannot send request " + formatRequest(request) + " as URL is not absolute and no HOST header is present"));
                 });
 
         HostAndPort host = HostAndPort.fromString(hostAndPort);

components/client/src/main/java/com/hotels/styx/client/netty/connectionpool/HttpRequestOperation.java

@@ -49,6 +49,7 @@
 import static com.google.common.base.Objects.toStringHelper;
 import static com.hotels.styx.api.HttpHeaderNames.HOST;
 import static com.hotels.styx.api.extension.service.BackendService.DEFAULT_RESPONSE_TIMEOUT_MILLIS;
+import static com.hotels.styx.common.format.HttpMessageFormatter.formatRequest;
 import static io.netty.handler.codec.http.LastHttpContent.EMPTY_LAST_CONTENT;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
@@ -159,7 +160,7 @@ public Observable<LiveHttpResponse> execute(NettyConnection nettyConnection) {
 
                                 if (requestIsOngoing(requestRequestBodyChunkSubscriber.get())) {
                                     LOGGER.warn("Origin responded too quickly to an ongoing request, or it was cancelled. Connection={}, Request={}.",
-                                            new Object[]{nettyConnection.channel(), this.request});
+                                            new Object[]{nettyConnection.channel(), formatRequest(this.request)});
                                     nettyConnection.close();
                                 }
                             }
@@ -192,7 +193,7 @@ private void removeProxyBridgeHandlers(NettyConnection connection) {
         } catch (NoSuchElementException cause) {
             long elapsedTime = System.currentTimeMillis() - requestTime;
             LOGGER.error("Failed to remove pipeline handlers from pooled connection. elapsedTime={}, request={}, terminationCount={}, executionCount={}, cause={}",
-                    new Object[]{elapsedTime, request, terminationCount.get(), executeCount.get(), cause});
+                    new Object[]{elapsedTime, formatRequest(request), terminationCount.get(), executeCount.get(), cause});
         }
     }
 
@@ -256,7 +257,7 @@ private ChannelFutureListener subscribeToRequestBody() {
                 } else {
                     String channelIdentifier = String.format("%s -> %s", nettyConnection.channel().localAddress(), nettyConnection.channel().remoteAddress());
                     LOGGER.error(format("Failed to send request headers. origin=%s connection=%s request=%s",
-                            nettyConnection.getOrigin(), channelIdentifier, request), headersFuture.cause());
+                            nettyConnection.getOrigin(), channelIdentifier, formatRequest(request)), headersFuture.cause());
                     responseFromOriginObserver.onError(new TransportLostException(nettyConnection.channel().remoteAddress(), nettyConnection.getOrigin()));
                 }
             };
@@ -311,7 +312,7 @@ public void onNext(ByteBuf chunk) {
                         } else {
                             String channelIdentifier = String.format("%s -> %s", nettyConnection.channel().localAddress(), nettyConnection.channel().remoteAddress());
                             LOGGER.error(format("Failed to send request body data. origin=%s connection=%s request=%s",
-                                    nettyConnection.getOrigin(), channelIdentifier, request), future.cause());
+                                    nettyConnection.getOrigin(), channelIdentifier, formatRequest(request)), future.cause());
                             this.onError(new TransportLostException(nettyConnection.channel().remoteAddress(), nettyConnection.getOrigin()));
                         }
                     });

components/client/src/test/unit/java/com/hotels/styx/client/netty/HttpRequestMessageLoggerTest.java

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2013-2018 Expedia Inc.
+  Copyright (C) 2013-2019 Expedia Inc.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -68,8 +68,9 @@ public void logsClientSideRequestShortFormat() {
         LiveHttpRequest styxRequest = get("http://www.hotels.com/foo/bar/request").build();
         new HttpRequestMessageLogger("com.hotels.styx.http-messages.outbound", false).logRequest(styxRequest, origin, true);
 
-        assertThat(log.lastMessage(), is(loggingEvent(INFO, format("requestId=%s, secure=true, request=\\{method=GET, uri=%s, origin=\"%s\"\\}",
-                styxRequest.id(), styxRequest.url(), origin.hostAndPortString()))));
+        assertThat(log.lastMessage(), is(loggingEvent(INFO,
+                format("requestId=%s, request=LiveHttpRequest\\{version=HTTP/1.1, method=GET, url=%s, id=%s}, secure=true, origin=MyApp:h1:%s",
+                    styxRequest.id(), styxRequest.url(), styxRequest.id(), origin.hostAndPortString()))));
     }
 
     @Test
@@ -78,8 +79,8 @@ public void logsClientSideRequestLongFormat() {
         new HttpRequestMessageLogger("com.hotels.styx.http-messages.outbound", true).logRequest(styxRequest, origin, true);
 
         assertThat(log.lastMessage(), is(loggingEvent(INFO,
-                format("requestId=%s, secure=true, request=\\{method=GET, uri=%s, origin=\"%s\", headers=\\[Host=www.hotels.com\\]\\}",
-                        styxRequest.id(), styxRequest.url(), origin.hostAndPortString()))));
+                format("requestId=%s, request=LiveHttpRequest\\{version=HTTP/1.1, method=GET, url=%s, headers=\\[Host:www.hotels.com\\], id=%s}, secure=true, origin=MyApp:h1:%s",
+                        styxRequest.id(), styxRequest.url(), styxRequest.id(), origin.hostAndPortString()))));
     }
 
     @Test
@@ -88,7 +89,7 @@ public void logsClientSideResponseDetailsShortFormat() {
         LiveHttpResponse styxResponse = response(OK).build();
         new HttpRequestMessageLogger("com.hotels.styx.http-messages.outbound", false).logResponse(styxRequest, styxResponse);
 
-        assertThat(log.lastMessage(), is(loggingEvent(INFO, format("requestId=%s, response=\\{status=200 OK\\}", styxRequest.id()))));
+        assertThat(log.lastMessage(), is(loggingEvent(INFO, format("requestId=%s, response=LiveHttpResponse\\{version=HTTP/1.1, status=200 OK\\}", styxRequest.id()))));
     }
 
     @Test
@@ -97,7 +98,7 @@ public void logsClientSideResponseDetailsLongFormat() {
         LiveHttpResponse styxResponse = response(OK).build();
         new HttpRequestMessageLogger("com.hotels.styx.http-messages.outbound", true).logResponse(styxRequest, styxResponse);
 
-        assertThat(log.lastMessage(), is(loggingEvent(INFO, format("requestId=%s, response=\\{status=200 OK\\, headers=\\[\\]}", styxRequest.id()))));
+        assertThat(log.lastMessage(), is(loggingEvent(INFO, format("requestId=%s, response=LiveHttpResponse\\{version=HTTP/1.1, status=200 OK, headers=\\[\\]\\}", styxRequest.id()))));
     }
 
     @Test
@@ -121,7 +122,7 @@ private Object[][] responseLogUnexpectedArguments() {
 
         return new Object[][]{
                 {normalRequest, null, WARN, "requestId=.*, response=null"},
-                {null, normalResponse, INFO, "requestId=null, response=\\{status=200 OK\\}"},
+                {null, normalResponse, INFO, "requestId=null, response=LiveHttpResponse\\{version=HTTP/1.1, status=200 OK\\}"},
                 {null, null, WARN, "requestId=null, response=null"},
         };
     }

components/common/src/main/java/com/hotels/styx/common/format/HttpHeaderFormatter.java

@@ -0,0 +1,89 @@
+/*
+  Copyright (C) 2013-2019 Expedia Inc.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+package com.hotels.styx.common.format;
+
+import com.hotels.styx.api.HttpHeader;
+import com.hotels.styx.api.HttpHeaders;
+import com.hotels.styx.api.RequestCookie;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static java.util.Collections.emptyList;
+import static java.util.Objects.requireNonNull;
+
+public class HttpHeaderFormatter {
+
+    private static final List<String> COOKIE_HEADER_NAMES = Arrays.asList("cookie", "set-cookie");
+    private static HttpHeaderFormatter instance;
+
+    public static void initialise(List<String> headerValuesToHide, List<String> cookieValuesToHide) {
+        headerValuesToHide.replaceAll(String::toLowerCase);
+        cookieValuesToHide.replaceAll(String::toLowerCase);
+        instance = new HttpHeaderFormatter(headerValuesToHide, cookieValuesToHide);
+    }
+
+    public static HttpHeaderFormatter instance() {
+        return instance == null
+                ? new HttpHeaderFormatter(emptyList(), emptyList())
+                : instance;
+    }
+
+    private List<String> headerValuesToHide;
+    private List<String> cookieValuesToHide;
+
+    private HttpHeaderFormatter(List<String> headerValuesToHide, List<String> cookieValuesToHide) {
+        this.headerValuesToHide = requireNonNull(headerValuesToHide);
+        this.cookieValuesToHide = requireNonNull(cookieValuesToHide);
+    }
+
+    public String format(HttpHeaders headers) {
+        String sanitisedHeaders = StreamSupport.stream(headers.spliterator(), false)
+                .map(this::hideOrFormatHeader)
+                .collect(Collectors.joining(", "));
+
+        return "[" + sanitisedHeaders + "]";
+    }
+
+    private String hideOrFormatHeader(HttpHeader header) {
+        return headerValuesToHide.contains(header.name().toLowerCase())
+                ? header.name() + ":****"
+                : formatHeaderAsCookieIfNecessary(header);
+    }
+
+    private String formatHeaderAsCookieIfNecessary(HttpHeader header) {
+        return COOKIE_HEADER_NAMES.contains(header.name().toLowerCase())
+                ? formatCookieHeader(header)
+                : header.toString();
+    }
+
+    private String formatCookieHeader(HttpHeader header) {
+        String cookies = RequestCookie.decode(header.value()).stream()
+                .map(this::hideOrFormatCookie)
+                .collect(Collectors.joining(";"));
+
+        return header.name() + ":" + cookies;
+    }
+
+    private String hideOrFormatCookie(RequestCookie cookie) {
+        return cookieValuesToHide.contains(cookie.name().toLowerCase())
+                ? cookie.name() + "=****"
+                : cookie.toString();
+    }
+
+}