From 5701dcb75655166fee98845093865f9233280a71 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 13 Feb 2017 10:09:34 -0700 Subject: [PATCH 01/13] WIP: Implement HTTPS interception detection by Durumeric, et. al. Special thanks to @FiloSottile for guidance with the custom listener. --- caddyhttp/httpserver/mitm.go | 488 +++++++++++++++++++++++++++++++++ caddyhttp/httpserver/server.go | 19 +- 2 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 caddyhttp/httpserver/mitm.go diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go new file mode 100644 index 00000000000..0bbe104f6d7 --- /dev/null +++ b/caddyhttp/httpserver/mitm.go @@ -0,0 +1,488 @@ +package httpserver + +import ( + "bytes" + "context" + "crypto/tls" + "io" + "net" + "net/http" + "strings" + "sync" + "time" +) + +// tlsHandler is a http.Handler that will inject a value +// into the request context indicating if the TLS +// connection is likely being intercepted. +type tlsHandler struct { + next http.Handler + listener *tlsHelloListener +} + +// ServeHTTP checks the User-Agent. For the four main browsers (Chrome, +// Edge, Firefox, and Safari) indicated by the User-Agent, the properties +// of the TLS Client Hello will be compared. The context value "mitm" will +// be set to a value indicating if it is likely that the underlying TLS +// connection is being intercepted. +// +// Note that due to Microsoft's decision to intentionally make IE/Edge +// user agents obscure (and look like other browsers), this may offer +// less accuracy for IE/Edge clients. +// +// This MITM detection capability is based on research done by Durumeric, +// Halderman, et. al. in "The Security Impact of HTTPS Interception" (NDSS '17): +// https://jhalderm.com/pub/papers/interception-ndss17.pdf +func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ua := r.Header.Get("User-Agent") + if strings.Contains(ua, "Edge") { + h.listener.helloInfosMu.RLock() + info := h.listener.helloInfos[r.RemoteAddr] + h.listener.helloInfosMu.RUnlock() + if info.advertisesHeartbeatSupport() || !info.looksLikeEdge() { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) + } else { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) + } + } else if strings.Contains(ua, "Chrome") { + h.listener.helloInfosMu.RLock() + info := h.listener.helloInfos[r.RemoteAddr] + h.listener.helloInfosMu.RUnlock() + if info.advertisesHeartbeatSupport() || !info.looksLikeChrome() { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) + } else { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) + } + } else if strings.Contains(ua, "Firefox") { + h.listener.helloInfosMu.RLock() + info := h.listener.helloInfos[r.RemoteAddr] + h.listener.helloInfosMu.RUnlock() + if info.advertisesHeartbeatSupport() || !info.looksLikeFirefox() { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) + } else { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) + } + } else if strings.Contains(ua, "Safari") { + h.listener.helloInfosMu.RLock() + info := h.listener.helloInfos[r.RemoteAddr] + h.listener.helloInfosMu.RUnlock() + if info.advertisesHeartbeatSupport() || !info.looksLikeSafari() { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) + } else { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) + } + } + h.next.ServeHTTP(w, r) +} + +// multiConn is a net.Conn that reads from the +// given reader instead of the wire directly. This +// is useful when some of the connection has already +// been read (like the TLS Client Hello) and the +// reader is a io.MultiReader that starts with +// the contents of the buffer. +type multiConn struct { + net.Conn + reader io.Reader +} + +// Read reads from mc.reader. +func (mc multiConn) Read(b []byte) (n int, err error) { + return mc.reader.Read(b) +} + +// parseRawClientHello parses data which contains the raw +// TLS Client Hello message. It extracts relevant information +// into info. Any error reading the Client Hello (such as +// insufficient length or invalid length values) results in +// a silent error and an incomplete info struct, since there +// is no good way to handle an error like this during Accept(). +// +// The majority of this code is borrowed from the Go standard +// library, which is (c) The Go Authors. It has been modified +// to fit this use case. +func parseRawClientHello(data []byte) (info rawHelloInfo) { + if len(data) < 42 { + return + } + sessionIdLen := int(data[38]) + if sessionIdLen > 32 || len(data) < 39+sessionIdLen { + return + } + data = data[39+sessionIdLen:] + if len(data) < 2 { + return + } + // cipherSuiteLen is the number of bytes of cipher suite numbers. Since + // they are uint16s, the number must be even. + cipherSuiteLen := int(data[0])<<8 | int(data[1]) + if cipherSuiteLen%2 == 1 || len(data) < 2+cipherSuiteLen { + return + } + numCipherSuites := cipherSuiteLen / 2 + // read in the cipher suites + info.cipherSuites = make([]uint16, numCipherSuites) + for i := 0; i < numCipherSuites; i++ { + info.cipherSuites[i] = uint16(data[2+2*i])<<8 | uint16(data[3+2*i]) + } + data = data[2+cipherSuiteLen:] + if len(data) < 1 { + return + } + // read in the compression methods + compressionMethodsLen := int(data[0]) + if len(data) < 1+compressionMethodsLen { + return + } + info.compressionMethods = data[1 : 1+compressionMethodsLen] + + data = data[1+compressionMethodsLen:] + + // ClientHello is optionally followed by extension data + if len(data) < 2 { + return + } + extensionsLength := int(data[0])<<8 | int(data[1]) + data = data[2:] + if extensionsLength != len(data) { + return + } + + // read in each extension, and extract any relevant information + // from extensions we care about + for len(data) != 0 { + if len(data) < 4 { + return + } + extension := uint16(data[0])<<8 | uint16(data[1]) + length := int(data[2])<<8 | int(data[3]) + data = data[4:] + if len(data) < length { + return + } + + // record that the client advertised support for this extension + info.extensions = append(info.extensions, extension) + + switch extension { + case extensionSupportedCurves: + // http://tools.ietf.org/html/rfc4492#section-5.5.1 + if length < 2 { + return + } + l := int(data[0])<<8 | int(data[1]) + if l%2 == 1 || length != l+2 { + return + } + numCurves := l / 2 + info.curves = make([]tls.CurveID, numCurves) + d := data[2:] + for i := 0; i < numCurves; i++ { + info.curves[i] = tls.CurveID(d[0])<<8 | tls.CurveID(d[1]) + d = d[2:] + } + case extensionSupportedPoints: + // http://tools.ietf.org/html/rfc4492#section-5.5.2 + if length < 1 { + return + } + l := int(data[0]) + if length != l+1 { + return + } + info.points = make([]uint8, l) + copy(info.points, data[1:]) + } + + data = data[length:] + } + + return +} + +// newTLSListener returns a new tlsHelloListener that wraps ln. +func newTLSListener(ln net.Listener, config *tls.Config, readTimeout time.Duration) *tlsHelloListener { + return &tlsHelloListener{ + Listener: ln, + config: config, + readTimeout: readTimeout, + helloInfos: make(map[string]rawHelloInfo), + } +} + +// tlsHelloListener is a TLS listener that is specially designed +// to read the ClientHello manually so we can extract necessary +// information from it. Each ClientHello message is mapped by +// the remote address of the client, which must be removed when +// the connection is closed (use ConnState). +type tlsHelloListener struct { + net.Listener + config *tls.Config + readTimeout time.Duration + helloInfos map[string]rawHelloInfo + helloInfosMu sync.RWMutex +} + +// Accept waits for and returns the next connection to the listener. +// After it accepts the underlying connection, it reads the +// ClientHello message and stores the parsed data into a map on l. +func (l *tlsHelloListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + // TODO: Reading from this connection in the same goroutine is blocking, is it not? + + // Be careful to limit the amount of time to allow reading from this connection. + conn.SetDeadline(time.Now().Add(l.readTimeout)) + + // Read the header bytes. + hdr := make([]byte, 5) + _, err = io.ReadFull(conn, hdr) + if err != nil { + // returning an error will terminate the Accept loop + // in net/http, which isn't what we want; we'll just + // let the error occur naturally when it tries to read. + return conn, nil + } + + // Get the length of the ClientHello message and read it as well. + length := uint16(hdr[3])<<8 | uint16(hdr[4]) + hello := make([]byte, int(length)) + _, err = io.ReadFull(conn, hello) + if err != nil { + return conn, nil + } + + // Parse the ClientHello and store it in the map. + rawParsed := parseRawClientHello(hello) + l.helloInfosMu.Lock() + l.helloInfos[conn.RemoteAddr().String()] = rawParsed + l.helloInfosMu.Unlock() + + // Since we buffered the header and ClientHello, pretend we were + // never here by lining up the buffered values to be read with a + // custom connection type, followed by the rest of the actual + // underlying connection. + mr := io.MultiReader(bytes.NewReader(hdr), bytes.NewReader(hello), conn) + mc := multiConn{Conn: conn, reader: mr} + + // Clear the read timeout and let the built-in TLS server take care of + // it. This may not be a perfect way to do timeouts, but meh, it works. + conn.SetDeadline(time.Time{}) + + // Let the built-in TLS server handle the connection now as usual. + return tls.Server(mc, l.config), nil +} + +// rawHelloInfo contains the "raw" data parsed from the TLS +// Client Hello. No interpretation is done on the raw data. +// +// The methods on this type implement heuristics described +// by Durumeric, Halderman, et. al. in +// "The Security Impact of HTTPS Interception": +// https://jhalderm.com/pub/papers/interception-ndss17.pdf +type rawHelloInfo struct { + cipherSuites []uint16 + extensions []uint16 + compressionMethods []byte + curves []tls.CurveID + points []uint8 +} + +// advertisesHeartbeatSupport returns true if info indicates +// that the client supports the Heartbeat extension. +func (info rawHelloInfo) advertisesHeartbeatSupport() bool { + for _, ext := range info.extensions { + if ext == extensionHeartbeat { + return true + } + } + return false +} + +// looksLikeFirefox returns true if info looks like a handshake +// from a modern version of Firefox. +func (info rawHelloInfo) looksLikeFirefox() bool { + // "To determine whether a Firefox session has been + // intercepted, we check for the presence and order + // of extensions, cipher suites, elliptic curves, + // EC point formats, and handshake compression methods." + + // We check for both the presence of extensions and their ordering. + // Note: Firefox will sometimes have 21 (padding) as first extension, + // and other times it will not have it at all (Feb. 2017). + if len(info.extensions) > 0 && info.extensions[0] == 21 { + info.extensions = info.extensions[1:] + } + expectedExtensions := []uint16{0, 23, 65281, 10, 11, 35, 16, 5, 65283, 13} + if len(info.extensions) != len(expectedExtensions) { + return false + } + for i := range expectedExtensions { + if info.extensions[i] != expectedExtensions[i] { + return false + } + } + + // We check for both presence of curves and their ordering. + expectedCurves := []tls.CurveID{29, 23, 24, 25} + if len(info.curves) != len(expectedCurves) { + return false + } + for i := range expectedCurves { + if info.curves[i] != expectedCurves[i] { + return false + } + } + + // We check for order of cipher suites but not presence, since + // according to the paper, cipher suites may be not be added + // or reordered by the user, but they may be disabled. + expectedCipherSuiteOrder := []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + 0xc02f, // tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + 0xcca9, // tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + 0x33, // tls.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, + 0x39, // tls.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + } + // this loop checks the order of cipher suites + // but tolerates missing ones + var j int + for _, cipherSuite := range info.cipherSuites { + var found bool + for j < len(expectedCipherSuiteOrder) { + if expectedCipherSuiteOrder[j] == cipherSuite { + found = true + break + } + j++ + } + if j == len(expectedCipherSuiteOrder)-1 && !found { + return false + } + } + + return true +} + +// looksLikeChrome returns true if info looks like a handshake +// from a modern version of Chrome. +func (info rawHelloInfo) looksLikeChrome() bool { + // "We check for ciphers and extensions that Chrome is known + // to not support, but do not check for the inclusion of + // specific ciphers or extensions, nor do we validate their + // order. When appropriate, we check the presence and order + // of elliptic curves, compression methods, and EC point formats." + + // Not in Chrome 56, but present in Safari 10 (Feb. 2017): + // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 (0xc024) + // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 (0xc023) + // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (0xc00a) + // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (0xc009) + // TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 (0xc028) + // TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 (0xc027) + // TLS_RSA_WITH_AES_256_CBC_SHA256 (0x3d) + // TLS_RSA_WITH_AES_128_CBC_SHA256 (0x3c) + + // Not in Chrome 56, but present in Firefox 51 (Feb. 2017): + // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (0xc00a) + // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (0xc009) + // TLS_DHE_RSA_WITH_AES_128_CBC_SHA (0x33) + // TLS_DHE_RSA_WITH_AES_256_CBC_SHA (0x39) + + chromeCipherExclusions := map[uint16]struct{}{ + 0xc024: {}, // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 + 0xc023: {}, // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: {}, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: {}, + 0xc028: {}, // TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: {}, + 0x3d: {}, // TLS_RSA_WITH_AES_256_CBC_SHA256 + tls.TLS_RSA_WITH_AES_128_CBC_SHA256: {}, + 0x33: {}, // TLS_DHE_RSA_WITH_AES_128_CBC_SHA + 0x39: {}, // TLS_DHE_RSA_WITH_AES_256_CBC_SHA + } + for _, ext := range info.cipherSuites { + if _, ok := chromeCipherExclusions[ext]; ok { + return false + } + } + + // Chrome does not include curve 25 (CurveP521). + for _, curve := range info.curves { + if curve == 25 { + return false + } + } + + return true +} + +// looksLikeEdge returns true if info looks like a handshake +// from a modern version of MS Edge. +func (info rawHelloInfo) looksLikeEdge() bool { + // "SChannel connections can by uniquely identified because SChannel + // is the only TLS library we tested that includes the OCSP status + // request extension before the supported groups and EC point formats + // extensions." + // NOTE - TODO: Chrome also puts 5 before 10 and 11... + var extPosOCSPStatusRequest, extPosSupportedGroups, extPosPointFormats int + for i, ext := range info.extensions { + switch ext { + case extensionOCSPStatusRequest: + extPosOCSPStatusRequest = i + case extensionSupportedCurves: + extPosSupportedGroups = i + case extensionSupportedPoints: + extPosPointFormats = i + } + } + return extPosOCSPStatusRequest < extPosSupportedGroups && + extPosOCSPStatusRequest < extPosPointFormats +} + +// looksLikeSafari returns true if info looks like a handshake +// from a modern version of MS Safari. +func (info rawHelloInfo) looksLikeSafari() bool { + // "One unique aspect of Secure Transport is that it includes + // the TLS_EMPTY_RENEGOTIATION_INFO_SCSV (0xff) cipher first, + // whereas the other libraries we investigated include the + // cipher last. Similar to Microsoft, Apple has changed + // TLS behavior in minor OS updates, which are not indicated + // in the HTTP User-Agent header. We allow for any of the + // updates when validating handshakes, and we check for the + // presence and ordering of ciphers, extensions, elliptic + // curves, and compression methods." + + // Note that any C lib (e.g. curl) compiled on macOS + // will probably use Secure Transport which will also + // share the TLS handshake characteristics of Safari. + + if len(info.cipherSuites) < 1 { + return false + } + return info.cipherSuites[0] == scsvRenegotiation + // TODO: Implement checking of presence and ordering + // of cipher suites etc. as described by the paper. +} + +const ( + extensionOCSPStatusRequest = 5 + extensionSupportedCurves = 10 // also called "SupportedGroups" + extensionSupportedPoints = 11 + extensionHeartbeat = 15 + + scsvRenegotiation = 0xff +) diff --git a/caddyhttp/httpserver/server.go b/caddyhttp/httpserver/server.go index f7e6efa2e64..29296762858 100644 --- a/caddyhttp/httpserver/server.go +++ b/caddyhttp/httpserver/server.go @@ -46,6 +46,7 @@ func NewServer(addr string, group []*SiteConfig) (*Server, error) { connTimeout: GracefulTimeout, } s.Server.Handler = s // this is weird, but whatever + tlsh := &tlsHandler{next: s.Server.Handler} s.Server.ConnState = func(c net.Conn, cs http.ConnState) { if cs == http.StateIdle { s.listenerMu.Lock() @@ -55,6 +56,15 @@ func NewServer(addr string, group []*SiteConfig) (*Server, error) { } s.listenerMu.Unlock() } + // when a connection closes or is hijacked, delete its entry + // in the map, because we are done with it. + if tlsh.listener != nil { + if cs == http.StateHijacked || cs == http.StateClosed { + tlsh.listener.helloInfosMu.Lock() + delete(tlsh.listener.helloInfos, c.RemoteAddr().String()) + tlsh.listener.helloInfosMu.Unlock() + } + } } // Disable HTTP/2 if desired @@ -92,6 +102,10 @@ func NewServer(addr string, group []*SiteConfig) (*Server, error) { s.Server.TLSConfig.NextProtos = []string{"h2"} } + if s.Server.TLSConfig != nil { + s.Server.Handler = tlsh + } + // Compile custom middleware for every site (enables virtual hosting) for _, site := range group { stack := Handler(staticfiles.FileServer{Root: http.Dir(site.Root), Hide: site.HiddenFiles}) @@ -175,7 +189,10 @@ func (s *Server) Serve(ln net.Listener) error { // not implement the File() method we need for graceful restarts // on POSIX systems. // TODO: Is this ^ still relevant anymore? Maybe we can now that it's a net.Listener... - ln = tls.NewListener(ln, s.Server.TLSConfig) + ln = newTLSListener(ln, s.Server.TLSConfig, s.Server.ReadTimeout) + if handler, ok := s.Server.Handler.(*tlsHandler); ok { + handler.listener = ln.(*tlsHelloListener) + } // Rotate TLS session ticket keys s.tlsGovChan = caddytls.RotateSessionTicketKeys(s.Server.TLSConfig) From 8a6ab5fd09ea220f602e9676c45fecd79663384d Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 13 Feb 2017 10:10:01 -0700 Subject: [PATCH 02/13] Add {{.IsMITM}} context action and {mitm} placeholder --- caddyhttp/httpserver/context.go | 11 +++++++++++ caddyhttp/httpserver/replacer.go | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/caddyhttp/httpserver/context.go b/caddyhttp/httpserver/context.go index b7d1065c29e..549ced886f1 100644 --- a/caddyhttp/httpserver/context.go +++ b/caddyhttp/httpserver/context.go @@ -321,3 +321,14 @@ func (c Context) Files(name string) ([]string, error) { return names, nil } + +// IsMITM returns true if it seems likely that the TLS connection +// is being intercepted. +func (c Context) IsMITM() bool { + if val, ok := c.Req.Context().Value(CtxKey("mitm")).(bool); ok { + return val + } + return false +} + +type CtxKey string diff --git a/caddyhttp/httpserver/replacer.go b/caddyhttp/httpserver/replacer.go index a3dc258a5d1..c1d8bd39e15 100644 --- a/caddyhttp/httpserver/replacer.go +++ b/caddyhttp/httpserver/replacer.go @@ -291,6 +291,15 @@ func (r *replacer) getSubstitution(key string) string { } } return requestReplacer.Replace(r.requestBody.String()) + case "{mitm}": + if val, ok := r.request.Context().Value(CtxKey("mitm")).(bool); ok { + if val { + return "likely" + } else { + return "unlikely" + } + } + return "unknown" case "{status}": if r.responseRecorder == nil { return r.emptyValue From cbf62a7e4f0870d7b1842d6895ee4f15ee72cf36 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 13 Feb 2017 23:49:37 -0700 Subject: [PATCH 03/13] Improve MITM detection heuristics for Firefox and Edge --- CONTRIBUTING.md | 4 +- caddy.go | 2 + caddyhttp/httpserver/mitm.go | 137 ++++++++++++++++++++--------------- 3 files changed, 85 insertions(+), 58 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb916c5566e..43a2ae8e667 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,7 +99,9 @@ one collaborator who did not open the pull request before merging. This will help ensure high code quality as new collaborators are added to the project. Read [CodeReviewComments](https://github.com/golang/go/wiki/CodeReviewComments) -on the Go wiki for an idea of what we look for in good, clean Go code. +on the Go wiki for an idea of what we look for in good, clean Go code, and +check out [what Linus suggests](https://gist.github.com/matthewhudson/1475276) +for good commit messages. diff --git a/caddy.go b/caddy.go index b55c302f340..0df59fb74b9 100644 --- a/caddy.go +++ b/caddy.go @@ -5,6 +5,8 @@ // 1. Set the AppName and AppVersion variables. // 2. Call LoadCaddyfile() to get the Caddyfile. // Pass in the name of the server type (like "http"). +// Make sure the server type's package is imported +// (import _ "github.com/mholt/caddy/caddyhttp"). // 3. Call caddy.Start() to start Caddy. You get back // an Instance, on which you can call Restart() to // restart it or Stop() to stop it. diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index 0bbe104f6d7..ae1f9fa03dd 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -97,6 +97,8 @@ func (mc multiConn) Read(b []byte) (n int, err error) { // insufficient length or invalid length values) results in // a silent error and an incomplete info struct, since there // is no good way to handle an error like this during Accept(). +// The data is expected to contain the whole ClientHello and +// ONLY the ClientHello. // // The majority of this code is borrowed from the Go standard // library, which is (c) The Go Authors. It has been modified @@ -310,21 +312,14 @@ func (info rawHelloInfo) looksLikeFirefox() bool { // of extensions, cipher suites, elliptic curves, // EC point formats, and handshake compression methods." - // We check for both the presence of extensions and their ordering. - // Note: Firefox will sometimes have 21 (padding) as first extension, - // and other times it will not have it at all (Feb. 2017). - if len(info.extensions) > 0 && info.extensions[0] == 21 { - info.extensions = info.extensions[1:] - } - expectedExtensions := []uint16{0, 23, 65281, 10, 11, 35, 16, 5, 65283, 13} - if len(info.extensions) != len(expectedExtensions) { + // We check for the presence and order of the extensions. + // Note: Sometimes padding (21) is present, sometimes not. + // Note: Firefox 51+ does not advertise 0x3374 (13172, NPN). + // Note: Firefox doesn't advertise 0x0 (0, SNI) when connecting to IP addresses. + requiredExtensionsOrder := []uint16{23, 65281, 10, 11, 35, 16, 5, 65283, 13} + if !assertPresenceAndOrdering(requiredExtensionsOrder, info.extensions, true) { return false } - for i := range expectedExtensions { - if info.extensions[i] != expectedExtensions[i] { - return false - } - } // We check for both presence of curves and their ordering. expectedCurves := []tls.CurveID{29, 23, 24, 25} @@ -341,39 +336,55 @@ func (info rawHelloInfo) looksLikeFirefox() bool { // according to the paper, cipher suites may be not be added // or reordered by the user, but they may be disabled. expectedCipherSuiteOrder := []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - 0xc02f, // tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - 0xcca9, // tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - 0x33, // tls.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - 0x39, // tls.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, // 0xc02b + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // 0xc02f + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, // 0xcca9 + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, // 0xcca8 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, // 0xc02c + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // 0xc030 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // 0xc00a + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // 0xc009 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // 0xc013 + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // 0xc014 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA, // 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA, // 0x39 + tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f + tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35 + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // 0xa + } + return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false) +} + +// assertPresenceAndOrdering will return true if candidateList contains +// the items in requiredItems in the same order as requiredItems. +// +// If requiredIsSubset is true, then all items in requiredItems must be +// present in candidateList. If requiredIsSubset is false, then requiredItems +// may contain items that are not in candidateList. +// +// In all cases, the order of requiredItems is enforced. +func assertPresenceAndOrdering(requiredItems, candidateList []uint16, requiredIsSubset bool) bool { + superset := requiredItems + subset := candidateList + if requiredIsSubset { + superset = candidateList + subset = requiredItems } - // this loop checks the order of cipher suites - // but tolerates missing ones + var j int - for _, cipherSuite := range info.cipherSuites { + for _, item := range superset { var found bool - for j < len(expectedCipherSuiteOrder) { - if expectedCipherSuiteOrder[j] == cipherSuite { + for j < len(subset) { + if subset[j] == item { found = true break } j++ } - if j == len(expectedCipherSuiteOrder)-1 && !found { + if j == len(subset)-1 && !found { return false } } - return true } @@ -402,17 +413,18 @@ func (info rawHelloInfo) looksLikeChrome() bool { // TLS_DHE_RSA_WITH_AES_128_CBC_SHA (0x33) // TLS_DHE_RSA_WITH_AES_256_CBC_SHA (0x39) + // Selected ciphers present in Chrome mobile (Feb. 2017): + // 0xc00a, 0xc014, 0xc009, 0x9c, 0x9d, 0x2f, 0x35, 0xa + chromeCipherExclusions := map[uint16]struct{}{ - 0xc024: {}, // TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 - 0xc023: {}, // TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: {}, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: {}, - 0xc028: {}, // TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: {}, - 0x3d: {}, // TLS_RSA_WITH_AES_256_CBC_SHA256 - tls.TLS_RSA_WITH_AES_128_CBC_SHA256: {}, - 0x33: {}, // TLS_DHE_RSA_WITH_AES_128_CBC_SHA - 0x39: {}, // TLS_DHE_RSA_WITH_AES_256_CBC_SHA + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384: {}, // 0xc024 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: {}, // 0xc023 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384: {}, // 0xc028 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: {}, // 0xc027 + TLS_RSA_WITH_AES_256_CBC_SHA256: {}, // 0x3d + tls.TLS_RSA_WITH_AES_128_CBC_SHA256: {}, // 0x3c + TLS_DHE_RSA_WITH_AES_128_CBC_SHA: {}, // 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA: {}, // 0x39 } for _, ext := range info.cipherSuites { if _, ok := chromeCipherExclusions[ext]; ok { @@ -420,7 +432,7 @@ func (info rawHelloInfo) looksLikeChrome() bool { } } - // Chrome does not include curve 25 (CurveP521). + // Chrome does not include curve 25 (CurveP521) (as of Chrome 56, Feb. 2017). for _, curve := range info.curves { if curve == 25 { return false @@ -437,20 +449,20 @@ func (info rawHelloInfo) looksLikeEdge() bool { // is the only TLS library we tested that includes the OCSP status // request extension before the supported groups and EC point formats // extensions." - // NOTE - TODO: Chrome also puts 5 before 10 and 11... - var extPosOCSPStatusRequest, extPosSupportedGroups, extPosPointFormats int + // + // More specifically, the OCSP status request extension appears + // *directly* before the other two extensions, which occur in that + // order. (I contacted the authors for clarification and verified it.) for i, ext := range info.extensions { - switch ext { - case extensionOCSPStatusRequest: - extPosOCSPStatusRequest = i - case extensionSupportedCurves: - extPosSupportedGroups = i - case extensionSupportedPoints: - extPosPointFormats = i + if ext == extensionOCSPStatusRequest { + if len(info.extensions) <= i+2 { + return false + } + return info.extensions[i+1] == extensionSupportedCurves && + info.extensions[i+2] == extensionSupportedPoints } } - return extPosOCSPStatusRequest < extPosSupportedGroups && - extPosOCSPStatusRequest < extPosPointFormats + return false } // looksLikeSafari returns true if info looks like a handshake @@ -485,4 +497,15 @@ const ( extensionHeartbeat = 15 scsvRenegotiation = 0xff + + // cipher suites missing from the crypto/tls package, + // in no particular order here + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xcca9 + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xcca8 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xc024 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xc023 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xc028 + TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x3d + TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x39 ) From fd323a3f652f07597dec017178e9110f341aa2c8 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 13 Feb 2017 23:50:02 -0700 Subject: [PATCH 04/13] Add tests for MITM detection heuristics --- caddyhttp/httpserver/mitm_test.go | 178 ++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 caddyhttp/httpserver/mitm_test.go diff --git a/caddyhttp/httpserver/mitm_test.go b/caddyhttp/httpserver/mitm_test.go new file mode 100644 index 00000000000..d1289e5e8a9 --- /dev/null +++ b/caddyhttp/httpserver/mitm_test.go @@ -0,0 +1,178 @@ +package httpserver + +import ( + "crypto/tls" + "encoding/hex" + "reflect" + "testing" +) + +func TestParseClientHello(t *testing.T) { + for i, test := range []struct { + inputHex string + expected rawHelloInfo + }{ + { + // curl 7.51.0 (x86_64-apple-darwin16.0) libcurl/7.51.0 SecureTransport zlib/1.2.8 + inputHex: `010000a6030358a28c73a71bdfc1f09dee13fecdc58805dcce42ac44254df548f14645f7dc2c00004400ffc02cc02bc024c023c00ac009c008c030c02fc028c027c014c013c012009f009e006b0067003900330016009d009c003d003c0035002f000a00af00ae008d008c008b01000039000a00080006001700180019000b00020100000d00120010040102010501060104030203050306030005000501000000000012000000170000`, + expected: rawHelloInfo{ + cipherSuites: []uint16{255, 49196, 49195, 49188, 49187, 49162, 49161, 49160, 49200, 49199, 49192, 49191, 49172, 49171, 49170, 159, 158, 107, 103, 57, 51, 22, 157, 156, 61, 60, 53, 47, 10, 175, 174, 141, 140, 139}, + extensions: []uint16{10, 11, 13, 5, 18, 23}, + compressionMethods: []byte{0}, + curves: []tls.CurveID{23, 24, 25}, + points: []uint8{0}, + }, + }, + { + // Chrome 56 + inputHex: `010000c003031dae75222dae1433a5a283ddcde8ddabaefbf16d84f250eee6fdff48cdfff8a00000201a1ac02bc02fc02cc030cca9cca8cc14cc13c013c014009c009d002f0035000a010000777a7a0000ff010001000000000e000c0000096c6f63616c686f73740017000000230000000d00140012040308040401050308050501080606010201000500050100000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a000a0008aaaa001d001700182a2a000100`, + expected: rawHelloInfo{ + cipherSuites: []uint16{6682, 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49171, 49172, 156, 157, 47, 53, 10}, + extensions: []uint16{31354, 65281, 0, 23, 35, 13, 5, 18, 16, 30032, 11, 10, 10794}, + compressionMethods: []byte{0}, + curves: []tls.CurveID{43690, 29, 23, 24}, + points: []uint8{0}, + }, + }, + { + // Firefox 51 + inputHex: `010000bd030375f9022fc3a6562467f3540d68013b2d0b961979de6129e944efe0b35531323500001ec02bc02fcca9cca8c02cc030c00ac009c013c01400330039002f0035000a010000760000000e000c0000096c6f63616c686f737400170000ff01000100000a000a0008001d001700180019000b00020100002300000010000e000c02683208687474702f312e31000500050100000000ff030000000d0020001e040305030603020308040805080604010501060102010402050206020202`, + expected: rawHelloInfo{ + cipherSuites: []uint16{49195, 49199, 52393, 52392, 49196, 49200, 49162, 49161, 49171, 49172, 51, 57, 47, 53, 10}, + extensions: []uint16{0, 23, 65281, 10, 11, 35, 16, 5, 65283, 13}, + compressionMethods: []byte{0}, + curves: []tls.CurveID{29, 23, 24, 25}, + points: []uint8{0}, + }, + }, + { + // openssl s_client (OpenSSL 0.9.8zh 14 Jan 2016) + inputHex: `0100012b03035d385236b8ca7b7946fa0336f164e76bf821ed90e8de26d97cc677671b6f36380000acc030c02cc028c024c014c00a00a500a300a1009f006b006a0069006800390038003700360088008700860085c032c02ec02ac026c00fc005009d003d00350084c02fc02bc027c023c013c00900a400a200a0009e00670040003f003e0033003200310030009a0099009800970045004400430042c031c02dc029c025c00ec004009c003c002f009600410007c011c007c00cc00200050004c012c008001600130010000dc00dc003000a00ff0201000055000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000f000101`, + expected: rawHelloInfo{ + cipherSuites: []uint16{49200, 49196, 49192, 49188, 49172, 49162, 165, 163, 161, 159, 107, 106, 105, 104, 57, 56, 55, 54, 136, 135, 134, 133, 49202, 49198, 49194, 49190, 49167, 49157, 157, 61, 53, 132, 49199, 49195, 49191, 49187, 49171, 49161, 164, 162, 160, 158, 103, 64, 63, 62, 51, 50, 49, 48, 154, 153, 152, 151, 69, 68, 67, 66, 49201, 49197, 49193, 49189, 49166, 49156, 156, 60, 47, 150, 65, 7, 49169, 49159, 49164, 49154, 5, 4, 49170, 49160, 22, 19, 16, 13, 49165, 49155, 10, 255}, + extensions: []uint16{11, 10, 35, 13, 15}, + compressionMethods: []byte{1, 0}, + curves: []tls.CurveID{23, 25, 28, 27, 24, 26, 22, 14, 13, 11, 12, 9, 10}, + points: []uint8{0, 1, 2}, + }, + }, + } { + data, err := hex.DecodeString(test.inputHex) + if err != nil { + t.Fatalf("Test %d: Could not decode hex data: %v", i, err) + } + actual := parseRawClientHello(data) + if !reflect.DeepEqual(test.expected, actual) { + t.Errorf("Test %d: Expected %+v; got %+v", i, test.expected, actual) + } + } +} + +func TestHeuristicFunctions(t *testing.T) { + // To test the heuristics, we assemble a collection of real + // ClientHello messages from various TLS clients. Please be + // sure to hex-encode them and document the User-Agent + // associated with the connection. + // + // If the TLS client used is not an HTTP client (e.g. s_client), + // you can leave the userAgent blank, but please use a comment + // to document crucial missing information such as client name, + // version, and platform, maybe even the date you collected + // the sample! Please group similar clients together, ordered + // by version for convenience. + + // clientHello pairs a User-Agent string to its ClientHello message. + type clientHello struct { + userAgent string + helloHex string + } + + // clientHellos groups samples of true (real) ClientHellos by the + // name of the browser that produced them. We limit the set of + // browsers to those we are programmed to protect, as well as a + // category for "Other" which contains real ClientHello messages + // from clients that we do not recognize, which may be used to + // test or imitate interception scenarios. + // + // Please group similar clients and order by version for convenience + // when adding to the test cases. + clientHellos := map[string][]clientHello{ + "Chrome": []clientHello{ + { + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + helloHex: `010000c003031dae75222dae1433a5a283ddcde8ddabaefbf16d84f250eee6fdff48cdfff8a00000201a1ac02bc02fc02cc030cca9cca8cc14cc13c013c014009c009d002f0035000a010000777a7a0000ff010001000000000e000c0000096c6f63616c686f73740017000000230000000d00140012040308040401050308050501080606010201000500050100000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a000a0008aaaa001d001700182a2a000100`, + }, + }, + "Firefox": []clientHello{ + { + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:51.0) Gecko/20100101 Firefox/51.0", + helloHex: `010000bd030375f9022fc3a6562467f3540d68013b2d0b961979de6129e944efe0b35531323500001ec02bc02fcca9cca8c02cc030c00ac009c013c01400330039002f0035000a010000760000000e000c0000096c6f63616c686f737400170000ff01000100000a000a0008001d001700180019000b00020100002300000010000e000c02683208687474702f312e31000500050100000000ff030000000d0020001e040305030603020308040805080604010501060102010402050206020202`, + }, + }, + // TODO... in the process of downloading a VM... + // "Edge": []clientHello{ + // { + // userAgent: "", + // helloHex: ``, + // }, + // }, + "Safari": []clientHello{ + { + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8", + helloHex: `010000d2030358a295b513c8140c6ff880f4a8a73cc830ed2dab2c4f2068eb365228d828732e00002600ffc02cc02bc024c023c00ac009c030c02fc028c027c014c013009d009c003d003c0035002f010000830000000e000c0000096c6f63616c686f7374000a00080006001700180019000b00020100000d00120010040102010501060104030203050306033374000000100030002e0268320568322d31360568322d31350568322d313408737064792f332e3106737064792f3308687474702f312e310005000501000000000012000000170000`, + }, + }, + "Other": []clientHello{ + { + // openssl s_client (OpenSSL 0.9.8zh 14 Jan 2016) + helloHex: `0100012b03035d385236b8ca7b7946fa0336f164e76bf821ed90e8de26d97cc677671b6f36380000acc030c02cc028c024c014c00a00a500a300a1009f006b006a0069006800390038003700360088008700860085c032c02ec02ac026c00fc005009d003d00350084c02fc02bc027c023c013c00900a400a200a0009e00670040003f003e0033003200310030009a0099009800970045004400430042c031c02dc029c025c00ec004009c003c002f009600410007c011c007c00cc00200050004c012c008001600130010000dc00dc003000a00ff0201000055000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000f000101`, + }, + { + // curl 7.51.0 (x86_64-apple-darwin16.0) libcurl/7.51.0 SecureTransport zlib/1.2.8 + userAgent: "curl/7.51.0", + helloHex: `010000a6030358a28c73a71bdfc1f09dee13fecdc58805dcce42ac44254df548f14645f7dc2c00004400ffc02cc02bc024c023c00ac009c008c030c02fc028c027c014c013c012009f009e006b0067003900330016009d009c003d003c0035002f000a00af00ae008d008c008b01000039000a00080006001700180019000b00020100000d00120010040102010501060104030203050306030005000501000000000012000000170000`, + }, + }, + } + + for client, chs := range clientHellos { + for i, ch := range chs { + hello, err := hex.DecodeString(ch.helloHex) + if err != nil { + t.Errorf("[%s] Test %d: Error decoding ClientHello: %v", client, i, err) + continue + } + parsed := parseRawClientHello(hello) + + isChrome := parsed.looksLikeChrome() + isFirefox := parsed.looksLikeFirefox() + isSafari := parsed.looksLikeSafari() + isEdge := parsed.looksLikeEdge() + + // we want each of the heuristic functions to be as + // exclusive but as low-maintenance as possible; + // in other words, if one returns true, the others + // should return false, with as little logic as possible, + // but with enough logic to force TLS proxies to do a + // good job preserving characterstics of the handshake. + var wrong bool + switch client { + case "Chrome": + wrong = !isChrome || isFirefox || isSafari || isEdge + case "Firefox": + wrong = isChrome || !isFirefox || isSafari || isEdge + case "Safari": + wrong = isChrome || isFirefox || !isSafari || isEdge + case "Edge": + wrong = isChrome || isFirefox || isSafari || !isEdge + case "Others": + wrong = isChrome || isFirefox || isSafari || isEdge + } + + if wrong { + t.Errorf("[%s] Test %d: Chrome=%v, Firefox=%v, Safari=%v, Edge=%v", + client, i, isChrome, isFirefox, isSafari, isEdge) + } + } + } +} From 7b7e1e4e3967a9a557c5dbe4c5b8c8806d5b0d3b Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 14 Feb 2017 00:19:34 -0700 Subject: [PATCH 05/13] Improve Safari heuristics for interception detection --- caddyhttp/httpserver/mitm.go | 40 +++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index ae1f9fa03dd..a3ef6d6b4f2 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -482,12 +482,45 @@ func (info rawHelloInfo) looksLikeSafari() bool { // will probably use Secure Transport which will also // share the TLS handshake characteristics of Safari. + // Let's do the easy check first... should be sufficient in many cases. if len(info.cipherSuites) < 1 { return false } - return info.cipherSuites[0] == scsvRenegotiation - // TODO: Implement checking of presence and ordering - // of cipher suites etc. as described by the paper. + if info.cipherSuites[0] != scsvRenegotiation { + return false + } + + // We check for the presence and order of the extensions. + requiredExtensionsOrder := []uint16{10, 11, 13, 13172, 16, 5, 18, 23} + if !assertPresenceAndOrdering(requiredExtensionsOrder, info.extensions, true) { + return false + } + + // We check for order of cipher suites but not presence, since + // according to the paper, cipher suites may be not be added + // or reordered by the user, but they may be disabled. + expectedCipherSuiteOrder := []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, // 0xc02c + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, // 0xc02b + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, // 0xc024 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // 0xc023 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // 0xc00a + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // 0xc009 + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // 0xc030 + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // 0xc02f + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, //0xc028 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // 0xc027 + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // 0xc014 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // 0xc013 + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // 0x9d + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // 0x9c + TLS_RSA_WITH_AES_256_CBC_SHA256, // 0x3d + TLS_RSA_WITH_AES_128_CBC_SHA256, // 0x3c + tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35 + tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f + } + return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false) + // TODO: Check curves: [23 24 25] } const ( @@ -505,6 +538,7 @@ const ( TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xc024 TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xc023 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xc028 + TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x3c TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x3d TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x33 TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x39 From dc1af8f1883aa9045e35d0b7c1c632e995f41f5d Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 14 Feb 2017 08:34:07 -0700 Subject: [PATCH 06/13] Read ClientHello during first Read() instead of during Accept() As far as I can tell, reading the ClientHello during Accept() prevents new connections from being accepted during the read. Since Read() should be called in its own goroutine, this keeps Accept() non-blocking. --- caddyhttp/httpserver/mitm.go | 90 ++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index a3ef6d6b4f2..d84d3250893 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -75,6 +75,49 @@ func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.next.ServeHTTP(w, r) } +type clientHelloConn struct { + net.Conn + readHello bool + listener *tlsHelloListener +} + +func (c *clientHelloConn) Read(b []byte) (n int, err error) { + if !c.readHello { + // Read the header bytes. + hdr := make([]byte, 5) + n, err := io.ReadFull(c.Conn, hdr) + if err != nil { + return n, err + } + + // Get the length of the ClientHello message and read it as well. + length := uint16(hdr[3])<<8 | uint16(hdr[4]) + hello := make([]byte, int(length)) + n, err = io.ReadFull(c.Conn, hello) + if err != nil { + return n, err + } + + // Parse the ClientHello and store it in the map. + rawParsed := parseRawClientHello(hello) + c.listener.helloInfosMu.Lock() + c.listener.helloInfos[c.Conn.RemoteAddr().String()] = rawParsed + c.listener.helloInfosMu.Unlock() + + // Since we buffered the header and ClientHello, pretend we were + // never here by lining up the buffered values to be read with a + // custom connection type, followed by the rest of the actual + // underlying connection. + mr := io.MultiReader(bytes.NewReader(hdr), bytes.NewReader(hello), c.Conn) + mc := multiConn{Conn: c.Conn, reader: mr} + + c.Conn = mc + + c.readHello = true + } + return c.Conn.Read(b) +} + // multiConn is a net.Conn that reads from the // given reader instead of the wire directly. This // is useful when some of the connection has already @@ -233,49 +276,8 @@ func (l *tlsHelloListener) Accept() (net.Conn, error) { if err != nil { return nil, err } - - // TODO: Reading from this connection in the same goroutine is blocking, is it not? - - // Be careful to limit the amount of time to allow reading from this connection. - conn.SetDeadline(time.Now().Add(l.readTimeout)) - - // Read the header bytes. - hdr := make([]byte, 5) - _, err = io.ReadFull(conn, hdr) - if err != nil { - // returning an error will terminate the Accept loop - // in net/http, which isn't what we want; we'll just - // let the error occur naturally when it tries to read. - return conn, nil - } - - // Get the length of the ClientHello message and read it as well. - length := uint16(hdr[3])<<8 | uint16(hdr[4]) - hello := make([]byte, int(length)) - _, err = io.ReadFull(conn, hello) - if err != nil { - return conn, nil - } - - // Parse the ClientHello and store it in the map. - rawParsed := parseRawClientHello(hello) - l.helloInfosMu.Lock() - l.helloInfos[conn.RemoteAddr().String()] = rawParsed - l.helloInfosMu.Unlock() - - // Since we buffered the header and ClientHello, pretend we were - // never here by lining up the buffered values to be read with a - // custom connection type, followed by the rest of the actual - // underlying connection. - mr := io.MultiReader(bytes.NewReader(hdr), bytes.NewReader(hello), conn) - mc := multiConn{Conn: conn, reader: mr} - - // Clear the read timeout and let the built-in TLS server take care of - // it. This may not be a perfect way to do timeouts, but meh, it works. - conn.SetDeadline(time.Time{}) - - // Let the built-in TLS server handle the connection now as usual. - return tls.Server(mc, l.config), nil + helloConn := &clientHelloConn{Conn: conn, listener: l} + return tls.Server(helloConn, l.config), nil } // rawHelloInfo contains the "raw" data parsed from the TLS @@ -508,7 +510,7 @@ func (info rawHelloInfo) looksLikeSafari() bool { tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // 0xc009 tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // 0xc030 tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // 0xc02f - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, //0xc028 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, // 0xc028 tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // 0xc027 tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // 0xc014 tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // 0xc013 From fbc1114fb9ba7cee5ff0bd28d026f2bd1252791e Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 14 Feb 2017 17:38:14 -0700 Subject: [PATCH 07/13] Clean up MITM detection handler; make possible to close connection --- caddyhttp/httpserver/mitm.go | 72 ++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index d84d3250893..e74362a8d41 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -16,8 +16,9 @@ import ( // into the request context indicating if the TLS // connection is likely being intercepted. type tlsHandler struct { - next http.Handler - listener *tlsHelloListener + next http.Handler + listener *tlsHelloListener + closeOnMITM bool // whether to close connection on MITM; TODO: expose through new directive } // ServeHTTP checks the User-Agent. For the four main browsers (Chrome, @@ -34,44 +35,45 @@ type tlsHandler struct { // Halderman, et. al. in "The Security Impact of HTTPS Interception" (NDSS '17): // https://jhalderm.com/pub/papers/interception-ndss17.pdf func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.listener.helloInfosMu.RLock() + info := h.listener.helloInfos[r.RemoteAddr] + h.listener.helloInfosMu.RUnlock() + + // detectInterception uses heuristics to try to detect HTTPS interception. + // It returns true if it thinks the connection is being MITM'ed. + // It adds a "mitm" value to the request context containing the results + // of the inspection either way. Only call this function if the User-Agent + // is a recognized browser. + detectInterception := func(helloCheckFn func() bool) bool { + mitm := info.advertisesHeartbeatSupport() || // no major browsers have ever implemented Heartbeat + r.Header.Get("X-BlueCoat-Via") != "" || // Blue Coat + r.Header.Get("X-FCCKV2") != "" || // Fortinet + !helloCheckFn() // check if ClientHello doesn't match client as we would expect + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), mitm)) + return mitm + } + ua := r.Header.Get("User-Agent") - if strings.Contains(ua, "Edge") { - h.listener.helloInfosMu.RLock() - info := h.listener.helloInfos[r.RemoteAddr] - h.listener.helloInfosMu.RUnlock() - if info.advertisesHeartbeatSupport() || !info.looksLikeEdge() { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) - } else { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) - } + var mitm bool + if strings.Contains(ua, "Edge") || strings.Contains(ua, "MSIE") { + mitm = detectInterception(info.looksLikeEdge) } else if strings.Contains(ua, "Chrome") { - h.listener.helloInfosMu.RLock() - info := h.listener.helloInfos[r.RemoteAddr] - h.listener.helloInfosMu.RUnlock() - if info.advertisesHeartbeatSupport() || !info.looksLikeChrome() { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) - } else { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) - } + mitm = detectInterception(info.looksLikeChrome) } else if strings.Contains(ua, "Firefox") { - h.listener.helloInfosMu.RLock() - info := h.listener.helloInfos[r.RemoteAddr] - h.listener.helloInfosMu.RUnlock() - if info.advertisesHeartbeatSupport() || !info.looksLikeFirefox() { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) - } else { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) - } + mitm = detectInterception(info.looksLikeFirefox) } else if strings.Contains(ua, "Safari") { - h.listener.helloInfosMu.RLock() - info := h.listener.helloInfos[r.RemoteAddr] - h.listener.helloInfosMu.RUnlock() - if info.advertisesHeartbeatSupport() || !info.looksLikeSafari() { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), true)) - } else { - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), false)) - } + mitm = detectInterception(info.looksLikeSafari) } + + if mitm && h.closeOnMITM { + // TODO: This termination might need to happen later in the middleware + // chain in order to be picked up by the log directive, in case the site + // owner still wants to log this event. It'll probably require a new + // directive. If this feature is useful, we can finish implementing this. + r.Close = true + return + } + h.next.ServeHTTP(w, r) } From 3328e94508f39be9a1edf0befa0da0fc959b2cf3 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 14 Feb 2017 17:41:22 -0700 Subject: [PATCH 08/13] Use standard lib cipher suite values when possible --- caddyhttp/httpserver/mitm.go | 46 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index e74362a8d41..1c457cabe42 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -340,21 +340,21 @@ func (info rawHelloInfo) looksLikeFirefox() bool { // according to the paper, cipher suites may be not be added // or reordered by the user, but they may be disabled. expectedCipherSuiteOrder := []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, // 0xc02b - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // 0xc02f - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, // 0xcca9 - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, // 0xcca8 - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, // 0xc02c - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // 0xc030 - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // 0xc00a - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // 0xc009 - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // 0xc013 - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // 0xc014 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA, // 0x33 - TLS_DHE_RSA_WITH_AES_256_CBC_SHA, // 0x39 - tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f - tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35 - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // 0xa + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, // 0xc02b + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, // 0xc02f + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, // 0xcca9 + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, // 0xcca8 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, // 0xc02c + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // 0xc030 + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // 0xc00a + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // 0xc009 + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // 0xc013 + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // 0xc014 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA, // 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA, // 0x39 + tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f + tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35 + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // 0xa } return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false) } @@ -537,13 +537,11 @@ const ( // cipher suites missing from the crypto/tls package, // in no particular order here - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xcca9 - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xcca8 - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xc024 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xc023 - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xc028 - TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x3c - TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x3d - TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x33 - TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x39 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xc024 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xc023 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xc028 + TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x3c + TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x3d + TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x33 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x39 ) From 479fa6a0a9af9aadf195e58d48927412c0642939 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 14 Feb 2017 23:04:23 -0700 Subject: [PATCH 09/13] Improve Edge heuristics and test cases --- caddyhttp/httpserver/mitm.go | 18 +++++++++++---- caddyhttp/httpserver/mitm_test.go | 38 +++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index 1c457cabe42..f6635d0d82f 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -55,7 +55,7 @@ func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ua := r.Header.Get("User-Agent") var mitm bool - if strings.Contains(ua, "Edge") || strings.Contains(ua, "MSIE") { + if strings.Contains(ua, "Edge") || strings.Contains(ua, "MSIE") { // check Edge first! mitm = detectInterception(info.looksLikeEdge) } else if strings.Contains(ua, "Chrome") { mitm = detectInterception(info.looksLikeChrome) @@ -462,11 +462,21 @@ func (info rawHelloInfo) looksLikeEdge() bool { if len(info.extensions) <= i+2 { return false } - return info.extensions[i+1] == extensionSupportedCurves && - info.extensions[i+2] == extensionSupportedPoints + if info.extensions[i+1] != extensionSupportedCurves || + info.extensions[i+2] != extensionSupportedPoints { + return false + } } } - return false + + // As of Feb. 2017, Edge does not have 0xff, but Avast adds it + for _, cs := range info.cipherSuites { + if cs == scsvRenegotiation { + return false + } + } + + return true } // looksLikeSafari returns true if info looks like a handshake diff --git a/caddyhttp/httpserver/mitm_test.go b/caddyhttp/httpserver/mitm_test.go index d1289e5e8a9..98d988f6346 100644 --- a/caddyhttp/httpserver/mitm_test.go +++ b/caddyhttp/httpserver/mitm_test.go @@ -109,13 +109,12 @@ func TestHeuristicFunctions(t *testing.T) { helloHex: `010000bd030375f9022fc3a6562467f3540d68013b2d0b961979de6129e944efe0b35531323500001ec02bc02fcca9cca8c02cc030c00ac009c013c01400330039002f0035000a010000760000000e000c0000096c6f63616c686f737400170000ff01000100000a000a0008001d001700180019000b00020100002300000010000e000c02683208687474702f312e31000500050100000000ff030000000d0020001e040305030603020308040805080604010501060102010402050206020202`, }, }, - // TODO... in the process of downloading a VM... - // "Edge": []clientHello{ - // { - // userAgent: "", - // helloHex: ``, - // }, - // }, + "Edge": []clientHello{ + { + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000bd030358a3c9bf05f734842e189fb6ce653b67b846e990bc1fc5fb8c397874d06020f1000038c02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f000a006a00400038003200130100005c000500050100000000000a00080006001d00170018000b00020100000d00140012040105010201040305030203020206010603002300000010000e000c02683208687474702f312e310017000055000006000100020002ff01000100`, + }, + }, "Safari": []clientHello{ { userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8", @@ -132,6 +131,31 @@ func TestHeuristicFunctions(t *testing.T) { userAgent: "curl/7.51.0", helloHex: `010000a6030358a28c73a71bdfc1f09dee13fecdc58805dcce42ac44254df548f14645f7dc2c00004400ffc02cc02bc024c023c00ac009c008c030c02fc028c027c014c013c012009f009e006b0067003900330016009d009c003d003c0035002f000a00af00ae008d008c008b01000039000a00080006001700180019000b00020100000d00120010040102010501060104030203050306030005000501000000000012000000170000`, }, + { + // Avast 17.1.2286 (Feb. 2017) on Windows 10 x64 build 14393, intercepting Edge + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000ce0303b418fdc4b6cf6436a5e2bfb06b96ed5faa7285c20c7b49341a78be962a9dc40000003ac02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f000a006a004000380032001300ff0100006b00000014001200000f66696e6572706978656c732e636f6d000b000403000102000a00080006001d0017001800230000000d001400120401050102010403050302030202060106030005000501000000000010000e000c02683208687474702f312e310016000000170000`, + }, + { + // Kaspersky Internet Security 17.0.0.611 on Windows 10 x64 build 14393, intercepting Edge + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000eb030361ce302bf4b0d5adf1ff30b2cf433c4a4b68f33e07b2651695e7ae6ec3cf126400003ac02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f000a006a004000380032001300ff0100008800000014001200000f66696e6572706978656c732e636f6d000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000500050100000000000f0001010010000e000c02683208687474702f312e31`, + }, + { + // Kaspersky Internet Security 17.0.0.611 on Windows 10 x64 build 14393, intercepting Firefox 51 + userAgent: "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0", + helloHex: `010001fc0303768e3f9ea75194c7cb03d23e8e6371b95fb696d339b797be57a634309ec98a42200f2a7554098364b7f05d21a8c7f43f31a893a4fc5670051020408c8e4dc234dd001cc02bc02fc02cc030c00ac009c013c01400330039002f0035000a00ff0100019700000014001200000f66696e6572706978656c732e636f6d000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230078bf4e244d4de3d53c6331edda9672dfc4a17aae92b671e86da1368b1b5ae5324372817d8f3b7ffe1a7a1537a5049b86cd7c44863978c1e615b005942755da20fc3a4e34a16f78034aa3b1cffcef95f81a0995c522a53b0e95a4f98db84c43359d93d8647b2de2a69f3ebdcfc6bca452730cbd00179226dedf000d0020001e060106020603050105020503040104020403030103020303020102020203000500050100000000000f0001010010000e000c02683208687474702f312e3100150093000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000`, + }, + { + // Kaspersky Internet Security 17.0.0.611 on Windows 10 x64 build 14393, intercepting Chrome 56 + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + helloHex: `010000c903033481e7af24e647ba5a79ec97e9264c1a1f990cf842f50effe22be52130d5af82000018c02bc02fc02cc030c013c014009c009d002f0035000a00ff0100008800000014001200000f66696e6572706978656c732e636f6d000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000500050100000000000f0001010010000e000c02683208687474702f312e31`, + }, + { + // AVG 17.1.3006 (build 17.1.3354.20) on Windows 10 x64 build 14393, intercepting Edge + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + helloHex: `010000ca0303fd83091207161eca6b4887db50587109c50e463beb190362736b1fcf9e05f807000036c02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f006a00400038003200ff0100006b00000014001200000f66696e6572706978656c732e636f6d000b000403000102000a00080006001d0017001800230000000d001400120401050102010403050302030202060106030005000501000000000010000e000c02683208687474702f312e310016000000170000`, + }, }, } From 30de2e47590bdd63219b259c87b281e0f5ec00d1 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 15 Feb 2017 14:44:27 -0700 Subject: [PATCH 10/13] Refactor MITM checking logic; add some debug statements for now --- caddyhttp/httpserver/mitm.go | 48 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index f6635d0d82f..b915076cb77 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/tls" + "fmt" "io" "net" "net/http" @@ -39,30 +40,36 @@ func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { info := h.listener.helloInfos[r.RemoteAddr] h.listener.helloInfosMu.RUnlock() - // detectInterception uses heuristics to try to detect HTTPS interception. - // It returns true if it thinks the connection is being MITM'ed. - // It adds a "mitm" value to the request context containing the results - // of the inspection either way. Only call this function if the User-Agent - // is a recognized browser. - detectInterception := func(helloCheckFn func() bool) bool { - mitm := info.advertisesHeartbeatSupport() || // no major browsers have ever implemented Heartbeat - r.Header.Get("X-BlueCoat-Via") != "" || // Blue Coat - r.Header.Get("X-FCCKV2") != "" || // Fortinet - !helloCheckFn() // check if ClientHello doesn't match client as we would expect - r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), mitm)) - return mitm - } - ua := r.Header.Get("User-Agent") - var mitm bool - if strings.Contains(ua, "Edge") || strings.Contains(ua, "MSIE") { // check Edge first! - mitm = detectInterception(info.looksLikeEdge) + + fmt.Printf("*******\nURI: %s\n", r.RequestURI) + fmt.Printf("User-Agent: %s\n", ua) + fmt.Printf("Headers: %+v\n", r.Header) + fmt.Printf("Parsed ClientHello: %+v\n", info) + + var checked, mitm bool + if r.Header.Get("X-BlueCoat-Via") != "" || // Blue Coat (masks User-Agent header to generic values) + r.Header.Get("X-FCCKV2") != "" || // Fortinet + info.advertisesHeartbeatSupport() { // no major browsers have ever implemented Heartbeat + checked = true + mitm = true + } else if strings.Contains(ua, "Edge") || strings.Contains(ua, "MSIE") || + strings.Contains(ua, "Trident") { + checked = true + mitm = !info.looksLikeEdge() } else if strings.Contains(ua, "Chrome") { - mitm = detectInterception(info.looksLikeChrome) + checked = true + mitm = !info.looksLikeChrome() } else if strings.Contains(ua, "Firefox") { - mitm = detectInterception(info.looksLikeFirefox) + checked = true + mitm = !info.looksLikeFirefox() } else if strings.Contains(ua, "Safari") { - mitm = detectInterception(info.looksLikeSafari) + checked = true + mitm = !info.looksLikeSafari() + } + + if checked { + r = r.WithContext(context.WithValue(r.Context(), CtxKey("mitm"), mitm)) } if mitm && h.closeOnMITM { @@ -99,6 +106,7 @@ func (c *clientHelloConn) Read(b []byte) (n int, err error) { if err != nil { return n, err } + fmt.Printf("RAW HELLO: %x\n", hello) // Parse the ClientHello and store it in the map. rawParsed := parseRawClientHello(hello) From 8122d7c2dc9828c1d9d4b1cd927c923c9444ab3d Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 16 Feb 2017 21:21:13 -0700 Subject: [PATCH 11/13] Fix bug in MITM heuristic tests and actual heuristic code --- caddyhttp/httpserver/mitm.go | 80 ++++++++++++++++--------------- caddyhttp/httpserver/mitm_test.go | 26 +++++----- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index b915076cb77..d5d0c6e404e 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -367,39 +367,6 @@ func (info rawHelloInfo) looksLikeFirefox() bool { return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false) } -// assertPresenceAndOrdering will return true if candidateList contains -// the items in requiredItems in the same order as requiredItems. -// -// If requiredIsSubset is true, then all items in requiredItems must be -// present in candidateList. If requiredIsSubset is false, then requiredItems -// may contain items that are not in candidateList. -// -// In all cases, the order of requiredItems is enforced. -func assertPresenceAndOrdering(requiredItems, candidateList []uint16, requiredIsSubset bool) bool { - superset := requiredItems - subset := candidateList - if requiredIsSubset { - superset = candidateList - subset = requiredItems - } - - var j int - for _, item := range superset { - var found bool - for j < len(subset) { - if subset[j] == item { - found = true - break - } - j++ - } - if j == len(subset)-1 && !found { - return false - } - } - return true -} - // looksLikeChrome returns true if info looks like a handshake // from a modern version of Chrome. func (info rawHelloInfo) looksLikeChrome() bool { @@ -477,11 +444,15 @@ func (info rawHelloInfo) looksLikeEdge() bool { } } - // As of Feb. 2017, Edge does not have 0xff, but Avast adds it for _, cs := range info.cipherSuites { + // As of Feb. 2017, Edge does not have 0xff, but Avast adds it if cs == scsvRenegotiation { return false } + // Edge and modern IE do not have 0x4 or 0x5, but Blue Coat does + if cs == TLS_RSA_WITH_RC4_128_MD5 || cs == tls.TLS_RSA_WITH_RC4_128_SHA { + return false + } } return true @@ -518,9 +489,7 @@ func (info rawHelloInfo) looksLikeSafari() bool { return false } - // We check for order of cipher suites but not presence, since - // according to the paper, cipher suites may be not be added - // or reordered by the user, but they may be disabled. + // We check for order and presence of cipher suites expectedCipherSuiteOrder := []uint16{ tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, // 0xc02c tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, // 0xc02b @@ -541,8 +510,40 @@ func (info rawHelloInfo) looksLikeSafari() bool { tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35 tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f } - return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false) - // TODO: Check curves: [23 24 25] + return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, true) +} + +// assertPresenceAndOrdering will return true if candidateList contains +// the items in requiredItems in the same order as requiredItems. +// +// If requiredIsSubset is true, then all items in requiredItems must be +// present in candidateList. If requiredIsSubset is false, then requiredItems +// may contain items that are not in candidateList. +// +// In all cases, the order of requiredItems is enforced. +func assertPresenceAndOrdering(requiredItems, candidateList []uint16, requiredIsSubset bool) bool { + superset := requiredItems + subset := candidateList + if requiredIsSubset { + superset = candidateList + subset = requiredItems + } + + var j int + for _, item := range subset { + var found bool + for j < len(superset) { + if superset[j] == item { + found = true + break + } + j++ + } + if j == len(superset) && !found { + return false + } + } + return true } const ( @@ -562,4 +563,5 @@ const ( TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x3d TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x33 TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x39 + TLS_RSA_WITH_RC4_128_MD5 = 0x4 ) diff --git a/caddyhttp/httpserver/mitm_test.go b/caddyhttp/httpserver/mitm_test.go index 98d988f6346..8b42616ed21 100644 --- a/caddyhttp/httpserver/mitm_test.go +++ b/caddyhttp/httpserver/mitm_test.go @@ -121,7 +121,7 @@ func TestHeuristicFunctions(t *testing.T) { helloHex: `010000d2030358a295b513c8140c6ff880f4a8a73cc830ed2dab2c4f2068eb365228d828732e00002600ffc02cc02bc024c023c00ac009c030c02fc028c027c014c013009d009c003d003c0035002f010000830000000e000c0000096c6f63616c686f7374000a00080006001700180019000b00020100000d00120010040102010501060104030203050306033374000000100030002e0268320568322d31360568322d31350568322d313408737064792f332e3106737064792f3308687474702f312e310005000501000000000012000000170000`, }, }, - "Other": []clientHello{ + "Other": []clientHello{ // these are either non-browser clients or intercepted client hellos { // openssl s_client (OpenSSL 0.9.8zh 14 Jan 2016) helloHex: `0100012b03035d385236b8ca7b7946fa0336f164e76bf821ed90e8de26d97cc677671b6f36380000acc030c02cc028c024c014c00a00a500a300a1009f006b006a0069006800390038003700360088008700860085c032c02ec02ac026c00fc005009d003d00350084c02fc02bc027c023c013c00900a400a200a0009e00670040003f003e0033003200310030009a0099009800970045004400430042c031c02dc029c025c00ec004009c003c002f009600410007c011c007c00cc00200050004c012c008001600130010000dc00dc003000a00ff0201000055000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000f000101`, @@ -156,6 +156,10 @@ func TestHeuristicFunctions(t *testing.T) { userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", helloHex: `010000ca0303fd83091207161eca6b4887db50587109c50e463beb190362736b1fcf9e05f807000036c02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f006a00400038003200ff0100006b00000014001200000f66696e6572706978656c732e636f6d000b000403000102000a00080006001d0017001800230000000d001400120401050102010403050302030202060106030005000501000000000010000e000c02683208687474702f312e310016000000170000`, }, + { + // IE 11 on Windows 7, this connection was intercepted by Blue Coat + helloHex: "010000b1030358a3f3bae627f464da8cb35976b88e9119640032d41e62a107d608ed8d3e62b9000034c028c027c014c013009f009e009d009cc02cc02bc024c023c00ac009003d003c0035002f006a004000380032000a0013000500040100005400000014001200000f66696e6572706978656c732e636f6d000500050100000000000a00080006001700180019000b00020100000d0014001206010603040105010201040305030203020200170000ff01000100", + }, }, } @@ -179,23 +183,23 @@ func TestHeuristicFunctions(t *testing.T) { // should return false, with as little logic as possible, // but with enough logic to force TLS proxies to do a // good job preserving characterstics of the handshake. - var wrong bool + var correct bool switch client { case "Chrome": - wrong = !isChrome || isFirefox || isSafari || isEdge + correct = isChrome && !isFirefox && !isSafari && !isEdge case "Firefox": - wrong = isChrome || !isFirefox || isSafari || isEdge + correct = !isChrome && isFirefox && !isSafari && !isEdge case "Safari": - wrong = isChrome || isFirefox || !isSafari || isEdge + correct = !isChrome && !isFirefox && isSafari && !isEdge case "Edge": - wrong = isChrome || isFirefox || isSafari || !isEdge - case "Others": - wrong = isChrome || isFirefox || isSafari || isEdge + correct = !isChrome && !isFirefox && !isSafari && isEdge + case "Other": + correct = !isChrome && !isFirefox && !isSafari && !isEdge } - if wrong { - t.Errorf("[%s] Test %d: Chrome=%v, Firefox=%v, Safari=%v, Edge=%v", - client, i, isChrome, isFirefox, isSafari, isEdge) + if !correct { + t.Errorf("[%s] Test %d: Chrome=%v, Firefox=%v, Safari=%v, Edge=%v; parsed hello: %+v", + client, i, isChrome, isFirefox, isSafari, isEdge, parsed) } } } From e5ab9ca5190a7140806b0a9c10fe0b8a5b815271 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 16 Feb 2017 21:27:28 -0700 Subject: [PATCH 12/13] Fix gofmt --- caddyhttp/httpserver/mitm_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/caddyhttp/httpserver/mitm_test.go b/caddyhttp/httpserver/mitm_test.go index 8b42616ed21..e5c75af8a8a 100644 --- a/caddyhttp/httpserver/mitm_test.go +++ b/caddyhttp/httpserver/mitm_test.go @@ -97,31 +97,31 @@ func TestHeuristicFunctions(t *testing.T) { // Please group similar clients and order by version for convenience // when adding to the test cases. clientHellos := map[string][]clientHello{ - "Chrome": []clientHello{ + "Chrome": { { userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", helloHex: `010000c003031dae75222dae1433a5a283ddcde8ddabaefbf16d84f250eee6fdff48cdfff8a00000201a1ac02bc02fc02cc030cca9cca8cc14cc13c013c014009c009d002f0035000a010000777a7a0000ff010001000000000e000c0000096c6f63616c686f73740017000000230000000d00140012040308040401050308050501080606010201000500050100000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a000a0008aaaa001d001700182a2a000100`, }, }, - "Firefox": []clientHello{ + "Firefox": { { userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:51.0) Gecko/20100101 Firefox/51.0", helloHex: `010000bd030375f9022fc3a6562467f3540d68013b2d0b961979de6129e944efe0b35531323500001ec02bc02fcca9cca8c02cc030c00ac009c013c01400330039002f0035000a010000760000000e000c0000096c6f63616c686f737400170000ff01000100000a000a0008001d001700180019000b00020100002300000010000e000c02683208687474702f312e31000500050100000000ff030000000d0020001e040305030603020308040805080604010501060102010402050206020202`, }, }, - "Edge": []clientHello{ + "Edge": { { userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", helloHex: `010000bd030358a3c9bf05f734842e189fb6ce653b67b846e990bc1fc5fb8c397874d06020f1000038c02cc02bc030c02f009f009ec024c023c028c027c00ac009c014c01300390033009d009c003d003c0035002f000a006a00400038003200130100005c000500050100000000000a00080006001d00170018000b00020100000d00140012040105010201040305030203020206010603002300000010000e000c02683208687474702f312e310017000055000006000100020002ff01000100`, }, }, - "Safari": []clientHello{ + "Safari": { { userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8", helloHex: `010000d2030358a295b513c8140c6ff880f4a8a73cc830ed2dab2c4f2068eb365228d828732e00002600ffc02cc02bc024c023c00ac009c030c02fc028c027c014c013009d009c003d003c0035002f010000830000000e000c0000096c6f63616c686f7374000a00080006001700180019000b00020100000d00120010040102010501060104030203050306033374000000100030002e0268320568322d31360568322d31350568322d313408737064792f332e3106737064792f3308687474702f312e310005000501000000000012000000170000`, }, }, - "Other": []clientHello{ // these are either non-browser clients or intercepted client hellos + "Other": { // these are either non-browser clients or intercepted client hellos { // openssl s_client (OpenSSL 0.9.8zh 14 Jan 2016) helloHex: `0100012b03035d385236b8ca7b7946fa0336f164e76bf821ed90e8de26d97cc677671b6f36380000acc030c02cc028c024c014c00a00a500a300a1009f006b006a0069006800390038003700360088008700860085c032c02ec02ac026c00fc005009d003d00350084c02fc02bc027c023c013c00900a400a200a0009e00670040003f003e0033003200310030009a0099009800970045004400430042c031c02dc029c025c00ec004009c003c002f009600410007c011c007c00cc00200050004c012c008001600130010000dc00dc003000a00ff0201000055000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000f000101`, From 3f49c57b53d3fc9b7b60b9904caeb0c5550f8a91 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 17 Feb 2017 10:59:18 -0700 Subject: [PATCH 13/13] Remove debug statements; preparing for merge --- caddyhttp/httpserver/mitm.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/caddyhttp/httpserver/mitm.go b/caddyhttp/httpserver/mitm.go index d5d0c6e404e..acb23244fd7 100644 --- a/caddyhttp/httpserver/mitm.go +++ b/caddyhttp/httpserver/mitm.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/tls" - "fmt" "io" "net" "net/http" @@ -42,11 +41,6 @@ func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ua := r.Header.Get("User-Agent") - fmt.Printf("*******\nURI: %s\n", r.RequestURI) - fmt.Printf("User-Agent: %s\n", ua) - fmt.Printf("Headers: %+v\n", r.Header) - fmt.Printf("Parsed ClientHello: %+v\n", info) - var checked, mitm bool if r.Header.Get("X-BlueCoat-Via") != "" || // Blue Coat (masks User-Agent header to generic values) r.Header.Get("X-FCCKV2") != "" || // Fortinet @@ -106,7 +100,6 @@ func (c *clientHelloConn) Read(b []byte) (n int, err error) { if err != nil { return n, err } - fmt.Printf("RAW HELLO: %x\n", hello) // Parse the ClientHello and store it in the map. rawParsed := parseRawClientHello(hello)