From 1874f7a7c2b643ca565b555b3f7a72409ab7675c Mon Sep 17 00:00:00 2001 From: kelmenhorst <45046038+kelmenhorst@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:10:27 +0200 Subject: [PATCH] enable utls for websteps (#442) This diff enables `websteps` to use uTLS for TLS parroting. It integrates the `oohttp.StdlibTransport` wrapper which uses the `ooni/oohttp` fork. `oohttp` supports TLS-like connections like `utls.Conn`. As a prototype, the testhelper and `websteps` code now uses the `utls.HelloChrome_Auto` fingerprint, i.e. the simulated TLS fingerprint of the Google Chrome browser. It is a further contribution for my GSoC project. Reference issue: https://github.com/ooni/probe/issues/1733 --- go.mod | 3 +- go.sum | 6 +- .../oohelperd/internal/websteps/explore.go | 9 +- .../oohelperd/internal/websteps/factory.go | 96 ------------------- .../oohelperd/internal/websteps/generate.go | 8 +- .../internal/websteps/generate_test.go | 10 +- .../engine/experiment/websteps/factory.go | 39 +++++--- internal/engine/experiment/websteps/tls.go | 15 ++- .../engine/experiment/websteps/websteps.go | 2 +- internal/netxlite/tlshandshaker.go | 1 + 10 files changed, 66 insertions(+), 123 deletions(-) delete mode 100644 internal/cmd/oohelperd/internal/websteps/factory.go diff --git a/go.mod b/go.mod index 7ccb0b311f..f7bcc72219 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/miekg/dns v1.1.42 github.com/mitchellh/go-wordwrap v1.0.1 github.com/montanaflynn/stats v0.6.6 + github.com/ooni/oohttp v0.0.0-20210818104219-f8ceac6f2622 github.com/ooni/probe-assets v0.3.1 github.com/ooni/psiphon v0.8.0 github.com/oschwald/geoip2-golang v1.5.0 @@ -41,7 +42,7 @@ require ( gitlab.com/yawning/obfs4.git v0.0.0-20210511220700-e330d1b7024b gitlab.com/yawning/utls.git v0.0.12-1 golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect - golang.org/x/net v0.0.0-20210510120150-4163338589ed + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 gopkg.in/AlecAivazis/survey.v1 v1.8.8 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect diff --git a/go.sum b/go.sum index 119f622e46..04b714debe 100644 --- a/go.sum +++ b/go.sum @@ -385,6 +385,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/ooni/oohttp v0.0.0-20210818104219-f8ceac6f2622 h1:Wpu4o3J3fLD4BPqA3CmrnbXVAAWKEjramvfhDUFZp+E= +github.com/ooni/oohttp v0.0.0-20210818104219-f8ceac6f2622/go.mod h1:kgtoj+Dn4bmx09hEUgbPI7YX0gkWlu+fz2I0S5auyX4= github.com/ooni/probe-assets v0.3.1 h1:6PDcoJTICJxL8PdeM0+a3ZfkTWrFfCn90fUqTWR0LDA= github.com/ooni/probe-assets v0.3.1/go.mod h1:N0PyNM3aadlYDDCFXAPzs54HC54+MZA/4/xnCtd9EAo= github.com/ooni/psiphon v0.8.0 h1:digldztBlINi3HWuxdK4gFhkiaheAoDVjZN/ApZHWBM= @@ -693,8 +695,8 @@ golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I= -golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/internal/cmd/oohelperd/internal/websteps/explore.go b/internal/cmd/oohelperd/internal/websteps/explore.go index 888cb7f78c..f3c6cc4d31 100644 --- a/internal/cmd/oohelperd/internal/websteps/explore.go +++ b/internal/cmd/oohelperd/internal/websteps/explore.go @@ -8,8 +8,10 @@ import ( "sort" "strings" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/websteps" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/runtimex" + utls "gitlab.com/yawning/utls.git" ) // Explore is the second step of the test helper algorithm. Its objective @@ -104,7 +106,10 @@ func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http. tlsConf := &tls.Config{ NextProtos: []string{"h2", "http/1.1"}, } - transport := netxlite.NewHTTPTransport(NewDialerResolver(e.resolver), tlsConf, &netxlite.TLSHandshakerConfigurable{}) + handshaker := &netxlite.TLSHandshakerConfigurable{ + NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto), + } + transport := websteps.NewTransportWithDialer(websteps.NewDialerResolver(e.resolver), tlsConf, handshaker) // TODO(bassosimone): here we should use runtimex.PanicOnError jarjar, _ := cookiejar.New(nil) clnt := &http.Client{ @@ -126,7 +131,7 @@ func (e *DefaultExplorer) get(URL *url.URL, headers map[string][]string) (*http. // getH3 uses HTTP/3 to get the given URL and returns the final // response after redirection, and an error. If the error is nil, the final response is valid. func (e *DefaultExplorer) getH3(h3URL *h3URL, headers map[string][]string) (*http.Response, error) { - dialer := NewQUICDialerResolver(e.resolver) + dialer := websteps.NewQUICDialerResolver(e.resolver) tlsConf := &tls.Config{ NextProtos: []string{h3URL.proto}, } diff --git a/internal/cmd/oohelperd/internal/websteps/factory.go b/internal/cmd/oohelperd/internal/websteps/factory.go deleted file mode 100644 index 8f9730c01c..0000000000 --- a/internal/cmd/oohelperd/internal/websteps/factory.go +++ /dev/null @@ -1,96 +0,0 @@ -package websteps - -import ( - "context" - "crypto/tls" - "errors" - "net" - "net/http" - "sync" - - "github.com/apex/log" - "github.com/lucas-clemente/quic-go" - "github.com/lucas-clemente/quic-go/http3" - "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" - "github.com/ooni/probe-cli/v3/internal/errorsx" - "github.com/ooni/probe-cli/v3/internal/netxlite" -) - -var ErrNoConnReuse = errors.New("cannot reuse connection") - -// NewDialerResolver contructs a new dialer for TCP connections, -// with default, errorwrapping and resolve functionalities -func NewDialerResolver(resolver netxlite.Resolver) netxlite.Dialer { - var d netxlite.Dialer = netxlite.DefaultDialer - d = &errorsx.ErrorWrapperDialer{Dialer: d} - d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d} - return d -} - -// NewQUICDialerResolver creates a new QUICDialerResolver -// with default, errorwrapping and resolve functionalities -func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDialer { - var ql quicdialer.QUICListener = &netxlite.QUICListenerStdlib{} - ql = &errorsx.ErrorWrapperQUICListener{QUICListener: ql} - var dialer netxlite.QUICContextDialer = &netxlite.QUICDialerQUICGo{ - QUICListener: ql, - } - dialer = &errorsx.ErrorWrapperQUICDialer{Dialer: dialer} - dialer = &netxlite.QUICDialerResolver{Resolver: resolver, Dialer: dialer} - return dialer -} - -// NewSingleH3Transport creates an http3.RoundTripper -func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper { - transport := &http3.RoundTripper{ - DisableCompression: true, - TLSClientConfig: tlscfg, - QuicConfig: qcfg, - Dial: (&SingleDialerH3{qsess: &qsess}).Dial, - } - return transport -} - -// NewSingleTransport determines the appropriate HTTP Transport from the ALPN -func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) { - singledialer := &SingleDialer{conn: &conn} - transport = http.DefaultTransport.(*http.Transport).Clone() - transport.(*http.Transport).DialContext = singledialer.DialContext - transport.(*http.Transport).DialTLSContext = singledialer.DialContext - transport.(*http.Transport).DisableCompression = true - transport.(*http.Transport).MaxConnsPerHost = 1 - transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)} - return transport -} - -type SingleDialer struct { - sync.Mutex - conn *net.Conn -} - -func (s *SingleDialer) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { - s.Lock() - defer s.Unlock() - if s.conn == nil { - return nil, ErrNoConnReuse - } - c := s.conn - s.conn = nil - return *c, nil -} - -type SingleDialerH3 struct { - sync.Mutex - qsess *quic.EarlySession -} - -func (s *SingleDialerH3) Dial(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlySession, error) { - s.Lock() - defer s.Unlock() - if s.qsess == nil { - return nil, ErrNoConnReuse - } - qs := s.qsess - s.qsess = nil - return *qs, nil -} diff --git a/internal/cmd/oohelperd/internal/websteps/generate.go b/internal/cmd/oohelperd/internal/websteps/generate.go index f67aa99610..430c239220 100644 --- a/internal/cmd/oohelperd/internal/websteps/generate.go +++ b/internal/cmd/oohelperd/internal/websteps/generate.go @@ -130,7 +130,7 @@ func (g *DefaultGenerator) GenerateHTTPEndpoint(ctx context.Context, rt *RoundTr URL: rt.Request.URL.String(), }, } - transport := NewSingleTransport(tcpConn) + transport := websteps.NewSingleTransport(tcpConn) if g.transport != nil { transport = g.transport } @@ -176,7 +176,7 @@ func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundT } defer tcpConn.Close() - tlsConn, err = TLSDo(tcpConn, rt.Request.URL.Hostname()) + tlsConn, err = TLSDo(ctx, tcpConn, rt.Request.URL.Hostname()) currentEndpoint.TLSHandshake = &TLSHandshakeMeasurement{ Failure: newfailure(err), } @@ -193,7 +193,7 @@ func (g *DefaultGenerator) GenerateHTTPSEndpoint(ctx context.Context, rt *RoundT URL: rt.Request.URL.String(), }, } - transport := NewSingleTransport(tlsConn) + transport := websteps.NewSingleTransport(tlsConn) if g.transport != nil { transport = g.transport } @@ -248,7 +248,7 @@ func (g *DefaultGenerator) GenerateH3Endpoint(ctx context.Context, rt *RoundTrip URL: rt.Request.URL.String(), }, } - var transport http.RoundTripper = NewSingleH3Transport(sess, tlsConf, &quic.Config{}) + var transport http.RoundTripper = websteps.NewSingleH3Transport(sess, tlsConf, &quic.Config{}) if g.transport != nil { transport = g.transport } diff --git a/internal/cmd/oohelperd/internal/websteps/generate_test.go b/internal/cmd/oohelperd/internal/websteps/generate_test.go index 8817d1f36a..0b0760e77e 100644 --- a/internal/cmd/oohelperd/internal/websteps/generate_test.go +++ b/internal/cmd/oohelperd/internal/websteps/generate_test.go @@ -252,7 +252,10 @@ func TestGenerateHTTPS(t *testing.T) { t.Fatal("TCPConnectMeasurement should not be nil") } if endpointMeasurement.TLSHandshake == nil { - t.Fatal("TCPConnectMeasurement should not be nil") + t.Fatal("TLSHandshakeMeasurement should not be nil") + } + if endpointMeasurement.TLSHandshake.Failure != nil { + t.Fatal("unexpected failure at TLSHandshakeMeasurement") } if endpointMeasurement.HTTPRoundTrip == nil { t.Fatal("HTTPRoundTripMeasurement should not be nil") @@ -283,7 +286,10 @@ func TestGenerateHTTPSTLSFailure(t *testing.T) { t.Fatal("TCPConnectMeasurement should not be nil") } if endpointMeasurement.TLSHandshake == nil { - t.Fatal("TCPConnectMeasurement should not be nil") + t.Fatal("TLSHandshakeMeasurement should not be nil") + } + if endpointMeasurement.TLSHandshake.Failure == nil { + t.Fatal("expected failure at TLSHandshakeMeasurement") } if endpointMeasurement.HTTPRoundTrip != nil { t.Fatal("HTTPRoundTripMeasurement should be nil") diff --git a/internal/engine/experiment/websteps/factory.go b/internal/engine/experiment/websteps/factory.go index 50693cd4a4..70a6581bd4 100644 --- a/internal/engine/experiment/websteps/factory.go +++ b/internal/engine/experiment/websteps/factory.go @@ -9,9 +9,9 @@ import ( "net/url" "sync" - "github.com/apex/log" "github.com/lucas-clemente/quic-go" "github.com/lucas-clemente/quic-go/http3" + oohttp "github.com/ooni/oohttp" "github.com/ooni/probe-cli/v3/internal/engine/netx/quicdialer" "github.com/ooni/probe-cli/v3/internal/errorsx" "github.com/ooni/probe-cli/v3/internal/netxlite" @@ -53,8 +53,8 @@ func NewQUICDialerResolver(resolver netxlite.Resolver) netxlite.QUICContextDiale return dialer } -// NewSingleH3Transport creates an http3.RoundTripper -func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) *http3.RoundTripper { +// NewSingleH3Transport creates an http3.RoundTripper. +func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *quic.Config) http.RoundTripper { transport := &http3.RoundTripper{ DisableCompression: true, TLSClientConfig: tlscfg, @@ -64,16 +64,33 @@ func NewSingleH3Transport(qsess quic.EarlySession, tlscfg *tls.Config, qcfg *qui return transport } -// NewSingleTransport determines the appropriate HTTP Transport from the ALPN -func NewSingleTransport(conn net.Conn) (transport http.RoundTripper) { +// NewSingleTransport creates a new HTTP transport with a single-use dialer. +func NewSingleTransport(conn net.Conn) http.RoundTripper { singledialer := &SingleDialer{conn: &conn} - transport = http.DefaultTransport.(*http.Transport).Clone() - transport.(*http.Transport).DialContext = singledialer.DialContext - transport.(*http.Transport).DialTLSContext = singledialer.DialContext - transport.(*http.Transport).DisableCompression = true - transport.(*http.Transport).MaxConnsPerHost = 1 + transport := newBaseTransport() + transport.DialContext = singledialer.DialContext + transport.DialTLSContext = singledialer.DialContext + return transport +} + +// NewSingleTransport creates a new HTTP transport with a custom dialer and handshaker. +func NewTransportWithDialer(dialer netxlite.Dialer, tlsConfig *tls.Config, handshaker netxlite.TLSHandshaker) http.RoundTripper { + transport := newBaseTransport() + transport.DialContext = dialer.DialContext + transport.DialTLSContext = (&netxlite.TLSDialer{ + Config: tlsConfig, + Dialer: dialer, + TLSHandshaker: handshaker, + }).DialTLSContext + return transport +} - transport = &netxlite.HTTPTransportLogger{Logger: log.Log, HTTPTransport: transport.(*http.Transport)} +// newBaseTransport creates a new HTTP transport with the default dialer. +func newBaseTransport() (transport *oohttp.StdlibTransport) { + base := oohttp.DefaultTransport.(*oohttp.Transport).Clone() + base.DisableCompression = true + base.MaxConnsPerHost = 1 + transport = &oohttp.StdlibTransport{Transport: base} return transport } diff --git a/internal/engine/experiment/websteps/tls.go b/internal/engine/experiment/websteps/tls.go index 1ed117678b..911c72580b 100644 --- a/internal/engine/experiment/websteps/tls.go +++ b/internal/engine/experiment/websteps/tls.go @@ -1,16 +1,23 @@ package websteps import ( + "context" "crypto/tls" "net" + + "github.com/ooni/probe-cli/v3/internal/netxlite" + utls "gitlab.com/yawning/utls.git" ) // TLSDo performs the TLS check. -func TLSDo(conn net.Conn, hostname string) (*tls.Conn, error) { - tlsConn := tls.Client(conn, &tls.Config{ +func TLSDo(ctx context.Context, conn net.Conn, hostname string) (net.Conn, error) { + tlsConf := &tls.Config{ ServerName: hostname, NextProtos: []string{"h2", "http/1.1"}, - }) - err := tlsConn.Handshake() + } + h := &netxlite.TLSHandshakerConfigurable{ + NewConn: netxlite.NewConnUTLS(&utls.HelloChrome_Auto), + } + tlsConn, _, err := h.Handshake(ctx, conn, tlsConf) return tlsConn, err } diff --git a/internal/engine/experiment/websteps/websteps.go b/internal/engine/experiment/websteps/websteps.go index 71bc1ed685..2cd285ed78 100644 --- a/internal/engine/experiment/websteps/websteps.go +++ b/internal/engine/experiment/websteps/websteps.go @@ -241,7 +241,7 @@ func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpo defer conn.Close() // TLS handshake step - tlsconn, err := TLSDo(conn, URL.Hostname()) + tlsconn, err := TLSDo(ctx, conn, URL.Hostname()) endpointMeasurement.TLSHandshake = &TLSHandshakeMeasurement{ Failure: archival.NewFailure(err), } diff --git a/internal/netxlite/tlshandshaker.go b/internal/netxlite/tlshandshaker.go index 4a08afb0e1..924eb951e1 100644 --- a/internal/netxlite/tlshandshaker.go +++ b/internal/netxlite/tlshandshaker.go @@ -135,6 +135,7 @@ func (c *UTLSConn) ConnectionState() tls.ConnectionState { DidResume: uState.DidResume, CipherSuite: uState.CipherSuite, NegotiatedProtocol: uState.NegotiatedProtocol, + NegotiatedProtocolIsMutual: true, ServerName: uState.ServerName, PeerCertificates: uState.PeerCertificates, VerifiedChains: uState.VerifiedChains,