refactor: use ooni/oocrypto instead of ooni/go (#751)

Rather than building for Android using ooni/go, we're now using
ooni/oocryto as the TLS dependency. Such a repository only forks
crypto/tls and some minor crypto packages and includes the
same set of patches that we have been using in ooni/go.

This new strategy should be better than the previous one in
terms of building for Android, because we can use the vanilla
go1.18.2 build. It also seems that it is easier to track and
merge from upstream with ooni/oocrypto than it is with ooni/go.

Should this assessment be wrong, we can revert back to the
previous scenario where we used ooni/go.

See ooni/probe#2106 for extra context.

go.mod

@@ -20,13 +20,13 @@ require (
  github.com/hexops/gotextdiff v1.0.3
  github.com/iancoleman/strcase v0.2.0
  github.com/lucas-clemente/quic-go v0.27.0
- github.com/marten-seemann/qtls-go1-17 v0.1.1
  github.com/marten-seemann/qtls-go1-18 v0.1.1
  github.com/mattn/go-colorable v0.1.12
  github.com/miekg/dns v1.1.49
  github.com/mitchellh/go-wordwrap v1.0.1
  github.com/montanaflynn/stats v0.6.6
  github.com/ooni/go-libtor v1.1.5
+ github.com/ooni/oocrypto v0.0.0-20220522151937-5069b5e3bf80
  github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e
  github.com/ooni/probe-assets v0.9.0
  github.com/ooni/psiphon/tunnel-core v0.0.0-20220519122549-9c044eb6bd83
@@ -41,7 +41,7 @@ require (
  gitlab.com/yawning/utls.git v0.0.12-1
  golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898
  golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
- golang.org/x/sys v0.0.0-20220519141025-dcacdad47464
+ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
 )
 
 require (
@@ -72,6 +72,7 @@ require (
  github.com/klauspost/reedsolomon v1.9.16 // indirect
  github.com/marten-seemann/qpack v0.2.1 // indirect
  github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+ github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
  github.com/mattn/go-isatty v0.0.14 // indirect
  github.com/mattn/go-sqlite3 v1.14.13 // indirect
  github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect

go.sum

@@ -614,6 +614,8 @@ github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/ooni/go-libtor v1.1.5 h1:YbwXR9aLuL37EwL7rksPCQQhcHwoxU+M/v+jwZR+n5Y=
 github.com/ooni/go-libtor v1.1.5/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI=
+github.com/ooni/oocrypto v0.0.0-20220522151937-5069b5e3bf80 h1:ubOaPXDY9TcGc8QfVTx+tAtcw44O2079H/juYnVuI6c=
+github.com/ooni/oocrypto v0.0.0-20220522151937-5069b5e3bf80/go.mod h1:tP8tp455ERRdy38YWJQtQHb5LB6I5DvI4ffrnouypnM=
 github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e h1:hM6+SmEh6aCzXZDIHTwA0UeyjXNy7EfK1NpE3zr3WBo=
 github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e/go.mod h1:p2VVLbs+BXBIgTHITV9Vw8Rv6G1u66JUWP/8KCgDGNo=
 github.com/ooni/probe-assets v0.9.0 h1:RBqouztuKP0kWJzYwBjY0fPiwl5MQ8Aosy6ks8j1hR4=
@@ -1182,8 +1184,8 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220519141025-dcacdad47464 h1:MpIuURY70f0iKp/oooEFtB2oENcHITo/z1b6u41pKCw=
-golang.org/x/sys v0.0.0-20220519141025-dcacdad47464/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=

internal/engine/netx/httptransport/system.go

@@ -1,8 +1,7 @@
 package httptransport
 
 import (
- "net/http"
-
+ oohttp "github.com/ooni/oohttp"
  "github.com/ooni/probe-cli/v3/internal/model"
 )
 
@@ -13,7 +12,7 @@ import (
 //
 // New code should use netxlite.NewHTTPTransport instead.
 func NewSystemTransport(config Config) model.HTTPTransport {
- txp := http.DefaultTransport.(*http.Transport).Clone()
+ txp := oohttp.DefaultTransport.(*oohttp.Transport).Clone()
  txp.DialContext = config.Dialer.DialContext
  txp.DialTLSContext = config.TLSDialer.DialTLSContext
  // Better for Cloudflare DNS and also better because we have less
@@ -24,12 +23,12 @@ func NewSystemTransport(config Config) model.HTTPTransport {
  // back the true headers, such as Content-Length. This change is
  // functional to OONI's goal of observing the network.
  txp.DisableCompression = true
- return &SystemTransportWrapper{txp}
+ return &SystemTransportWrapper{&oohttp.StdlibTransport{Transport: txp}}
 }
 
 // SystemTransportWrapper adapts *http.Transport to have the .Network method
 type SystemTransportWrapper struct {
- *http.Transport
+ *oohttp.StdlibTransport
 }
 
 func (txp *SystemTransportWrapper) Network() string {

internal/engine/netx/tlsdialer/integration_test.go

@@ -5,6 +5,7 @@ import (
  "testing"
 
  "github.com/apex/log"
+ oohttp "github.com/ooni/oohttp"
  "github.com/ooni/probe-cli/v3/internal/netxlite"
 )
 
@@ -19,9 +20,11 @@ func TestTLSDialerSuccess(t *testing.T) {
  DebugLogger: log.Log,
  },
  }
- txp := &http.Transport{
- DialTLSContext: dialer.DialTLSContext,
- ForceAttemptHTTP2: true,
+ txp := &oohttp.StdlibTransport{
+ Transport: &oohttp.Transport{
+ DialTLSContext: dialer.DialTLSContext,
+ ForceAttemptHTTP2: true,
+ },
  }
  client := &http.Client{Transport: txp}
  resp, err := client.Get("https://www.google.com")

internal/netxlite/tls.go

@@ -13,6 +13,7 @@ import (
  "net"
  "time"
 
+ ootls "github.com/ooni/oocrypto/tls"
  oohttp "github.com/ooni/oohttp"
  "github.com/ooni/probe-cli/v3/internal/model"
  "github.com/ooni/probe-cli/v3/internal/runtimex"
@@ -161,7 +162,7 @@ func newTLSHandshaker(th model.TLSHandshaker, logger model.DebugLogger) model.TL
 type tlsHandshakerConfigurable struct {
  // NewConn is the OPTIONAL factory for creating a new connection. If
  // this factory is not set, we'll use the stdlib.
- NewConn func(conn net.Conn, config *tls.Config) TLSConn
+ NewConn func(conn net.Conn, config *tls.Config) (TLSConn, error)
 
  // Timeout is the OPTIONAL timeout imposed on the TLS handshake. If zero
  // or negative, we will use default timeout of 10 seconds.
@@ -190,19 +191,22 @@ func (h *tlsHandshakerConfigurable) Handshake(
  config = config.Clone()
  config.RootCAs = defaultCertPool
  }
- tlsconn := h.newConn(conn, config)
+ tlsconn, err := h.newConn(conn, config)
+ if err != nil {
+ return nil, tls.ConnectionState{}, err
+ }
  if err := tlsconn.HandshakeContext(ctx); err != nil {
  return nil, tls.ConnectionState{}, err
  }
  return tlsconn, tlsconn.ConnectionState(), nil
 }
 
 // newConn creates a new TLSConn.
-func (h *tlsHandshakerConfigurable) newConn(conn net.Conn, config *tls.Config) TLSConn {
+func (h *tlsHandshakerConfigurable) newConn(conn net.Conn, config *tls.Config) (TLSConn, error) {
  if h.NewConn != nil {
  return h.NewConn(conn, config)
  }
- return tls.Client(conn, config)
+ return ootls.NewClientConnStdlib(conn, config)
 }
 
 // defaultTLSHandshaker is the default TLS handshaker.

internal/netxlite/tls_test.go

@@ -202,13 +202,13 @@ func TestTLSHandshakerConfigurable(t *testing.T) {
  expected := errors.New("mocked error")
  var gotTLSConfig *tls.Config
  handshaker := &tlsHandshakerConfigurable{
- NewConn: func(conn net.Conn, config *tls.Config) TLSConn {
+ NewConn: func(conn net.Conn, config *tls.Config) (TLSConn, error) {
  gotTLSConfig = config
  return &mocks.TLSConn{
  MockHandshakeContext: func(ctx context.Context) error {
  return expected
  },
- }
+ }, nil
  },
  }
  ctx := context.Background()
@@ -235,6 +235,32 @@ func TestTLSHandshakerConfigurable(t *testing.T) {
  t.Fatal("gotTLSConfig.RootCAs has not been correctly set")
  }
  })
+
+ t.Run("we cannot create a new conn", func(t *testing.T) {
+ expected := errors.New("mocked error")
+ handshaker := &tlsHandshakerConfigurable{
+ NewConn: func(conn net.Conn, config *tls.Config) (TLSConn, error) {
+ return nil, expected
+ },
+ }
+ ctx := context.Background()
+ config := &tls.Config{}
+ conn := &mocks.Conn{
+ MockSetDeadline: func(t time.Time) error {
+ return nil
+ },
+ }
+ tlsConn, connState, err := handshaker.Handshake(ctx, conn, config)
+ if !errors.Is(err, expected) {
+ t.Fatal("not the error we expected", err)
+ }
+ if !reflect.ValueOf(connState).IsZero() {
+ t.Fatal("expected zero connState here")
+ }
+ if tlsConn != nil {
+ t.Fatal("expected nil tlsConn here")
+ }
+ })
  })
 }
 

internal/netxlite/utls.go

@@ -8,7 +8,9 @@ import (
  "context"
  "crypto/tls"
  "errors"
+ "fmt"
  "net"
+ "reflect"
 
  "github.com/ooni/probe-cli/v3/internal/model"
  utls "gitlab.com/yawning/utls.git"
@@ -42,18 +44,48 @@ type utlsConn struct {
 var _ TLSConn = &utlsConn{}
 
 // newConnUTLS returns a NewConn function for creating utlsConn instances.
-func newConnUTLS(clientHello *utls.ClientHelloID) func(conn net.Conn, config *tls.Config) TLSConn {
- return func(conn net.Conn, config *tls.Config) TLSConn {
- uConfig := &utls.Config{
- RootCAs: config.RootCAs,
- NextProtos: config.NextProtos,
- ServerName: config.ServerName,
- InsecureSkipVerify: config.InsecureSkipVerify,
- DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled,
+func newConnUTLS(clientHello *utls.ClientHelloID) func(conn net.Conn, config *tls.Config) (TLSConn, error) {
+ return func(conn net.Conn, config *tls.Config) (TLSConn, error) {
+ return newConnUTLSWithHelloID(conn, config, clientHello)
+ }
+}
+
+// errUTLSIncompatibleStdlibConfig indicates that the stdlib config you passed to
+// newConnUTLSWithHelloID contains some fields we don't support.
+var errUTLSIncompatibleStdlibConfig = errors.New("utls: incompatible stdlib config")
+
+// newConnUTLSWithHelloID creates a new connection with the given client hello ID.
+func newConnUTLSWithHelloID(conn net.Conn, config *tls.Config, cid *utls.ClientHelloID) (TLSConn, error) {
+ supportedFields := map[string]bool{
+ "DynamicRecordSizingDisabled": true,
+ "InsecureSkipVerify": true,
+ "NextProtos": true,
+ "RootCAs": true,
+ "ServerName": true,
+ }
+ value := reflect.ValueOf(config).Elem()
+ kind := value.Type()
+ for idx := 0; idx < value.NumField(); idx++ {
+ field := value.Field(idx)
+ if field.IsZero() {
+ continue
+ }
+ fieldKind := kind.Field(idx)
+ if supportedFields[fieldKind.Name] {
+ continue
  }
- tlsConn := utls.UClient(conn, uConfig, *clientHello)
- return &utlsConn{UConn: tlsConn}
+ err := fmt.Errorf("%w: field %s is nonzero", errUTLSIncompatibleStdlibConfig, fieldKind.Name)
+ return nil, err
+ }
+ uConfig := &utls.Config{
+ DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled,
+ InsecureSkipVerify: config.InsecureSkipVerify,
+ RootCAs: config.RootCAs,
+ NextProtos: config.NextProtos,
+ ServerName: config.ServerName,
  }
+ tlsConn := utls.UClient(conn, uConfig, *cid)
+ return &utlsConn{UConn: tlsConn}, nil
 }
 
 // ErrUTLSHandshakePanic indicates that there was panic handshaking

internal/netxlite/utls_test.go

@@ -2,7 +2,9 @@ package netxlite
 
 import (
  "context"
+ "crypto/tls"
  "errors"
+ "net"
  "sync"
  "testing"
  "time"
@@ -92,3 +94,51 @@ func TestUTLSConn(t *testing.T) {
  })
  })
 }
+
+func Test_newConnUTLSWithHelloID(t *testing.T) {
+ tests := []struct {
+ name string
+ config *tls.Config
+ cid *utls.ClientHelloID
+ wantNilConn bool
+ wantErr error
+ }{{
+ name: "with only supported fields",
+ config: &tls.Config{
+ DynamicRecordSizingDisabled: true,
+ InsecureSkipVerify: true,
+ NextProtos: []string{"h3"},
+ RootCAs: NewDefaultCertPool(),
+ ServerName: "ooni.org",
+ },
+ cid: &utls.HelloFirefox_55,
+ wantNilConn: false,
+ wantErr: nil,
+ }, {
+ name: "with unsupported fields",
+ config: &tls.Config{
+ Time: func() time.Time {
+ return time.Now()
+ },
+ },
+ cid: &utls.HelloChrome_58,
+ wantNilConn: true,
+ wantErr: errUTLSIncompatibleStdlibConfig,
+ }}
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ conn, err := net.Dial("udp", "8.8.8.8:443") // we just need a conn
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+ got, err := newConnUTLSWithHelloID(conn, tt.config, tt.cid)
+ if !errors.Is(err, tt.wantErr) {
+ t.Fatal("unexpected err", err)
+ }
+ if got != nil && tt.wantNilConn {
+ t.Fatal("expected nil conn here")
+ }
+ })
+ }
+}