From 19f3787a8a0b336dffb08ebb1088b304285ead2b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 21 Sep 2023 10:16:00 +0200 Subject: [PATCH] feat(enginenetx): introduce loadable TLSDialer policy (#1290) This commit introduces a loadable TLSDialer policy. We will use this feature to load the policy from disk. While there, note that the "null" tactic could be implemented using the loadable tactic structure and its wrapper. Part of https://github.com/ooni/probe-cli/pull/1288 --- .../enginenetx/httpsdialer_internal_test.go | 24 ++- internal/enginenetx/httpsdialer_test.go | 194 ++++++++++++++++++ internal/enginenetx/httpsdialercore.go | 9 + internal/enginenetx/httpsdialerloadable.go | 109 ++++++++++ internal/enginenetx/httpsdialernull.go | 84 +------- 5 files changed, 339 insertions(+), 81 deletions(-) create mode 100644 internal/enginenetx/httpsdialerloadable.go diff --git a/internal/enginenetx/httpsdialer_internal_test.go b/internal/enginenetx/httpsdialer_internal_test.go index 4ec6766494..849584f5c8 100644 --- a/internal/enginenetx/httpsdialer_internal_test.go +++ b/internal/enginenetx/httpsdialer_internal_test.go @@ -28,10 +28,13 @@ func TestHTTPSDialerTacticsEmitter(t *testing.T) { var tactics []HTTPSDialerTactic for idx := 0; idx < 255; idx++ { - tactics = append(tactics, &httpsDialerNullTactic{ - Address: fmt.Sprintf("10.0.0.%d", idx), - Delay: 0, - Domain: "www.example.com", + tactics = append(tactics, &HTTPSDialerLoadableTacticWrapper{ + Tactic: &HTTPSDialerLoadableTactic{ + IPAddr: fmt.Sprintf("10.0.0.%d", idx), + InitialDelay: 0, + SNI: "www.example.com", + VerifyHostname: "www.example.com", + }, }) } @@ -65,4 +68,17 @@ func TestHTTPSDialerVerifyCertificateChain(t *testing.T) { t.Fatal("unexpected error", err) } }) + + t.Run("with an empty hostname", func(t *testing.T) { + tlsConn := &mocks.TLSConn{ + MockConnectionState: func() tls.ConnectionState { + return tls.ConnectionState{} // empty but should not be an issue + }, + } + certPool := netxlite.NewMozillaCertPool() + err := httpsDialerVerifyCertificateChain("", tlsConn, certPool) + if !errors.Is(err, errEmptyVerifyHostname) { + t.Fatal("unexpected error", err) + } + }) } diff --git a/internal/enginenetx/httpsdialer_test.go b/internal/enginenetx/httpsdialer_test.go index a330e54081..7804864829 100644 --- a/internal/enginenetx/httpsdialer_test.go +++ b/internal/enginenetx/httpsdialer_test.go @@ -2,7 +2,9 @@ package enginenetx_test import ( "context" + "encoding/json" "testing" + "time" "github.com/apex/log" "github.com/google/go-cmp/cmp" @@ -11,6 +13,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netemx" "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/testingx" ) @@ -439,3 +442,194 @@ func TestHTTPSDialerWAI(t *testing.T) { }) } } + +func TestLoadHTTPSDialerPolicy(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the test case name + name string + + // input contains the serialized input bytes + input []byte + + // expectErr contains the expected error string or the empty string on success + expectErr string + + // expectPolicy contains the expected policy we loaded or nil + expectedPolicy *enginenetx.HTTPSDialerLoadablePolicy + } + + cases := []testcase{{ + name: "with nil input", + input: nil, + expectErr: "unexpected end of JSON input", + expectedPolicy: nil, + }, { + name: "with invalid serialized JSON", + input: []byte(`{`), + expectErr: "unexpected end of JSON input", + expectedPolicy: nil, + }, { + name: "with empty serialized JSON", + input: []byte(`{}`), + expectErr: "", + expectedPolicy: &enginenetx.HTTPSDialerLoadablePolicy{}, + }, { + name: "with real serialized policy", + input: (func() []byte { + return runtimex.Try1(json.Marshal(&enginenetx.HTTPSDialerLoadablePolicy{ + Domains: map[string][]*enginenetx.HTTPSDialerLoadableTactic{ + "api.ooni.io": {{ + IPAddr: "162.55.247.208", + InitialDelay: 0, + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "46.101.82.151", + InitialDelay: 300 * time.Millisecond, + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 600 * time.Millisecond, + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "46.101.82.151", + InitialDelay: 3000 * time.Millisecond, + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 3300 * time.Millisecond, + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }}, + }, + })) + })(), + expectErr: "", + expectedPolicy: &enginenetx.HTTPSDialerLoadablePolicy{ + Domains: map[string][]*enginenetx.HTTPSDialerLoadableTactic{ + "api.ooni.io": {{ + IPAddr: "162.55.247.208", + InitialDelay: 0, + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "46.101.82.151", + InitialDelay: 300 * time.Millisecond, + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 600 * time.Millisecond, + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "46.101.82.151", + InitialDelay: 3000 * time.Millisecond, + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, { + IPAddr: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 3300 * time.Millisecond, + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }}, + }, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + policy, err := enginenetx.LoadHTTPSDialerPolicy(tc.input) + + switch { + case err != nil && tc.expectErr == "": + t.Fatal("expected", tc.expectErr, "got", err) + + case err == nil && tc.expectErr != "": + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != "": + if diff := cmp.Diff(tc.expectErr, err.Error()); diff != "" { + t.Fatal(diff) + } + + case err == nil && tc.expectErr == "": + // all good + } + + if diff := cmp.Diff(tc.expectedPolicy, policy); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestHTTPSDialerLoadableTacticWrapper(t *testing.T) { + t.Run("IPAddr", func(t *testing.T) { + expected := "10.0.0.1" + ldt := &enginenetx.HTTPSDialerLoadableTacticWrapper{ + Tactic: &enginenetx.HTTPSDialerLoadableTactic{ + IPAddr: expected, + }, + } + if got := ldt.IPAddr(); got != expected { + t.Fatal("expected", expected, "got", got) + } + }) + + t.Run("InitialDelay", func(t *testing.T) { + expected := time.Millisecond + ldt := &enginenetx.HTTPSDialerLoadableTacticWrapper{ + Tactic: &enginenetx.HTTPSDialerLoadableTactic{ + InitialDelay: expected, + }, + } + if got := ldt.InitialDelay(); got != expected { + t.Fatal("expected", expected, "got", got) + } + }) + + t.Run("SNI", func(t *testing.T) { + expected := "x.org" + ldt := &enginenetx.HTTPSDialerLoadableTacticWrapper{ + Tactic: &enginenetx.HTTPSDialerLoadableTactic{ + SNI: expected, + }, + } + if got := ldt.SNI(); got != expected { + t.Fatal("expected", expected, "got", got) + } + }) + + t.Run("String", func(t *testing.T) { + expected := "&{IPAddr:162.55.247.208 InitialDelay:150ms SNI:www.example.com VerifyHostname:api.ooni.io}" + ldt := &enginenetx.HTTPSDialerLoadableTacticWrapper{ + Tactic: &enginenetx.HTTPSDialerLoadableTactic{ + IPAddr: "162.55.247.208", + InitialDelay: 150 * time.Millisecond, + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + } + got := ldt.String() + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("VerifyHostname", func(t *testing.T) { + expected := "x.org" + ldt := &enginenetx.HTTPSDialerLoadableTacticWrapper{ + Tactic: &enginenetx.HTTPSDialerLoadableTactic{ + VerifyHostname: expected, + }, + } + if got := ldt.VerifyHostname(); got != expected { + t.Fatal("expected", expected, "got", got) + } + }) +} diff --git a/internal/enginenetx/httpsdialercore.go b/internal/enginenetx/httpsdialercore.go index 64304170d9..aa7e7272d3 100644 --- a/internal/enginenetx/httpsdialercore.go +++ b/internal/enginenetx/httpsdialercore.go @@ -381,6 +381,9 @@ func httpsDialerTacticWaitReady(ctx context.Context, tactic HTTPSDialerTactic) e // errNoPeerCertificate is an internal error returned when we don't have any peer certificate. var errNoPeerCertificate = errors.New("no peer certificate") +// errEmptyVerifyHostname indicates there is no hostname to verify against +var errEmptyVerifyHostname = errors.New("empty VerifyHostname") + // httpsDialerVerifyCertificateChain verifies the certificate chain with the given hostname. func httpsDialerVerifyCertificateChain(hostname string, conn model.TLSConn, rootCAs *x509.CertPool) error { // This code comes from the example in the Go source tree that shows @@ -399,6 +402,12 @@ func httpsDialerVerifyCertificateChain(hostname string, conn model.TLSConn, root // // See https://github.com/golang/go/blob/go1.21.0/src/crypto/tls/handshake_client.go#L962. + // Protect against a programming or configuration error where the + // programmer or user has not set the hostname. + if hostname == "" { + return errEmptyVerifyHostname + } + state := conn.ConnectionState() opts := x509.VerifyOptions{ DNSName: hostname, // note: here we're using the real hostname diff --git a/internal/enginenetx/httpsdialerloadable.go b/internal/enginenetx/httpsdialerloadable.go new file mode 100644 index 0000000000..3fc76a5cce --- /dev/null +++ b/internal/enginenetx/httpsdialerloadable.go @@ -0,0 +1,109 @@ +package enginenetx + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// HTTPSDialerLoadablePolicy is an [HTTPSDialerPolicy] that you +// can load from its JSON serialization on disk. +type HTTPSDialerLoadablePolicy struct { + // Domains maps each domain to its policy. When there is + // no domain, the code falls back to the default "null" policy + // implemented by HTTPSDialerNullPolicy. + Domains map[string][]*HTTPSDialerLoadableTactic +} + +// LoadHTTPSDialerPolicy loads the [HTTPSDialerPolicy] from +// the given bytes containing a serialized JSON object. +func LoadHTTPSDialerPolicy(data []byte) (*HTTPSDialerLoadablePolicy, error) { + var p HTTPSDialerLoadablePolicy + if err := json.Unmarshal(data, &p); err != nil { + return nil, err + } + return &p, nil +} + +// HTTPSDialerLoadableTactic is an [HTTPSDialerTactic] that you +// can load from JSON as part of [HTTPSDialerLoadablePolicy]. +type HTTPSDialerLoadableTactic struct { + // IPAddr is the IP address to use for dialing. + IPAddr string + + // InitialDelay is the time in nanoseconds after which + // you would like to start this policy. + InitialDelay time.Duration + + // SNI is the TLS ServerName to send over the wire. + SNI string + + // VerifyHostname is the hostname using during + // the X.509 certificate verification. + VerifyHostname string +} + +// HTTPSDialerLoadableTacticWrapper wraps [HTTPSDialerLoadableTactic] +// to make it implements the [HTTPSDialerTactic] interface. +type HTTPSDialerLoadableTacticWrapper struct { + Tactic *HTTPSDialerLoadableTactic +} + +// IPAddr implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) IPAddr() string { + return ldt.Tactic.IPAddr +} + +// InitialDelay implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) InitialDelay() time.Duration { + return ldt.Tactic.InitialDelay +} + +// NewTLSHandshaker implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) NewTLSHandshaker(netx *netxlite.Netx, logger model.Logger) model.TLSHandshaker { + return netxlite.NewTLSHandshakerStdlib(logger) +} + +// OnStarting implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) OnStarting() { + // TODO(bassosimone): implement +} + +// OnSuccess implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) OnSuccess() { + // TODO(bassosimone): implement +} + +// OnTCPConnectError implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) OnTCPConnectError(ctx context.Context, err error) { + // TODO(bassosimone): implement +} + +// OnTLSHandshakeError implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) OnTLSHandshakeError(ctx context.Context, err error) { + // TODO(bassosimone): implement +} + +// OnTLSVerifyError implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) OnTLSVerifyError(ctz context.Context, err error) { + // TODO(bassosimone): implement +} + +// SNI implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) SNI() string { + return ldt.Tactic.SNI +} + +// String implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) String() string { + return fmt.Sprintf("%+v", ldt.Tactic) +} + +// VerifyHostname implements HTTPSDialerTactic. +func (ldt *HTTPSDialerLoadableTacticWrapper) VerifyHostname() string { + return ldt.Tactic.VerifyHostname +} diff --git a/internal/enginenetx/httpsdialernull.go b/internal/enginenetx/httpsdialernull.go index bfe23608fe..cfaa80adbe 100644 --- a/internal/enginenetx/httpsdialernull.go +++ b/internal/enginenetx/httpsdialernull.go @@ -2,11 +2,9 @@ package enginenetx import ( "context" - "fmt" "time" "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/netxlite" ) // HTTPSDialerNullPolicy is the default "null" policy where we use the default @@ -34,10 +32,13 @@ func (*HTTPSDialerNullPolicy) LookupTactics( const delay = 300 * time.Millisecond var tactics []HTTPSDialerTactic for idx, addr := range addrs { - tactics = append(tactics, &httpsDialerNullTactic{ - Address: addr, - Delay: time.Duration(idx) * delay, // zero for the first dial - Domain: domain, + tactics = append(tactics, &HTTPSDialerLoadableTacticWrapper{ + Tactic: &HTTPSDialerLoadableTactic{ + IPAddr: addr, + InitialDelay: time.Duration(idx) * delay, // zero for the first dial + SNI: domain, + VerifyHostname: domain, + }, }) } @@ -48,74 +49,3 @@ func (*HTTPSDialerNullPolicy) LookupTactics( func (*HTTPSDialerNullPolicy) Parallelism() int { return 16 } - -// httpsDialerNullTactic is the default "null" tactic where we use the -// resolved IP addresses with the domain as the SNI value. -// -// We say that this is the "null" tactic because this is what you would get -// by default if you were not using any tactic. -type httpsDialerNullTactic struct { - // Address is the IP address we resolved. - Address string - - // Delay is the delay after which we start dialing. - Delay time.Duration - - // Domain is the related IP address. - Domain string -} - -// IPAddr implements HTTPSDialerTactic. -func (dt *httpsDialerNullTactic) IPAddr() string { - return dt.Address -} - -// InitialDelay implements HTTPSDialerTactic. -func (dt *httpsDialerNullTactic) InitialDelay() time.Duration { - return dt.Delay -} - -// NewTLSHandshaker implements HTTPSDialerTactic. -func (*httpsDialerNullTactic) NewTLSHandshaker(netx *netxlite.Netx, logger model.Logger) model.TLSHandshaker { - return netx.NewTLSHandshakerStdlib(logger) -} - -// OnStarting implements HTTPSDialerTactic. -func (*httpsDialerNullTactic) OnStarting() { - // nothing -} - -// OnSuccess implements HTTPSDialerTactic. -func (*httpsDialerNullTactic) OnSuccess() { - // nothing -} - -// OnTCPConnectError implements HTTPSDialerTactic. -func (*httpsDialerNullTactic) OnTCPConnectError(ctx context.Context, err error) { - // nothing -} - -// OnTLSHandshakeError implements HTTPSDialerTactic. -func (*httpsDialerNullTactic) OnTLSHandshakeError(ctx context.Context, err error) { - // nothing -} - -// OnTLSVerifyError implements HTTPSDialerTactic. -func (*httpsDialerNullTactic) OnTLSVerifyError(ctx context.Context, err error) { - // nothing -} - -// SNI implements HTTPSDialerTactic. -func (dt *httpsDialerNullTactic) SNI() string { - return dt.Domain -} - -// String implements fmt.Stringer. -func (dt *httpsDialerNullTactic) String() string { - return fmt.Sprintf("NullTactic{Address:\"%s\" Domain:\"%s\"}", dt.Address, dt.Domain) -} - -// VerifyHostname implements HTTPSDialerTactic. -func (dt *httpsDialerNullTactic) VerifyHostname() string { - return dt.Domain -}