feat(testingx): introduce more comprehensive HTTP(S) proxy

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/montanaflynn/stats v0.7.1
 	github.com/ooni/go-libtor v1.1.8
-	github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3
+	github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014
 	github.com/ooni/oocrypto v0.5.3
 	github.com/ooni/oohttp v0.6.3
 	github.com/ooni/probe-assets v0.18.0

go.sum

@@ -483,8 +483,8 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl
 github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
 github.com/ooni/go-libtor v1.1.8 h1:Wo3V3DVTxl5vZdxtQakqYP+DAHx7pPtAFSl1bnAa08w=
 github.com/ooni/go-libtor v1.1.8/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI=
-github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3 h1:zpTbzNzpo00cKbjLLnWMKjZeGLdoNC81vMiBDiur7NU=
-github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3/go.mod h1:3LJOzTIu2O4ADDJN2ILG4ViJOqyH/u9fKY8QT2Rma8Y=
+github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014 h1:4kOSV4D6mwrdoNUkAbGz1XoFUPcjsuNlLhZMc2CoHGg=
+github.com/ooni/netem v0.0.0-20230915101649-ab0dc13be014/go.mod h1:3LJOzTIu2O4ADDJN2ILG4ViJOqyH/u9fKY8QT2Rma8Y=
 github.com/ooni/oocrypto v0.5.3 h1:CAb0Ze6q/EWD1PRGl9KqpzMfkut4O3XMaiKYsyxrWOs=
 github.com/ooni/oocrypto v0.5.3/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk=
 github.com/ooni/oohttp v0.6.3 h1:MHydpeAPU/LSDSI/hIFJwZm4afBhd2Yo+rNxxFdeMCY=

internal/testingproxy/dialer.go

@@ -0,0 +1,49 @@
+package testingproxy
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+)
+
+// dialerWithAssertions ensures that we're dialing with the proxy address.
+type dialerWithAssertions struct {
+	// ExpectAddress is the expected IP address to dial
+	ExpectAddress string
+
+	// Dialer is the underlying dialer to use
+	Dialer model.Dialer
+}
+
+var _ model.Dialer = &dialerWithAssertions{}
+
+// CloseIdleConnections implements model.Dialer.
+func (d *dialerWithAssertions) CloseIdleConnections() {
+	d.Dialer.CloseIdleConnections()
+}
+
+// DialContext implements model.Dialer.
+func (d *dialerWithAssertions) DialContext(ctx context.Context, network string, address string) (net.Conn, error) {
+	// make sure the network is tcp
+	const expectNetwork = "tcp"
+	runtimex.Assert(
+		network == expectNetwork,
+		fmt.Sprintf("dialerWithAssertions: expected %s, got %s", expectNetwork, network),
+	)
+	log.Printf("dialerWithAssertions: verified that the network is %s as expected", expectNetwork)
+
+	// make sure the IP address is the expected one
+	ipAddr, _ := runtimex.Try2(net.SplitHostPort(address))
+	runtimex.Assert(
+		ipAddr == d.ExpectAddress,
+		fmt.Sprintf("dialerWithAssertions: expected %s, got %s", d.ExpectAddress, ipAddr),
+	)
+	log.Printf("dialerWithAssertions: verified that the address is %s as expected", d.ExpectAddress)
+
+	// now that we're sure we're using the proxy, we can actually dial
+	return d.Dialer.DialContext(ctx, network, address)
+}

internal/testingproxy/doc.go

@@ -0,0 +1,2 @@
+// Package testingproxy contains shared test cases for the  proxies.
+package testingproxy

internal/testingproxy/hosthttp.go

@@ -0,0 +1,74 @@
+package testingproxy
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+// WithHostNetworkHTTPProxyAndURL returns a [TestCase] where:
+//
+// - we fetch a URL;
+//
+// - using the host network;
+//
+// - and an HTTP proxy.
+//
+// Because this [TestCase] uses the host network, it does not run in -short mode.
+func WithHostNetworkHTTPProxyAndURL(URL string) TestCase {
+	return &hostNetworkTestCaseWithHTTP{
+		TargetURL: URL,
+	}
+}
+
+type hostNetworkTestCaseWithHTTP struct {
+	TargetURL string
+}
+
+var _ TestCase = &hostNetworkTestCaseWithHTTP{}
+
+// Name implements TestCase.
+func (tc *hostNetworkTestCaseWithHTTP) Name() string {
+	return fmt.Sprintf("fetching %s using the host network and an HTTP proxy", tc.TargetURL)
+}
+
+// Run implements TestCase.
+func (tc *hostNetworkTestCaseWithHTTP) Run(t *testing.T) {
+	// create an instance of Netx where the underlying network is nil,
+	// which means we're using the host's network
+	netx := &netxlite.Netx{Underlying: nil}
+
+	// create the proxy server using the host network
+	proxyServer := testingx.MustNewHTTPServer(testingx.NewHTTPProxyHandler(log.Log, netx))
+	defer proxyServer.Close()
+
+	//log.SetLevel(log.DebugLevel)
+
+	// create an HTTP client configured to use the given proxy
+	//
+	// note how we use a dialer that asserts that we're using the proxy IP address
+	// rather than the host address, so we're sure that we're using the proxy
+	dialer := &dialerWithAssertions{
+		ExpectAddress: "127.0.0.1",
+		Dialer:        netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)),
+	}
+	tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(log.Log))
+	txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer,
+		netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))))
+	client := &http.Client{Transport: txp}
+	defer client.CloseIdleConnections()
+
+	// get the homepage and assert we're getting a succesful response
+	httpCheckResponse(t, client, tc.TargetURL)
+}
+
+// Short implements TestCase.
+func (tc *hostNetworkTestCaseWithHTTP) Short() bool {
+	return false
+}

internal/testingproxy/hosthttps.go

@@ -0,0 +1,83 @@
+package testingproxy
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+// WithHostNetworkHTTPWithTLSProxyAndURL returns a [TestCase] where:
+//
+// - we fetch a URL;
+//
+// - using the host network;
+//
+// - and an HTTPS proxy.
+//
+// Because this [TestCase] uses the host network, it does not run in -short mode.
+func WithHostNetworkHTTPWithTLSProxyAndURL(URL string) TestCase {
+	return &hostNetworkTestCaseWithHTTPWithTLS{
+		TargetURL: URL,
+	}
+}
+
+type hostNetworkTestCaseWithHTTPWithTLS struct {
+	TargetURL string
+}
+
+var _ TestCase = &hostNetworkTestCaseWithHTTPWithTLS{}
+
+// Name implements TestCase.
+func (tc *hostNetworkTestCaseWithHTTPWithTLS) Name() string {
+	return fmt.Sprintf("fetching %s using the host network and an HTTPS proxy", tc.TargetURL)
+}
+
+// Run implements TestCase.
+func (tc *hostNetworkTestCaseWithHTTPWithTLS) Run(t *testing.T) {
+	// create an instance of Netx where the underlying network is nil,
+	// which means we're using the host's network
+	netx := &netxlite.Netx{Underlying: nil}
+
+	// create the proxy server using the host network
+	proxyServer := testingx.MustNewHTTPServerTLS(testingx.NewHTTPProxyHandler(log.Log, netx))
+	defer proxyServer.Close()
+
+	//log.SetLevel(log.DebugLevel)
+
+	// extend the default cert pool with the proxy's own CA
+	pool := netxlite.NewMozillaCertPool()
+	pool.AddCert(proxyServer.CACert)
+	tlsConfig := &tls.Config{RootCAs: pool}
+
+	// create an HTTP client configured to use the given proxy
+	//
+	// note how we use a dialer that asserts that we're using the proxy IP address
+	// rather than the host address, so we're sure that we're using the proxy
+	dialer := &dialerWithAssertions{
+		ExpectAddress: "127.0.0.1",
+		Dialer:        netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)),
+	}
+	tlsDialer := netxlite.NewTLSDialerWithConfig(
+		dialer, netxlite.NewTLSHandshakerStdlib(log.Log),
+		tlsConfig,
+	)
+	txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer,
+		netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))))
+	client := &http.Client{Transport: txp}
+	defer client.CloseIdleConnections()
+
+	// get the homepage and assert we're getting a succesful response
+	httpCheckResponse(t, client, tc.TargetURL)
+}
+
+// Short implements TestCase.
+func (tc *hostNetworkTestCaseWithHTTPWithTLS) Short() bool {
+	return false
+}

internal/testingproxy/httputils.go

@@ -0,0 +1,53 @@
+package testingproxy
+
+import "net/http"
+
+type httpClient interface {
+	Get(URL string) (*http.Response, error)
+}
+
+type httpClientMock struct {
+	MockGet func(URL string) (*http.Response, error)
+}
+
+var _ httpClient = &httpClientMock{}
+
+// Get implements httpClient.
+func (c *httpClientMock) Get(URL string) (*http.Response, error) {
+	return c.MockGet(URL)
+}
+
+type httpTestingT interface {
+	Logf(format string, v ...any)
+	Fatal(v ...any)
+}
+
+type httpTestingTMock struct {
+	MockLogf  func(format string, v ...any)
+	MockFatal func(v ...any)
+}
+
+var _ httpTestingT = &httpTestingTMock{}
+
+// Fatal implements httpTestingT.
+func (t *httpTestingTMock) Fatal(v ...any) {
+	t.MockFatal(v...)
+}
+
+// Logf implements httpTestingT.
+func (t *httpTestingTMock) Logf(format string, v ...any) {
+	t.MockLogf(format, v...)
+}
+
+func httpCheckResponse(t httpTestingT, client httpClient, targetURL string) {
+	// get the homepage and assert we're getting a succesful response
+	resp, err := client.Get(targetURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	t.Logf("%+v", resp)
+	if resp.StatusCode != 200 {
+		t.Fatal("invalid status code")
+	}
+}