feat(netemx): support simulating the oohelperd (ooni#1204)

This diff extends netemx to add support for simulating the oohelperd
using netem.

We extracted this diff from ooni#1185.

The reference issue is ooni/probe#2461.

We're now well positioned to write netemx-based tests for Web
Connectivity 🥳 🥳 🥳 🥳 !

---------

Co-authored-by: kelmenhorst <k.elmenhorst@mailbox.org>

internal/netemx/oohelperd.go

@@ -0,0 +1,66 @@
+package netemx
+
+import (
+ "net/http"
+ "net/http/cookiejar"
+
+ "github.com/ooni/netem"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+ "github.com/ooni/probe-cli/v3/internal/oohelperd"
+ "golang.org/x/net/publicsuffix"
+)
+
+// OOHelperDFactory is the factory to create an [http.Handler] implementing the OONI Web Connectivity
+// test helper using a specific [netem.UnderlyingNetwork].
+type OOHelperDFactory struct{}
+
+var _ QAEnvHTTPHandlerFactory = &OOHelperDFactory{}
+
+// NewHandler implements QAEnvHTTPHandlerFactory.NewHandler.
+func (f *OOHelperDFactory) NewHandler(unet netem.UnderlyingNetwork) http.Handler {
+ netx := netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: unet}}
+ handler := oohelperd.NewHandler()
+
+ handler.NewDialer = func(logger model.Logger) model.Dialer {
+ return netx.NewDialerWithResolver(logger, netx.NewStdlibResolver(logger))
+ }
+
+ handler.NewQUICDialer = func(logger model.Logger) model.QUICDialer {
+ return netx.NewQUICDialerWithResolver(
+ netx.NewQUICListener(),
+ logger,
+ netx.NewStdlibResolver(logger),
+ )
+ }
+
+ handler.NewResolver = func(logger model.Logger) model.Resolver {
+ return netx.NewStdlibResolver(logger)
+ }
+
+ handler.NewHTTPClient = func(logger model.Logger) model.HTTPClient {
+ cookieJar, _ := cookiejar.New(&cookiejar.Options{
+ PublicSuffixList: publicsuffix.List,
+ })
+ return &http.Client{
+ Transport: netx.NewHTTPTransportStdlib(logger),
+ CheckRedirect: nil,
+ Jar: cookieJar,
+ Timeout: 0,
+ }
+ }
+
+ handler.NewHTTP3Client = func(logger model.Logger) model.HTTPClient {
+ cookieJar, _ := cookiejar.New(&cookiejar.Options{
+ PublicSuffixList: publicsuffix.List,
+ })
+ return &http.Client{
+ Transport: netx.NewHTTP3TransportStdlib(logger),
+ CheckRedirect: nil,
+ Jar: cookieJar,
+ Timeout: 0,
+ }
+ }
+
+ return handler
+}

internal/netemx/oohelperd_test.go

@@ -0,0 +1,140 @@
+package netemx
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+ "testing"
+
+ "github.com/apex/log"
+ "github.com/google/go-cmp/cmp"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+ "github.com/ooni/probe-cli/v3/internal/runtimex"
+)
+
+func TestOOHelperDHandler(t *testing.T) {
+ // we use completely unrelated IP addresses such that, in the unlikely event in
+ // which we're not using netem, the test is poised to fail.
+ //
+ // (These were two IP addresses assigned to me when I was at polito.it.)
+ const (
+ zeroThOONIOrgAddr = "130.192.91.211"
+ exampleComAddr = "130.192.91.231"
+ )
+
+ env := NewQAEnv(
+ QAEnvOptionHTTPServer(zeroThOONIOrgAddr, &OOHelperDFactory{}),
+ QAEnvOptionHTTPServer(exampleComAddr, ExampleWebPageHandlerFactory()),
+ )
+ env.AddRecordToAllResolvers("example.com", "web01.example.com", exampleComAddr)
+ env.AddRecordToAllResolvers("0.th.ooni.org", "0-th.ooni.org", zeroThOONIOrgAddr)
+ defer env.Close()
+
+ env.Do(func() {
+ thReq := &model.THRequest{
+ HTTPRequest: "https://example.com/",
+ HTTPRequestHeaders: map[string][]string{
+ "accept": {model.HTTPHeaderAccept},
+ "accept-language": {model.HTTPHeaderAcceptLanguage},
+ "user-agent": {model.HTTPHeaderUserAgent},
+ },
+ TCPConnect: []string{exampleComAddr},
+ XQUICEnabled: true,
+ }
+ thReqRaw := runtimex.Try1(json.Marshal(thReq))
+
+ //log.SetLevel(log.DebugLevel)
+
+ httpClient := netxlite.NewHTTPClientStdlib(log.Log)
+
+ req, err := http.NewRequest(http.MethodPost, "https://0.th.ooni.org/", bytes.NewReader(thReqRaw))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ t.Fatal("unexpected status code", resp.StatusCode)
+ }
+ body, err := netxlite.ReadAllContext(context.Background(), resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ //t.Log(string(body))
+
+ var thResp model.THResponse
+ if err := json.Unmarshal(body, &thResp); err != nil {
+ t.Fatal(err)
+ }
+
+ expectedTHResp := &model.THResponse{
+ TCPConnect: map[string]model.THTCPConnectResult{
+ "130.192.91.231:443": {
+ Status: true,
+ Failure: nil,
+ },
+ },
+ TLSHandshake: map[string]model.THTLSHandshakeResult{
+ "130.192.91.231:443": {
+ ServerName: "example.com",
+ Status: true,
+ Failure: nil,
+ },
+ },
+ QUICHandshake: map[string]model.THTLSHandshakeResult{
+ "130.192.91.231:443": {
+ ServerName: "example.com",
+ Status: true,
+ Failure: nil,
+ },
+ },
+ HTTPRequest: model.THHTTPRequestResult{
+ BodyLength: 203,
+ DiscoveredH3Endpoint: "example.com:443",
+ Failure: nil,
+ Title: "Default Web Page",
+ Headers: map[string]string{
+ "Alt-Svc": `h3=":443"`,
+ "Content-Length": "203",
+ "Content-Type": "text/html; charset=utf-8",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ },
+ StatusCode: 200,
+ },
+ HTTP3Request: &model.THHTTPRequestResult{
+ BodyLength: 203,
+ DiscoveredH3Endpoint: "",
+ Failure: nil,
+ Title: "Default Web Page",
+ Headers: map[string]string{
+ "Alt-Svc": `h3=":443"`,
+ "Content-Type": "text/html; charset=utf-8",
+ "Date": "Thu, 24 Aug 2023 14:35:29 GMT",
+ },
+ StatusCode: 200,
+ },
+ DNS: model.THDNSResult{
+ Failure: nil,
+ Addrs: []string{"130.192.91.231"},
+ ASNs: nil,
+ },
+ IPInfo: map[string]*model.THIPInfo{
+ "130.192.91.231": {
+ ASN: 137,
+ Flags: 10,
+ },
+ },
+ }
+
+ if diff := cmp.Diff(expectedTHResp, &thResp); diff != "" {
+ t.Fatal(diff)
+ }
+ })
+}

internal/netemx/web.go

@@ -25,6 +25,8 @@ const ExampleWebPage = `<!doctype html>
 func ExampleWebPageHandlerFactory() QAEnvHTTPHandlerFactory {
  return QAEnvHTTPHandlerFactoryFunc(func(_ netem.UnderlyingNetwork) http.Handler {
  return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("Alt-Svc", `h3=":443"`)
+ w.Header().Add("Date", "Thu, 24 Aug 2023 14:35:29 GMT")
  w.Write([]byte(ExampleWebPage))
  })
  })

internal/oohelperd/handler.go

@@ -22,9 +22,9 @@ import (
  "golang.org/x/net/publicsuffix"
 )
 
-// maxAcceptableBodySize is the maximum acceptable body size for incoming
+// MaxAcceptableBodySize is the maximum acceptable body size for incoming
 // API requests as well as when we're measuring webpages.
-const maxAcceptableBodySize = 1 << 24
+const MaxAcceptableBodySize = 1 << 24
 
 // Handler is an [http.Handler] implementing the Web
 // Connectivity test helper HTTP API.
@@ -68,7 +68,7 @@ func NewHandler() *Handler {
  return &Handler{
  BaseLogger: log.Log,
  Indexer: &atomic.Int64{},
- MaxAcceptableBody: maxAcceptableBodySize,
+ MaxAcceptableBody: MaxAcceptableBodySize,
  Measure: measure,
 
  NewHTTPClient: func(logger model.Logger) model.HTTPClient {