From 6b99d2a0d991978ac673cd34c505a4505e41b7a3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 24 Aug 2023 16:49:00 +0200 Subject: [PATCH] feat(netemx): support simulating the oohelperd (#1204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This diff extends netemx to add support for simulating the oohelperd using netem. We extracted this diff from https://github.com/ooni/probe-cli/pull/1185. The reference issue is https://github.com/ooni/probe/issues/2461. We're now well positioned to write netemx-based tests for Web Connectivity 🥳 🥳 🥳 🥳 ! --------- Co-authored-by: kelmenhorst --- internal/netemx/oohelperd.go | 66 ++++++++++++++ internal/netemx/oohelperd_test.go | 140 ++++++++++++++++++++++++++++++ internal/netemx/web.go | 2 + internal/oohelperd/handler.go | 6 +- 4 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 internal/netemx/oohelperd.go create mode 100644 internal/netemx/oohelperd_test.go diff --git a/internal/netemx/oohelperd.go b/internal/netemx/oohelperd.go new file mode 100644 index 0000000000..7277573e47 --- /dev/null +++ b/internal/netemx/oohelperd.go @@ -0,0 +1,66 @@ +package netemx + +import ( + "net/http" + "net/http/cookiejar" + + "github.com/ooni/netem" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/oohelperd" + "golang.org/x/net/publicsuffix" +) + +// OOHelperDFactory is the factory to create an [http.Handler] implementing the OONI Web Connectivity +// test helper using a specific [netem.UnderlyingNetwork]. +type OOHelperDFactory struct{} + +var _ QAEnvHTTPHandlerFactory = &OOHelperDFactory{} + +// NewHandler implements QAEnvHTTPHandlerFactory.NewHandler. +func (f *OOHelperDFactory) NewHandler(unet netem.UnderlyingNetwork) http.Handler { + netx := netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: unet}} + handler := oohelperd.NewHandler() + + handler.NewDialer = func(logger model.Logger) model.Dialer { + return netx.NewDialerWithResolver(logger, netx.NewStdlibResolver(logger)) + } + + handler.NewQUICDialer = func(logger model.Logger) model.QUICDialer { + return netx.NewQUICDialerWithResolver( + netx.NewQUICListener(), + logger, + netx.NewStdlibResolver(logger), + ) + } + + handler.NewResolver = func(logger model.Logger) model.Resolver { + return netx.NewStdlibResolver(logger) + } + + handler.NewHTTPClient = func(logger model.Logger) model.HTTPClient { + cookieJar, _ := cookiejar.New(&cookiejar.Options{ + PublicSuffixList: publicsuffix.List, + }) + return &http.Client{ + Transport: netx.NewHTTPTransportStdlib(logger), + CheckRedirect: nil, + Jar: cookieJar, + Timeout: 0, + } + } + + handler.NewHTTP3Client = func(logger model.Logger) model.HTTPClient { + cookieJar, _ := cookiejar.New(&cookiejar.Options{ + PublicSuffixList: publicsuffix.List, + }) + return &http.Client{ + Transport: netx.NewHTTP3TransportStdlib(logger), + CheckRedirect: nil, + Jar: cookieJar, + Timeout: 0, + } + } + + return handler +} diff --git a/internal/netemx/oohelperd_test.go b/internal/netemx/oohelperd_test.go new file mode 100644 index 0000000000..08d66150c0 --- /dev/null +++ b/internal/netemx/oohelperd_test.go @@ -0,0 +1,140 @@ +package netemx + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func TestOOHelperDHandler(t *testing.T) { + // we use completely unrelated IP addresses such that, in the unlikely event in + // which we're not using netem, the test is poised to fail. + // + // (These were two IP addresses assigned to me when I was at polito.it.) + const ( + zeroThOONIOrgAddr = "130.192.91.211" + exampleComAddr = "130.192.91.231" + ) + + env := NewQAEnv( + QAEnvOptionHTTPServer(zeroThOONIOrgAddr, &OOHelperDFactory{}), + QAEnvOptionHTTPServer(exampleComAddr, ExampleWebPageHandlerFactory()), + ) + env.AddRecordToAllResolvers("example.com", "web01.example.com", exampleComAddr) + env.AddRecordToAllResolvers("0.th.ooni.org", "0-th.ooni.org", zeroThOONIOrgAddr) + defer env.Close() + + env.Do(func() { + thReq := &model.THRequest{ + HTTPRequest: "https://example.com/", + HTTPRequestHeaders: map[string][]string{ + "accept": {model.HTTPHeaderAccept}, + "accept-language": {model.HTTPHeaderAcceptLanguage}, + "user-agent": {model.HTTPHeaderUserAgent}, + }, + TCPConnect: []string{exampleComAddr}, + XQUICEnabled: true, + } + thReqRaw := runtimex.Try1(json.Marshal(thReq)) + + //log.SetLevel(log.DebugLevel) + + httpClient := netxlite.NewHTTPClientStdlib(log.Log) + + req, err := http.NewRequest(http.MethodPost, "https://0.th.ooni.org/", bytes.NewReader(thReqRaw)) + if err != nil { + t.Fatal(err) + } + + resp, err := httpClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatal("unexpected status code", resp.StatusCode) + } + body, err := netxlite.ReadAllContext(context.Background(), resp.Body) + if err != nil { + t.Fatal(err) + } + + //t.Log(string(body)) + + var thResp model.THResponse + if err := json.Unmarshal(body, &thResp); err != nil { + t.Fatal(err) + } + + expectedTHResp := &model.THResponse{ + TCPConnect: map[string]model.THTCPConnectResult{ + "130.192.91.231:443": { + Status: true, + Failure: nil, + }, + }, + TLSHandshake: map[string]model.THTLSHandshakeResult{ + "130.192.91.231:443": { + ServerName: "example.com", + Status: true, + Failure: nil, + }, + }, + QUICHandshake: map[string]model.THTLSHandshakeResult{ + "130.192.91.231:443": { + ServerName: "example.com", + Status: true, + Failure: nil, + }, + }, + HTTPRequest: model.THHTTPRequestResult{ + BodyLength: 203, + DiscoveredH3Endpoint: "example.com:443", + Failure: nil, + Title: "Default Web Page", + Headers: map[string]string{ + "Alt-Svc": `h3=":443"`, + "Content-Length": "203", + "Content-Type": "text/html; charset=utf-8", + "Date": "Thu, 24 Aug 2023 14:35:29 GMT", + }, + StatusCode: 200, + }, + HTTP3Request: &model.THHTTPRequestResult{ + BodyLength: 203, + DiscoveredH3Endpoint: "", + Failure: nil, + Title: "Default Web Page", + Headers: map[string]string{ + "Alt-Svc": `h3=":443"`, + "Content-Type": "text/html; charset=utf-8", + "Date": "Thu, 24 Aug 2023 14:35:29 GMT", + }, + StatusCode: 200, + }, + DNS: model.THDNSResult{ + Failure: nil, + Addrs: []string{"130.192.91.231"}, + ASNs: nil, + }, + IPInfo: map[string]*model.THIPInfo{ + "130.192.91.231": { + ASN: 137, + Flags: 10, + }, + }, + } + + if diff := cmp.Diff(expectedTHResp, &thResp); diff != "" { + t.Fatal(diff) + } + }) +} diff --git a/internal/netemx/web.go b/internal/netemx/web.go index 9596680f29..4db9724624 100644 --- a/internal/netemx/web.go +++ b/internal/netemx/web.go @@ -25,6 +25,8 @@ const ExampleWebPage = ` func ExampleWebPageHandlerFactory() QAEnvHTTPHandlerFactory { return QAEnvHTTPHandlerFactoryFunc(func(_ netem.UnderlyingNetwork) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Alt-Svc", `h3=":443"`) + w.Header().Add("Date", "Thu, 24 Aug 2023 14:35:29 GMT") w.Write([]byte(ExampleWebPage)) }) }) diff --git a/internal/oohelperd/handler.go b/internal/oohelperd/handler.go index 15eda5435f..1bc6ad27ed 100644 --- a/internal/oohelperd/handler.go +++ b/internal/oohelperd/handler.go @@ -22,9 +22,9 @@ import ( "golang.org/x/net/publicsuffix" ) -// maxAcceptableBodySize is the maximum acceptable body size for incoming +// MaxAcceptableBodySize is the maximum acceptable body size for incoming // API requests as well as when we're measuring webpages. -const maxAcceptableBodySize = 1 << 24 +const MaxAcceptableBodySize = 1 << 24 // Handler is an [http.Handler] implementing the Web // Connectivity test helper HTTP API. @@ -68,7 +68,7 @@ func NewHandler() *Handler { return &Handler{ BaseLogger: log.Log, Indexer: &atomic.Int64{}, - MaxAcceptableBody: maxAcceptableBodySize, + MaxAcceptableBody: MaxAcceptableBodySize, Measure: measure, NewHTTPClient: func(logger model.Logger) model.HTTPClient {