refactor(netemx): allow running the oohelperd using netem (ooni#1201)

The oohelperd `http.Handler` constructor needs a reference to a
`model.UnderlyingNetwork` that will be used for fetching websites and
verifying that they're accessible. This diff changes how `netemx` works
such that we can construct an `http.Handler` that depends on a
`model.UnderlyingNetwork`, thus enabling this use case.

This diff has been adapted from
ooni#1185.

The reference issue is ooni/probe#2461.

---------

Co-authored-by: kelmenhorst <k.elmenhorst@mailbox.org>

internal/experiment/fbmessenger/fbmessenger_test.go

@@ -74,7 +74,7 @@ func TestMeasurerRun(t *testing.T) {
  }
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // configure the DNS for all resolvers
@@ -125,7 +125,7 @@ func TestMeasurerRun(t *testing.T) {
  }
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // configure the DNS for all resolvers
@@ -185,7 +185,7 @@ func TestMeasurerRun(t *testing.T) {
  }()
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // configure the DNS for all resolvers
@@ -245,7 +245,7 @@ func TestMeasurerRun(t *testing.T) {
  }
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // configure all DNS servers but the ISP's one

internal/experiment/simplequicping/simplequicping_test.go

@@ -104,7 +104,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: without DPI: expect success", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  env.Do(func() {
@@ -140,7 +140,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: with DPI that drops UDP datagrams to 8.8.8.8:443: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition

internal/experiment/sniblocking/sniblocking_test.go

@@ -214,7 +214,7 @@ func configureDNSWithDefaults(config *netem.DNSConfig) {
 func TestMeasurerWithInvalidInput(t *testing.T) {
  t.Run("with no measurement input: expect input error", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -237,7 +237,7 @@ func TestMeasurerWithInvalidInput(t *testing.T) {
 
  t.Run("with invalid MeasurementInput: expect parsing error", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -269,7 +269,7 @@ func TestMeasurerWithInvalidInput(t *testing.T) {
 func TestMeasurerRun(t *testing.T) {
  t.Run("without DPI: expect success", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -424,7 +424,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with cache: expect to see cached entry", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -480,7 +480,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with DPI that blocks target SNI", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -529,7 +529,7 @@ func TestMeasurerRun(t *testing.T) {
 
 func TestMeasureonewithcacheWorks(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here

internal/experiment/tcpping/tcpping_test.go

@@ -90,7 +90,7 @@ func TestMeasurer_run(t *testing.T) {
  })
 
  t.Run("with netem: without DPI: expect success", func(t *testing.T) {
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  env.Do(func() {
@@ -132,7 +132,7 @@ func TestMeasurer_run(t *testing.T) {
 
  t.Run("with netem: with DPI that drops TCP segments to 8.8.8.8:443: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition

internal/experiment/telegram/telegram_test.go

@@ -357,7 +357,7 @@ func newQAEnvironment(ipaddrs ...string) *netemx.QAEnv {
 
  // add handler for telegram web (we're using a different-from-reality HTTP handler
  // but we're not testing for the returned webpage, so we should be fine)
- options = append(options, netemx.QAEnvOptionHTTPServer(telegramWebAddr, netemx.QAEnvDefaultHTTPHandler()))
+ options = append(options, netemx.QAEnvOptionHTTPServer(telegramWebAddr, netemx.QAEnvDefaultHTTPHandlerFactory()))
 
  // create the environment proper with all the options
  env := netemx.NewQAEnv(options...)

internal/experiment/tlsping/tlsping_test.go

@@ -105,7 +105,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: without DPI: expect success", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  env.Do(func() {
@@ -147,7 +147,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: with DPI that drops TCP segments to 8.8.8.8:443: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition
@@ -205,7 +205,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: with DPI that resets TLS to SNI blocked.com: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandler()))
+ env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.QAEnvDefaultHTTPHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition

internal/experiment/webconnectivitylte/qa_test.go

@@ -25,81 +25,83 @@ const qaWebServerAddress = "93.184.216.34"
 // qaZeroTHOoniOrg is the address of 0.th.ooni.org.
 const qaZeroTHOoniOrg = "104.248.30.161"
 
-// qaNewMockedTestHelper returns an [http.Handler] that returns the expected TH response
-// based on the configuration we setup in [qaNewEnvironment].
-func qaNewMockedTestHelper() http.Handler {
+// qaNewMockedTestHelperFactory returns a [netemx.QAEnvHTTPHandlerFactory] that returns the expected
+// TH response based on the configuration we setup in [qaNewEnvironment].
+func qaNewMockedTestHelperFactory() netemx.QAEnvHTTPHandlerFactory {
  // TODO(bassosimone,kelmenhorst): we should use the real TH code rather than this fragile mock
 
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- // read raw request body
- rawRequest, err := netxlite.ReadAllContext(r.Context(), r.Body)
- if err != nil {
- // it does not make sense to send a response here because the connection
- // has been closed while reading the body
- return
- }
-
- // parse raw request body
- var request model.THRequest
- if err := json.Unmarshal(rawRequest, &request); err != nil {
- w.WriteHeader(http.StatusBadRequest)
- return
- }
-
- // parse the raw request URL
- URL, err := url.Parse(request.HTTPRequest)
- if err != nil {
- w.WriteHeader(http.StatusBadRequest)
- return
- }
-
- // create a fake response
- response := &model.THResponse{
- TCPConnect: map[string]model.THTCPConnectResult{
- net.JoinHostPort(qaWebServerAddress, "80"): {
- Status: true,
- Failure: nil,
+ return netemx.QAEnvHTTPHandlerFactoryFunc(func(_ netem.UnderlyingNetwork) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // read raw request body
+ rawRequest, err := netxlite.ReadAllContext(r.Context(), r.Body)
+ if err != nil {
+ // it does not make sense to send a response here because the connection
+ // has been closed while reading the body
+ return
+ }
+
+ // parse raw request body
+ var request model.THRequest
+ if err := json.Unmarshal(rawRequest, &request); err != nil {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // parse the raw request URL
+ URL, err := url.Parse(request.HTTPRequest)
+ if err != nil {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+
+ // create a fake response
+ response := &model.THResponse{
+ TCPConnect: map[string]model.THTCPConnectResult{
+ net.JoinHostPort(qaWebServerAddress, "80"): {
+ Status: true,
+ Failure: nil,
+ },
  },
- },
- TLSHandshake: map[string]model.THTLSHandshakeResult{
- net.JoinHostPort(qaWebServerAddress, "443"): {
- ServerName: URL.Hostname(),
- Status:  true,
- Failure: nil,
+  TLSHandshake: map[string]model.THTLSHandshakeResult{
+  net.JoinHostPort(qaWebServerAddress, "443"): {
+  ServerName: URL.Hostname(),
+  Status: true,
+  Failure: nil,
+ },
  },
- },
- QUICHandshake: map[string]model.THTLSHandshakeResult{},
- HTTPRequest: model.THHTTPRequestResult{
- BodyLength: int64(len(netemx.QAEnvDefaultWebPage)),
- DiscoveredH3Endpoint: "",
- Failure: nil,
- Title: "Default Web Page",
- Headers: map[string]string{},
- StatusCode: 200,
- },
- HTTP3Request: nil,
- DNS: model.THDNSResult{
- Failure: nil,
- Addrs: []string{qaWebServerAddress},
- ASNs: []int64{15133},
- },
- IPInfo: map[string]*model.THIPInfo{
- qaWebServerAddress: {
- ASN: 15133,
- Flags: model.THIPInfoFlagResolvedByTH | model.THIPInfoFlagResolvedByProbe,
+ QUICHandshake: map[string]model.THTLSHandshakeResult{},
+ HTTPRequest: model.THHTTPRequestResult{
+ BodyLength: int64(len(netemx.QAEnvDefaultWebPage)),
+ DiscoveredH3Endpoint: "",
+ Failure: nil,
+ Title: "Default Web Page",
+ Headers: map[string]string{},
+ StatusCode: 200,
  },
- },
- }
-
- // serialize the response
- rawResponse, err := json.Marshal(response)
- if err != nil {
- w.WriteHeader(http.StatusInternalServerError)
- return
- }
-
- // write the response
- w.Write(rawResponse)
+ HTTP3Request: nil,
+ DNS: model.THDNSResult{
+ Failure: nil,
+ Addrs: []string{qaWebServerAddress},
+ ASNs: []int64{15133},
+ },
+ IPInfo: map[string]*model.THIPInfo{
+ qaWebServerAddress: {
+ ASN: 15133,
+ Flags: model.THIPInfoFlagResolvedByTH | model.THIPInfoFlagResolvedByProbe,
+ },
+ },
+ }
+
+ // serialize the response
+ rawResponse, err := json.Marshal(response)
+ if err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+
+ // write the response
+ w.Write(rawResponse)
+ })
  })
 }
 
@@ -118,8 +120,8 @@ func qaAddTHDomains(config *netem.DNSConfig) {
 func qaNewEnvironment() *netemx.QAEnv {
  return netemx.NewQAEnv(
  netemx.QAEnvOptionDNSOverUDPResolvers("8.8.4.4"),
- netemx.QAEnvOptionHTTPServer(qaWebServerAddress, netemx.QAEnvDefaultHTTPHandler()),
- netemx.QAEnvOptionHTTPServer(qaZeroTHOoniOrg, qaNewMockedTestHelper()),
+ netemx.QAEnvOptionHTTPServer(qaWebServerAddress, netemx.QAEnvDefaultHTTPHandlerFactory()),
+ netemx.QAEnvOptionHTTPServer(qaZeroTHOoniOrg, qaNewMockedTestHelperFactory()),
  )
 }
 

internal/experiment/whatsapp/whatsapp_test.go

@@ -73,7 +73,7 @@ func newQAEnvironment() *netemx.QAEnv {
  // - TCP listeners for endpoints on 443 and 5222
  env := netemx.NewQAEnv(
  netemx.QAEnvOptionLogger(log.Log),
- netemx.QAEnvOptionHTTPServer(whatsappWebAddr, netemx.QAEnvDefaultHTTPHandler()),
+ netemx.QAEnvOptionHTTPServer(whatsappWebAddr, netemx.QAEnvDefaultHTTPHandlerFactory()),
  netemx.QAEnvOptionNetStack(whatsappEndpointAddr, endpointsNetStack),
  )
 

internal/netemx/example_test.go

@@ -33,7 +33,7 @@ func exampleNewEnvironment() *netemx.QAEnv {
  netemx.QAEnvOptionDNSOverUDPResolvers("8.8.4.4", "9.9.9.9"),
  netemx.QAEnvOptionClientAddress(exampleClientAddress),
  netemx.QAEnvOptionISPResolverAddress(exampleISPResolverAddress),
- netemx.QAEnvOptionHTTPServer(exampleExampleComAddress, netemx.QAEnvDefaultHTTPHandler()),
+ netemx.QAEnvOptionHTTPServer(exampleExampleComAddress, netemx.QAEnvDefaultHTTPHandlerFactory()),
  netemx.QAEnvOptionLogger(log.Log),
  )
 }