feat(netemx): create "internet" netem-based emulation (ooni#1205)

This diff introduces code to emulate a large-scale scenario with several
servers. This scenario is suitable for writing Web Connectivity
integration tests because it includes a test helper, the OONI API, a
GeoIP service, etc.

This work derives from ooni#1185 and
is part of ooni/probe#2461.

---------

Co-authored-by: kelmenhorst <k.elmenhorst@mailbox.org>

internal/experiment/dnsping/dnsping_test.go

@@ -99,7 +99,7 @@ func TestMeasurer_run(t *testing.T) {
 
  t.Run("with netem: without DPI: expect success", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionDNSOverUDPResolvers("8.8.8.8"))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionDNSOverUDPResolvers("8.8.8.8"))
  defer env.Close()
 
  // we use the same configuration for all resolvers
@@ -146,7 +146,7 @@ func TestMeasurer_run(t *testing.T) {
 
  t.Run("with netem: with DNS spoofing: expect to see delayed responses", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionDNSOverUDPResolvers("8.8.8.8"))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionDNSOverUDPResolvers("8.8.8.8"))
  defer env.Close()
 
  // we use the same configuration for all resolvers

internal/experiment/fbmessenger/fbmessenger_test.go

@@ -74,7 +74,7 @@ func TestMeasurerRun(t *testing.T) {
  }
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // configure the DNS for all resolvers
@@ -125,7 +125,7 @@ func TestMeasurerRun(t *testing.T) {
  }
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // configure the DNS for all resolvers
@@ -185,7 +185,7 @@ func TestMeasurerRun(t *testing.T) {
  }()
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // configure the DNS for all resolvers
@@ -245,7 +245,7 @@ func TestMeasurerRun(t *testing.T) {
  }
 
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // configure all DNS servers but the ISP's one

internal/experiment/simplequicping/simplequicping_test.go

@@ -104,7 +104,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: without DPI: expect success", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  env.Do(func() {
@@ -140,7 +140,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: with DPI that drops UDP datagrams to 8.8.8.8:443: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition

internal/experiment/sniblocking/sniblocking_test.go

@@ -214,7 +214,7 @@ func configureDNSWithDefaults(config *netem.DNSConfig) {
 func TestMeasurerWithInvalidInput(t *testing.T) {
  t.Run("with no measurement input: expect input error", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -237,7 +237,7 @@ func TestMeasurerWithInvalidInput(t *testing.T) {
 
  t.Run("with invalid MeasurementInput: expect parsing error", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -269,7 +269,7 @@ func TestMeasurerWithInvalidInput(t *testing.T) {
 func TestMeasurerRun(t *testing.T) {
  t.Run("without DPI: expect success", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -424,7 +424,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with cache: expect to see cached entry", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -480,7 +480,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with DPI that blocks target SNI", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here
@@ -529,7 +529,7 @@ func TestMeasurerRun(t *testing.T) {
 
 func TestMeasureonewithcacheWorks(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // we use the same valid DNS config for client and servers here

internal/experiment/tcpping/tcpping_test.go

@@ -90,7 +90,7 @@ func TestMeasurer_run(t *testing.T) {
  })
 
  t.Run("with netem: without DPI: expect success", func(t *testing.T) {
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  env.Do(func() {
@@ -132,7 +132,7 @@ func TestMeasurer_run(t *testing.T) {
 
  t.Run("with netem: with DPI that drops TCP segments to 8.8.8.8:443: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition

internal/experiment/telegram/telegram_test.go

@@ -360,7 +360,7 @@ func newQAEnvironment(ipaddrs ...string) *netemx.QAEnv {
  options = append(options, netemx.QAEnvOptionHTTPServer(telegramWebAddr, netemx.ExampleWebPageHandlerFactory()))
 
  // create the environment proper with all the options
- env := netemx.NewQAEnv(options...)
+ env := netemx.MustNewQAEnv(options...)
 
  // register with all the possible resolvers the correct DNS records - registering again
  // inside individual tests will override the values we're setting here

internal/experiment/tlsping/tlsping_test.go

@@ -105,7 +105,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: without DPI: expect success", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  env.Do(func() {
@@ -147,7 +147,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: with DPI that drops TCP segments to 8.8.8.8:443: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition
@@ -205,7 +205,7 @@ func TestMeasurerRun(t *testing.T) {
 
  t.Run("with netem: with DPI that resets TLS to SNI blocked.com: expect failure", func(t *testing.T) {
  // create a new test environment
- env := netemx.NewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
+ env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()))
  defer env.Close()
 
  // add DPI engine to emulate the censorship condition

internal/experiment/webconnectivitylte/qa_test.go

@@ -118,7 +118,7 @@ func qaAddTHDomains(config *netem.DNSConfig) {
 
 // qaNewEnvironment creates a new environment for running QA.
 func qaNewEnvironment() *netemx.QAEnv {
- return netemx.NewQAEnv(
+ return netemx.MustNewQAEnv(
  netemx.QAEnvOptionDNSOverUDPResolvers("8.8.4.4"),
  netemx.QAEnvOptionHTTPServer(qaWebServerAddress, netemx.ExampleWebPageHandlerFactory()),
  netemx.QAEnvOptionHTTPServer(qaZeroTHOoniOrg, qaNewMockedTestHelperFactory()),

internal/experiment/whatsapp/whatsapp_test.go

@@ -71,7 +71,7 @@ func newQAEnvironment() *netemx.QAEnv {
  // - HTTPS listeners for whatsappWebAddr on port 443
  //
  // - TCP listeners for endpoints on 443 and 5222
- env := netemx.NewQAEnv(
+ env := netemx.MustNewQAEnv(
  netemx.QAEnvOptionLogger(log.Log),
  netemx.QAEnvOptionHTTPServer(whatsappWebAddr, netemx.ExampleWebPageHandlerFactory()),
  netemx.QAEnvOptionNetStack(whatsappEndpointAddr, endpointsNetStack),

internal/netemx/dnsoverhttps.go

@@ -0,0 +1,20 @@
+package netemx
+
+import (
+ "net/http"
+
+ "github.com/ooni/netem"
+ "github.com/ooni/probe-cli/v3/internal/testingx"
+)
+
+// DNSOverHTTPSHandlerFactory is a [QAEnvHTTPHandlerFactory] for [testingx.GeoIPHandlerUbuntu].
+type DNSOverHTTPSHandlerFactory struct {
+ Config *netem.DNSConfig
+}
+
+var _ QAEnvHTTPHandlerFactory = &DNSOverHTTPSHandlerFactory{}
+
+// NewHandler implements QAEnvHTTPHandlerFactory.
+func (f *DNSOverHTTPSHandlerFactory) NewHandler(unet netem.UnderlyingNetwork) http.Handler {
+ return &testingx.DNSOverHTTPSHandler{Config: f.Config}
+}

internal/netemx/doc.go

@@ -1,4 +1,2 @@
-// Package netemx contains ooni/netem adapters. This code could have been part
-// of the [netxlite] package, but we keep it separate such that you do not link
-// with [netem] unless you need to use [netem] (presumably for testing).
+// Package netemx contains code to run integration tests using netem.
 package netemx