sniblocking: classify failures to produce a result

Since in OONI we typically use a three class system (i.e. accessible,
anomaly, or blocking), I am using these three classes.

I am also adding further information that help in understanding what
is the real root cause of the failure.

Part of #309.

experiment/sniblocking/sniblocking.go

@@ -52,10 +52,49 @@ type Subresult struct {
 
 // TestKeys contains sniblocking test keys.
 type TestKeys struct {
+	Result  string    `json:"result"`
 	Control Subresult `json:"control"`
 	Target  Subresult `json:"target"`
 }
 
+const (
+	classAccessibleInvalidHostname = "accessible_invalid_hostname"
+	classAccessibleValidHostname   = "accessible_valid_hostname"
+	classAnomalySSLError           = "anomaly_ssl_error"
+	classAnomalyTestHelperBlocked  = "anomaly_test_helper_blocked"
+	classAnomalyTimeout            = "anomaly_timeout"
+	classAnomalyUnexpectedFailure  = "anomaly_unexpected_failure"
+	classBlockedTCPIPError         = "blocked_tcpip_error"
+)
+
+func (tk *TestKeys) classify() string {
+	if tk.Target.Failure == nil {
+		return classAccessibleValidHostname
+	}
+	switch *tk.Target.Failure {
+	case modelx.FailureConnectionRefused:
+		return classAnomalyTestHelperBlocked
+	case modelx.FailureDNSNXDOMAINError:
+		return classAnomalyTestHelperBlocked
+	case modelx.FailureConnectionReset:
+		return classBlockedTCPIPError
+	case modelx.FailureEOFError:
+		return classBlockedTCPIPError
+	case modelx.FailureSSLInvalidHostname:
+		return classAccessibleInvalidHostname
+	case modelx.FailureSSLUnknownAuthority:
+		return classAnomalySSLError
+	case modelx.FailureSSLInvalidCertificate:
+		return classAnomalySSLError
+	case modelx.FailureGenericTimeoutError:
+		if tk.Control.Failure != nil {
+			return classAnomalyTestHelperBlocked
+		}
+		return classAnomalyTimeout
+	}
+	return classAnomalyUnexpectedFailure
+}
+
 type measurer struct {
 	cache  map[string]Subresult
 	config Config
@@ -83,7 +122,7 @@ func (m *measurer) measureone(
 	select {
 	case <-time.After(sleeptime):
 	case <-ctx.Done():
-		s := "generic_timeout_error"
+		s := modelx.FailureGenericTimeoutError
 		return Subresult{
 			Failure: &s,
 			SNI:     sni,
@@ -188,6 +227,8 @@ func processall(
 			break
 		}
 	}
+	testkeys.Result = testkeys.classify()
+	sess.Logger().Infof("sni_blocking: result: %s", testkeys.Result)
 	callbacks.OnDataUsage(
 		float64(receivedBytes)/1024.0, // downloaded
 		float64(sentBytes)/1024.0,     // uploaded

experiment/sniblocking/sniblocking_test.go

@@ -10,13 +10,97 @@ import (
 	"github.com/ooni/probe-engine/internal/mockable"
 	"github.com/ooni/probe-engine/internal/netxlogger"
 	"github.com/ooni/probe-engine/model"
+	"github.com/ooni/probe-engine/netx/modelx"
 )
 
 const (
 	softwareName    = "ooniprobe-example"
 	softwareVersion = "0.0.1"
 )
 
+func TestUnitTestKeysClassify(t *testing.T) {
+	asStringPtr := func(s string) *string {
+		return &s
+	}
+	t.Run("with tk.Target.Failure == nil", func(t *testing.T) {
+		tk := new(TestKeys)
+		if tk.classify() != "accessible_valid_hostname" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == connection_refused", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureConnectionRefused)
+		if tk.classify() != "anomaly_test_helper_blocked" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == dns_nxdomain_error", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureDNSNXDOMAINError)
+		if tk.classify() != "anomaly_test_helper_blocked" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == connection_reset", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureConnectionReset)
+		if tk.classify() != "blocked_tcpip_error" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == eof_error", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureEOFError)
+		if tk.classify() != "blocked_tcpip_error" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == ssl_invalid_hostname", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureSSLInvalidHostname)
+		if tk.classify() != "accessible_invalid_hostname" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == ssl_unknown_authority", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureSSLUnknownAuthority)
+		if tk.classify() != "anomaly_ssl_error" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == ssl_invalid_certificate", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureSSLInvalidCertificate)
+		if tk.classify() != "anomaly_ssl_error" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == generic_timeout_error #1", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureGenericTimeoutError)
+		if tk.classify() != "anomaly_timeout" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == generic_timeout_error #2", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr(modelx.FailureGenericTimeoutError)
+		tk.Control.Failure = asStringPtr(modelx.FailureGenericTimeoutError)
+		if tk.classify() != "anomaly_test_helper_blocked" {
+			t.Fatal("unexpected result")
+		}
+	})
+	t.Run("with tk.Target.Failure == unknown_failure", func(t *testing.T) {
+		tk := new(TestKeys)
+		tk.Target.Failure = asStringPtr("unknown_failure")
+		if tk.classify() != "anomaly_unexpected_failure" {
+			t.Fatal("unexpected result")
+		}
+	})
+}
+
 func TestUnitNewExperimentMeasurer(t *testing.T) {
 	measurer := NewExperimentMeasurer(Config{})
 	if measurer.ExperimentName() != "sni_blocking" {
@@ -105,7 +189,7 @@ func TestUnitMeasureoneCancelledContext(t *testing.T) {
 		"kernel.org",
 		"example.com:443",
 	)
-	if *result.Failure != "generic_timeout_error" {
+	if *result.Failure != modelx.FailureGenericTimeoutError {
 		t.Fatal("unexpected failure")
 	}
 	if result.SNI != "kernel.org" {
@@ -127,7 +211,7 @@ func TestUnitMeasureoneSuccess(t *testing.T) {
 		"kernel.org",
 		"example.com:443",
 	)
-	if *result.Failure != "ssl_invalid_hostname" {
+	if *result.Failure != modelx.FailureSSLInvalidHostname {
 		t.Fatal("unexpected failure")
 	}
 	if result.SNI != "kernel.org" {
@@ -159,7 +243,7 @@ func TestUnitMeasureonewithcacheWorks(t *testing.T) {
 		if result.Cached != expected {
 			t.Fatal("unexpected cached")
 		}
-		if *result.Failure != "ssl_invalid_hostname" {
+		if *result.Failure != modelx.FailureSSLInvalidHostname {
 			t.Fatal("unexpected failure")
 		}
 		if result.SNI != "kernel.org" {

internal/oonidatamodel/oonidatamodel_test.go

@@ -66,7 +66,7 @@ func TestUnitNewTCPConnectListFailure(t *testing.T) {
 		Connects: []*modelx.ConnectEvent{
 			&modelx.ConnectEvent{
 				RemoteAddress: "8.8.8.8:53",
-				Error:         errors.New("connection_reset"),
+				Error:         errors.New(modelx.FailureConnectionReset),
 			},
 		},
 	})
@@ -79,7 +79,7 @@ func TestUnitNewTCPConnectListFailure(t *testing.T) {
 	if out[0].Port != 53 {
 		t.Fatal("unexpected out[0].Port")
 	}
-	if *out[0].Status.Failure != "connection_reset" {
+	if *out[0].Status.Failure != modelx.FailureConnectionReset {
 		t.Fatal("unexpected out[0].Failure")
 	}
 	if out[0].Status.Success != false {
@@ -92,7 +92,7 @@ func TestUnitNewTCPConnectListInvalidInput(t *testing.T) {
 		Connects: []*modelx.ConnectEvent{
 			&modelx.ConnectEvent{
 				RemoteAddress: "8.8.8.8",
-				Error:         errors.New("connection_reset"),
+				Error:         errors.New(modelx.FailureConnectionReset),
 			},
 		},
 	})
@@ -105,7 +105,7 @@ func TestUnitNewTCPConnectListInvalidInput(t *testing.T) {
 	if out[0].Port != 0 {
 		t.Fatal("unexpected out[0].Port")
 	}
-	if *out[0].Status.Failure != "connection_reset" {
+	if *out[0].Status.Failure != modelx.FailureConnectionReset {
 		t.Fatal("unexpected out[0].Failure")
 	}
 	if out[0].Status.Success != false {
@@ -648,7 +648,7 @@ func TestUnitNewDNSQueriesListSuccess(t *testing.T) {
 				TransportNetwork: "system",
 			},
 			&modelx.ResolveDoneEvent{
-				Error:            errors.New("dns_nxdomain_error"),
+				Error:            errors.New(modelx.FailureDNSNXDOMAINError),
 				Hostname:         "dns.googlex",
 				TransportNetwork: "system",
 			},
@@ -767,7 +767,7 @@ func dnscheckbad(e DNSQueryEntry) error {
 	if e.Engine != "system" {
 		return errors.New("invalid engine")
 	}
-	if *e.Failure != "dns_nxdomain_error" {
+	if *e.Failure != modelx.FailureDNSNXDOMAINError {
 		return errors.New("invalid failure")
 	}
 	if e.Hostname != "dns.googlex" {

internal/oonitemplates/oonitemplates_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	goptlib "git.torproject.org/pluggable-transports/goptlib.git"
+	"github.com/ooni/probe-engine/netx/modelx"
 	"gitlab.com/yawning/obfs4.git/transports"
 	obfs4base "gitlab.com/yawning/obfs4.git/transports/base"
 )
@@ -37,7 +38,7 @@ func TestIntegrationDNSLookupCancellation(t *testing.T) {
 	if results.Error == nil {
 		t.Fatal("expected an error here")
 	}
-	if results.Error.Error() != "generic_timeout_error" {
+	if results.Error.Error() != modelx.FailureGenericTimeoutError {
 		t.Fatal("not the error we expected")
 	}
 	if len(results.Addresses) > 0 {
@@ -152,7 +153,7 @@ func TestIntegrationTLSConnectCancellation(t *testing.T) {
 	if results.Error == nil {
 		t.Fatal("expected an error here")
 	}
-	if results.Error.Error() != "generic_timeout_error" {
+	if results.Error.Error() != modelx.FailureGenericTimeoutError {
 		t.Fatal("not the error we expected")
 	}
 }

netx/http_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/ooni/probe-engine/netx"
+	"github.com/ooni/probe-engine/netx/modelx"
 )
 
 func dowithclient(t *testing.T, client *netx.HTTPClient) {
@@ -147,7 +148,7 @@ func TestIntegrationHTTPTransportTimeout(t *testing.T) {
 	if err == nil {
 		t.Fatal("expected an error here")
 	}
-	if !strings.HasSuffix(err.Error(), "generic_timeout_error") {
+	if !strings.HasSuffix(err.Error(), modelx.FailureGenericTimeoutError) {
 		t.Fatal("not the error we expected")
 	}
 	if resp != nil {

netx/internal/dialer/dnsdialer/dnsdialer_test.go

@@ -101,11 +101,11 @@ func TestReduceErrors(t *testing.T) {
 			Failure: "unknown_error: antani",
 		}
 		err3 := &modelx.ErrWrapper{
-			Failure: "connection_refused",
+			Failure: modelx.FailureConnectionRefused,
 		}
 		err4 := errors.New("mocked error #3")
 		result := reduceErrors([]error{err1, err2, err3, err4})
-		if result.Error() != "connection_refused" {
+		if result.Error() != modelx.FailureConnectionRefused {
 			t.Fatal("wrong result")
 		}
 	})

netx/internal/errwrapper/errwrapper.go

@@ -55,50 +55,50 @@ func toFailureString(err error) string {
 	}
 
 	if errors.Is(err, modelx.ErrDNSBogon) {
-		return "dns_bogon_error" // not in MK
+		return modelx.FailureDNSBogonError // not in MK
 	}
 
 	var x509HostnameError x509.HostnameError
 	if errors.As(err, &x509HostnameError) {
 		// Test case: https://wrong.host.badssl.com/
-		return "ssl_invalid_hostname"
+		return modelx.FailureSSLInvalidHostname
 	}
 	var x509UnknownAuthorityError x509.UnknownAuthorityError
 	if errors.As(err, &x509UnknownAuthorityError) {
 		// Test case: https://self-signed.badssl.com/. This error has
 		// never been among the ones returned by MK.
-		return "ssl_unknown_authority"
+		return modelx.FailureSSLUnknownAuthority
 	}
 	var x509CertificateInvalidError x509.CertificateInvalidError
 	if errors.As(err, &x509CertificateInvalidError) {
 		// Test case: https://expired.badssl.com/
-		return "ssl_invalid_certificate"
+		return modelx.FailureSSLInvalidCertificate
 	}
 
 	s := err.Error()
 	if strings.HasSuffix(s, "EOF") {
-		return "eof_error"
+		return modelx.FailureEOFError
 	}
 	if strings.HasSuffix(s, "connection refused") {
-		return "connection_refused"
+		return modelx.FailureConnectionRefused
 	}
 	if strings.HasSuffix(s, "connection reset by peer") {
-		return "connection_reset"
+		return modelx.FailureConnectionReset
 	}
 	if strings.HasSuffix(s, "context deadline exceeded") {
-		return "generic_timeout_error"
+		return modelx.FailureGenericTimeoutError
 	}
 	if strings.HasSuffix(s, "i/o timeout") {
-		return "generic_timeout_error"
+		return modelx.FailureGenericTimeoutError
 	}
 	if strings.HasSuffix(s, "TLS handshake timeout") {
-		return "generic_timeout_error"
+		return modelx.FailureGenericTimeoutError
 	}
 	if strings.HasSuffix(s, "no such host") {
 		// This is dns_lookup_error in MK but such error is used as a
 		// generic "hey, the lookup failed" error. Instead, this error
 		// that we return here is significantly more specific.
-		return "dns_nxdomain_error"
+		return modelx.FailureDNSNXDOMAINError
 	}
 
 	return fmt.Sprintf("unknown_failure: %s", s)