urlgetter: add tcp_connect to test_keys

This was planned inside #656.

It actually also helps with #646.

experiment/urlgetter/getter.go

@@ -40,6 +40,9 @@ func (g Getter) Get(ctx context.Context) (TestKeys, error) {
 	tk.Requests = append(
 		tk.Requests, archival.NewRequestList(begin, events)...,
 	)
+	tk.TCPConnect = append(
+		tk.TCPConnect, archival.NewTCPConnectList(begin, events)...,
+	)
 	tk.TLSHandshakes = append(
 		tk.TLSHandshakes, archival.NewTLSHandshakesList(begin, events)...,
 	)

experiment/urlgetter/getter_test.go

@@ -48,6 +48,9 @@ func TestGetterWithCancelledContextVanilla(t *testing.T) {
 	if len(tk.Queries) != 0 {
 		t.Fatal("not the Queries we expected")
 	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
 	if len(tk.Requests) != 1 {
 		t.Fatal("not the Requests we expected")
 	}
@@ -106,6 +109,9 @@ func TestGetterWithCancelledContextNoFollowRedirects(t *testing.T) {
 	if len(tk.Queries) != 0 {
 		t.Fatal("not the Queries we expected")
 	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
 	if len(tk.Requests) != 1 {
 		t.Fatal("not the Requests we expected")
 	}
@@ -154,6 +160,9 @@ func TestGetterWithCancelledContextCannotStartTunnel(t *testing.T) {
 	if len(tk.Queries) != 0 {
 		t.Fatal("not the Queries we expected")
 	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
 	if len(tk.Requests) != 0 {
 		t.Fatal("not the Requests we expected")
 	}
@@ -210,6 +219,9 @@ func TestGetterWithCancelledContextWithTunnel(t *testing.T) {
 	if len(tk.Queries) != 0 {
 		t.Fatal("not the Queries we expected")
 	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
 	if len(tk.Requests) != 1 {
 		t.Fatal("not the Requests we expected")
 	}
@@ -259,6 +271,9 @@ func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) {
 	if len(tk.Queries) != 0 {
 		t.Fatal("not the Queries we expected")
 	}
+	if len(tk.TCPConnect) != 0 {
+		t.Fatal("not the TCPConnect we expected")
+	}
 	if len(tk.Requests) != 0 {
 		t.Fatal("not the Requests we expected")
 	}
@@ -272,3 +287,113 @@ func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) {
 		t.Fatal("not the Tunnel we expected")
 	}
 }
+
+func TestGetterIntegration(t *testing.T) {
+	ctx := context.Background()
+	g := urlgetter.Getter{
+		Config: urlgetter.Config{
+			NoFollowRedirects: true, // reduce number of events
+		},
+		Session: &mockable.ExperimentSession{},
+		Target:  "https://www.google.com",
+	}
+	tk, err := g.Get(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tk.Agent != "agent" {
+		t.Fatal("not the Agent we expected")
+	}
+	if tk.BootstrapTime != 0 {
+		t.Fatal("not the BootstrapTime we expected")
+	}
+	if tk.Failure != nil {
+		t.Fatal("not the Failure we expected")
+	}
+	var (
+		httpTransactionStart     bool
+		httpRequestMetadata      bool
+		resolveStart             bool
+		resolveDone              bool
+		connect                  bool
+		tlsHandshakeStart        bool
+		tlsHandshakeDone         bool
+		httpWroteHeaders         bool
+		httpWroteRequest         bool
+		httpFirstResponseByte    bool
+		httpResponseMetadata     bool
+		httpResponseBodySnapshot bool
+		httpTransactionDone      bool
+	)
+	for _, ev := range tk.NetworkEvents {
+		switch ev.Operation {
+		case "http_transaction_start":
+			httpTransactionStart = true
+		case "http_request_metadata":
+			httpRequestMetadata = true
+		case "resolve_start":
+			resolveStart = true
+		case "resolve_done":
+			resolveDone = true
+		case "connect":
+			connect = true
+		case "tls_handshake_start":
+			tlsHandshakeStart = true
+		case "tls_handshake_done":
+			tlsHandshakeDone = true
+		case "http_wrote_headers":
+			httpWroteHeaders = true
+		case "http_wrote_request":
+			httpWroteRequest = true
+		case "http_first_response_byte":
+			httpFirstResponseByte = true
+		case "http_response_metadata":
+			httpResponseMetadata = true
+		case "http_response_body_snapshot":
+			httpResponseBodySnapshot = true
+		case "http_transaction_done":
+			httpTransactionDone = true
+		}
+	}
+	ok := true
+	ok = ok && httpTransactionStart
+	ok = ok && httpRequestMetadata
+	ok = ok && resolveStart
+	ok = ok && resolveDone
+	ok = ok && connect
+	ok = ok && tlsHandshakeStart
+	ok = ok && tlsHandshakeDone
+	ok = ok && httpWroteHeaders
+	ok = ok && httpWroteRequest
+	ok = ok && httpFirstResponseByte
+	ok = ok && httpResponseMetadata
+	ok = ok && httpResponseBodySnapshot
+	ok = ok && httpTransactionDone
+	if !ok {
+		t.Fatal("not the NetworkEvents we expected")
+	}
+	if len(tk.Queries) != 2 {
+		t.Fatal("not the Queries we expected")
+	}
+	if len(tk.TCPConnect) != 1 {
+		t.Fatal("not the TCPConnect we expected")
+	}
+	if len(tk.Requests) != 1 {
+		t.Fatal("not the Requests we expected")
+	}
+	if tk.Requests[0].Request.Method != "GET" {
+		t.Fatal("not the Method we expected")
+	}
+	if tk.Requests[0].Request.URL != "https://www.google.com" {
+		t.Fatal("not the URL we expected")
+	}
+	if tk.SOCKSProxy != "" {
+		t.Fatal("not the SOCKSProxy we expected")
+	}
+	if len(tk.TLSHandshakes) != 1 {
+		t.Fatal("not the TLSHandshakes we expected")
+	}
+	if tk.Tunnel != "" {
+		t.Fatal("not the Tunnel we expected")
+	}
+}

experiment/urlgetter/urlgetter.go

@@ -30,16 +30,17 @@ type Config struct {
 
 // TestKeys contains the experiment's result.
 type TestKeys struct {
-	Agent         string                   `json:"agent"`
-	BootstrapTime float64                  `json:"bootstrap_time,omitempty"`
-	DNSCache      []string                 `json:"dns_cache,omitempty"`
-	Failure       *string                  `json:"failure"`
-	NetworkEvents []archival.NetworkEvent  `json:"network_events"`
-	Queries       []archival.DNSQueryEntry `json:"queries"`
-	Requests      []archival.RequestEntry  `json:"requests"`
-	SOCKSProxy    string                   `json:"socksproxy,omitempty"`
-	TLSHandshakes []archival.TLSHandshake  `json:"tls_handshakes"`
-	Tunnel        string                   `json:"tunnel,omitempty"`
+	Agent         string                     `json:"agent"`
+	BootstrapTime float64                    `json:"bootstrap_time,omitempty"`
+	DNSCache      []string                   `json:"dns_cache,omitempty"`
+	Failure       *string                    `json:"failure"`
+	NetworkEvents []archival.NetworkEvent    `json:"network_events"`
+	Queries       []archival.DNSQueryEntry   `json:"queries"`
+	Requests      []archival.RequestEntry    `json:"requests"`
+	TCPConnect    []archival.TCPConnectEntry `json:"tcp_connect"`
+	SOCKSProxy    string                     `json:"socksproxy,omitempty"`
+	TLSHandshakes []archival.TLSHandshake    `json:"tls_handshakes"`
+	Tunnel        string                     `json:"tunnel,omitempty"`
 }
 
 // RegisterExtensions registers the extensions used by the urlgetter