wip(httpext): refactor cleanURL

js/modules/k6/http/request_test.go

@@ -750,7 +750,7 @@ func TestRequestAndBatch(t *testing.T) {
 		t.Run("auth", func(t *testing.T) {
 			t.Run("basic", func(t *testing.T) {
 				url := sr("http://bob:pass@HTTPBIN_IP:HTTPBIN_PORT/basic-auth/bob/pass")
-				urlExpected := sr("http://HTTPBIN_IP:HTTPBIN_PORT/basic-auth/bob/pass")
+				urlExpected := sr("http://****:****@HTTPBIN_IP:HTTPBIN_PORT/basic-auth/bob/pass")
 
 				_, err := common.RunString(rt, fmt.Sprintf(`
 				let res = http.request("GET", "%s", null, {});
@@ -762,7 +762,7 @@ func TestRequestAndBatch(t *testing.T) {
 			t.Run("digest", func(t *testing.T) {
 				t.Run("success", func(t *testing.T) {
 					url := sr("http://bob:pass@HTTPBIN_IP:HTTPBIN_PORT/digest-auth/auth/bob/pass")
-					urlExpected := sr("http://HTTPBIN_IP:HTTPBIN_PORT/digest-auth/auth/bob/pass")
+					urlExpected := sr("http://****:****@HTTPBIN_IP:HTTPBIN_PORT/digest-auth/auth/bob/pass")
 
 					_, err := common.RunString(rt, fmt.Sprintf(`
 					let res = http.request("GET", "%s", null, { auth: "digest" });
@@ -1925,8 +1925,12 @@ func TestDigestAuthWithBody(t *testing.T) {
 	`, urlWithCreds))
 	require.NoError(t, err)
 
-	expectedURL := tb.Replacer.Replace("http://HTTPBIN_IP:HTTPBIN_PORT/digest-auth-with-post/auth/testuser/testpwd")
+	expectedURL := tb.Replacer.Replace(
+		"http://HTTPBIN_IP:HTTPBIN_PORT/digest-auth-with-post/auth/testuser/testpwd")
+	expectedName := tb.Replacer.Replace(
+		"http://****:****@HTTPBIN_IP:HTTPBIN_PORT/digest-auth-with-post/auth/testuser/testpwd")
+
 	sampleContainers := stats.GetBufferedSamples(samples)
-	assertRequestMetricsEmitted(t, sampleContainers[0:1], "POST", expectedURL, expectedURL, 401, "")
-	assertRequestMetricsEmitted(t, sampleContainers[1:2], "POST", expectedURL, expectedURL, 200, "")
+	assertRequestMetricsEmitted(t, sampleContainers[0:1], "POST", expectedURL, expectedName, 401, "")
+	assertRequestMetricsEmitted(t, sampleContainers[1:2], "POST", expectedURL, expectedName, 200, "")
 }

js/modules/k6/ws/ws.go

@@ -147,7 +147,7 @@ func (*WS) Connect(ctx context.Context, url string, args ...goja.Value) (*WSHTTP
 	}
 
 	if state.Options.SystemTags["url"] {
-		tags["url"] = url
+		tags["url"] = lib.CleanURL(url)
 	}
 	if state.Options.SystemTags["group"] {
 		tags["group"] = state.Group.Path

lib/netext/httpext/request.go

@@ -23,7 +23,6 @@ package httpext
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
@@ -67,19 +66,6 @@ func (u URL) GetURL() *url.URL {
 	return u.u
 }
 
-// Return a sanitized URL, clean of e.g. user credentials
-func cleanURL(urlString string) string {
-	idxEnd := strings.Index(urlString, "@")
-	if idxEnd == -1 {
-		return urlString
-	}
-	idxStart := strings.Index(urlString[:idxEnd], "/")
-	if idxStart != -1 {
-		return fmt.Sprintf("%s%s", urlString[0:idxStart+2], urlString[idxEnd+1:])
-	}
-	return urlString
-}
-
 // Request represent an http request
 type Request struct {
 	Method  string                          `json:"method"`
@@ -226,12 +212,12 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
 		tags["method"] = preq.Req.Method
 	}
 	if state.Options.SystemTags["url"] {
-		tags["url"] = cleanURL(preq.URL.URL)
+		tags["url"] = lib.CleanURL(preq.URL.URL)
 	}
 
 	// Only set the name system tag if the user didn't explicitly set it beforehand
 	if _, ok := tags["name"]; !ok && state.Options.SystemTags["name"] {
-		tags["name"] = cleanURL(preq.URL.Name)
+		tags["name"] = lib.CleanURL(preq.URL.Name)
 	}
 	if state.Options.SystemTags["group"] {
 		tags["group"] = state.Group.Path

lib/netext/httpext/request_test.go

@@ -82,26 +82,3 @@ func TestMakeRequestError(t *testing.T) {
 		require.Equal(t, err.Error(), "unknown compressionType CompressionType(13)")
 	})
 }
-
-func TestCleanURL(t *testing.T) {
-	testCases := []struct {
-		url      string
-		expected string
-	}{
-		{"https://example.com/", "https://example.com/"},
-		{"https://example.com/${}", "https://example.com/${}"},
-		{"https://user:pass@example.com/", "https://example.com/"},
-		{"https://user:pass@example.com/path?a=1&b=2", "https://example.com/path?a=1&b=2"},
-		{"https://user:pass@example.com/${}", "https://example.com/${}"},
-		{"@malformed/url", "@malformed/url"},
-		{"not a url", "not a url"},
-	}
-
-	for _, tc := range testCases {
-		tc := tc
-		t.Run(tc.url, func(t *testing.T) {
-			res := cleanURL(tc.url)
-			require.Equal(t, tc.expected, res)
-		})
-	}
-}

lib/netext/httpext/transport.go

@@ -111,7 +111,7 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe
 		}
 	} else {
 		if enabledTags["url"] {
-			tags["url"] = cleanURL(unfReq.request.URL.String())
+			tags["url"] = lib.CleanURL(unfReq.request.URL.String())
 		}
 		if enabledTags["status"] {
 			tags["status"] = strconv.Itoa(unfReq.response.StatusCode)

lib/util.go

@@ -21,6 +21,8 @@
 package lib
 
 import (
+	"fmt"
+	"net/url"
 	"strings"
 
 	"github.com/loadimpact/k6/lib/types"
@@ -64,6 +66,24 @@ func Clampf(val, min, max float64) float64 {
 	}
 }
 
+// CleanURL returns a sanitized URL, clean of e.g. user credentials
+func CleanURL(urlString string) string {
+	u, err := url.ParseRequestURI(urlString)
+	if err == nil && u.User != nil {
+		// it's a proper URL, mask credentials
+		old := fmt.Sprintf("%s@", u.User.String())
+		new := "****@"
+		_, passSet := u.User.Password()
+		if passSet {
+			new = fmt.Sprintf("****:%s", new)
+		}
+		clean := strings.Replace(u.String(), old, new, 1)
+		// Replace back any literal `${}` placeholders escaped by `u.String()`
+		return strings.Replace(clean, `$%7B%7D`, `${}`, -1)
+	}
+	return urlString
+}
+
 // Returns the maximum value of a and b.
 func Max(a, b int64) int64 {
 	if a > b {

lib/util_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/loadimpact/k6/lib/types"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestSumStages(t *testing.T) {
@@ -165,3 +166,27 @@ func TestMax(t *testing.T) {
 	assert.Equal(t, int64(100), Max(10, 100))
 	assert.Equal(t, int64(100), Max(100, 10))
 }
+
+func TestCleanURL(t *testing.T) {
+	testCases := []struct {
+		url      string
+		expected string
+	}{
+		{"https://example.com/", "https://example.com/"},
+		{"https://example.com/${}", "https://example.com/${}"},
+		{"https://user@example.com/", "https://****@example.com/"},
+		{"https://user:pass@example.com/", "https://****:****@example.com/"},
+		{"https://user:pass@example.com/path?a=1&b=2", "https://****:****@example.com/path?a=1&b=2"},
+		{"https://user:pass@example.com/${}/${}", "https://****:****@example.com/${}/${}"},
+		{"@malformed/url", "@malformed/url"},
+		{"not a url", "not a url"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.url, func(t *testing.T) {
+			res := CleanURL(tc.url)
+			require.Equal(t, tc.expected, res)
+		})
+	}
+}