GDPR-friendly /setuid (prebid#513)

* Implemented a GDPR-aware /setuid endpoitn.

* Made the gdpr module return a special error type if the cause of the error was a bad consent string.

* Added docs.

* Better docs.

* Added metrics for syncs prevented by GDPR.

* Added a test for the new metrics code.

* Moved the TODO in pbs_light.go so that it overlaps with the other PR, and will cause conflcits.

* Fixed vet error.

docs/endpoints/setuid.md

@@ -10,8 +10,17 @@ This endpoint saves a UserID for a Bidder in the Cookie. Saved IDs will be recog
 ### Query Params
 
 - `bidder`: The FamilyName of the [Usersyncer](../../usersync/usersync.go) which is being synced.
-- `uid`: The ID which the Bidder uses to recognize this user.
+- `uid`: The ID which the Bidder uses to recognize this user. If undefined, the UID for `bidder` will be deleted.
+- `gdpr`: This should be `1` if GDPR is in effect, `0` if not, and undefined if the caller isn't sure
+- `gdpr_consent`: This is required if `gdpr` is one, and optional (but encouraged) otherwise. If present, it should be an [unpadded base64-URL](https://tools.ietf.org/html/rfc4648#page-7) encoded [Vendor Consent String](https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/Consent%20string%20and%20vendor%20list%20formats%20v1.1%20Final.md#vendor-consent-string-format-).
+
+If the `gdpr` and `gdpr_consent` params are included, this endpoint will _not_ write a cookie unless:
+
+1. The Vendor ID set by the Prebid Server host company has permission to save cookies for that user.
+2. The Prebid Server host company did not configure it to run with GDPR support.
+
+If in doubt, contact the company hosting Prebid Server and ask if they're GDPR-ready.
 
 ### Sample request
 
-`GET http://prebid.site.com/setuid?bidder=adnxs&uid=12345`
+`GET http://prebid.site.com/setuid?bidder=adnxs&uid=12345&gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw`

endpoints/setuid.go

@@ -1,19 +1,20 @@
 package endpoints
 
 import (
+	"context"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/julienschmidt/httprouter"
 	"github.com/prebid/prebid-server/analytics"
 	"github.com/prebid/prebid-server/config"
+	"github.com/prebid/prebid-server/gdpr"
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/pbsmetrics"
 	"github.com/prebid/prebid-server/usersync"
 )
 
-func NewSetUIDEndpoint(cfg config.HostCookie, pbsanalytics analytics.PBSAnalyticsModule, metrics pbsmetrics.MetricsEngine) httprouter.Handle {
+func NewSetUIDEndpoint(cfg config.HostCookie, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metrics pbsmetrics.MetricsEngine) httprouter.Handle {
 	cookieTTL := time.Duration(cfg.TTL) * 24 * time.Hour
 	return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		so := analytics.SetUIDObject{
@@ -31,17 +32,26 @@ func NewSetUIDEndpoint(cfg config.HostCookie, pbsanalytics analytics.PBSAnalytic
 			return
 		}
 
-		query := getRawQueryMap(r.URL.RawQuery)
-		bidder := query["bidder"]
+		query := r.URL.Query()
+		if shouldReturn, status, body := preventSyncsGDPR(query.Get("gdpr"), query.Get("gdpr_consent"), perms); shouldReturn {
+			w.WriteHeader(status)
+			w.Write([]byte(body))
+			metrics.RecordUserIDSet(pbsmetrics.UserLabels{Action: pbsmetrics.RequestActionGDPR})
+			so.Status = status
+			return
+		}
+
+		bidder := query.Get("bidder")
 		if bidder == "" {
 			w.WriteHeader(http.StatusBadRequest)
+			w.Write([]byte(`"bidder" query param is required`))
 			metrics.RecordUserIDSet(pbsmetrics.UserLabels{Action: pbsmetrics.RequestActionErr})
 			so.Status = http.StatusBadRequest
 			return
 		}
 		so.Bidder = bidder
 
-		uid := query["uid"]
+		uid := query.Get("uid")
 		so.UID = uid
 
 		var err error = nil
@@ -64,16 +74,32 @@ func NewSetUIDEndpoint(cfg config.HostCookie, pbsanalytics analytics.PBSAnalytic
 	})
 }
 
-func getRawQueryMap(query string) map[string]string {
-	m := make(map[string]string)
-	for _, kv := range strings.SplitN(query, "&", -1) {
-		if len(kv) == 0 {
-			continue
+func preventSyncsGDPR(gdprEnabled string, gdprConsent string, perms gdpr.Permissions) (bool, int, string) {
+	switch gdprEnabled {
+	case "0":
+		return false, 0, ""
+	case "1":
+		if gdprConsent == "" {
+			return true, http.StatusBadRequest, "gdpr_consent is required when gdpr=1"
 		}
-		pair := strings.SplitN(kv, "=", 2)
-		if len(pair) == 2 {
-			m[pair[0]] = pair[1]
+		fallthrough
+	case "":
+		if allowed, err := perms.HostCookiesAllowed(context.Background(), gdprConsent); err != nil {
+			if _, ok := err.(*gdpr.ErrorMalformedConsent); ok {
+				return true, http.StatusBadRequest, "gdpr_consent was invalid. " + err.Error()
+			} else {
+				// We can't really distinguish between requests that are for a new version of the global vendor list, and
+				// ones which are simply malformed (version number is much too large).
+				// Since we try to fetch new versions as requests come in for them, PBS *should* self-correct
+				// rather quickly, meaning that most of these will be malformed strings.
+				return true, http.StatusBadRequest, "No global vendor list was available to interpret this consent string. If this is a new, valid version, it should become available soon."
+			}
+		} else if !allowed {
+			return true, http.StatusOK, "The gdpr_consent string prevents cookies from being saved"
+		} else {
+			return false, 0, ""
 		}
+	default:
+		return true, http.StatusBadRequest, "the gdpr query param must be either 0 or 1. You gave " + gdprEnabled
 	}
-	return m
 }

endpoints/setuid_test.go

@@ -1,6 +1,8 @@
 package endpoints
 
 import (
+	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"regexp"
@@ -18,50 +20,95 @@ import (
 )
 
 func TestNormalSet(t *testing.T) {
-	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", nil))
+	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", nil), true, false)
 	assertIntsMatch(t, http.StatusOK, response.Code)
-
-	cookie := parseCookieString(t, response)
-	assertIntsMatch(t, 1, cookie.LiveSyncCount())
-	assertBoolsMatch(t, true, cookie.HasLiveSync("pubmatic"))
-	assertSyncValue(t, cookie, "pubmatic", "123")
+	assertHasSyncs(t, response, map[string]string{
+		"pubmatic": "123",
+	})
 }
 
 func TestUnset(t *testing.T) {
-	response := doRequest(makeRequest("/setuid?bidder=pubmatic", map[string]string{"pubmatic": "1234"}))
+	response := doRequest(makeRequest("/setuid?bidder=pubmatic", map[string]string{"pubmatic": "1234"}), true, false)
 	assertIntsMatch(t, http.StatusOK, response.Code)
-
-	cookie := parseCookieString(t, response)
-	assertIntsMatch(t, 0, cookie.LiveSyncCount())
+	assertHasSyncs(t, response, nil)
 }
 
 func TestMergeSet(t *testing.T) {
-	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", map[string]string{"rubicon": "def"}))
+	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", map[string]string{"rubicon": "def"}), true, false)
 	assertIntsMatch(t, http.StatusOK, response.Code)
+	assertHasSyncs(t, response, map[string]string{
+		"pubmatic": "123",
+		"rubicon":  "def",
+	})
+}
 
-	cookie := parseCookieString(t, response)
-	assertIntsMatch(t, 2, cookie.LiveSyncCount())
-	assertBoolsMatch(t, true, cookie.HasLiveSync("pubmatic"))
-	assertBoolsMatch(t, true, cookie.HasLiveSync("rubicon"))
-	assertSyncValue(t, cookie, "pubmatic", "123")
-	assertSyncValue(t, cookie, "rubicon", "def")
+func TestGDPRPrevention(t *testing.T) {
+	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", nil), false, false)
+	assertIntsMatch(t, http.StatusOK, response.Code)
+	assertStringsMatch(t, "The gdpr_consent string prevents cookies from being saved", response.Body.String())
+	assertNoCookie(t, response)
 }
 
-func TestNoBidder(t *testing.T) {
-	response := doRequest(makeRequest("/setuid?uid=123", nil))
+func TestGDPRConsentError(t *testing.T) {
+	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", nil), false, true)
 	assertIntsMatch(t, http.StatusBadRequest, response.Code)
+	assertStringsMatch(t, "No global vendor list was available to interpret this consent string. If this is a new, valid version, it should become available soon.", response.Body.String())
+	assertNoCookie(t, response)
+}
+
+func TestInapplicableGDPR(t *testing.T) {
+	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123&gdpr=0", nil), false, false)
+	assertIntsMatch(t, http.StatusOK, response.Code)
+	assertHasSyncs(t, response, map[string]string{
+		"pubmatic": "123",
+	})
+}
+
+func TestExplicitGDPRPrevention(t *testing.T) {
+	response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", nil), false, false)
+	assertIntsMatch(t, http.StatusOK, response.Code)
+	assertStringsMatch(t, "The gdpr_consent string prevents cookies from being saved", response.Body.String())
+	assertNoCookie(t, response)
+}
+
+func assertNoCookie(t *testing.T, resp *httptest.ResponseRecorder) {
+	t.Helper()
+	assertStringsMatch(t, "", resp.Header().Get("Set-Cookie"))
+}
+
+func TestBadRequests(t *testing.T) {
+	assertBadRequest(t, "/setuid?uid=123", `"bidder" query param is required`)
+	assertBadRequest(t, "/setuid?bidder=appnexus&uid=123&gdpr=2", "the gdpr query param must be either 0 or 1. You gave 2")
+	assertBadRequest(t, "/setuid?bidder=appnexus&uid=123&gdpr=1", "gdpr_consent is required when gdpr=1")
 }
 
 func TestOptedOut(t *testing.T) {
 	request := httptest.NewRequest("GET", "/setuid?bidder=pubmatic&uid=123", nil)
 	cookie := usersync.NewPBSCookie()
 	cookie.SetPreference(false)
 	addCookie(request, cookie)
-	response := doRequest(request)
+	response := doRequest(request, true, false)
 
 	assertIntsMatch(t, http.StatusUnauthorized, response.Code)
 }
 
+func assertHasSyncs(t *testing.T, resp *httptest.ResponseRecorder, syncs map[string]string) {
+	t.Helper()
+	cookie := parseCookieString(t, resp)
+	assertIntsMatch(t, len(syncs), cookie.LiveSyncCount())
+	for bidder, value := range syncs {
+		assertBoolsMatch(t, true, cookie.HasLiveSync(bidder))
+		assertSyncValue(t, cookie, bidder, value)
+	}
+}
+
+func assertBadRequest(t *testing.T, uri string, errMsg string) {
+	t.Helper()
+	response := doRequest(makeRequest(uri, nil), true, false)
+	assertIntsMatch(t, http.StatusBadRequest, response.Code)
+	assertStringsMatch(t, errMsg, response.Body.String())
+}
+
 func makeRequest(uri string, existingSyncs map[string]string) *http.Request {
 	request := httptest.NewRequest("GET", uri, nil)
 	if len(existingSyncs) > 0 {
@@ -74,9 +121,13 @@ func makeRequest(uri string, existingSyncs map[string]string) *http.Request {
 	return request
 }
 
-func doRequest(req *http.Request) *httptest.ResponseRecorder {
+func doRequest(req *http.Request, gdprAllowsHostCookies bool, gdprReturnsError bool) *httptest.ResponseRecorder {
+	perms := &mockPermsSetUID{
+		allowHost: gdprAllowsHostCookies,
+		errorHost: gdprReturnsError,
+	}
 	cfg := config.Configuration{}
-	endpoint := NewSetUIDEndpoint(cfg.HostCookie, analyticsConf.NewPBSAnalytics(&cfg.Analytics), pbsmetrics.NewMetricsEngine(&cfg, openrtb_ext.BidderList()))
+	endpoint := NewSetUIDEndpoint(cfg.HostCookie, perms, analyticsConf.NewPBSAnalytics(&cfg.Analytics), pbsmetrics.NewMetricsEngine(&cfg, openrtb_ext.BidderList()))
 	response := httptest.NewRecorder()
 	endpoint(response, req, nil)
 	return response
@@ -123,3 +174,20 @@ func assertSyncValue(t *testing.T, cookie *usersync.PBSCookie, family string, ex
 	got, _, _ := cookie.GetUID(family)
 	assertStringsMatch(t, expectedValue, got)
 }
+
+type mockPermsSetUID struct {
+	allowHost bool
+	errorHost bool
+}
+
+func (g *mockPermsSetUID) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
+	var err error
+	if g.errorHost {
+		err = errors.New("something went wrong")
+	}
+	return g.allowHost, err
+}
+
+func (g *mockPermsSetUID) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error) {
+	return false, nil
+}

gdpr/gdpr.go

@@ -10,9 +10,13 @@ import (
 
 type Permissions interface {
 	// Determines whether or not the host company is allowed to read/write cookies.
+	//
+	// If the consent string was nonsenical, the returned error will be an ErrorMalformedConsent.
 	HostCookiesAllowed(ctx context.Context, consent string) (bool, error)
 
 	// Determines whether or not the given bidder is allowed to user personal info for ad targeting.
+	//
+	// If the consent string was nonsenical, the returned error will be an ErrorMalformedConsent.
 	BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error)
 }
 
@@ -29,3 +33,14 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
 		fetchVendorList: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker),
 	}
 }
+
+// An ErrorMalformedConsent will be returned by the Permissions interface if
+// the consent string argument was the reason for the failure.
+type ErrorMalformedConsent struct {
+	consent string
+	cause   error
+}
+
+func (e *ErrorMalformedConsent) Error() string {
+	return "malformed consent string " + e.consent + ": " + e.cause.Error()
+}

gdpr/impl.go

@@ -28,12 +28,7 @@ func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, consent string
 		return p.cfg.UsersyncIfAmbiguous, nil
 	}
 
-	data, err := base64.RawURLEncoding.DecodeString(consent)
-	if err != nil {
-		return false, err
-	}
-
-	parsedConsent, err := vendorconsent.Parse([]byte(data))
+	parsedConsent, err := parseConsent(consent)
 	if err != nil {
 		return false, err
 	}
@@ -58,22 +53,36 @@ func (p *permissionsImpl) BidderSyncAllowed(ctx context.Context, bidder openrtb_
 		return false, nil
 	}
 
-	data, err := base64.RawURLEncoding.DecodeString(consent)
+	parsedConsent, err := parseConsent(consent)
 	if err != nil {
 		return false, err
 	}
 
-	parsedConsent, err := vendorconsent.Parse([]byte(data))
+	vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
 	if err != nil {
 		return false, err
 	}
 
-	vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
+	return hasPermissions(parsedConsent, vendorList, id, consentconstants.AdSelectionDeliveryReporting), nil
+}
+
+func parseConsent(consent string) (vendorconsent.VendorConsents, error) {
+	data, err := base64.RawURLEncoding.DecodeString(consent)
 	if err != nil {
-		return false, err
+		return nil, &ErrorMalformedConsent{
+			consent: consent,
+			cause:   err,
+		}
 	}
 
-	return hasPermissions(parsedConsent, vendorList, id, consentconstants.AdSelectionDeliveryReporting), nil
+	parsedConsent, err := vendorconsent.Parse([]byte(data))
+	if err != nil {
+		return nil, &ErrorMalformedConsent{
+			consent: consent,
+			cause:   err,
+		}
+	}
+	return parsedConsent, nil
 }
 
 func hasPermissions(consent vendorconsent.VendorConsents, vendorList vendorlist.VendorList, vendorID uint16, purpose consentconstants.Purpose) bool {

gdpr/impl_test.go

@@ -139,6 +139,19 @@ func TestProhibitedVendors(t *testing.T) {
 	assertBoolsEqual(t, false, allowSync)
 }
 
+func TestMalformedConsent(t *testing.T) {
+	perms := permissionsImpl{
+		cfg: config.GDPR{
+			HostVendorID: 2,
+		},
+		fetchVendorList: listFetcher(nil),
+	}
+
+	sync, err := perms.HostCookiesAllowed(context.Background(), "BON")
+	assertErr(t, err, true)
+	assertBoolsEqual(t, false, sync)
+}
+
 func parseVendorListData(t *testing.T, data string) vendorlist.VendorList {
 	t.Helper()
 	parsed, err := vendorlist.ParseEagerly([]byte(data))
@@ -170,11 +183,14 @@ func assertNilErr(t *testing.T, err error) {
 	}
 }
 
-func assertErr(t *testing.T, err error) {
+func assertErr(t *testing.T, err error, badConsent bool) {
 	t.Helper()
 	if err == nil {
 		t.Errorf("Expected error did not occur.")
+		return
 	}
+	_, isBadConsent := err.(*ErrorMalformedConsent)
+	assertBoolsEqual(t, badConsent, isBadConsent)
 }
 
 func assertBoolsEqual(t *testing.T, expected bool, actual bool) {