GDPR-friendly /setuid (prebid#513)

* Implemented a GDPR-aware /setuid endpoitn.

* Made the gdpr module return a special error type if the cause of the error was a bad consent string.

* Added docs.

* Better docs.

* Added metrics for syncs prevented by GDPR.

* Added a test for the new metrics code.

* Moved the TODO in pbs_light.go so that it overlaps with the other PR, and will cause conflcits.

* Fixed vet error.

docs/endpoints/setuid.md

@@ -10,8 +10,17 @@ This endpoint saves a UserID for a Bidder in the Cookie. Saved IDs will be recog
 ### Query Params
 
 - `bidder`: The FamilyName of the [Usersyncer](../../usersync/usersync.go) which is being synced.
-- `uid`: The ID which the Bidder uses to recognize this user.
+- `uid`: The ID which the Bidder uses to recognize this user. If undefined, the UID for `bidder` will be deleted.
+- `gdpr`: This should be `1` if GDPR is in effect, `0` if not, and undefined if the caller isn't sure
+- `gdpr_consent`: This is required if `gdpr` is one, and optional (but encouraged) otherwise. If present, it should be an [unpadded base64-URL](https://tools.ietf.org/html/rfc4648#page-7) encoded [Vendor Consent String](https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/Consent%20string%20and%20vendor%20list%20formats%20v1.1%20Final.md#vendor-consent-string-format-).
+
+If the `gdpr` and `gdpr_consent` params are included, this endpoint will _not_ write a cookie unless:
+
+1. The Vendor ID set by the Prebid Server host company has permission to save cookies for that user.
+2. The Prebid Server host company did not configure it to run with GDPR support.
+
+If in doubt, contact the company hosting Prebid Server and ask if they're GDPR-ready.
 
 ### Sample request
 
-`GET http://prebid.site.com/setuid?bidder=adnxs&uid=12345`
+`GET http://prebid.site.com/setuid?bidder=adnxs&uid=12345&gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw`

endpoints/setuid.go

@@ -1,19 +1,20 @@
 package endpoints
 
 import (
+ "context"
  "net/http"
- "strings"
  "time"
 
  "github.com/julienschmidt/httprouter"
  "github.com/prebid/prebid-server/analytics"
  "github.com/prebid/prebid-server/config"
+ "github.com/prebid/prebid-server/gdpr"
  "github.com/prebid/prebid-server/openrtb_ext"
  "github.com/prebid/prebid-server/pbsmetrics"
  "github.com/prebid/prebid-server/usersync"
 )
 
-func NewSetUIDEndpoint(cfg config.HostCookie, pbsanalytics analytics.PBSAnalyticsModule, metrics pbsmetrics.MetricsEngine) httprouter.Handle {
+func NewSetUIDEndpoint(cfg config.HostCookie, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metrics pbsmetrics.MetricsEngine) httprouter.Handle {
  cookieTTL := time.Duration(cfg.TTL) * 24 * time.Hour
  return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
  so := analytics.SetUIDObject{
@@ -31,17 +32,26 @@ func NewSetUIDEndpoint(cfg config.HostCookie, pbsanalytics analytics.PBSAnalytic
  return
  }
 
- query := getRawQueryMap(r.URL.RawQuery)
- bidder := query["bidder"]
+ query := r.URL.Query()
+ if shouldReturn, status, body := preventSyncsGDPR(query.Get("gdpr"), query.Get("gdpr_consent"), perms); shouldReturn {
+ w.WriteHeader(status)
+ w.Write([]byte(body))
+ metrics.RecordUserIDSet(pbsmetrics.UserLabels{Action: pbsmetrics.RequestActionGDPR})
+ so.Status = status
+ return
+ }
+
+ bidder := query.Get("bidder")
  if bidder == "" {
  w.WriteHeader(http.StatusBadRequest)
+ w.Write([]byte(`"bidder" query param is required`))
  metrics.RecordUserIDSet(pbsmetrics.UserLabels{Action: pbsmetrics.RequestActionErr})
  so.Status = http.StatusBadRequest
  return
  }
  so.Bidder = bidder
 
- uid := query["uid"]
+ uid := query.Get("uid")
  so.UID = uid
 
  var err error = nil
@@ -64,16 +74,32 @@ func NewSetUIDEndpoint(cfg config.HostCookie, pbsanalytics analytics.PBSAnalytic
  })
 }
 
-func getRawQueryMap(query string) map[string]string {
- m := make(map[string]string)
- for _, kv := range strings.SplitN(query, "&", -1) {
- if len(kv) == 0 {
- continue
+func preventSyncsGDPR(gdprEnabled string, gdprConsent string, perms gdpr.Permissions) (bool, int, string) {
+ switch gdprEnabled {
+ case "0":
+ return false, 0, ""
+ case "1":
+ if gdprConsent == "" {
+ return true, http.StatusBadRequest, "gdpr_consent is required when gdpr=1"
  }
- pair := strings.SplitN(kv, "=", 2)
- if len(pair) == 2 {
- m[pair[0]] = pair[1]
+ fallthrough
+ case "":
+ if allowed, err := perms.HostCookiesAllowed(context.Background(), gdprConsent); err != nil {
+ if _, ok := err.(*gdpr.ErrorMalformedConsent); ok {
+ return true, http.StatusBadRequest, "gdpr_consent was invalid. " + err.Error()
+ } else {
+ // We can't really distinguish between requests that are for a new version of the global vendor list, and
+ // ones which are simply malformed (version number is much too large).
+ // Since we try to fetch new versions as requests come in for them, PBS *should* self-correct
+ // rather quickly, meaning that most of these will be malformed strings.
+ return true, http.StatusBadRequest, "No global vendor list was available to interpret this consent string. If this is a new, valid version, it should become available soon."
+ }
+ } else if !allowed {
+ return true, http.StatusOK, "The gdpr_consent string prevents cookies from being saved"
+ } else {
+ return false, 0, ""
  }
+ default:
+ return true, http.StatusBadRequest, "the gdpr query param must be either 0 or 1. You gave " + gdprEnabled
  }
- return m
 }

endpoints/setuid_test.go

@@ -1,6 +1,8 @@
 package endpoints
 
 import (
+ "context"
+ "errors"
  "net/http"
  "net/http/httptest"
  "regexp"
@@ -18,50 +20,95 @@ import (
 )
 
 func TestNormalSet(t *testing.T) {
- response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", nil))
+ response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", nil), true, false)
  assertIntsMatch(t, http.StatusOK, response.Code)
-
- cookie := parseCookieString(t, response)
- assertIntsMatch(t, 1, cookie.LiveSyncCount())
- assertBoolsMatch(t, true, cookie.HasLiveSync("pubmatic"))
- assertSyncValue(t, cookie, "pubmatic", "123")
+ assertHasSyncs(t, response, map[string]string{
+ "pubmatic": "123",
+ })
 }
 
 func TestUnset(t *testing.T) {
- response := doRequest(makeRequest("/setuid?bidder=pubmatic", map[string]string{"pubmatic": "1234"}))
+ response := doRequest(makeRequest("/setuid?bidder=pubmatic", map[string]string{"pubmatic": "1234"}), true, false)
  assertIntsMatch(t, http.StatusOK, response.Code)
-
- cookie := parseCookieString(t, response)
- assertIntsMatch(t, 0, cookie.LiveSyncCount())
+ assertHasSyncs(t, response, nil)
 }
 
 func TestMergeSet(t *testing.T) {
- response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", map[string]string{"rubicon": "def"}))
+ response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", map[string]string{"rubicon": "def"}), true, false)
  assertIntsMatch(t, http.StatusOK, response.Code)
+ assertHasSyncs(t, response, map[string]string{
+ "pubmatic": "123",
+ "rubicon": "def",
+ })
+}
 
- cookie := parseCookieString(t, response)
- assertIntsMatch(t, 2, cookie.LiveSyncCount())
- assertBoolsMatch(t, true, cookie.HasLiveSync("pubmatic"))
- assertBoolsMatch(t, true, cookie.HasLiveSync("rubicon"))
- assertSyncValue(t, cookie, "pubmatic", "123")
- assertSyncValue(t, cookie, "rubicon", "def")
+func TestGDPRPrevention(t *testing.T) {
+ response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123", nil), false, false)
+ assertIntsMatch(t, http.StatusOK, response.Code)
+ assertStringsMatch(t, "The gdpr_consent string prevents cookies from being saved", response.Body.String())
+ assertNoCookie(t, response)
 }
 
-func TestNoBidder(t *testing.T) {
- response := doRequest(makeRequest("/setuid?uid=123", nil))
+func TestGDPRConsentError(t *testing.T) {
+ response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", nil), false, true)
  assertIntsMatch(t, http.StatusBadRequest, response.Code)
+ assertStringsMatch(t, "No global vendor list was available to interpret this consent string. If this is a new, valid version, it should become available soon.", response.Body.String())
+ assertNoCookie(t, response)
+}
+
+func TestInapplicableGDPR(t *testing.T) {
+ response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123&gdpr=0", nil), false, false)
+ assertIntsMatch(t, http.StatusOK, response.Code)
+ assertHasSyncs(t, response, map[string]string{
+ "pubmatic": "123",
+ })
+}
+
+func TestExplicitGDPRPrevention(t *testing.T) {
+ response := doRequest(makeRequest("/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", nil), false, false)
+ assertIntsMatch(t, http.StatusOK, response.Code)
+ assertStringsMatch(t, "The gdpr_consent string prevents cookies from being saved", response.Body.String())
+ assertNoCookie(t, response)
+}
+
+func assertNoCookie(t *testing.T, resp *httptest.ResponseRecorder) {
+ t.Helper()
+ assertStringsMatch(t, "", resp.Header().Get("Set-Cookie"))
+}
+
+func TestBadRequests(t *testing.T) {
+ assertBadRequest(t, "/setuid?uid=123", `"bidder" query param is required`)
+ assertBadRequest(t, "/setuid?bidder=appnexus&uid=123&gdpr=2", "the gdpr query param must be either 0 or 1. You gave 2")
+ assertBadRequest(t, "/setuid?bidder=appnexus&uid=123&gdpr=1", "gdpr_consent is required when gdpr=1")
 }
 
 func TestOptedOut(t *testing.T) {
  request := httptest.NewRequest("GET", "/setuid?bidder=pubmatic&uid=123", nil)
  cookie := usersync.NewPBSCookie()
  cookie.SetPreference(false)
  addCookie(request, cookie)
- response := doRequest(request)
+ response := doRequest(request, true, false)
 
  assertIntsMatch(t, http.StatusUnauthorized, response.Code)
 }
 
+func assertHasSyncs(t *testing.T, resp *httptest.ResponseRecorder, syncs map[string]string) {
+ t.Helper()
+ cookie := parseCookieString(t, resp)
+ assertIntsMatch(t, len(syncs), cookie.LiveSyncCount())
+ for bidder, value := range syncs {
+ assertBoolsMatch(t, true, cookie.HasLiveSync(bidder))
+ assertSyncValue(t, cookie, bidder, value)
+ }
+}
+
+func assertBadRequest(t *testing.T, uri string, errMsg string) {
+ t.Helper()
+ response := doRequest(makeRequest(uri, nil), true, false)
+ assertIntsMatch(t, http.StatusBadRequest, response.Code)
+ assertStringsMatch(t, errMsg, response.Body.String())
+}
+
 func makeRequest(uri string, existingSyncs map[string]string) *http.Request {
  request := httptest.NewRequest("GET", uri, nil)
  if len(existingSyncs) > 0 {
@@ -74,9 +121,13 @@ func makeRequest(uri string, existingSyncs map[string]string) *http.Request {
  return request
 }
 
-func doRequest(req *http.Request) *httptest.ResponseRecorder {
+func doRequest(req *http.Request, gdprAllowsHostCookies bool, gdprReturnsError bool) *httptest.ResponseRecorder {
+ perms := &mockPermsSetUID{
+ allowHost: gdprAllowsHostCookies,
+ errorHost: gdprReturnsError,
+ }
  cfg := config.Configuration{}
- endpoint := NewSetUIDEndpoint(cfg.HostCookie, analyticsConf.NewPBSAnalytics(&cfg.Analytics), pbsmetrics.NewMetricsEngine(&cfg, openrtb_ext.BidderList()))
+ endpoint := NewSetUIDEndpoint(cfg.HostCookie, perms, analyticsConf.NewPBSAnalytics(&cfg.Analytics), pbsmetrics.NewMetricsEngine(&cfg, openrtb_ext.BidderList()))
  response := httptest.NewRecorder()
  endpoint(response, req, nil)
  return response
@@ -123,3 +174,20 @@ func assertSyncValue(t *testing.T, cookie *usersync.PBSCookie, family string, ex
  got, _, _ := cookie.GetUID(family)
  assertStringsMatch(t, expectedValue, got)
 }
+
+type mockPermsSetUID struct {
+ allowHost bool
+ errorHost bool
+}
+
+func (g *mockPermsSetUID) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
+ var err error
+ if g.errorHost {
+ err = errors.New("something went wrong")
+ }
+ return g.allowHost, err
+}
+
+func (g *mockPermsSetUID) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error) {
+ return false, nil
+}

gdpr/gdpr.go

@@ -10,9 +10,13 @@ import (
 
 type Permissions interface {
  // Determines whether or not the host company is allowed to read/write cookies.
+ //
+ // If the consent string was nonsenical, the returned error will be an ErrorMalformedConsent.
  HostCookiesAllowed(ctx context.Context, consent string) (bool, error)
 
  // Determines whether or not the given bidder is allowed to user personal info for ad targeting.
+ //
+ // If the consent string was nonsenical, the returned error will be an ErrorMalformedConsent.
  BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error)
 }
 
@@ -29,3 +33,14 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
  fetchVendorList: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker),
  }
 }
+
+// An ErrorMalformedConsent will be returned by the Permissions interface if
+// the consent string argument was the reason for the failure.
+type ErrorMalformedConsent struct {
+ consent string
+ cause error
+}
+
+func (e *ErrorMalformedConsent) Error() string {
+ return "malformed consent string " + e.consent + ": " + e.cause.Error()
+}

gdpr/impl.go

@@ -28,12 +28,7 @@ func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, consent string
  return p.cfg.UsersyncIfAmbiguous, nil
  }
 
- data, err := base64.RawURLEncoding.DecodeString(consent)
- if err != nil {
- return false, err
- }
-
- parsedConsent, err := vendorconsent.Parse([]byte(data))
+ parsedConsent, err := parseConsent(consent)
  if err != nil {
  return false, err
  }
@@ -58,22 +53,36 @@ func (p *permissionsImpl) BidderSyncAllowed(ctx context.Context, bidder openrtb_
  return false, nil
  }
 
- data, err := base64.RawURLEncoding.DecodeString(consent)
+ parsedConsent, err := parseConsent(consent)
  if err != nil {
  return false, err
  }
 
- parsedConsent, err := vendorconsent.Parse([]byte(data))
+ vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
  if err != nil {
  return false, err
  }
 
- vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
+ return hasPermissions(parsedConsent, vendorList, id, consentconstants.AdSelectionDeliveryReporting), nil
+}
+
+func parseConsent(consent string) (vendorconsent.VendorConsents, error) {
+ data, err := base64.RawURLEncoding.DecodeString(consent)
  if err != nil {
- return false, err
+ return nil, &ErrorMalformedConsent{
+ consent: consent,
+ cause: err,
+ }
  }
 
- return hasPermissions(parsedConsent, vendorList, id, consentconstants.AdSelectionDeliveryReporting), nil
+ parsedConsent, err := vendorconsent.Parse([]byte(data))
+ if err != nil {
+ return nil, &ErrorMalformedConsent{
+ consent: consent,
+ cause: err,
+ }
+ }
+ return parsedConsent, nil
 }
 
 func hasPermissions(consent vendorconsent.VendorConsents, vendorList vendorlist.VendorList, vendorID uint16, purpose consentconstants.Purpose) bool {

gdpr/impl_test.go

@@ -139,6 +139,19 @@ func TestProhibitedVendors(t *testing.T) {
  assertBoolsEqual(t, false, allowSync)
 }
 
+func TestMalformedConsent(t *testing.T) {
+ perms := permissionsImpl{
+ cfg: config.GDPR{
+ HostVendorID: 2,
+ },
+ fetchVendorList: listFetcher(nil),
+ }
+
+ sync, err := perms.HostCookiesAllowed(context.Background(), "BON")
+ assertErr(t, err, true)
+ assertBoolsEqual(t, false, sync)
+}
+
 func parseVendorListData(t *testing.T, data string) vendorlist.VendorList {
  t.Helper()
  parsed, err := vendorlist.ParseEagerly([]byte(data))
@@ -170,11 +183,14 @@ func assertNilErr(t *testing.T, err error) {
  }
 }
 
-func assertErr(t *testing.T, err error) {
+func assertErr(t *testing.T, err error, badConsent bool) {
  t.Helper()
  if err == nil {
  t.Errorf("Expected error did not occur.")
+ return
  }
+ _, isBadConsent := err.(*ErrorMalformedConsent)
+ assertBoolsEqual(t, badConsent, isBadConsent)
 }
 
 func assertBoolsEqual(t *testing.T, expected bool, actual bool) {