GDPR-friendly /cookie_sync (#512)

* Added a basic gdpr module for consent management. No tests yet, and probably still lots of bugs

* Updated gopkg.lock.

* Moved the URL creation into a function.

* Made the URL-maker function a method arg, so it could be mocked out during testing.

* Started some test helper code.

* Added lots of tests.

* Updated the /cookie_sync endpoint to recognize GDPR.

* Patched up the gdpr module to respect configurable timeouts.

* Added two new tests for untested code.

* Making some cookie_sync code a bit more readable.

* Added a test for a missing case.

* Updated the /cookie_sync docs

* Code review comments.

config/config.go

@@ -3,6 +3,7 @@ package config
 import (
  "fmt"
  "strings"
+ "time"
 
  "github.com/spf13/viper"
 )
@@ -43,8 +44,22 @@ func (cfg *Configuration) validate() error {
 }
 
 type GDPR struct {
- HostVendorID int `mapstructure:"host_vendor_id"`
- UsersyncIfAmbiguous bool `mapstructure:"usersync_if_ambiguous"`
+ HostVendorID int `mapstructure:"host_vendor_id"`
+ UsersyncIfAmbiguous bool `mapstructure:"usersync_if_ambiguous"`
+ Timeouts GDPRTimeouts `mapstructure:"timeouts_ms"`
+}
+
+type GDPRTimeouts struct {
+ InitVendorlistFetch int `mapstructure:"init_vendorlist_fetches"`
+ ActiveVendorlistFetch int `mapstructure:"active_vendorlist_fetch"`
+}
+
+func (t *GDPRTimeouts) InitTimeout() time.Duration {
+ return time.Duration(t.InitVendorlistFetch) * time.Millisecond
+}
+
+func (t *GDPRTimeouts) ActiveTimeout() time.Duration {
+ return time.Duration(t.ActiveVendorlistFetch) * time.Millisecond
 }
 
 func (cfg *GDPR) validate() error {

docs/endpoints/cookieSync.md

@@ -11,10 +11,22 @@ must supply a JSON object to define the list of bidders that may need to be sync
 
 ```
 {
- "bidders": ["appnexus", "rubicon"]
+ "bidders": ["appnexus", "rubicon"],
+ "gdpr": 1,
+ "gdpr_consent": "BONV8oqONXwgmADACHENAO7pqzAAppY"
 }
 ```
 
+`bidders` is optional. If present, it limits the endpoint to return syncs for bidders defined in the list.
+
+`gdpr` is optional. It should be 1 if GDPR is in effect, 0 if not, and omitted if the caller is unsure.
+
+`gdpr_consent` is optional. If present, it should be an [unpadded base64-URL](https://tools.ietf.org/html/rfc4648#page-7) encoded [Vendor Consent String](https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/Consent%20string%20and%20vendor%20list%20formats%20v1.1%20Final.md#vendor-consent-string-format-).
+
+If `gdpr` is omitted, callers are still encouraged to send `gdpr_consent` if they have it.
+Depending on how the Prebid Server host company has configured their servers, they may or may not require it for cookie syncs.
+
+
 If the `bidders` field is an empty list, it will not supply any syncs. If the `bidders` field is omitted completely, it will attempt
 to sync all bidders.
 

endpoints/cookie_sync.go

@@ -1,112 +1,167 @@
 package endpoints
 
 import (
+ "context"
  "encoding/json"
+ "errors"
  "fmt"
+ "io/ioutil"
  "net/http"
 
- "github.com/golang/glog"
+ "github.com/buger/jsonparser"
+
  "github.com/julienschmidt/httprouter"
  "github.com/prebid/prebid-server/analytics"
  "github.com/prebid/prebid-server/config"
+ "github.com/prebid/prebid-server/gdpr"
  "github.com/prebid/prebid-server/openrtb_ext"
  "github.com/prebid/prebid-server/pbsmetrics"
  "github.com/prebid/prebid-server/usersync"
 )
 
-func NewCookieSyncEndpoint(syncers map[openrtb_ext.BidderName]usersync.Usersyncer, optOutCookie *config.Cookie, metrics pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule) httprouter.Handle {
- return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
- //CookieSyncObject makes a log of requests and responses to /cookie_sync endpoint
- co := analytics.CookieSyncObject{
- Status: http.StatusOK,
- Errors: make([]error, 0),
- BidderStatus: make([]*usersync.CookieSyncBidders, 0),
- }
+func NewCookieSyncEndpoint(syncers map[openrtb_ext.BidderName]usersync.Usersyncer, optOutCookie *config.Cookie, syncPermissions gdpr.Permissions, metrics pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule) httprouter.Handle {
+ deps := &cookieSyncDeps{
+ syncers: syncers,
+ optOutCookie: optOutCookie,
+ syncPermissions: syncPermissions,
+ metrics: metrics,
+ pbsAnalytics: pbsAnalytics,
+ }
+ return deps.Endpoint
+}
 
- defer pbsAnalytics.LogCookieSyncObject(&co)
+type cookieSyncDeps struct {
+ syncers map[openrtb_ext.BidderName]usersync.Usersyncer
+ optOutCookie *config.Cookie
+ syncPermissions gdpr.Permissions
+ metrics pbsmetrics.MetricsEngine
+ pbsAnalytics analytics.PBSAnalyticsModule
+}
 
- metrics.RecordCookieSync(pbsmetrics.Labels{})
- userSyncCookie := usersync.ParsePBSCookieFromRequest(r, optOutCookie)
- if !userSyncCookie.AllowSyncs() {
- http.Error(w, "User has opted out", http.StatusUnauthorized)
- co.Status = http.StatusUnauthorized
- co.Errors = append(co.Errors, fmt.Errorf("user has opted out"))
- return
- }
+func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ //CookieSyncObject makes a log of requests and responses to /cookie_sync endpoint
+ co := analytics.CookieSyncObject{
+ Status: http.StatusOK,
+ Errors: make([]error, 0),
+ BidderStatus: make([]*usersync.CookieSyncBidders, 0),
+ }
 
- defer r.Body.Close()
-
- csReq := &cookieSyncRequest{}
- csReqRaw := map[string]json.RawMessage{}
- err := json.NewDecoder(r.Body).Decode(&csReqRaw)
- if err != nil {
- if glog.V(2) {
- glog.Infof("Failed to parse /cookie_sync request body: %v", err)
- }
- co.Status = http.StatusBadRequest
- co.Errors = append(co.Errors, fmt.Errorf("JSON parse failed"))
- http.Error(w, "JSON parse failed", http.StatusBadRequest)
- return
- }
- biddersOmitted := true
- if biddersRaw, ok := csReqRaw["bidders"]; ok {
- biddersOmitted = false
- err := json.Unmarshal(biddersRaw, &csReq.Bidders)
- if err != nil {
- if glog.V(2) {
- glog.Infof("Failed to parse /cookie_sync request body (bidders list): %v", err)
- }
- co.Status = http.StatusBadRequest
- co.Errors = append(co.Errors, fmt.Errorf("JSON parse failed (bidders"))
- http.Error(w, "JSON parse failed (bidders)", http.StatusBadRequest)
- return
- }
- }
+ defer deps.pbsAnalytics.LogCookieSyncObject(&co)
 
- csResp := cookieSyncResponse{
- BidderStatus: make([]*usersync.CookieSyncBidders, 0, len(csReq.Bidders)),
- }
+ deps.metrics.RecordCookieSync(pbsmetrics.Labels{})
+ userSyncCookie := usersync.ParsePBSCookieFromRequest(r, deps.optOutCookie)
+ if !userSyncCookie.AllowSyncs() {
+ http.Error(w, "User has opted out", http.StatusUnauthorized)
+ co.Status = http.StatusUnauthorized
+ co.Errors = append(co.Errors, fmt.Errorf("user has opted out"))
+ return
+ }
 
- if userSyncCookie.LiveSyncCount() == 0 {
- csResp.Status = "no_cookie"
- } else {
- csResp.Status = "ok"
- }
+ defer r.Body.Close()
+ bodyBytes, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ co.Status = http.StatusBadRequest
+ co.Errors = append(co.Errors, errors.New("Failed to read request body"))
+ http.Error(w, "Failed to read request body", http.StatusBadRequest)
+ return
+ }
+ biddersJSON, err := parseBidders(bodyBytes)
+ if err != nil {
+ co.Status = http.StatusBadRequest
+ co.Errors = append(co.Errors, errors.New("Failed to check request.bidders in request body. Was your JSON well-formed?"))
+ http.Error(w, "Failed to check request.bidders in request body. Was your JSON well-formed?", http.StatusBadRequest)
+ return
+ }
 
-  // If at the end (After possibly reading stored bidder lists) there still are no bidders,
-  // and "bidders" is not found in the JSON, sync all bidders
- if len(csReq.Bidders) == 0 && biddersOmitted {
-  for bidder := range syncers {
-  csReq.Bidders = append(csReq.Bidders, string(bidder))
-  }
-  }
+ parsedReq := &cookieSyncRequest{}
+ if err := json.Unmarshal(bodyBytes, parsedReq); err != nil {
+ co.Status = http.StatusBadRequest
+ co.Errors = append(co.Errors, fmt.Errorf("JSON parsing failed: %v", err))
+ http.Error(w, "JSON parsing failed: "+err.Error(), http.StatusBadRequest)
+ return
+ }
 
- for _, bidder := range csReq.Bidders {
- if syncer, ok := syncers[openrtb_ext.BidderName(bidder)]; ok {
- if !userSyncCookie.HasLiveSync(syncer.FamilyName()) {
- b := usersync.CookieSyncBidders{
- BidderCode: bidder,
- NoCookie: true,
- UsersyncInfo: syncer.GetUsersyncInfo(),
- }
- csResp.BidderStatus = append(csResp.BidderStatus, &b)
- }
- }
+ if len(biddersJSON) == 0 {
+ parsedReq.Bidders = make([]string, 0, len(deps.syncers))
+ for bidder := range deps.syncers {
+ parsedReq.Bidders = append(parsedReq.Bidders, string(bidder))
  }
+ }
+
+ parsedReq.filterExistingSyncs(deps.syncers, userSyncCookie)
+ parsedReq.filterForGDPR(deps.syncPermissions)
 
- if len(csResp.BidderStatus) > 0 {
- co.BidderStatus = append(co.BidderStatus, csResp.BidderStatus...)
+ csResp := cookieSyncResponse{
+ Status: cookieSyncStatus(userSyncCookie.LiveSyncCount()),
+ BidderStatus: make([]*usersync.CookieSyncBidders, len(parsedReq.Bidders)),
+ }
+ for i := 0; i < len(parsedReq.Bidders); i++ {
+ bidder := parsedReq.Bidders[i]
+ csResp.BidderStatus[i] = &usersync.CookieSyncBidders{
+ BidderCode: bidder,
+ NoCookie: true,
+ UsersyncInfo: deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(),
  }
+ }
 
- enc := json.NewEncoder(w)
- enc.SetEscapeHTML(false)
- //enc.SetIndent("", " ")
- enc.Encode(csResp)
+ if len(csResp.BidderStatus) > 0 {
+ co.BidderStatus = append(co.BidderStatus, csResp.BidderStatus...)
  }
+
+ enc := json.NewEncoder(w)
+ enc.SetEscapeHTML(false)
+ enc.Encode(csResp)
+}
+
+func parseBidders(request []byte) ([]byte, error) {
+ value, valueType, _, err := jsonparser.Get(request, "bidders")
+ if err == nil && valueType != jsonparser.NotExist {
+ return value, nil
+ } else if err != jsonparser.KeyPathNotFoundError {
+ return nil, err
+ }
+ return nil, nil
+}
+
+func cookieSyncStatus(syncCount int) string {
+ if syncCount == 0 {
+ return "no_cookie"
+ }
+ return "ok"
 }
 
 type cookieSyncRequest struct {
  Bidders []string `json:"bidders"`
+ GDPR *int `json:"gdpr"`
+ Consent string `json:"gdpr_consent"`
+}
+
+func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderName]usersync.Usersyncer, cookie *usersync.PBSCookie) {
+ for i := 0; i < len(req.Bidders); i++ {
+ thisBidder := req.Bidders[i]
+ if syncer, isValid := valid[openrtb_ext.BidderName(thisBidder)]; !isValid || cookie.HasLiveSync(syncer.FamilyName()) {
+ req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...)
+ i--
+ }
+ }
+}
+
+func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) {
+ if req.GDPR != nil && *req.GDPR == 0 {
+ return
+ }
+
+ if allowSync, err := permissions.HostCookiesAllowed(context.Background(), req.Consent); err != nil || !allowSync {
+ req.Bidders = nil
+ return
+ }
+
+ for i := 0; i < len(req.Bidders); i++ {
+ if allowSync, err := permissions.BidderSyncAllowed(context.Background(), openrtb_ext.BidderName(req.Bidders[i]), req.Consent); err != nil || !allowSync {
+ req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...)
+ i--
+ }
+ }
 }
 
 type cookieSyncResponse struct {