GDPR-friendly /cookie_sync

config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/spf13/viper"
 )
@@ -43,8 +44,22 @@ func (cfg *Configuration) validate() error {
 }
 
 type GDPR struct {
-	HostVendorID        int  `mapstructure:"host_vendor_id"`
-	UsersyncIfAmbiguous bool `mapstructure:"usersync_if_ambiguous"`
+	HostVendorID        int          `mapstructure:"host_vendor_id"`
+	UsersyncIfAmbiguous bool         `mapstructure:"usersync_if_ambiguous"`
+	Timeouts            GDPRTimeouts `mapstructure:"timeouts_ms"`
+}
+
+type GDPRTimeouts struct {
+	InitVendorlistFetch   int `mapstructure:"init_vendorlist_fetches"`
+	ActiveVendorlistFetch int `mapstructure:"active_vendorlist_fetch"`
+}
+
+func (t *GDPRTimeouts) InitTimeout() time.Duration {
+	return time.Duration(t.InitVendorlistFetch) * time.Millisecond
+}
+
+func (t *GDPRTimeouts) ActiveTimeout() time.Duration {
+	return time.Duration(t.ActiveVendorlistFetch) * time.Millisecond
 }
 
 func (cfg *GDPR) validate() error {

docs/endpoints/cookieSync.md

@@ -11,10 +11,22 @@ must supply a JSON object to define the list of bidders that may need to be sync
 
 ```
 {
-    "bidders": ["appnexus", "rubicon"]
+    "bidders": ["appnexus", "rubicon"],
+    "gdpr": 1,
+    "gdpr_consent": "BONV8oqONXwgmADACHENAO7pqzAAppY"
 }
 ```
 
+`bidders` is optional. If present, it limits the endpoint to return syncs for bidders defined in the list.
+
+`gdpr` is optional. It should be 1 if GDPR is in effect, 0 if not, and omitted if the caller is unsure.
+
+`gdpr_consent` is optional. If present, it should be an [unpadded base64-URL](https://tools.ietf.org/html/rfc4648#page-7) encoded [Vendor Consent String](https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/Consent%20string%20and%20vendor%20list%20formats%20v1.1%20Final.md#vendor-consent-string-format-).
+
+If `gdpr` is  omitted, callers are still encouraged to send `gdpr_consent` if they have it.
+Depending on how the Prebid Server host company has configured their servers, they may or may not require it for cookie syncs.
+
+
 If the `bidders` field is an empty list, it will not supply any syncs. If the `bidders` field is omitted completely, it will attempt
 to sync all bidders.
 

endpoints/cookie_sync.go

@@ -1,112 +1,167 @@
 package endpoints
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 
-	"github.com/golang/glog"
+	"github.com/buger/jsonparser"
+
 	"github.com/julienschmidt/httprouter"
 	"github.com/prebid/prebid-server/analytics"
 	"github.com/prebid/prebid-server/config"
+	"github.com/prebid/prebid-server/gdpr"
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/pbsmetrics"
 	"github.com/prebid/prebid-server/usersync"
 )
 
-func NewCookieSyncEndpoint(syncers map[openrtb_ext.BidderName]usersync.Usersyncer, optOutCookie *config.Cookie, metrics pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule) httprouter.Handle {
-	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		//CookieSyncObject makes a log of requests and responses to  /cookie_sync endpoint
-		co := analytics.CookieSyncObject{
-			Status:       http.StatusOK,
-			Errors:       make([]error, 0),
-			BidderStatus: make([]*usersync.CookieSyncBidders, 0),
-		}
+func NewCookieSyncEndpoint(syncers map[openrtb_ext.BidderName]usersync.Usersyncer, optOutCookie *config.Cookie, syncPermissions gdpr.Permissions, metrics pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule) httprouter.Handle {
+	deps := &cookieSyncDeps{
+		syncers:         syncers,
+		optOutCookie:    optOutCookie,
+		syncPermissions: syncPermissions,
+		metrics:         metrics,
+		pbsAnalytics:    pbsAnalytics,
+	}
+	return deps.Endpoint
+}
 
-		defer pbsAnalytics.LogCookieSyncObject(&co)
+type cookieSyncDeps struct {
+	syncers         map[openrtb_ext.BidderName]usersync.Usersyncer
+	optOutCookie    *config.Cookie
+	syncPermissions gdpr.Permissions
+	metrics         pbsmetrics.MetricsEngine
+	pbsAnalytics    analytics.PBSAnalyticsModule
+}
 
-		metrics.RecordCookieSync(pbsmetrics.Labels{})
-		userSyncCookie := usersync.ParsePBSCookieFromRequest(r, optOutCookie)
-		if !userSyncCookie.AllowSyncs() {
-			http.Error(w, "User has opted out", http.StatusUnauthorized)
-			co.Status = http.StatusUnauthorized
-			co.Errors = append(co.Errors, fmt.Errorf("user has opted out"))
-			return
-		}
+func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	//CookieSyncObject makes a log of requests and responses to  /cookie_sync endpoint
+	co := analytics.CookieSyncObject{
+		Status:       http.StatusOK,
+		Errors:       make([]error, 0),
+		BidderStatus: make([]*usersync.CookieSyncBidders, 0),
+	}
 
-		defer r.Body.Close()
-
-		csReq := &cookieSyncRequest{}
-		csReqRaw := map[string]json.RawMessage{}
-		err := json.NewDecoder(r.Body).Decode(&csReqRaw)
-		if err != nil {
-			if glog.V(2) {
-				glog.Infof("Failed to parse /cookie_sync request body: %v", err)
-			}
-			co.Status = http.StatusBadRequest
-			co.Errors = append(co.Errors, fmt.Errorf("JSON parse failed"))
-			http.Error(w, "JSON parse failed", http.StatusBadRequest)
-			return
-		}
-		biddersOmitted := true
-		if biddersRaw, ok := csReqRaw["bidders"]; ok {
-			biddersOmitted = false
-			err := json.Unmarshal(biddersRaw, &csReq.Bidders)
-			if err != nil {
-				if glog.V(2) {
-					glog.Infof("Failed to parse /cookie_sync request body (bidders list): %v", err)
-				}
-				co.Status = http.StatusBadRequest
-				co.Errors = append(co.Errors, fmt.Errorf("JSON parse failed (bidders"))
-				http.Error(w, "JSON parse failed (bidders)", http.StatusBadRequest)
-				return
-			}
-		}
+	defer deps.pbsAnalytics.LogCookieSyncObject(&co)
 
-		csResp := cookieSyncResponse{
-			BidderStatus: make([]*usersync.CookieSyncBidders, 0, len(csReq.Bidders)),
-		}
+	deps.metrics.RecordCookieSync(pbsmetrics.Labels{})
+	userSyncCookie := usersync.ParsePBSCookieFromRequest(r, deps.optOutCookie)
+	if !userSyncCookie.AllowSyncs() {
+		http.Error(w, "User has opted out", http.StatusUnauthorized)
+		co.Status = http.StatusUnauthorized
+		co.Errors = append(co.Errors, fmt.Errorf("user has opted out"))
+		return
+	}
 
-		if userSyncCookie.LiveSyncCount() == 0 {
-			csResp.Status = "no_cookie"
-		} else {
-			csResp.Status = "ok"
-		}
+	defer r.Body.Close()
+	bodyBytes, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		co.Status = http.StatusBadRequest
+		co.Errors = append(co.Errors, errors.New("Failed to read request body"))
+		http.Error(w, "Failed to read request body", http.StatusBadRequest)
+		return
+	}
+	biddersJSON, err := parseBidders(bodyBytes)
+	if err != nil {
+		co.Status = http.StatusBadRequest
+		co.Errors = append(co.Errors, errors.New("Failed to check request.bidders in request body. Was your JSON well-formed?"))
+		http.Error(w, "Failed to check request.bidders in request body. Was your JSON well-formed?", http.StatusBadRequest)
+		return
+	}
 
-		// If at the end (After possibly reading stored bidder lists) there still are no bidders,
-		// and "bidders" is not found in the JSON, sync all bidders
-		if len(csReq.Bidders) == 0 && biddersOmitted {
-			for bidder := range syncers {
-				csReq.Bidders = append(csReq.Bidders, string(bidder))
-			}
-		}
+	parsedReq := &cookieSyncRequest{}
+	if err := json.Unmarshal(bodyBytes, parsedReq); err != nil {
+		co.Status = http.StatusBadRequest
+		co.Errors = append(co.Errors, fmt.Errorf("JSON parsing failed: %v", err))
+		http.Error(w, "JSON parsing failed: "+err.Error(), http.StatusBadRequest)
+		return
+	}
 
-		for _, bidder := range csReq.Bidders {
-			if syncer, ok := syncers[openrtb_ext.BidderName(bidder)]; ok {
-				if !userSyncCookie.HasLiveSync(syncer.FamilyName()) {
-					b := usersync.CookieSyncBidders{
-						BidderCode:   bidder,
-						NoCookie:     true,
-						UsersyncInfo: syncer.GetUsersyncInfo(),
-					}
-					csResp.BidderStatus = append(csResp.BidderStatus, &b)
-				}
-			}
+	if len(biddersJSON) == 0 {
+		parsedReq.Bidders = make([]string, 0, len(deps.syncers))
+		for bidder := range deps.syncers {
+			parsedReq.Bidders = append(parsedReq.Bidders, string(bidder))
 		}
+	}
+
+	parsedReq.filterExistingSyncs(deps.syncers, userSyncCookie)
+	parsedReq.filterForGDPR(deps.syncPermissions)
 
-		if len(csResp.BidderStatus) > 0 {
-			co.BidderStatus = append(co.BidderStatus, csResp.BidderStatus...)
+	csResp := cookieSyncResponse{
+		Status:       cookieSyncStatus(userSyncCookie.LiveSyncCount()),
+		BidderStatus: make([]*usersync.CookieSyncBidders, len(parsedReq.Bidders)),
+	}
+	for i := 0; i < len(parsedReq.Bidders); i++ {
+		bidder := parsedReq.Bidders[i]
+		csResp.BidderStatus[i] = &usersync.CookieSyncBidders{
+			BidderCode:   bidder,
+			NoCookie:     true,
+			UsersyncInfo: deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(),
 		}
+	}
 
-		enc := json.NewEncoder(w)
-		enc.SetEscapeHTML(false)
-		//enc.SetIndent("", "  ")
-		enc.Encode(csResp)
+	if len(csResp.BidderStatus) > 0 {
+		co.BidderStatus = append(co.BidderStatus, csResp.BidderStatus...)
 	}
+
+	enc := json.NewEncoder(w)
+	enc.SetEscapeHTML(false)
+	enc.Encode(csResp)
+}
+
+func parseBidders(request []byte) ([]byte, error) {
+	value, valueType, _, err := jsonparser.Get(request, "bidders")
+	if err == nil && valueType != jsonparser.NotExist {
+		return value, nil
+	} else if err != jsonparser.KeyPathNotFoundError {
+		return nil, err
+	}
+	return nil, nil
+}
+
+func cookieSyncStatus(syncCount int) string {
+	if syncCount == 0 {
+		return "no_cookie"
+	}
+	return "ok"
 }
 
 type cookieSyncRequest struct {
 	Bidders []string `json:"bidders"`
+	GDPR    *int     `json:"gdpr"`
+	Consent string   `json:"gdpr_consent"`
+}
+
+func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderName]usersync.Usersyncer, cookie *usersync.PBSCookie) {
+	for i := 0; i < len(req.Bidders); i++ {
+		thisBidder := req.Bidders[i]
+		if syncer, isValid := valid[openrtb_ext.BidderName(thisBidder)]; !isValid || cookie.HasLiveSync(syncer.FamilyName()) {
+			req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...)
+			i--
+		}
+	}
+}
+
+func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) {
+	if req.GDPR != nil && *req.GDPR == 0 {
+		return
+	}
+
+	if allowSync, err := permissions.HostCookiesAllowed(context.Background(), req.Consent); err != nil || !allowSync {
+		req.Bidders = nil
+		return
+	}
+
+	for i := 0; i < len(req.Bidders); i++ {
+		if allowSync, err := permissions.BidderSyncAllowed(context.Background(), openrtb_ext.BidderName(req.Bidders[i]), req.Consent); err != nil || !allowSync {
+			req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...)
+			i--
+		}
+	}
 }
 
 type cookieSyncResponse struct {