From 39516e9b0a8dc1c62ed42921e5c668e46cb8a82a Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 26 May 2020 02:28:15 -0400 Subject: [PATCH 01/27] CCPA NoSale Publisher Override --- endpoints/cookie_sync.go | 23 ++-- exchange/utils.go | 8 +- exchange/utils_test.go | 14 +++ openrtb_ext/request.go | 4 + privacy/ccpa/policy.go | 63 ++++++++++- privacy/ccpa/policy_test.go | 219 ++++++++++++++++++++++++++++++++++-- 6 files changed, 310 insertions(+), 21 deletions(-) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 9787a8f78f2..f16756a4148 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -122,7 +122,8 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h for _, b := range parsedReq.Bidders { adapterSyncs[openrtb_ext.BidderName(b)] = true } - parsedReq.filterForPrivacy(deps.syncPermissions, privacyPolicy, deps.enforceCCPA) + parsedReq.filterForGDPR(privacyPolicy, deps.syncPermissions) + parsedReq.filterForCCPA(privacyPolicy, deps.enforceCCPA) // surviving bidders are not privacy blocked for _, b := range parsedReq.Bidders { adapterSyncs[openrtb_ext.BidderName(b)] = false @@ -223,12 +224,7 @@ func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderNa } } -func (req *cookieSyncRequest) filterForPrivacy(permissions gdpr.Permissions, privacyPolicies privacy.Policies, enforceCCPA bool) { - if enforceCCPA && privacyPolicies.CCPA.ShouldEnforce() { - req.Bidders = nil - return - } - +func (req *cookieSyncRequest) filterForGDPR(privacyPolicies privacy.Policies, permissions gdpr.Permissions) { if req.GDPR != nil && *req.GDPR == 0 { return } @@ -246,6 +242,19 @@ func (req *cookieSyncRequest) filterForPrivacy(permissions gdpr.Permissions, pri } } +func (req *cookieSyncRequest) filterForCCPA(privacyPolicies privacy.Policies, enforceCCPA bool) { + if !enforceCCPA { + return + } + + for i := 0; i < len(req.Bidders); i++ { + if privacyPolicies.CCPA.ShouldEnforce(req.Bidders[i]) { + req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) + i-- + } + } +} + // filterToLimit will enforce a max limit on cookiesyncs supplied, picking a random subset of syncs to get to the limit if over. func (req *cookieSyncRequest) filterToLimit() { if req.Limit <= 0 { diff --git a/exchange/utils.go b/exchange/utils.go index f09b11513f1..a080dd61460 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -48,13 +48,17 @@ func cleanOpenRTBRequests(ctx context.Context, COPPA: orig.Regs != nil && orig.Regs.COPPA == 1, } + var ccpaPolicy ccpa.Policy if enforceCCPA { - ccpaPolicy, _ := ccpa.ReadPolicy(orig) - privacyEnforcement.CCPA = ccpaPolicy.ShouldEnforce() + ccpaPolicy, _ = ccpa.ReadPolicy(orig) } for bidder, bidReq := range requestsByBidder { + // CCPA + privacyEnforcement.CCPA = ccpaPolicy.ShouldEnforce(bidder.String()) + + // GDPR if gdpr == 1 { coreBidder := resolveBidder(bidder.String(), aliases) diff --git a/exchange/utils_test.go b/exchange/utils_test.go index edbe04a0d0f..0d810fe921a 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -79,6 +79,7 @@ func TestCleanOpenRTBRequests(t *testing.T) { func TestCleanOpenRTBRequestsCCPA(t *testing.T) { testCases := []struct { description string + reqExt json.RawMessage enforceCCPA bool expectDataScrub bool }{ @@ -87,6 +88,18 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { enforceCCPA: true, expectDataScrub: true, }, + { + description: "Feature Flag Enabled - No Sale Bidder - Scrub", + reqExt: json.RawMessage(`{"prebid":{"nosale":["appnexus"]}}`), + enforceCCPA: true, + expectDataScrub: false, + }, + { + description: "Feature Flag Enabled - No Sale Bidder - Doesn't Scrub", + reqExt: json.RawMessage(`{"prebid":{"nosale":["rubicon"]}}`), + enforceCCPA: true, + expectDataScrub: true, + }, { description: "Feature Flag Disabled", enforceCCPA: false, @@ -96,6 +109,7 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { for _, test := range testCases { req := newCCPABidRequest(t) + req.Ext = test.reqExt results, _, errs := cleanOpenRTBRequests(context.Background(), req, &emptyUsersync{}, map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels{}, pbsmetrics.Labels{}, &permissionsMock{}, true, test.enforceCCPA) result := results["appnexus"] diff --git a/openrtb_ext/request.go b/openrtb_ext/request.go index 25b5c881408..a92388a50cc 100644 --- a/openrtb_ext/request.go +++ b/openrtb_ext/request.go @@ -18,6 +18,10 @@ type ExtRequestPrebid struct { StoredRequest *ExtStoredRequest `json:"storedrequest,omitempty"` Targeting *ExtRequestTargeting `json:"targeting,omitempty"` SupportDeals bool `json:"supportdeals,omitempty"` + + // NoSale allows publishers to explicitly declare relationships with bidders which do not constitute + // a sale per CCPA law. Values are bidder names or a star ('*') character to represent all bidders. + NoSale []string `json:"nosale,omitempty"` } // ExtRequestPrebidCache defines the contract for bidrequest.ext.prebid.cache diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 8b50e1112a9..40af256ee1b 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "github.com/buger/jsonparser" "github.com/mxmCherry/openrtb" @@ -12,7 +13,8 @@ import ( // Policy represents the CCPA regulation for an OpenRTB bid request. type Policy struct { - Value string + Value string + NoSaleBidders []string } // ReadPolicy extracts the CCPA regulation policy from an OpenRTB regs ext. @@ -22,16 +24,40 @@ func ReadPolicy(req *openrtb.BidRequest) (Policy, error) { if req != nil && req.Regs != nil && len(req.Regs.Ext) > 0 { var ext openrtb_ext.ExtRegs if err := json.Unmarshal(req.Regs.Ext, &ext); err != nil { - return policy, err + return Policy{}, err } policy.Value = ext.USPrivacy } + if req != nil && len(req.Ext) > 0 { + var ext openrtb_ext.ExtRequest + if err := json.Unmarshal(req.Ext, &ext); err != nil { + return Policy{}, err + } + policy.NoSaleBidders = ext.Prebid.NoSale + } + return policy, nil } // Write mutates an OpenRTB bid request with the context of the CCPA policy. func (p Policy) Write(req *openrtb.BidRequest) error { + var err error + + err = p.writeRegsExt(req) + if err != nil { + return err + } + + err = p.writeExt(req) + if err != nil { + return err + } + + return nil +} + +func (p Policy) writeRegsExt(req *openrtb.BidRequest) error { if p.Value == "" { return nil } @@ -50,6 +76,31 @@ func (p Policy) Write(req *openrtb.BidRequest) error { return err } +func (p Policy) writeExt(req *openrtb.BidRequest) error { + if len(p.NoSaleBidders) == 0 { + return nil + } + + var ext openrtb_ext.ExtRequest + if len(req.Ext) == 0 { + ext = openrtb_ext.ExtRequest{} + } else { + err := json.Unmarshal(req.Ext, &ext) + if err != nil { + return err + } + } + + ext.Prebid.NoSale = p.NoSaleBidders + extJSON, err := json.Marshal(ext) + if err != nil { + return err + } + req.Ext = extJSON + + return nil +} + // Validate returns an error if the CCPA policy does not adhere to the IAB spec. func (p Policy) Validate() error { if err := ValidateConsent(p.Value); err != nil { @@ -94,10 +145,16 @@ func ValidateConsent(consent string) error { } // ShouldEnforce returns true when the opt-out signal is explicitly detected. -func (p Policy) ShouldEnforce() bool { +func (p Policy) ShouldEnforce(bidder string) bool { if err := p.Validate(); err != nil { return false } + for _, b := range p.NoSaleBidders { + if b == "*" || strings.EqualFold(b, bidder) { + return false + } + } + return p.Value != "" && p.Value[2] == 'Y' } diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index 54613c89880..aef63ed61d9 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -21,38 +21,45 @@ func TestRead(t *testing.T) { Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), }, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "ABC", + Value: "ABC", + NoSaleBidders: []string{"a", "b"}, }, }, { description: "Empty - No Request", request: nil, expectedPolicy: Policy{ - Value: "", + Value: "", + NoSaleBidders: nil, }, }, { description: "Empty - No Regs", request: &openrtb.BidRequest{ Regs: nil, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "", + Value: "", + NoSaleBidders: []string{"a", "b"}, }, }, { - description: "Empty - No Ext", + description: "Empty - No Regs.Ext", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{}, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "", + Value: "", + NoSaleBidders: []string{"a", "b"}, }, }, { - description: "Empty - No Value", + description: "Empty - No Regs.Ext Value", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"anythingElse":"42"}`), @@ -63,11 +70,48 @@ func TestRead(t *testing.T) { }, }, { - description: "Serialization Issue", + description: "Empty - No Ext", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"ABC"}`), + }, + Ext: nil, + }, + expectedPolicy: Policy{ + Value: "ABC", + NoSaleBidders: nil, + }, + }, + { + description: "Empty - No Ext Value", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"ABC"}`), + }, + Ext: json.RawMessage(`{"anythingElse":"42"}`), + }, + expectedPolicy: Policy{ + Value: "ABC", + NoSaleBidders: nil, + }, + }, + { + description: "Malformed Regs.Ext", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`malformed`), }, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + }, + expectedError: true, + }, + { + description: "Malformed Ext", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"ABC"}`), + }, + Ext: json.RawMessage(`malformed`), }, expectedError: true, }, @@ -88,6 +132,57 @@ func TestRead(t *testing.T) { } func TestWrite(t *testing.T) { + testCases := []struct { + description string + policy Policy + request *openrtb.BidRequest + expected *openrtb.BidRequest + expectedError bool + }{ + { + description: "Success", + policy: Policy{Value: "anyValue", NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"anyValue"}`), + }, + Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), + }, + }, + { + description: "Error Regs.Ext", + policy: Policy{Value: "anyValue", NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`malformed}`), + }, + }, + expectedError: true, + }, + { + description: "Error Ext", + policy: Policy{Value: "anyValue", NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`malformed}`), + }, + expectedError: true, + }, + } + + for _, test := range testCases { + err := test.policy.Write(test.request) + + if test.expectedError { + assert.Error(t, err, test.description) + } else { + assert.NoError(t, err, test.description) + assert.Equal(t, test.expected, test.request, test.description) + } + } +} + +func TestWriteRegsExt(t *testing.T) { testCases := []struct { description string policy Policy @@ -141,7 +236,89 @@ func TestWrite(t *testing.T) { } for _, test := range testCases { - err := test.policy.Write(test.request) + err := test.policy.writeRegsExt(test.request) + + if test.expectedError { + assert.Error(t, err, test.description) + } else { + assert.NoError(t, err, test.description) + assert.Equal(t, test.expected, test.request, test.description) + } + } +} + +func TestWriteExt(t *testing.T) { + testCases := []struct { + description string + policy Policy + request *openrtb.BidRequest + expected *openrtb.BidRequest + expectedError bool + }{ + { + description: "Nil", + policy: Policy{NoSaleBidders: nil}, + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{}, + }, + { + description: "Empty", + policy: Policy{NoSaleBidders: []string{}}, + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{}, + }, + { + description: "Values - Nil Ext", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: nil, + }, + expected: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), + }, + }, + { + description: "Values - Empty Prebid", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{}}`), + }, + expected: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), + }, + }, + { + description: "Values - Existing - Persists Other Values", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{"supportdeals":true}}`), + }, + expected: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{"supportdeals":true,"nosale":["a","b"]}}`), + }, + }, + { + description: "Values - Existing - Overwrites Same Value", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{"nosale":["1","2"]}}`), + }, + expected: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), + }, + }, + { + description: "Values - Malformed", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`malformed`), + }, + expectedError: true, + }, + } + + for _, test := range testCases { + err := test.policy.writeExt(test.request) if test.expectedError { assert.Error(t, err, test.description) @@ -303,38 +480,62 @@ func TestValidateConsent(t *testing.T) { func TestShouldEnforce(t *testing.T) { testCases := []struct { description string + bidder string policy Policy expected bool }{ { description: "Enforceable", + bidder: "a", policy: Policy{Value: "1-Y-"}, expected: true, }, + { + description: "Enforceable - No Sale For Different Bidder", + bidder: "a", + policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"b"}}, + expected: true, + }, { description: "Not Enforceable - Not Present", + bidder: "a", policy: Policy{Value: ""}, expected: false, }, { description: "Not Enforceable - Opt-Out Unknown", + bidder: "a", policy: Policy{Value: "1---"}, expected: false, }, { description: "Not Enforceable - Opt-Out Explicitly No", + bidder: "a", policy: Policy{Value: "1-N-"}, expected: false, }, + { + description: "Not Enforceable - No Sale All Bidders", + bidder: "a", + policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"*"}}, + expected: false, + }, + { + description: "Not Enforceable - No Sale Specific Bidder", + bidder: "a", + policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"a"}}, + expected: false, + }, { description: "Invalid", + bidder: "a", policy: Policy{Value: "2---"}, expected: false, }, } for _, test := range testCases { - result := test.policy.ShouldEnforce() + result := test.policy.ShouldEnforce(test.bidder) assert.Equal(t, test.expected, result, test.description) } } From 68e0eb68da7a796a89422deb6bee71c00226b3a0 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 26 May 2020 02:37:53 -0400 Subject: [PATCH 02/27] Added JSON Tests --- .../exchangetest/ccpa-nosale-any-bidder.json | 75 +++++++++++++++++++ .../ccpa-nosale-specific-bidder.json | 75 +++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 exchange/exchangetest/ccpa-nosale-any-bidder.json create mode 100644 exchange/exchangetest/ccpa-nosale-specific-bidder.json diff --git a/exchange/exchangetest/ccpa-nosale-any-bidder.json b/exchange/exchangetest/ccpa-nosale-any-bidder.json new file mode 100644 index 00000000000..f7abd91f512 --- /dev/null +++ b/exchange/exchangetest/ccpa-nosale-any-bidder.json @@ -0,0 +1,75 @@ +{ + "enforceCcpa": true, + "incomingRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [{ + "id": "my-imp-id", + "video": { + "mimes": ["video/mp4"] + }, + "ext": { + "appnexus": { + "placementId": 1 + } + } + }], + "regs": { + "ext": { + "us_privacy": "1-Y-" + } + }, + "ext": { + "prebid": { + "nosale": ["*"] + } + }, + "user": { + "buyeruid": "some-buyer-id" + } + } + }, + "outgoingRequests": { + "appnexus": { + "expectRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [{ + "id": "my-imp-id", + "video": { + "mimes": ["video/mp4"] + }, + "ext": { + "bidder": { + "placementId": 1 + } + } + }], + "regs": { + "ext": { + "us_privacy": "1-Y-" + } + }, + "ext": { + "prebid": { + "nosale": ["*"] + } + }, + "user": { + "buyeruid": "some-buyer-id" + } + }, + "bidAdjustment": 1.0 + }, + "mockResponse": { + "errors": ["appnexus-error"] + } + } + } +} \ No newline at end of file diff --git a/exchange/exchangetest/ccpa-nosale-specific-bidder.json b/exchange/exchangetest/ccpa-nosale-specific-bidder.json new file mode 100644 index 00000000000..b89e29aea01 --- /dev/null +++ b/exchange/exchangetest/ccpa-nosale-specific-bidder.json @@ -0,0 +1,75 @@ +{ + "enforceCcpa": true, + "incomingRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [{ + "id": "my-imp-id", + "video": { + "mimes": ["video/mp4"] + }, + "ext": { + "appnexus": { + "placementId": 1 + } + } + }], + "regs": { + "ext": { + "us_privacy": "1-Y-" + } + }, + "ext": { + "prebid": { + "nosale": ["appnexus"] + } + }, + "user": { + "buyeruid": "some-buyer-id" + } + } + }, + "outgoingRequests": { + "appnexus": { + "expectRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [{ + "id": "my-imp-id", + "video": { + "mimes": ["video/mp4"] + }, + "ext": { + "bidder": { + "placementId": 1 + } + } + }], + "regs": { + "ext": { + "us_privacy": "1-Y-" + } + }, + "ext": { + "prebid": { + "nosale": ["appnexus"] + } + }, + "user": { + "buyeruid": "some-buyer-id" + } + }, + "bidAdjustment": 1.0 + }, + "mockResponse": { + "errors": ["appnexus-error"] + } + } + } +} \ No newline at end of file From c3482be30cf50028d7dd525144cf0ad0f9b6f3c6 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 27 May 2020 14:31:05 -0400 Subject: [PATCH 03/27] Expanded Test Cases --- privacy/ccpa/policy_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index aef63ed61d9..114ade890f3 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -520,12 +520,24 @@ func TestShouldEnforce(t *testing.T) { policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"*"}}, expected: false, }, + { + description: "Not Enforceable - No Sale All Bidders Mixed With Specific Bidders", + bidder: "a", + policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"b", "*", "c"}}, + expected: false, + }, { description: "Not Enforceable - No Sale Specific Bidder", bidder: "a", policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"a"}}, expected: false, }, + { + description: "Not Enforceable - No Sale Specific Bidder Case Insensitive", + bidder: "a", + policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"A"}}, + expected: false, + }, { description: "Invalid", bidder: "a", From 2cdc53801d3b114621c85cda402d0fd1dc45a7b7 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 28 May 2020 01:17:23 -0400 Subject: [PATCH 04/27] Allow Non-Schema JSON In Req.Ext --- privacy/ccpa/policy.go | 33 +++++++++++++++++++++----- privacy/ccpa/policy_test.go | 46 +++++++++++++++++++++++++++++-------- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 40af256ee1b..4915d2c4b5f 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -81,18 +81,39 @@ func (p Policy) writeExt(req *openrtb.BidRequest) error { return nil } - var ext openrtb_ext.ExtRequest if len(req.Ext) == 0 { - ext = openrtb_ext.ExtRequest{} - } else { - err := json.Unmarshal(req.Ext, &ext) + ext := openrtb_ext.ExtRequest{} + ext.Prebid.NoSale = p.NoSaleBidders + + extJSON, err := json.Marshal(ext) if err != nil { return err } + + req.Ext = extJSON + return nil + } + + var extMap map[string]interface{} + if err := json.Unmarshal(req.Ext, &extMap); err != nil { + return err + } + + var extMapPrebid map[string]interface{} + if v, exists := extMap["prebid"]; !exists { + extMapPrebid = make(map[string]interface{}) + extMap["prebid"] = extMapPrebid + } else { + vCasted, ok := v.(map[string]interface{}) + if !ok { + return errors.New("invalid data type for ext.prebid") + } + extMapPrebid = vCasted } - ext.Prebid.NoSale = p.NoSaleBidders - extJSON, err := json.Marshal(ext) + extMapPrebid["nosale"] = p.NoSaleBidders + + extJSON, err := json.Marshal(extMap) if err != nil { return err } diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index 114ade890f3..733134d4955 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -256,19 +256,19 @@ func TestWriteExt(t *testing.T) { expectedError bool }{ { - description: "Nil", + description: "Nil NoSaleBidders", policy: Policy{NoSaleBidders: nil}, request: &openrtb.BidRequest{}, expected: &openrtb.BidRequest{}, }, { - description: "Empty", + description: "Empty NoSaleBidders", policy: Policy{NoSaleBidders: []string{}}, request: &openrtb.BidRequest{}, expected: &openrtb.BidRequest{}, }, { - description: "Values - Nil Ext", + description: "Nil Ext", policy: Policy{NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: nil, @@ -278,7 +278,17 @@ func TestWriteExt(t *testing.T) { }, }, { - description: "Values - Empty Prebid", + description: "Empty Ext", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`{}`), + }, + expected: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), + }, + }, + { + description: "Empty Ext.Prebid", policy: Policy{NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: json.RawMessage(`{"prebid":{}}`), @@ -288,17 +298,27 @@ func TestWriteExt(t *testing.T) { }, }, { - description: "Values - Existing - Persists Other Values", + description: "Existing Values In Ext", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"existing":true,"prebid":{}}`), + }, + expected: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"existing":true,"prebid":{"nosale":["a","b"]}}`), + }, + }, + { + description: "Existing Values In Ext.Prebid", policy: Policy{NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"supportdeals":true}}`), + Ext: json.RawMessage(`{"prebid":{"existing":true}}`), }, expected: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"supportdeals":true,"nosale":["a","b"]}}`), + Ext: json.RawMessage(`{"prebid":{"existing":true,"nosale":["a","b"]}}`), }, }, { - description: "Values - Existing - Overwrites Same Value", + description: "Overwrite Existing In Ext.Prebid", policy: Policy{NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: json.RawMessage(`{"prebid":{"nosale":["1","2"]}}`), @@ -308,13 +328,21 @@ func TestWriteExt(t *testing.T) { }, }, { - description: "Values - Malformed", + description: "Malformed Ext", policy: Policy{NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: json.RawMessage(`malformed`), }, expectedError: true, }, + { + description: "Invalid Ext.Prebid", + policy: Policy{NoSaleBidders: []string{"a", "b"}}, + request: &openrtb.BidRequest{ + Ext: json.RawMessage(`{"prebid":42}`), + }, + expectedError: true, + }, } for _, test := range testCases { From 15a81a61cf3b68ed36fa0ba7ab54d77c09e35202 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 28 May 2020 01:30:39 -0400 Subject: [PATCH 05/27] Enhanced Read Tests --- privacy/ccpa/policy_test.go | 52 ++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index 733134d4955..ec7324c5914 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -29,7 +29,7 @@ func TestRead(t *testing.T) { }, }, { - description: "Empty - No Request", + description: "No Request", request: nil, expectedPolicy: Policy{ Value: "", @@ -37,7 +37,7 @@ func TestRead(t *testing.T) { }, }, { - description: "Empty - No Regs", + description: "No Regs", request: &openrtb.BidRequest{ Regs: nil, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), @@ -48,7 +48,7 @@ func TestRead(t *testing.T) { }, }, { - description: "Empty - No Regs.Ext", + description: "No Regs.Ext", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{}, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), @@ -59,18 +59,33 @@ func TestRead(t *testing.T) { }, }, { - description: "Empty - No Regs.Ext Value", + description: "Empty Regs.Ext", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{}`), + }, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + }, + expectedPolicy: Policy{ + Value: "", + NoSaleBidders: []string{"a", "b"}, + }, + }, + { + description: "No Regs.Ext Value", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"anythingElse":"42"}`), }, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "", + Value: "", + NoSaleBidders: []string{"a", "b"}, }, }, { - description: "Empty - No Ext", + description: "No Ext", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), @@ -83,7 +98,20 @@ func TestRead(t *testing.T) { }, }, { - description: "Empty - No Ext Value", + description: "Empty Ext", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"ABC"}`), + }, + Ext: json.RawMessage(`{}`), + }, + expectedPolicy: Policy{ + Value: "ABC", + NoSaleBidders: nil, + }, + }, + { + description: "No Ext NoSale", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), @@ -115,6 +143,16 @@ func TestRead(t *testing.T) { }, expectedError: true, }, + { + description: "Incorrect Ext.Prebid.NoSale Type", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"ABC"}`), + }, + Ext: json.RawMessage(`{"prebid":{"nosale":"wrongtype"}}`), + }, + expectedError: true, + }, } for _, test := range testCases { From 6f422cc9f7624789ee11069eeaa55c10a7e64cfd Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Mon, 1 Jun 2020 11:02:02 -0400 Subject: [PATCH 06/27] Minor Refactor + Test Update --- exchange/utils_test.go | 10 ++++++-- openrtb_ext/request.go | 2 +- privacy/ccpa/policy.go | 50 ++++++++++++++++++++++++------------- privacy/ccpa/policy_test.go | 30 +++++++++++----------- 4 files changed, 56 insertions(+), 36 deletions(-) diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 2dbac7f5770..2637b998e68 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -93,13 +93,19 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { expectDataScrub: true, }, { - description: "Feature Flag Enabled - No Sale Bidder - Scrub", + description: "Feature Flag Enabled - No Sale Star - Doesn't Scrub", + reqExt: json.RawMessage(`{"prebid":{"nosale":["*"]}}`), + enforceCCPA: true, + expectDataScrub: false, + }, + { + description: "Feature Flag Enabled - No Sale Specific Bidder - Doesn't Scrub", reqExt: json.RawMessage(`{"prebid":{"nosale":["appnexus"]}}`), enforceCCPA: true, expectDataScrub: false, }, { - description: "Feature Flag Enabled - No Sale Bidder - Doesn't Scrub", + description: "Feature Flag Enabled - No Sale Different Bidder - Scrubs", reqExt: json.RawMessage(`{"prebid":{"nosale":["rubicon"]}}`), enforceCCPA: true, expectDataScrub: true, diff --git a/openrtb_ext/request.go b/openrtb_ext/request.go index a92388a50cc..d7fcd3d412e 100644 --- a/openrtb_ext/request.go +++ b/openrtb_ext/request.go @@ -20,7 +20,7 @@ type ExtRequestPrebid struct { SupportDeals bool `json:"supportdeals,omitempty"` // NoSale allows publishers to explicitly declare relationships with bidders which do not constitute - // a sale per CCPA law. Values are bidder names or a star ('*') character to represent all bidders. + // a sale per CCPA law. Values are either bidder names or a star ('*') to represent all bidders. NoSale []string `json:"nosale,omitempty"` } diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 4f625d24f1a..7d3f89032a1 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -10,6 +10,22 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" ) +const ( + ccpaVersion1 = '1' + ccpaNo = 'N' + ccpaYes = 'Y' + ccpaNotApplicable = '-' +) + +const ( + indexVersion = 0 + indexExplicitNotice = 1 + indexOptOutSale = 2 + indexLSPACoveredTransaction = 3 +) + +const allBidders = "*" + // Policy represents the CCPA regulation for an OpenRTB bid request. type Policy struct { Value string @@ -57,7 +73,7 @@ func (p Policy) Write(req *openrtb.BidRequest) error { } func (p Policy) writeRegsExt(req *openrtb.BidRequest) error { - if p.Value == "" { + if len(p.Value) == 0 { return nil } @@ -95,12 +111,11 @@ func (p Policy) writeExt(req *openrtb.BidRequest) error { ext.Prebid.NoSale = p.NoSaleBidders extJSON, err := json.Marshal(ext) - if err != nil { - return err + if err == nil { + req.Ext = extJSON } - req.Ext = extJSON - return nil + return err } var extMap map[string]interface{} @@ -123,12 +138,11 @@ func (p Policy) writeExt(req *openrtb.BidRequest) error { extMapPrebid["nosale"] = p.NoSaleBidders extJSON, err := json.Marshal(extMap) - if err != nil { - return err + if err == nil { + req.Ext = extJSON } - req.Ext = extJSON - return nil + return err } // Validate returns an error if the CCPA policy does not adhere to the IAB spec. @@ -150,24 +164,24 @@ func ValidateConsent(consent string) error { return errors.New("must contain 4 characters") } - if consent[0] != '1' { + if consent[indexVersion] != ccpaVersion1 { return errors.New("must specify version 1") } var c byte - c = consent[1] - if c != 'N' && c != 'Y' && c != '-' { + c = consent[indexExplicitNotice] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { return errors.New("must specify 'N', 'Y', or '-' for the explicit notice") } - c = consent[2] - if c != 'N' && c != 'Y' && c != '-' { + c = consent[indexOptOutSale] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { return errors.New("must specify 'N', 'Y', or '-' for the opt-out sale") } - c = consent[3] - if c != 'N' && c != 'Y' && c != '-' { + c = consent[indexLSPACoveredTransaction] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { return errors.New("must specify 'N', 'Y', or '-' for the limited service provider agreement") } @@ -181,10 +195,10 @@ func (p Policy) ShouldEnforce(bidder string) bool { } for _, b := range p.NoSaleBidders { - if b == "*" || strings.EqualFold(b, bidder) { + if b == allBidders || strings.EqualFold(b, bidder) { return false } } - return p.Value != "" && p.Value[2] == 'Y' + return p.Value != "" && p.Value[indexOptOutSale] == ccpaYes } diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index d9465400324..dd090ee24e6 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -29,7 +29,7 @@ func TestRead(t *testing.T) { }, }, { - description: "No Request", + description: "Nil Request", request: nil, expectedPolicy: Policy{ Value: "", @@ -37,7 +37,7 @@ func TestRead(t *testing.T) { }, }, { - description: "No Regs", + description: "Nil Regs", request: &openrtb.BidRequest{ Regs: nil, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), @@ -48,7 +48,7 @@ func TestRead(t *testing.T) { }, }, { - description: "No Regs.Ext", + description: "Nil Regs.Ext", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{}, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), @@ -72,7 +72,7 @@ func TestRead(t *testing.T) { }, }, { - description: "No Regs.Ext Value", + description: "Nil Regs.Ext Value", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"anythingElse":"42"}`), @@ -85,7 +85,7 @@ func TestRead(t *testing.T) { }, }, { - description: "No Ext", + description: "Nil Ext", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), @@ -111,7 +111,7 @@ func TestRead(t *testing.T) { }, }, { - description: "No Ext NoSale", + description: "Nil Ext.Prebid.NoSale", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), @@ -246,21 +246,21 @@ func TestWriteRegsExt(t *testing.T) { expected: &openrtb.BidRequest{}, }, { - description: "Enabled With Nil Request Regs Object", + description: "Enabled - Nil Regs", policy: Policy{Value: "anyValue"}, request: &openrtb.BidRequest{}, expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"anyValue"}`)}}, }, { - description: "Enabled With Nil Request Regs Ext Object", + description: "Enabled - Nil Regs.Ext", policy: Policy{Value: "anyValue"}, request: &openrtb.BidRequest{Regs: &openrtb.Regs{}}, expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"anyValue"}`)}}, }, { - description: "Enabled With Existing Request Regs Ext Object - Doesn't Overwrite", + description: "Enabled - Existing Regs.Ext - Doesn't Overwrite", policy: Policy{Value: "anyValue"}, request: &openrtb.BidRequest{Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"existing":"any"}`)}}, @@ -268,7 +268,7 @@ func TestWriteRegsExt(t *testing.T) { Ext: json.RawMessage(`{"existing":"any","us_privacy":"anyValue"}`)}}, }, { - description: "Enabled With Existing Request Regs Ext Object - Overwrites", + description: "Enabled - Existing Regs.Ext.US_Privacy - Overwrites", policy: Policy{Value: "anyValue"}, request: &openrtb.BidRequest{Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"existing":"any","us_privacy":"toBeOverwritten"}`)}}, @@ -276,14 +276,14 @@ func TestWriteRegsExt(t *testing.T) { Ext: json.RawMessage(`{"existing":"any","us_privacy":"anyValue"}`)}}, }, { - description: "Enabled With Existing Malformed Request Regs Ext Object", + description: "Enabled - Malformed Regs.Ext", policy: Policy{Value: "anyValue"}, request: &openrtb.BidRequest{Regs: &openrtb.Regs{ Ext: json.RawMessage(`malformed`)}}, expectedError: true, }, { - description: "Injection Attack With Nil Request Regs Object", + description: "Injection Attack - Nil Regs", policy: Policy{Value: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, request: &openrtb.BidRequest{}, expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ @@ -291,7 +291,7 @@ func TestWriteRegsExt(t *testing.T) { }}, }, { - description: "Injection Attack With Nil Request Regs Ext Object", + description: "Injection Attack - Nil Regs.Ext", policy: Policy{Value: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, request: &openrtb.BidRequest{Regs: &openrtb.Regs{}}, expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ @@ -299,7 +299,7 @@ func TestWriteRegsExt(t *testing.T) { }}, }, { - description: "Injection Attack With Existing Request Regs Ext Object", + description: "Injection Attack - Existing Regs.Ext", policy: Policy{Value: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, request: &openrtb.BidRequest{Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"existing":"any"}`), @@ -411,7 +411,7 @@ func TestWriteExt(t *testing.T) { expectedError: true, }, { - description: "Invalid Ext.Prebid", + description: "Invalid Ext.Prebid Type", policy: Policy{NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: json.RawMessage(`{"prebid":42}`), From 41dff0cb4330747ee250ddc588e887f503307b5a Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 2 Jun 2020 16:51:25 -0400 Subject: [PATCH 07/27] Code Review Feedback --- exchange/utils.go | 4 ++++ privacy/ccpa/policy.go | 19 +++++++++++++------ privacy/ccpa/policy_test.go | 20 ++++++++++++++++---- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/exchange/utils.go b/exchange/utils.go index a9973167bf1..6333c127d8d 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -41,6 +41,10 @@ func cleanOpenRTBRequests(ctx context.Context, requestsByBidder, errs = splitBidRequest(orig, impsByBidder, aliases, usersyncs, blables, labels) + if len(requestsByBidder) == 0 { + return + } + gdpr := extractGDPR(orig, usersyncIfAmbiguous) consent := extractConsent(orig) ampGDPRException := (labels.RType == pbsmetrics.ReqTypeAMP) && gDPR.AMPException() diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 7d3f89032a1..f62f459bbb5 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -36,20 +36,27 @@ type Policy struct { func ReadPolicy(req *openrtb.BidRequest) (Policy, error) { policy := Policy{} - if req != nil && req.Regs != nil && len(req.Regs.Ext) > 0 { + if req == nil { + return policy, nil + } + + if req.Regs != nil && len(req.Regs.Ext) > 0 { var ext openrtb_ext.ExtRegs if err := json.Unmarshal(req.Regs.Ext, &ext); err != nil { - return Policy{}, err + return policy, err } policy.Value = ext.USPrivacy } - if req != nil && len(req.Ext) > 0 { + if len(req.Ext) > 0 { var ext openrtb_ext.ExtRequest - if err := json.Unmarshal(req.Ext, &ext); err != nil { - return Policy{}, err + + // Errors with reading the NoSaleBidders list shouldn't block enforcement of CCPA, so take a + // 'best effort' approach here and ignore problems unmarshalling the Prebid extension. Failure + // here is very unlikely due to request validation happening early in the auction endpoint. + if err := json.Unmarshal(req.Ext, &ext); err == nil { + policy.NoSaleBidders = ext.Prebid.NoSale } - policy.NoSaleBidders = ext.Prebid.NoSale } return policy, nil diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index dd090ee24e6..bfdd75efff7 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -134,24 +134,30 @@ func TestRead(t *testing.T) { expectedError: true, }, { - description: "Malformed Ext", + description: "Malformed Ext - NoSale Ignored", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), }, Ext: json.RawMessage(`malformed`), }, - expectedError: true, + expectedPolicy: Policy{ + Value: "ABC", + NoSaleBidders: nil, + }, }, { - description: "Incorrect Ext.Prebid.NoSale Type", + description: "Incorrect Ext.Prebid.NoSale Type - NoSale Ignored", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), }, Ext: json.RawMessage(`{"prebid":{"nosale":"wrongtype"}}`), }, - expectedError: true, + expectedPolicy: Policy{ + Value: "ABC", + NoSaleBidders: nil, + }, }, { description: "Injection Attack", @@ -635,6 +641,12 @@ func TestShouldEnforce(t *testing.T) { policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"a"}}, expected: false, }, + { + description: "Not Enforceable - No Sale Specific Bidder Mixed", + bidder: "a", + policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"b", "a", "c"}}, + expected: false, + }, { description: "Not Enforceable - No Sale Specific Bidder Case Insensitive", bidder: "a", From c4e04339ae7cb5f37d951c50a74b3c2bea44476f Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 3 Jun 2020 13:38:21 -0400 Subject: [PATCH 08/27] WIP --- privacy/ccpa/policy.go | 62 ++++++++++++++++++++++++++++++------------ privacy/policies.go | 2 +- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index f62f459bbb5..c8c27f68822 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -26,42 +26,52 @@ const ( const allBidders = "*" -// Policy represents the CCPA regulation for an OpenRTB bid request. -type Policy struct { +// RawPolicy represents the user provided CCPA regulation values. +type RawPolicy struct { Value string NoSaleBidders []string } -// ReadPolicy extracts the CCPA regulation policy from an OpenRTB regs ext. -func ReadPolicy(req *openrtb.BidRequest) (Policy, error) { - policy := Policy{} +// ParsedPolicy represents the parsed and validated CCPA regulation values. +type ParsedPolicy struct { + OptOutSaleYes bool + NoSaleAllBidders bool + NoSaleSpecificBidders map[string]struct{} +} +// ReadPolicy extracts the user provided CCPA regulation values from an OpenRTB request. +func ReadPolicy(req *openrtb.BidRequest) (RawPolicy, error) { if req == nil { - return policy, nil + return RawPolicy{}, nil } + var value string if req.Regs != nil && len(req.Regs.Ext) > 0 { var ext openrtb_ext.ExtRegs if err := json.Unmarshal(req.Regs.Ext, &ext); err != nil { - return policy, err + return RawPolicy{}, err } - policy.Value = ext.USPrivacy + value = ext.USPrivacy } + var noSaleBidders []string if len(req.Ext) > 0 { var ext openrtb_ext.ExtRequest - - // Errors with reading the NoSaleBidders list shouldn't block enforcement of CCPA, so take a - // 'best effort' approach here and ignore problems unmarshalling the Prebid extension. Failure - // here is very unlikely due to request validation happening early in the auction endpoint. - if err := json.Unmarshal(req.Ext, &ext); err == nil { - policy.NoSaleBidders = ext.Prebid.NoSale + if err := json.Unmarshal(req.Ext, &ext); err != nil { + return RawPolicy{}, err } + noSaleBidders = ext.Prebid.NoSale } - return policy, nil + result := RawPolicy{{ + Value: value, + NoSaleBidders: noSaleBidders, + } + return result, nil } + + // Write mutates an OpenRTB bid request with the context of the CCPA policy. func (p Policy) Write(req *openrtb.BidRequest) error { var err error @@ -152,12 +162,16 @@ func (p Policy) writeExt(req *openrtb.BidRequest) error { return err } -// Validate returns an error if the CCPA policy does not adhere to the IAB spec. -func (p Policy) Validate() error { +// Validate returns an error if the CCPA policy does not adhere to the IAB spec or the NoSale list is invalid. +func (p Policy) Validate(bidders []string) error { if err := ValidateConsent(p.Value); err != nil { return fmt.Errorf("request.regs.ext.us_privacy %s", err.Error()) } + if err := ValidateNoSaleBidders(p.NoSaleBidders); err != nil { + return fmt.Errorf("request.ext.prebid.nosale %s", err.Error()) + } + return nil } @@ -195,6 +209,20 @@ func ValidateConsent(consent string) error { return nil } +func ValidateNoSaleBidders(noSaleBidders []string, bidders map[string]openrtb_ext.BidderName, aliases map[string]string) error { + if len(noSaleBidders) == 1 && noSaleBidders[0] == allBidders { + return nil + } + + for _, bidder := range noSaleBidders { + if !validBidders.cotains[bidder] { + return fmt.Errorf("unrecognized bidder '%s'", bidder) + } + } + + return nil +} + // ShouldEnforce returns true when the opt-out signal is explicitly detected. func (p Policy) ShouldEnforce(bidder string) bool { if err := p.Validate(); err != nil { diff --git a/privacy/policies.go b/privacy/policies.go index cb11c6d03a6..a5c038b9033 100644 --- a/privacy/policies.go +++ b/privacy/policies.go @@ -10,7 +10,7 @@ import ( // Policies represents the privacy regulations for an OpenRTB bid request. type Policies struct { GDPR gdpr.Policy - CCPA ccpa.Policy + CCPA ccpa.ParsedPolicy } type policyWriter interface { From d00742f5b2a1a0ab4bb8547d9737acc8a54a9de4 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 18 Aug 2020 16:29:08 -0400 Subject: [PATCH 09/27] Updated CCPA Policy --- privacy/ccpa/validatedpolicy.go | 112 ++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 privacy/ccpa/validatedpolicy.go diff --git a/privacy/ccpa/validatedpolicy.go b/privacy/ccpa/validatedpolicy.go new file mode 100644 index 00000000000..956aac3c3b5 --- /dev/null +++ b/privacy/ccpa/validatedpolicy.go @@ -0,0 +1,112 @@ +package ccpa + +import ( + "errors" + "fmt" + "strings" + + "github.com/prebid/prebid-server/openrtb_ext" +) + +const ( + ccpaVersion1 = '1' + ccpaNo = 'N' + ccpaYes = 'Y' + ccpaNotApplicable = '-' +) + +const ( + indexVersion = 0 + indexExplicitNotice = 1 + indexOptOutSale = 2 + indexLSPACoveredTransaction = 3 +) + +const allBidders = "*" + +type ValidatedPolicy struct { + Policy + OptOutSaleYes bool + NoSaleAllBidders bool + NoSaleSpecificBidders map[string]struct{} +} + +type ValidationErrors struct { + Consent error + NoSaleBidders error +} + +func (p Policy) Validate() (ValidatedPolicy, error) { + if err := ValidateConsent(p.Value); err != nil { + return fmt.Errorf("request.regs.ext.us_privacy %s", err.Error()) + } + + if err := ValidateNoSaleBidders(p.NoSaleBidders); err != nil { + return fmt.Errorf("request.ext.prebid.nosale %s", err.Error()) + } + + return nil +} + +// ValidateConsent returns an error if the CCPA consent string does not adhere to the IAB spec. +func ValidateConsent(consent string) (ValidatedPolicy, error) { + if consent == "" { + return nil + } + + if len(consent) != 4 { + return errors.New("must contain 4 characters") + } + + if consent[indexVersion] != ccpaVersion1 { + return errors.New("must specify version 1") + } + + var c byte + + c = consent[indexExplicitNotice] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { + return errors.New("must specify 'N', 'Y', or '-' for the explicit notice") + } + + c = consent[indexOptOutSale] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { + return errors.New("must specify 'N', 'Y', or '-' for the opt-out sale") + } + + c = consent[indexLSPACoveredTransaction] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { + return errors.New("must specify 'N', 'Y', or '-' for the limited service provider agreement") + } + + return nil +} + +func ValidateNoSaleBidders(noSaleBidders []string, bidders map[string]openrtb_ext.BidderName, aliases map[string]string) error { + if len(noSaleBidders) == 1 && noSaleBidders[0] == allBidders { + return nil + } + + for _, bidder := range noSaleBidders { + if !validBidders.cotains[bidder] { + return fmt.Errorf("unrecognized bidder '%s'", bidder) + } + } + + return nil +} + +// ShouldEnforce returns true when the opt-out signal is explicitly detected. +func (p ValidatedPolicy) ShouldEnforce(bidder string) bool { + if err := p.Validate(); err != nil { + return false + } + + for _, b := range p.NoSaleBidders { + if b == allBidders || strings.EqualFold(b, bidder) { + return false + } + } + + return p.Value != "" && p.Value[indexOptOutSale] == ccpaYes +} From fa6488f91a57475c4fb6393aa58e73c3c74df2dc Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 18 Aug 2020 16:29:31 -0400 Subject: [PATCH 10/27] Updated CCPA Policy (Missing Files) --- endpoints/openrtb2/auction.go | 16 + privacy/ccpa/policy.go | 272 ++++++------- privacy/ccpa/policy_test.go | 700 ++++++++++++++------------------ privacy/ccpa/validatedpolicy.go | 112 ----- 4 files changed, 456 insertions(+), 644 deletions(-) delete mode 100644 privacy/ccpa/validatedpolicy.go diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 86186fa8373..201da12dc1d 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -336,6 +336,22 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { policy.Value = "" if err := policy.Write(req); err != nil { errL = append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) + errL = append(errL, err) + return errL + } else { + _, err := policy.Validate(); err != nil { + if err.ValueError != nil { + policy.Value = "" + if err := ccpaPolicy.Write(req); err != nil { + errL = append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) + } + errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) + } + + if err.NoSaleBidderError != nil { + errL = append(errL, err.NoSaleBidderError) + return errL + } } } diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 5101151ab05..c41b70427c2 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -3,210 +3,210 @@ package ccpa import ( "encoding/json" "errors" - "fmt" - "strings" "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/openrtb_ext" ) -const ( - ccpaVersion1 = '1' - ccpaNo = 'N' - ccpaYes = 'Y' - ccpaNotApplicable = '-' -) - -const ( - indexVersion = 0 - indexExplicitNotice = 1 - indexOptOutSale = 2 - indexLSPACoveredTransaction = 3 -) - -const allBidders = "*" - -// RawPolicy represents the user provided CCPA regulation values. -type RawPolicy struct { - Value string +// Policy represents the CCPA regulatory information from the OpenRTB bid request. +type Policy struct { + Consent string NoSaleBidders []string } -// ParsedPolicy represents the parsed and validated CCPA regulation values. -type ParsedPolicy struct { - OptOutSaleYes bool - NoSaleAllBidders bool - NoSaleSpecificBidders map[string]struct{} -} - -// ReadPolicy extracts the CCPA regulation policy from an OpenRTB request. +// ReadPolicy extracts the CCPA regulatory information from the OpenRTB bid request. func ReadPolicy(req *openrtb.BidRequest) (Policy, error) { - policy := Policy{} + var consent string + var noSaleBidders []string + + if req == nil { + return Policy{}, nil + } - var value string + // Read consent from request.regs.ext if req.Regs != nil && len(req.Regs.Ext) > 0 { var ext openrtb_ext.ExtRegs if err := json.Unmarshal(req.Regs.Ext, &ext); err != nil { - return RawPolicy{}, err + return Policy{}, err } - value = ext.USPrivacy + consent = ext.USPrivacy } - var noSaleBidders []string + // Read no sale bidders from request.ext.prebid if len(req.Ext) > 0 { var ext openrtb_ext.ExtRequest if err := json.Unmarshal(req.Ext, &ext); err != nil { - return RawPolicy{}, err + return Policy{}, err } noSaleBidders = ext.Prebid.NoSale } - result := RawPolicy{{ - Value: value, - NoSaleBidders: noSaleBidders, - } - return result, nil + return Policy{consent, noSaleBidders}, nil } - - -// Write mutates an OpenRTB bid request with the context of the CCPA policy. -func (p Policy) Write(req *openrtb.BidRequest) error { - if p.Value == "" { - return clearPolicy(req) - } - +// Write mutates an OpenRTB bid request with the CCPA regulatory information. +func (p Policy) Write(req *openrtb.BidRequest) (err error) { if req == nil { - return nil + return } - if req.Regs == nil { - req.Regs = &openrtb.Regs{} + regs, err := buildRegs(p.Consent, req.Regs) + if err != nil { + return } - - if req.Regs.Ext == nil { - ext, err := json.Marshal(openrtb_ext.ExtRegs{USPrivacy: p.Value}) - if err == nil { - req.Regs.Ext = ext - } - return err + ext, err := buildExt(p.NoSaleBidders, req.Ext) + if err != nil { + return } - var extMap map[string]interface{} - err := json.Unmarshal(req.Regs.Ext, &extMap) - if err == nil { - extMap["us_privacy"] = p.Value - ext, err := json.Marshal(extMap) - if err == nil { - req.Regs.Ext = ext - } - } - return err -} + req.Regs = regs + req.Ext = ext -func clearPolicy(req *openrtb.BidRequest) error { - if req == nil { - return nil - } + return +} - if req.Regs == nil { - return nil +func buildRegs(consent string, regs *openrtb.Regs) (*openrtb.Regs, error) { + if consent == "" { + return buildRegsClear(regs) } + return buildRegsWrite(consent, regs) +} - if len(req.Regs.Ext) == 0 { - return nil +func buildRegsClear(regs *openrtb.Regs) (*openrtb.Regs, error) { + if regs == nil || len(regs.Ext) == 0 { + return regs, nil } var extMap map[string]interface{} - err := json.Unmarshal(req.Regs.Ext, &extMap) + err := json.Unmarshal(regs.Ext, &extMap) if err == nil { + regsCopy := *regs + + // Remove CCPA consent delete(extMap, "us_privacy") + + // Remove entire ext if it's empty if len(extMap) == 0 { - req.Regs.Ext = nil - } else { - ext, err := json.Marshal(extMap) - if err == nil { - req.Regs.Ext = ext - } - return err + regsCopy.Ext = nil + return ®sCopy, nil } - } - return err -} + ext, err := json.Marshal(extMap) + if err != nil { + return nil, err + } -// Validate returns an error if the CCPA policy does not adhere to the IAB spec or the NoSale list is invalid. -func (p Policy) Validate(bidders []string) error { - if err := ValidateConsent(p.Value); err != nil { - return fmt.Errorf("request.regs.ext.us_privacy %s", err.Error()) + regsCopy.Ext = ext + return ®sCopy, nil } + return nil, err +} + +func buildRegsWrite(consent string, regs *openrtb.Regs) (*openrtb.Regs, error) { + var regsCopy openrtb.Regs - if err := ValidateNoSaleBidders(p.NoSaleBidders); err != nil { - return fmt.Errorf("request.ext.prebid.nosale %s", err.Error()) + if regs == nil { + regsCopy = openrtb.Regs{} + } else { + regsCopy = *regs } - return nil -} + if regsCopy.Ext == nil { + ext, err := json.Marshal(openrtb_ext.ExtRegs{USPrivacy: consent}) + if err != nil { + return nil, err + } -// ValidateConsent returns an error if the CCPA consent string does not adhere to the IAB spec. -func ValidateConsent(consent string) error { - if consent == "" { - return nil + regsCopy.Ext = ext + return ®sCopy, nil } - if len(consent) != 4 { - return errors.New("must contain 4 characters") - } + var extMap map[string]interface{} + err := json.Unmarshal(regsCopy.Ext, &extMap) + if err == nil { + // Set CCPA consent + extMap["us_privacy"] = consent + + ext, err := json.Marshal(extMap) + if err != nil { + return nil, err + } - if consent[indexVersion] != ccpaVersion1 { - return errors.New("must specify version 1") + regsCopy.Ext = ext + return ®sCopy, nil } - var c byte + return nil, err +} - c = consent[indexExplicitNotice] - if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { - return errors.New("must specify 'N', 'Y', or '-' for the explicit notice") +func buildExt(noSaleBidders []string, ext json.RawMessage) (json.RawMessage, error) { + if len(noSaleBidders) == 0 { + return buildExtClear(ext) } + return buildExtWrite(noSaleBidders, ext) +} - c = consent[indexOptOutSale] - if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { - return errors.New("must specify 'N', 'Y', or '-' for the opt-out sale") +func buildExtClear(ext json.RawMessage) (json.RawMessage, error) { + if len(ext) == 0 { + return ext, nil } - c = consent[indexLSPACoveredTransaction] - if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { - return errors.New("must specify 'N', 'Y', or '-' for the limited service provider agreement") - } + var extMap map[string]interface{} + err := json.Unmarshal(ext, &extMap) + if err == nil { + prebidExt, exists := extMap["prebid"] - return nil -} + // If there's no prebid, there's nothing to do + if !exists { + return ext, nil + } -func ValidateNoSaleBidders(noSaleBidders []string, bidders map[string]openrtb_ext.BidderName, aliases map[string]string) error { - if len(noSaleBidders) == 1 && noSaleBidders[0] == allBidders { - return nil - } + // Verify prebid is an object + prebidExtMap, ok := prebidExt.(map[string]interface{}) + if !ok { + return nil, errors.New("prebi is not a map") + } - for _, bidder := range noSaleBidders { - if !validBidders.cotains[bidder] { - return fmt.Errorf("unrecognized bidder '%s'", bidder) + // Remove no sale member + delete(prebidExtMap, "nosale") + if len(prebidExtMap) == 0 { + delete(extMap, "prebid") } - } - return nil + // Remove entire ext if it's empty + if len(extMap) == 0 { + return nil, nil + } + + return json.Marshal(extMap) + } + return nil, err } -// ShouldEnforce returns true when the opt-out signal is explicitly detected. -func (p Policy) ShouldEnforce(bidder string) bool { - if err := p.Validate(); err != nil { - return false +func buildExtWrite(noSaleBidders []string, ext json.RawMessage) (json.RawMessage, error) { + if len(ext) == 0 { + return json.Marshal(openrtb_ext.ExtRequest{Prebid: openrtb_ext.ExtRequestPrebid{NoSale: noSaleBidders}}) } - for _, b := range p.NoSaleBidders { - if b == allBidders || strings.EqualFold(b, bidder) { - return false + var extMap map[string]interface{} + err := json.Unmarshal(ext, &extMap) + if err == nil { + var prebidExt map[string]interface{} + if prebidExtInterface, exists := extMap["prebid"]; exists { + // Reference Existing Prebid Ext Map + if prebidExtMap, ok := prebidExtInterface.(map[string]interface{}); ok { + prebidExt = prebidExtMap + } else { + return nil, errors.New("prebid invalid") + } + } else { + // Create New Empty Prebid Ext Map + prebidExt = make(map[string]interface{}) + extMap["prebid"] = prebidExt } - } - return p.Value != "" && p.Value[indexOptOutSale] == ccpaYes + prebidExt["nosale"] = noSaleBidders + return json.Marshal(extMap) + } + return nil, err } diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index a7d4e3563ca..da2cf3339ea 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -15,6 +15,14 @@ func TestRead(t *testing.T) { expectedPolicy Policy expectedError bool }{ + { + description: "Nil Request", + request: nil, + expectedPolicy: Policy{ + Consent: "", + NoSaleBidders: nil, + }, + }, { description: "Success", request: &openrtb.BidRequest{ @@ -24,18 +32,10 @@ func TestRead(t *testing.T) { Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "ABC", + Consent: "ABC", NoSaleBidders: []string{"a", "b"}, }, }, - { - description: "Nil Request", - request: nil, - expectedPolicy: Policy{ - Value: "", - NoSaleBidders: nil, - }, - }, { description: "Nil Regs", request: &openrtb.BidRequest{ @@ -43,7 +43,7 @@ func TestRead(t *testing.T) { Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "", + Consent: "", NoSaleBidders: []string{"a", "b"}, }, }, @@ -54,7 +54,7 @@ func TestRead(t *testing.T) { Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "", + Consent: "", NoSaleBidders: []string{"a", "b"}, }, }, @@ -67,12 +67,12 @@ func TestRead(t *testing.T) { Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "", + Consent: "", NoSaleBidders: []string{"a", "b"}, }, }, { - description: "Nil Regs.Ext Value", + description: "Missing Regs.Ext USPrivacy Value", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"anythingElse":"42"}`), @@ -80,10 +80,30 @@ func TestRead(t *testing.T) { Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedPolicy: Policy{ - Value: "", + Consent: "", NoSaleBidders: []string{"a", "b"}, }, }, + { + description: "Malformed Regs.Ext", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`malformed`), + }, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + }, + expectedError: true, + }, + { + description: "Invalid Regs.Ext Type", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":123`), + }, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + }, + expectedError: true, + }, { description: "Nil Ext", request: &openrtb.BidRequest{ @@ -93,7 +113,7 @@ func TestRead(t *testing.T) { Ext: nil, }, expectedPolicy: Policy{ - Value: "ABC", + Consent: "ABC", NoSaleBidders: nil, }, }, @@ -106,12 +126,12 @@ func TestRead(t *testing.T) { Ext: json.RawMessage(`{}`), }, expectedPolicy: Policy{ - Value: "ABC", + Consent: "ABC", NoSaleBidders: nil, }, }, { - description: "Nil Ext.Prebid.NoSale", + description: "Missing Ext.Prebid No Sale Value", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), @@ -119,45 +139,29 @@ func TestRead(t *testing.T) { Ext: json.RawMessage(`{"anythingElse":"42"}`), }, expectedPolicy: Policy{ - Value: "ABC", + Consent: "ABC", NoSaleBidders: nil, }, }, { - description: "Malformed Regs.Ext", - request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`malformed`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), - }, - expectedError: true, - }, - { - description: "Malformed Ext - NoSale Ignored", + description: "Malformed Ext", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), }, Ext: json.RawMessage(`malformed`), }, - expectedPolicy: Policy{ - Value: "ABC", - NoSaleBidders: nil, - }, + expectedError: true, }, { - description: "Incorrect Ext.Prebid.NoSale Type - NoSale Ignored", + description: "Invalid Ext.Prebid.NoSale Type", request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"ABC"}`), }, Ext: json.RawMessage(`{"prebid":{"nosale":"wrongtype"}}`), }, - expectedPolicy: Policy{ - Value: "ABC", - NoSaleBidders: nil, - }, + expectedError: true, }, { description: "Injection Attack", @@ -167,22 +171,15 @@ func TestRead(t *testing.T) { }, }, expectedPolicy: Policy{ - Value: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"", + Consent: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"", }, }, } for _, test := range testCases { - - p, e := ReadPolicy(test.request) - - if test.expectedError { - assert.Error(t, e, test.description) - } else { - assert.NoError(t, e, test.description) - } - - assert.Equal(t, test.expectedPolicy, p, test.description) + result, err := ReadPolicy(test.request) + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expectedPolicy, result, test.description) } } @@ -194,518 +191,429 @@ func TestWrite(t *testing.T) { expected *openrtb.BidRequest expectedError bool }{ + { + description: "Nil Request", + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + request: nil, + expected: nil, + }, { description: "Success", - policy: Policy{Value: "anyValue", NoSaleBidders: []string{"a", "b"}}, + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{}, expected: &openrtb.BidRequest{ Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"anyValue"}`), + Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`), }, Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, }, { - description: "Error Regs.Ext", - policy: Policy{Value: "anyValue", NoSaleBidders: []string{"a", "b"}}, + description: "Error Regs.Ext - No Partial Update To Request", + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Regs: &openrtb.Regs{ Ext: json.RawMessage(`malformed}`), }, }, expectedError: true, + expected: &openrtb.BidRequest{ + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`malformed}`), + }, + }, }, { - description: "Error Ext", - policy: Policy{Value: "anyValue", NoSaleBidders: []string{"a", "b"}}, + description: "Error Ext - No Partial Update To Request", + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: json.RawMessage(`malformed}`), }, expectedError: true, + expected: &openrtb.BidRequest{ + Ext: json.RawMessage(`malformed}`), + }, }, } for _, test := range testCases { err := test.policy.Write(test.request) - - if test.expectedError { - assert.Error(t, err, test.description) - } else { - assert.NoError(t, err, test.description) - assert.Equal(t, test.expected, test.request, test.description) - } + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, test.request, test.description) } } -func TestWriteRegsExt(t *testing.T) { +func TestBuildRegs(t *testing.T) { testCases := []struct { description string - policy Policy - request *openrtb.BidRequest - expected *openrtb.BidRequest + consent string + regs *openrtb.Regs + expected *openrtb.Regs expectedError bool }{ { - description: "Disabled", - policy: Policy{Value: ""}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{}, - }, - { - description: "Enabled - Nil Regs", - description: "Disabled - Nil Request", - policy: Policy{Value: ""}, - request: nil, - expected: nil, - }, - { - description: "Disabled - Empty Regs.Ext", - policy: Policy{Value: ""}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{}}, + description: "Clear", + consent: "", + regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"ABC"}`), + }, + expected: &openrtb.Regs{}, }, { - description: "Disabled - Remove From Request", - policy: Policy{Value: ""}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"toBeRemoved"}`)}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{}}, + description: "Clear - Error", + consent: "", + regs: &openrtb.Regs{ + Ext: json.RawMessage(`malformed`), + }, + expectedError: true, }, { - description: "Disabled - Remove From Request, Leave Other req Values", - policy: Policy{Value: ""}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{ - COPPA: 42, - Ext: json.RawMessage(`{"us_privacy":"toBeRemoved"}`)}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - COPPA: 42}}, + description: "Write", + consent: "anyConsent", + regs: nil, + expected: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`), + }, }, { - description: "Disabled - Remove From Request, Leave Other req.ext Values", - policy: Policy{Value: ""}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any","us_privacy":"toBeRemoved"}`)}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any"}`)}}, + description: "Write - Error", + consent: "anyConsent", + regs: &openrtb.Regs{ + Ext: json.RawMessage(`malformed`), + }, + expectedError: true, }, + } + + for _, test := range testCases { + result, err := buildRegs(test.consent, test.regs) + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, result, test.description) + } +} + +func TestBuildRegsClear(t *testing.T) { + testCases := []struct { + description string + regs *openrtb.Regs + expected *openrtb.Regs + expectedError bool + }{ { - description: "Enabled - Nil Request", - policy: Policy{Value: "anyValue"}, - request: nil, + description: "Nil Regs", + regs: nil, expected: nil, }, { - description: "Enabled With Nil Request Regs Object", - policy: Policy{Value: "anyValue"}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"anyValue"}`)}}, - }, - { - description: "Enabled - Nil Regs.Ext", - policy: Policy{Value: "anyValue"}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"anyValue"}`)}}, - }, - { - description: "Enabled - Existing Regs.Ext - Doesn't Overwrite", - policy: Policy{Value: "anyValue"}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any"}`)}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any","us_privacy":"anyValue"}`)}}, + description: "Nil Regs.Ext", + regs: &openrtb.Regs{Ext: nil}, + expected: &openrtb.Regs{Ext: nil}, }, { - description: "Enabled - Existing Regs.Ext.US_Privacy - Overwrites", - policy: Policy{Value: "anyValue"}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any","us_privacy":"toBeOverwritten"}`)}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any","us_privacy":"anyValue"}`)}}, + description: "Empty Regs.Ext", + regs: &openrtb.Regs{Ext: json.RawMessage(`{}`)}, + expected: &openrtb.Regs{}, }, { - description: "Enabled - Malformed Regs.Ext", - policy: Policy{Value: "anyValue"}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`malformed`)}}, - expectedError: true, + description: "Removes Regs.Ext Entirely", + regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + expected: &openrtb.Regs{}, }, { - description: "Injection Attack - Nil Regs", - policy: Policy{Value: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), - }}, + description: "Leaves Other Regs.Ext Values", + regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC", "other":"any"}`)}, + expected: &openrtb.Regs{Ext: json.RawMessage(`{"other":"any"}`)}, }, { - description: "Injection Attack - Nil Regs.Ext", - policy: Policy{Value: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{}}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), - }}, + description: "Invalid Regs.Ext Type - Still Cleared", + regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":123}`)}, + expected: &openrtb.Regs{}, }, { - description: "Injection Attack - Existing Regs.Ext", - policy: Policy{Value: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, - request: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any"}`), - }}, - expected: &openrtb.BidRequest{Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"existing":"any","us_privacy":"1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), - }}, + description: "Malformed Regs.Ext", + regs: &openrtb.Regs{Ext: json.RawMessage(`malformed`)}, + expectedError: true, }, } for _, test := range testCases { - err := test.policy.writeRegsExt(test.request) - - if test.expectedError { - assert.Error(t, err, test.description) - } else { - assert.NoError(t, err, test.description) - assert.Equal(t, test.expected, test.request, test.description) - } + result, err := buildRegsClear(test.regs) + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, result, test.description) } } -func TestWriteExt(t *testing.T) { +func TestBuildRegsWrite(t *testing.T) { testCases := []struct { description string - policy Policy - request *openrtb.BidRequest - expected *openrtb.BidRequest + consent string + regs *openrtb.Regs + expected *openrtb.Regs expectedError bool }{ { - description: "Nil NoSaleBidders", - policy: Policy{NoSaleBidders: nil}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{}, - }, - { - description: "Empty NoSaleBidders", - policy: Policy{NoSaleBidders: []string{}}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{}, - }, - { - description: "Nil Ext", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: nil, - }, - expected: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), - }, + description: "Nil Regs", + consent: "anyConsent", + regs: nil, + expected: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, }, { - description: "Empty Ext", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: json.RawMessage(`{}`), - }, - expected: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), - }, + description: "Nil Regs.Ext", + consent: "anyConsent", + regs: &openrtb.Regs{Ext: nil}, + expected: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, }, { - description: "Empty Ext.Prebid", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{}}`), - }, - expected: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), - }, + description: "Empty Regs.Ext", + consent: "anyConsent", + regs: &openrtb.Regs{Ext: json.RawMessage(`{}`)}, + expected: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, }, { - description: "Existing Values In Ext", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"existing":true,"prebid":{}}`), - }, - expected: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"existing":true,"prebid":{"nosale":["a","b"]}}`), - }, + description: "Overwrites Existing", + consent: "anyConsent", + regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + expected: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, }, { - description: "Existing Values In Ext.Prebid", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"existing":true}}`), - }, - expected: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"existing":true,"nosale":["a","b"]}}`), - }, + description: "Leaves Other Ext Values", + consent: "anyConsent", + regs: &openrtb.Regs{Ext: json.RawMessage(`{"other":"any"}`)}, + expected: &openrtb.Regs{Ext: json.RawMessage(`{"other":"any","us_privacy":"anyConsent"}`)}, }, { - description: "Overwrite Existing In Ext.Prebid", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"nosale":["1","2"]}}`), - }, - expected: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), - }, + description: "Invalid Regs.Ext Type - Still Overwrites", + consent: "anyConsent", + regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":123}`)}, + expected: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, }, { - description: "Malformed Ext", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: json.RawMessage(`malformed`), - }, - expectedError: true, - }, - { - description: "Invalid Ext.Prebid Type", - policy: Policy{NoSaleBidders: []string{"a", "b"}}, - request: &openrtb.BidRequest{ - Ext: json.RawMessage(`{"prebid":42}`), - }, + description: "Malformed Regs.Ext", + consent: "anyConsent", + regs: &openrtb.Regs{Ext: json.RawMessage(`malformed`)}, expectedError: true, }, } for _, test := range testCases { - err := test.policy.writeExt(test.request) - - if test.expectedError { - assert.Error(t, err, test.description) - } else { - assert.NoError(t, err, test.description) - assert.Equal(t, test.expected, test.request, test.description) - } + result, err := buildRegsWrite(test.consent, test.regs) + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, result, test.description) } } -func TestValidate(t *testing.T) { +func TestBuildExt(t *testing.T) { testCases := []struct { description string - policy Policy - expectedError string + noSaleBidders []string + ext json.RawMessage + expected json.RawMessage + expectedError bool }{ { - description: "Valid", - policy: Policy{Value: "1NYN"}, - expectedError: "", + description: "Clear - Nil", + noSaleBidders: nil, + ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + expected: nil, }, { - description: "Valid - Not Applicable", - policy: Policy{Value: "1---"}, - expectedError: "", + description: "Clear - Empty", + noSaleBidders: []string{}, + ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + expected: nil, }, { - description: "Valid - Empty", - policy: Policy{Value: ""}, - expectedError: "", - }, - { - description: "Invalid Length", - policy: Policy{Value: "1NY"}, - expectedError: "request.regs.ext.us_privacy must contain 4 characters", - }, - { - description: "Invalid Version", - policy: Policy{Value: "2---"}, - expectedError: "request.regs.ext.us_privacy must specify version 1", - }, - { - description: "Invalid Explicit Notice Char", - policy: Policy{Value: "1X--"}, - expectedError: "request.regs.ext.us_privacy must specify 'N', 'Y', or '-' for the explicit notice", - }, - { - description: "Invalid Explicit Notice Case", - policy: Policy{Value: "1y--"}, - expectedError: "request.regs.ext.us_privacy must specify 'N', 'Y', or '-' for the explicit notice", - }, - { - description: "Invalid Opt-Out Sale Char", - policy: Policy{Value: "1-X-"}, - expectedError: "request.regs.ext.us_privacy must specify 'N', 'Y', or '-' for the opt-out sale", - }, - { - description: "Invalid Opt-Out Sale Case", - policy: Policy{Value: "1-y-"}, - expectedError: "request.regs.ext.us_privacy must specify 'N', 'Y', or '-' for the opt-out sale", + description: "Clear - Error", + noSaleBidders: []string{}, + ext: json.RawMessage(`malformed`), + expectedError: true, }, { - description: "Invalid LSPA Char", - policy: Policy{Value: "1--X"}, - expectedError: "request.regs.ext.us_privacy must specify 'N', 'Y', or '-' for the limited service provider agreement", + description: "Write", + noSaleBidders: []string{"a", "b"}, + ext: nil, + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, { - description: "Invalid LSPA Case", - policy: Policy{Value: "1--y"}, - expectedError: "request.regs.ext.us_privacy must specify 'N', 'Y', or '-' for the limited service provider agreement", + description: "Write - Error", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`malformed`), + expectedError: true, }, } for _, test := range testCases { - result := test.policy.Validate() - - if test.expectedError == "" { - assert.NoError(t, result, test.description) - } else { - assert.EqualError(t, result, test.expectedError, test.description) - } + result, err := buildExt(test.noSaleBidders, test.ext) + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, result, test.description) } } -func TestValidateConsent(t *testing.T) { +func TestBuildExtClear(t *testing.T) { testCases := []struct { description string - consent string - expectedError string + ext json.RawMessage + expected json.RawMessage + expectedError bool }{ { - description: "Valid", - consent: "1NYN", - expectedError: "", + description: "Nil Ext", + ext: nil, + expected: nil, }, { - description: "Valid - Not Applicable", - consent: "1---", - expectedError: "", + description: "Empty Ext", + ext: json.RawMessage(``), + expected: json.RawMessage(``), }, { - description: "Invalid Empty", - consent: "", - expectedError: "", + description: "Empty Ext Object", + ext: json.RawMessage(`{}`), + expected: json.RawMessage(`{}`), }, { - description: "Invalid Length", - consent: "1NY", - expectedError: "must contain 4 characters", + description: "Empty Ext.Prebid", + ext: json.RawMessage(`{"prebid":{}}`), + expected: nil, }, { - description: "Invalid Version", - consent: "2---", - expectedError: "must specify version 1", + description: "Removes Ext Entirely", + ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), + expected: nil, }, { - description: "Invalid Explicit Notice Char", - consent: "1X--", - expectedError: "must specify 'N', 'Y', or '-' for the explicit notice", + description: "Leaves Other Ext Values", + ext: json.RawMessage(`{"other":"any","prebid":{"nosale":["a","b"]}}`), + expected: json.RawMessage(`{"other":"any"}`), }, { - description: "Invalid Explicit Notice Case", - consent: "1y--", - expectedError: "must specify 'N', 'Y', or '-' for the explicit notice", + description: "Leaves Other Ext.Prebid Values", + ext: json.RawMessage(`{"prebid":{"nosale":["a","b"],"other":"any"}}`), + expected: json.RawMessage(`{"prebid":{"other":"any"}}`), }, { - description: "Invalid Opt-Out Sale Char", - consent: "1-X-", - expectedError: "must specify 'N', 'Y', or '-' for the opt-out sale", + description: "Leaves All Other Values", + ext: json.RawMessage(`{"other":"ABC","prebid":{"nosale":["a","b"],"other":"123"}}`), + expected: json.RawMessage(`{"other":"ABC","prebid":{"other":"123"}}`), }, { - description: "Invalid Opt-Out Sale Case", - consent: "1-y-", - expectedError: "must specify 'N', 'Y', or '-' for the opt-out sale", + description: "Malformed Ext", + ext: json.RawMessage(`malformed`), + expectedError: true, }, { - description: "Invalid LSPA Char", - consent: "1--X", - expectedError: "must specify 'N', 'Y', or '-' for the limited service provider agreement", + description: "Malformed Ext.Prebid", + ext: json.RawMessage(`{"prebid":malformed}`), + expectedError: true, }, { - description: "Invalid LSPA Case", - consent: "1--y", - expectedError: "must specify 'N', 'Y', or '-' for the limited service provider agreement", + description: "Invalid Ext.Prebid Type", + ext: json.RawMessage(`{"prebid":123}`), + expectedError: true, }, } for _, test := range testCases { - result := ValidateConsent(test.consent) - - if test.expectedError == "" { - assert.NoError(t, result, test.description) - } else { - assert.EqualError(t, result, test.expectedError, test.description) - } + result, err := buildExtClear(test.ext) + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, result, test.description) } } -func TestShouldEnforce(t *testing.T) { +func TestBuildExtWrite(t *testing.T) { testCases := []struct { - description string - bidder string - policy Policy - expected bool + description string + noSaleBidders []string + ext json.RawMessage + expected json.RawMessage + expectedError bool }{ { - description: "Enforceable", - bidder: "a", - policy: Policy{Value: "1-Y-"}, - expected: true, + description: "Nil Ext", + noSaleBidders: []string{"a", "b"}, + ext: nil, + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, { - description: "Enforceable - No Sale For Different Bidder", - bidder: "a", - policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"b"}}, - expected: true, + description: "Empty Ext", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(``), + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, { - description: "Not Enforceable - Not Present", - bidder: "a", - policy: Policy{Value: ""}, - expected: false, + description: "Empty Ext Object", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{}`), + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, { - description: "Not Enforceable - Opt-Out Unknown", - bidder: "a", - policy: Policy{Value: "1---"}, - expected: false, + description: "Empty Ext.Prebid", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"prebid":{}}`), + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, { - description: "Not Enforceable - Opt-Out Explicitly No", - bidder: "a", - policy: Policy{Value: "1-N-"}, - expected: false, + description: "Overwrites Existing", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"prebid":{"nosale":["x","y"]}}`), + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, { - description: "Not Enforceable - No Sale All Bidders", - bidder: "a", - policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"*"}}, - expected: false, + description: "Leaves Other Ext Values", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"other":"any"}`), + expected: json.RawMessage(`{"other":"any","prebid":{"nosale":["a","b"]}}`), }, { - description: "Not Enforceable - No Sale All Bidders Mixed With Specific Bidders", - bidder: "a", - policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"b", "*", "c"}}, - expected: false, + description: "Leaves Other Ext.Prebid Values", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"prebid":{"other":"any"}}`), + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"],"other":"any"}}`), }, { - description: "Not Enforceable - No Sale Specific Bidder", - bidder: "a", - policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"a"}}, - expected: false, + description: "Leaves All Other Values", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"other":"ABC","prebid":{"other":"123"}}`), + expected: json.RawMessage(`{"other":"ABC","prebid":{"nosale":["a","b"],"other":"123"}}`), }, { - description: "Not Enforceable - No Sale Specific Bidder Mixed", - bidder: "a", - policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"b", "a", "c"}}, - expected: false, + description: "Invalid Ext.Prebid No Sale Type - Still Overrides", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"prebid":{"nosale":123}}`), + expected: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, { - description: "Not Enforceable - No Sale Specific Bidder Case Insensitive", - bidder: "a", - policy: Policy{Value: "1-Y-", NoSaleBidders: []string{"A"}}, - expected: false, + description: "Invalid Ext.Prebid Type ", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"prebid":"wrongtype"}`), + expectedError: true, }, { - description: "Invalid", - bidder: "a", - policy: Policy{Value: "2---"}, - expected: false, + description: "Malformed Ext", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{malformed`), + expectedError: true, + }, + { + description: "Malformed Ext.Prebid", + noSaleBidders: []string{"a", "b"}, + ext: json.RawMessage(`{"prebid":malformed}`), + expectedError: true, }, } for _, test := range testCases { - result := test.policy.ShouldEnforce(test.bidder) + result, err := buildExtWrite(test.noSaleBidders, test.ext) + assertError(t, test.expectedError, err, test.description) assert.Equal(t, test.expected, result, test.description) } } + +func assertError(t *testing.T, expectError bool, err error, description string) { + t.Helper() + if expectError { + assert.Error(t, err, description) + } else { + assert.NoError(t, err, description) + } +} diff --git a/privacy/ccpa/validatedpolicy.go b/privacy/ccpa/validatedpolicy.go deleted file mode 100644 index 956aac3c3b5..00000000000 --- a/privacy/ccpa/validatedpolicy.go +++ /dev/null @@ -1,112 +0,0 @@ -package ccpa - -import ( - "errors" - "fmt" - "strings" - - "github.com/prebid/prebid-server/openrtb_ext" -) - -const ( - ccpaVersion1 = '1' - ccpaNo = 'N' - ccpaYes = 'Y' - ccpaNotApplicable = '-' -) - -const ( - indexVersion = 0 - indexExplicitNotice = 1 - indexOptOutSale = 2 - indexLSPACoveredTransaction = 3 -) - -const allBidders = "*" - -type ValidatedPolicy struct { - Policy - OptOutSaleYes bool - NoSaleAllBidders bool - NoSaleSpecificBidders map[string]struct{} -} - -type ValidationErrors struct { - Consent error - NoSaleBidders error -} - -func (p Policy) Validate() (ValidatedPolicy, error) { - if err := ValidateConsent(p.Value); err != nil { - return fmt.Errorf("request.regs.ext.us_privacy %s", err.Error()) - } - - if err := ValidateNoSaleBidders(p.NoSaleBidders); err != nil { - return fmt.Errorf("request.ext.prebid.nosale %s", err.Error()) - } - - return nil -} - -// ValidateConsent returns an error if the CCPA consent string does not adhere to the IAB spec. -func ValidateConsent(consent string) (ValidatedPolicy, error) { - if consent == "" { - return nil - } - - if len(consent) != 4 { - return errors.New("must contain 4 characters") - } - - if consent[indexVersion] != ccpaVersion1 { - return errors.New("must specify version 1") - } - - var c byte - - c = consent[indexExplicitNotice] - if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { - return errors.New("must specify 'N', 'Y', or '-' for the explicit notice") - } - - c = consent[indexOptOutSale] - if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { - return errors.New("must specify 'N', 'Y', or '-' for the opt-out sale") - } - - c = consent[indexLSPACoveredTransaction] - if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { - return errors.New("must specify 'N', 'Y', or '-' for the limited service provider agreement") - } - - return nil -} - -func ValidateNoSaleBidders(noSaleBidders []string, bidders map[string]openrtb_ext.BidderName, aliases map[string]string) error { - if len(noSaleBidders) == 1 && noSaleBidders[0] == allBidders { - return nil - } - - for _, bidder := range noSaleBidders { - if !validBidders.cotains[bidder] { - return fmt.Errorf("unrecognized bidder '%s'", bidder) - } - } - - return nil -} - -// ShouldEnforce returns true when the opt-out signal is explicitly detected. -func (p ValidatedPolicy) ShouldEnforce(bidder string) bool { - if err := p.Validate(); err != nil { - return false - } - - for _, b := range p.NoSaleBidders { - if b == allBidders || strings.EqualFold(b, bidder) { - return false - } - } - - return p.Value != "" && p.Value[indexOptOutSale] == ccpaYes -} From 8994fc135d363b22e0e0917bf264ae93061d4184 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 18 Aug 2020 16:31:56 -0400 Subject: [PATCH 11/27] Better Error Messages --- privacy/ccpa/policy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index c41b70427c2..1ce05a92c62 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -164,7 +164,7 @@ func buildExtClear(ext json.RawMessage) (json.RawMessage, error) { // Verify prebid is an object prebidExtMap, ok := prebidExt.(map[string]interface{}) if !ok { - return nil, errors.New("prebi is not a map") + return nil, errors.New("ext.prebid is not a json object") } // Remove no sale member @@ -197,7 +197,7 @@ func buildExtWrite(noSaleBidders []string, ext json.RawMessage) (json.RawMessage if prebidExtMap, ok := prebidExtInterface.(map[string]interface{}); ok { prebidExt = prebidExtMap } else { - return nil, errors.New("prebid invalid") + return nil, errors.New("ext.prebid is not a json object") } } else { // Create New Empty Prebid Ext Map From b17d3458fe59bd86ccfa3ccd209518ebd7c0b0f1 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 18 Aug 2020 17:15:34 -0400 Subject: [PATCH 12/27] Added Parsed Policy --- privacy/ccpa/parsedpolicy.go | 130 +++++++++++++++++++++++++++++++++++ privacy/ccpa/policy.go | 4 +- 2 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 privacy/ccpa/parsedpolicy.go diff --git a/privacy/ccpa/parsedpolicy.go b/privacy/ccpa/parsedpolicy.go new file mode 100644 index 00000000000..99223c4917a --- /dev/null +++ b/privacy/ccpa/parsedpolicy.go @@ -0,0 +1,130 @@ +package ccpa + +import ( + "errors" + "fmt" +) + +const ( + ccpaVersion1 = '1' + ccpaYes = 'Y' + ccpaNo = 'N' + ccpaNotApplicable = '-' +) + +const ( + indexVersion = 0 + indexExplicitNotice = 1 + indexOptOutSale = 2 + indexLSPACoveredTransaction = 3 +) + +const allBiddersMarker = "*" + +// ParsedPolicy represents parsed and validated CCPA regulatory information from the OpenRTB bid request. +type ParsedPolicy struct { + Policy + isValid bool + consentOptOut bool + noSaleForAllBidders bool + noSaleSpecificBidders map[string]struct{} +} + +// Parse returns a parsed and validated ParsedPolicy which can be used for enforcement checks. +func (p Policy) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { + consentOptOut, err := parseConsent(p.Consent) + if err != nil { + return ParsedPolicy{isValid: false}, fmt.Errorf("request.regs.ext.us_privacy is invalid. %s", err.Error()) + } + + noSaleForAllBidders, noSaleSpecificBidders, err := parseNoSaleBidders(p.NoSaleBidders, validBidders) + if err != nil { + return ParsedPolicy{isValid: false}, fmt.Errorf("request.ext.prebid.nosale is invalid. %s", err.Error()) + } + + return ParsedPolicy{ + Policy: p, + isValid: true, + consentOptOut: consentOptOut, + noSaleForAllBidders: noSaleForAllBidders, + noSaleSpecificBidders: noSaleSpecificBidders, + }, nil +} + +func parseConsent(consent string) (consentOptOut bool, err error) { + if consent == "" { + return false, nil + } + + if len(consent) != 4 { + return false, errors.New("must contain 4 characters") + } + + if consent[indexVersion] != ccpaVersion1 { + return false, errors.New("must specify version 1") + } + + var c byte + + c = consent[indexExplicitNotice] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { + return false, errors.New("must specify 'N', 'Y', or '-' for the explicit notice") + } + + c = consent[indexOptOutSale] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { + return false, errors.New("must specify 'N', 'Y', or '-' for the opt-out sale") + } + + c = consent[indexLSPACoveredTransaction] + if c != ccpaNo && c != ccpaYes && c != ccpaNotApplicable { + return false, errors.New("must specify 'N', 'Y', or '-' for the limited service provider agreement") + } + + return consent[indexOptOutSale] == ccpaYes, nil +} + +func parseNoSaleBidders(noSaleBidders []string, validBidders map[string]struct{}) (noSaleForAllBidders bool, noSaleSpecificBidders map[string]struct{}, err error) { + if len(noSaleBidders) == 1 && noSaleBidders[0] == allBiddersMarker { + noSaleForAllBidders = true + return + } + + for _, bidder := range noSaleBidders { + if bidder == allBiddersMarker { + err = errors.New("err") + return + } + + if _, exists := validBidders[bidder]; exists { + noSaleSpecificBidders[bidder] = struct{}{} + } else { + err = fmt.Errorf("unrecognized bidder '%s'", bidder) + return + } + } + + return +} + +func (p ParsedPolicy) isNoSaleForBidder(bidder string) bool { + if p.noSaleForAllBidders { + return true + } + + _, exists := p.noSaleSpecificBidders[bidder] + return exists +} + +// ShouldEnforce returns true when the opt-out signal is explicitly detected. +func (p ParsedPolicy) ShouldEnforce(bidder string) bool { + if !p.isValid { + return false + } + + if p.isNoSaleForBidder(bidder) { + return false + } + + return p.consentOptOut +} diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 1ce05a92c62..52372503e86 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -164,7 +164,7 @@ func buildExtClear(ext json.RawMessage) (json.RawMessage, error) { // Verify prebid is an object prebidExtMap, ok := prebidExt.(map[string]interface{}) if !ok { - return nil, errors.New("ext.prebid is not a json object") + return nil, errors.New("request.ext.prebid is not a json object") } // Remove no sale member @@ -197,7 +197,7 @@ func buildExtWrite(noSaleBidders []string, ext json.RawMessage) (json.RawMessage if prebidExtMap, ok := prebidExtInterface.(map[string]interface{}); ok { prebidExt = prebidExtMap } else { - return nil, errors.New("ext.prebid is not a json object") + return nil, errors.New("request.ext.prebid is not a json object") } } else { // Create New Empty Prebid Ext Map From fb2a11492686c75ea9ec47ae03df5c808ca7baa1 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Fri, 21 Aug 2020 15:17:23 -0400 Subject: [PATCH 13/27] Refactored Policy + Implemented ParsedPolicy --- endpoints/openrtb2/amp_auction.go | 4 +- privacy/ccpa/parsedpolicy.go | 27 +++-- privacy/ccpa/policy.go | 167 +++++++++++++++--------------- privacy/ccpa/policy_test.go | 24 +++++ privacy/policies.go | 47 ++------- 5 files changed, 132 insertions(+), 137 deletions(-) diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 8efba5a926c..63754114d3f 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -403,8 +403,8 @@ func (deps *endpointDeps) overrideWithParams(httpRequest *http.Request, req *ope consent := readConsent(httpRequest.URL) if consent != "" { - if policies, ok := privacy.ReadPoliciesFromConsent(consent); ok { - if err := policies.Write(req); err != nil { + if policy, ok := privacy.ReadPolicyFromConsent(consent); ok { + if err := policy.Write(req); err != nil { return []error{err} } } else { diff --git a/privacy/ccpa/parsedpolicy.go b/privacy/ccpa/parsedpolicy.go index 99223c4917a..fec0ef98174 100644 --- a/privacy/ccpa/parsedpolicy.go +++ b/privacy/ccpa/parsedpolicy.go @@ -3,6 +3,8 @@ package ccpa import ( "errors" "fmt" + + "github.com/mxmCherry/openrtb" ) const ( @@ -23,28 +25,31 @@ const allBiddersMarker = "*" // ParsedPolicy represents parsed and validated CCPA regulatory information from the OpenRTB bid request. type ParsedPolicy struct { - Policy - isValid bool + policy Policy consentOptOut bool noSaleForAllBidders bool noSaleSpecificBidders map[string]struct{} } +// Write mutates an OpenRTB bid request with the CCPA regulatory information. +func (p ParsedPolicy) Write(req *openrtb.BidRequest) error { + return p.policy.Write(req) +} + // Parse returns a parsed and validated ParsedPolicy which can be used for enforcement checks. func (p Policy) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { consentOptOut, err := parseConsent(p.Consent) if err != nil { - return ParsedPolicy{isValid: false}, fmt.Errorf("request.regs.ext.us_privacy is invalid. %s", err.Error()) + return ParsedPolicy{}, fmt.Errorf("request.regs.ext.us_privacy is invalid. %s", err.Error()) } noSaleForAllBidders, noSaleSpecificBidders, err := parseNoSaleBidders(p.NoSaleBidders, validBidders) if err != nil { - return ParsedPolicy{isValid: false}, fmt.Errorf("request.ext.prebid.nosale is invalid. %s", err.Error()) + return ParsedPolicy{}, fmt.Errorf("request.ext.prebid.nosale is invalid. %s", err.Error()) } return ParsedPolicy{ - Policy: p, - isValid: true, + policy: p.Clone(), consentOptOut: consentOptOut, noSaleForAllBidders: noSaleForAllBidders, noSaleSpecificBidders: noSaleSpecificBidders, @@ -118,13 +123,15 @@ func (p ParsedPolicy) isNoSaleForBidder(bidder string) bool { // ShouldEnforce returns true when the opt-out signal is explicitly detected. func (p ParsedPolicy) ShouldEnforce(bidder string) bool { - if !p.isValid { - return false - } - if p.isNoSaleForBidder(bidder) { return false } return p.consentOptOut } + +// p := ccpa.Read(req) +// p.Clone() +// p.Write() // amp query params -> p -> write +// pp := p.Parse(bidders) +// pp.ShouldEnforce(bidder) diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 52372503e86..1f2f8bf71cc 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -44,6 +44,16 @@ func ReadPolicy(req *openrtb.BidRequest) (Policy, error) { return Policy{consent, noSaleBidders}, nil } +// Clone makes a deep copy the Policy. +func (p Policy) Clone() Policy { + if p.NoSaleBidders != nil { + noSaleBiddersCopy := make([]string, len(p.NoSaleBidders)) + copy(noSaleBiddersCopy, p.NoSaleBidders) + p.NoSaleBidders = noSaleBiddersCopy + } + return p +} + // Write mutates an OpenRTB bid request with the CCPA regulatory information. func (p Policy) Write(req *openrtb.BidRequest) (err error) { if req == nil { @@ -61,7 +71,6 @@ func (p Policy) Write(req *openrtb.BidRequest) (err error) { req.Regs = regs req.Ext = ext - return } @@ -78,65 +87,52 @@ func buildRegsClear(regs *openrtb.Regs) (*openrtb.Regs, error) { } var extMap map[string]interface{} - err := json.Unmarshal(regs.Ext, &extMap) - if err == nil { - regsCopy := *regs + if err := json.Unmarshal(regs.Ext, &extMap); err != nil { + return nil, err + } - // Remove CCPA consent - delete(extMap, "us_privacy") + delete(extMap, "us_privacy") - // Remove entire ext if it's empty - if len(extMap) == 0 { - regsCopy.Ext = nil - return ®sCopy, nil - } - - ext, err := json.Marshal(extMap) - if err != nil { - return nil, err - } + // Remove entire ext if it's now empty + if len(extMap) == 0 { + regsResult := *regs + regsResult.Ext = nil + return ®sResult, nil + } - regsCopy.Ext = ext - return ®sCopy, nil + // Marshal ext if there are still other fields + var regsResult openrtb.Regs + ext, err := json.Marshal(extMap) + if err == nil { + regsResult = *regs + regsResult.Ext = ext } - return nil, err + return ®sResult, err } func buildRegsWrite(consent string, regs *openrtb.Regs) (*openrtb.Regs, error) { - var regsCopy openrtb.Regs - if regs == nil { - regsCopy = openrtb.Regs{} - } else { - regsCopy = *regs + return marshalRegsExt(openrtb.Regs{}, openrtb_ext.ExtRegs{USPrivacy: consent}) } - if regsCopy.Ext == nil { - ext, err := json.Marshal(openrtb_ext.ExtRegs{USPrivacy: consent}) - if err != nil { - return nil, err - } - - regsCopy.Ext = ext - return ®sCopy, nil + if regs.Ext == nil { + return marshalRegsExt(*regs, openrtb_ext.ExtRegs{USPrivacy: consent}) } var extMap map[string]interface{} - err := json.Unmarshal(regsCopy.Ext, &extMap) - if err == nil { - // Set CCPA consent - extMap["us_privacy"] = consent - - ext, err := json.Marshal(extMap) - if err != nil { - return nil, err - } - - regsCopy.Ext = ext - return ®sCopy, nil + if err := json.Unmarshal(regs.Ext, &extMap); err != nil { + return nil, err } + extMap["us_privacy"] = consent + return marshalRegsExt(*regs, extMap) +} - return nil, err +func marshalRegsExt(regs openrtb.Regs, ext interface{}) (*openrtb.Regs, error) { + extJSON, err := json.Marshal(ext) + if err == nil { + regs.Ext = extJSON + } + return ®s, err } func buildExt(noSaleBidders []string, ext json.RawMessage) (json.RawMessage, error) { @@ -152,35 +148,33 @@ func buildExtClear(ext json.RawMessage) (json.RawMessage, error) { } var extMap map[string]interface{} - err := json.Unmarshal(ext, &extMap) - if err == nil { - prebidExt, exists := extMap["prebid"] - - // If there's no prebid, there's nothing to do - if !exists { - return ext, nil - } + if err := json.Unmarshal(ext, &extMap); err != nil { + return nil, err + } - // Verify prebid is an object - prebidExtMap, ok := prebidExt.(map[string]interface{}) - if !ok { - return nil, errors.New("request.ext.prebid is not a json object") - } + prebidExt, exists := extMap["prebid"] + if !exists { + return ext, nil + } - // Remove no sale member - delete(prebidExtMap, "nosale") - if len(prebidExtMap) == 0 { - delete(extMap, "prebid") - } + // Verify prebid is an object + prebidExtMap, ok := prebidExt.(map[string]interface{}) + if !ok { + return nil, errors.New("request.ext.prebid is not a json object") + } - // Remove entire ext if it's empty - if len(extMap) == 0 { - return nil, nil - } + // Remove no sale member + delete(prebidExtMap, "nosale") + if len(prebidExtMap) == 0 { + delete(extMap, "prebid") + } - return json.Marshal(extMap) + // Remove entire ext if it's empty + if len(extMap) == 0 { + return nil, nil } - return nil, err + + return json.Marshal(extMap) } func buildExtWrite(noSaleBidders []string, ext json.RawMessage) (json.RawMessage, error) { @@ -189,24 +183,25 @@ func buildExtWrite(noSaleBidders []string, ext json.RawMessage) (json.RawMessage } var extMap map[string]interface{} - err := json.Unmarshal(ext, &extMap) - if err == nil { - var prebidExt map[string]interface{} - if prebidExtInterface, exists := extMap["prebid"]; exists { - // Reference Existing Prebid Ext Map - if prebidExtMap, ok := prebidExtInterface.(map[string]interface{}); ok { - prebidExt = prebidExtMap - } else { - return nil, errors.New("request.ext.prebid is not a json object") - } + if err := json.Unmarshal(ext, &extMap); err != nil { + return nil, err + } + + var prebidExt map[string]interface{} + if prebidExtInterface, exists := extMap["prebid"]; exists { + // Reference Existing Prebid Ext Map + if prebidExtMap, ok := prebidExtInterface.(map[string]interface{}); ok { + prebidExt = prebidExtMap } else { - // Create New Empty Prebid Ext Map - prebidExt = make(map[string]interface{}) - extMap["prebid"] = prebidExt + return nil, errors.New("request.ext.prebid is not a json object") } - - prebidExt["nosale"] = noSaleBidders - return json.Marshal(extMap) + } else { + // Create New Empty Prebid Ext Map + prebidExt = make(map[string]interface{}) + extMap["prebid"] = prebidExt } - return nil, err + + prebidExt["nosale"] = noSaleBidders + return json.Marshal(extMap) + } diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index da2cf3339ea..9cd009d6f4d 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -183,6 +183,30 @@ func TestRead(t *testing.T) { } } +func TestClone(t *testing.T) { + policy := Policy{ + Consent: "anyConsent", + NoSaleBidders: []string{"a", "b"}, + } + + clone := policy.Clone() + clone.NoSaleBidders[0] = "1" + + assert.ElementsMatch(t, []string{"a", "b"}, policy.NoSaleBidders, "original") + assert.ElementsMatch(t, []string{"1", "b"}, clone.NoSaleBidders, "clone") +} + +func TestCloneNilNoSale(t *testing.T) { + policy := Policy{ + Consent: "anyConsent", + NoSaleBidders: nil, + } + + clone := policy.Clone() + + assert.Nil(t, clone.NoSaleBidders) +} + func TestWrite(t *testing.T) { testCases := []struct { description string diff --git a/privacy/policies.go b/privacy/policies.go index a5c038b9033..23ce738dc7f 100644 --- a/privacy/policies.go +++ b/privacy/policies.go @@ -7,54 +7,23 @@ import ( "github.com/prebid/prebid-server/privacy/gdpr" ) -// Policies represents the privacy regulations for an OpenRTB bid request. -type Policies struct { - GDPR gdpr.Policy - CCPA ccpa.ParsedPolicy -} - -type policyWriter interface { +type PolicyWriter interface { Write(req *openrtb.BidRequest) error } -// Write mutates an OpenRTB bid request with the policies applied. -func (p Policies) Write(req *openrtb.BidRequest) error { - return writePolicies(req, []policyWriter{ - p.GDPR, p.CCPA, - }) -} - -func writePolicies(req *openrtb.BidRequest, writers []policyWriter) error { - for _, writer := range writers { - if err := writer.Write(req); err != nil { - return err - } - } - - return nil -} - -// ReadPoliciesFromConsent inspects the consent string kind and sets the corresponding values in a new Policies object. -func ReadPoliciesFromConsent(consent string) (Policies, bool) { +// ReadPolicyFromConsent inspects the consent string and returns a validated policy writer. +func ReadPolicyFromConsent(consent string) (PolicyWriter, bool) { if len(consent) == 0 { - return Policies{}, false + return nil, false } if err := gdpr.ValidateConsent(consent); err == nil { - return Policies{ - GDPR: gdpr.Policy{ - Consent: consent, - }, - }, true + return gdpr.Policy{Consent: consent}, true } - if err := ccpa.ValidateConsent(consent); err == nil { - return Policies{ - CCPA: ccpa.Policy{ - Value: consent, - }, - }, true + if p, err := ccpa.Parse(ccpa.Policy{Consent: consent}, nil); err == nil { + return p, true } - return Policies{}, false + return nil, false } From ee6aebb1d7508fa7a389e4cea6a5eeb54cbd7bf7 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 26 Aug 2020 11:06:58 -0400 Subject: [PATCH 14/27] WIP - Change In Direction For Writes --- endpoints/auction.go | 22 +- endpoints/cookie_sync.go | 38 +-- endpoints/openrtb2/amp_auction.go | 41 +++- endpoints/openrtb2/auction.go | 64 +++-- endpoints/setuid_test.go | 3 +- exchange/utils.go | 18 +- privacy/ccpa/parsedpolicy.go | 56 ++--- privacy/ccpa/parsedpolicy_test.go | 382 ++++++++++++++++++++++++++++++ privacy/ccpa/policy.go | 70 ++++-- privacy/ccpa/policy_test.go | 204 +++++++++------- privacy/policies.go | 27 +-- privacy/policies_test.go | 119 ---------- usersync/usersync.go | 10 +- 13 files changed, 702 insertions(+), 352 deletions(-) create mode 100644 privacy/ccpa/parsedpolicy_test.go delete mode 100644 privacy/policies_test.go diff --git a/endpoints/auction.go b/endpoints/auction.go index bf592e43b02..ca6711d31c5 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -23,8 +23,6 @@ import ( "github.com/prebid/prebid-server/pbs" "github.com/prebid/prebid-server/pbsmetrics" pbc "github.com/prebid/prebid-server/prebid_cache_client" - "github.com/prebid/prebid-server/privacy" - gdprPolicy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync" ) @@ -190,20 +188,20 @@ func (a *auction) recoverSafely(inner func(*pbs.PBSBidder, pbsmetrics.AdapterLab } } -func (a *auction) shouldUsersync(ctx context.Context, bidder openrtb_ext.BidderName, gdprPrivacyPolicy gdprPolicy.Policy) bool { - switch gdprPrivacyPolicy.Signal { +func (a *auction) shouldUsersync(ctx context.Context, bidder openrtb_ext.BidderName, privacyPolicies usersync.PrivacyPolicies) bool { + switch privacyPolicies.GDPRSignal { case "0": return true case "1": - if gdprPrivacyPolicy.Consent == "" { + if privacyPolicies.GDPRConsent == "" { return false } fallthrough default: - if canSync, err := a.gdprPerms.HostCookiesAllowed(ctx, gdprPrivacyPolicy.Consent); !canSync || err != nil { + if canSync, err := a.gdprPerms.HostCookiesAllowed(ctx, privacyPolicies.GDPRConsent); !canSync || err != nil { return false } - canSync, err := a.gdprPerms.BidderSyncAllowed(ctx, bidder, gdprPrivacyPolicy.Consent) + canSync, err := a.gdprPerms.BidderSyncAllowed(ctx, bidder, privacyPolicies.GDPRConsent) return canSync && err == nil } } @@ -510,13 +508,11 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl uid, _, _ := req.Cookie.GetUID(syncer.FamilyName()) if uid == "" { bidder.NoCookie = true - privacyPolicies := privacy.Policies{ - GDPR: gdprPolicy.Policy{ - Signal: req.ParseGDPR(), - Consent: req.ParseConsent(), - }, + privacyPolicies := usersync.PrivacyPolicies{ + GDPRSignal: req.ParseGDPR(), + GDPRConsent: req.ParseConsent(), } - if a.shouldUsersync(*ctx, openrtb_ext.BidderName(syncerCode), privacyPolicies.GDPR) { + if a.shouldUsersync(*ctx, openrtb_ext.BidderName(syncerCode), privacyPolicies) { syncInfo, err := syncer.GetUsersyncInfo(privacyPolicies) if err == nil { bidder.UsersyncInfo = syncInfo diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index f16756a4148..b6f8ffb1149 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -18,9 +18,7 @@ import ( "github.com/prebid/prebid-server/gdpr" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" - "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/privacy/ccpa" - gdprPolicy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync" ) @@ -105,16 +103,6 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h } } - privacyPolicy := privacy.Policies{ - GDPR: gdprPolicy.Policy{ - Signal: gdprToString(parsedReq.GDPR), - Consent: parsedReq.Consent, - }, - CCPA: ccpa.Policy{ - Value: parsedReq.USPrivacy, - }, - } - parsedReq.filterExistingSyncs(deps.syncers, userSyncCookie, needSyncupForSameSite) adapterSyncs := make(map[openrtb_ext.BidderName]bool) @@ -122,8 +110,20 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h for _, b := range parsedReq.Bidders { adapterSyncs[openrtb_ext.BidderName(b)] = true } - parsedReq.filterForGDPR(privacyPolicy, deps.syncPermissions) - parsedReq.filterForCCPA(privacyPolicy, deps.enforceCCPA) + + privacyPolicy := usersync.PrivacyPolicies{ + GDPRSignal: gdprToString(parsedReq.GDPR), + GDPRConsent: parsedReq.Consent, + CCPAConsent: parsedReq.USPrivacy, + } + + parsedReq.filterForGDPR(deps.syncPermissions) + + ccpaPolicy := ccpa.ReadFromConsent(privacyPolicy.CCPAConsent) + if ccpaParsedPolicy, err := ccpaPolicy.Parse(); err != nil { + parsedReq.filterForCCPA(ccpaParsedPolicy, deps.enforceCCPA) + } + // surviving bidders are not privacy blocked for _, b := range parsedReq.Bidders { adapterSyncs[openrtb_ext.BidderName(b)] = false @@ -139,7 +139,7 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h } for i := 0; i < len(parsedReq.Bidders); i++ { bidder := parsedReq.Bidders[i] - syncInfo, err := deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(privacyPolicy) + syncInfo, err := deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(privacyPolicy) //PrivacyPolicies if err == nil { newSync := &usersync.CookieSyncBidders{ BidderCode: bidder, @@ -224,7 +224,7 @@ func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderNa } } -func (req *cookieSyncRequest) filterForGDPR(privacyPolicies privacy.Policies, permissions gdpr.Permissions) { +func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) { if req.GDPR != nil && *req.GDPR == 0 { return } @@ -242,13 +242,15 @@ func (req *cookieSyncRequest) filterForGDPR(privacyPolicies privacy.Policies, pe } } -func (req *cookieSyncRequest) filterForCCPA(privacyPolicies privacy.Policies, enforceCCPA bool) { +func (req *cookieSyncRequest) filterForCCPA(policy ccpa.ParsedPolicy, enforceCCPA bool) { if !enforceCCPA { return } + // let's do the parsing here instead? + for i := 0; i < len(req.Bidders); i++ { - if privacyPolicies.CCPA.ShouldEnforce(req.Bidders[i]) { + if policy.ShouldEnforce(req.Bidders[i]) { req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) i-- } diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 63754114d3f..628bdda54f3 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -23,6 +23,8 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" "github.com/prebid/prebid-server/privacy" + "github.com/prebid/prebid-server/privacy/ccpa" + "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/usersync" @@ -401,17 +403,12 @@ func (deps *endpointDeps) overrideWithParams(httpRequest *http.Request, req *ope req.Imp[0].TagID = slot } - consent := readConsent(httpRequest.URL) - if consent != "" { - if policy, ok := privacy.ReadPolicyFromConsent(consent); ok { - if err := policy.Write(req); err != nil { - return []error{err} - } - } else { - return []error{&errortypes.InvalidPrivacyConsent{ - Message: fmt.Sprintf("Consent '%s' is not recognized as either CCPA or GDPR TCF.", consent), - }} - } + policyWriter, policyWriterErr := readConsent(httpRequest.URL) + if policyWriterErr != nil { + return []error{policyWriterErr} + } + if err := policyWriter.Write(req); err != nil { + return []error{err} } if timeout, err := strconv.ParseInt(httpRequest.FormValue("timeout"), 10, 64); err == nil { @@ -556,7 +553,27 @@ func setAmpExt(site *openrtb.Site, value string) { } } -func readConsent(url *url.URL) string { +func readConsent(url *url.URL) (privacy.PolicyWriter, error) { + consent := readConsentFromURL(url) + + if len(consent) == 0 { + return privacy.NilPolicyWriter{}, nil + } + + if err := gdpr.ValidateConsent(consent); err == nil { + return gdpr.Policy{Consent: consent}, nil + } + + if ccpa.ValidateConsent(consent) { + return ccpa.NewConsentWriter(consent), nil + } + + return privacy.NilPolicyWriter{}, &errortypes.InvalidPrivacyConsent{ + Message: fmt.Sprintf("Consent '%s' is not recognized as either CCPA or GDPR TCF.", consent), + } +} + +func readConsentFromURL(url *url.URL) string { if v := url.Query().Get("consent_string"); v != "" { return v } diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 201da12dc1d..ae83447426d 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -28,6 +28,7 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" "github.com/prebid/prebid-server/prebid_cache_client" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/privacy/ccpa" "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" @@ -303,55 +304,38 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { } if (req.Site == nil && req.App == nil) || (req.Site != nil && req.App != nil) { - errL = append(errL, errors.New("request.site or request.app must be defined, but not both.")) - return errL + return append(errL, errors.New("request.site or request.app must be defined, but not both.")) } if err := deps.validateSite(req.Site); err != nil { - errL = append(errL, err) - return errL + return append(errL, err) } if err := deps.validateApp(req.App); err != nil { - errL = append(errL, err) - return errL + return append(errL, err) } if err := validateUser(req.User, aliases); err != nil { - errL = append(errL, err) - return errL + return append(errL, err) } if err := validateRegs(req.Regs); err != nil { - errL = append(errL, err) - return errL + return append(errL, err) } - if policy, err := ccpa.ReadPolicy(req); err != nil { - errL = append(errL, errL...) - return errL - } else if err := policy.Validate(); err != nil { - errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) + if ccpaPolicy, err := ccpa.ReadFromRequest(req); err != nil { + return append(errL, errL...) + } else if _, err := ccpaPolicy.Parse(getValidBidders(aliases)); err == nil { + if _, isInvalidConsent := err.(*privacy.InvalidConsentError); isInvalidConsent { + errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) - policy.Value = "" - if err := policy.Write(req); err != nil { - errL = append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) - errL = append(errL, err) - return errL - } else { - _, err := policy.Validate(); err != nil { - if err.ValueError != nil { - policy.Value = "" - if err := ccpaPolicy.Write(req); err != nil { - errL = append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) - } - errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) - } - - if err.NoSaleBidderError != nil { - errL = append(errL, err.NoSaleBidderError) - return errL + // remove invalid consent from request + ccpaPolicy.Consent = "" + if err := ccpaPolicy.Write(req); err != nil { + return append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) } + } else { + return append(errL, errL...) } } @@ -1309,3 +1293,17 @@ func validateAccount(cfg *config.Configuration, pubID string) error { } return err } + +func getValidBidders(aliases map[string]string) map[string]struct{} { + validBidders := make(map[string]struct{}) + + for _, v := range openrtb_ext.BidderMap { + validBidders[v.String()] = struct{}{} + } + + for k := range aliases { + validBidders[k] = struct{}{} + } + + return validBidders +} diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 3f47b257d2e..2ad1cc8b2bb 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -12,7 +12,6 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/pbsmetrics" - "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" @@ -461,7 +460,7 @@ func (s fakeSyncer) FamilyName() string { } // GetUsersyncInfo implements the Usersyncer interface with a no-op. -func (s fakeSyncer) GetUsersyncInfo(privacyPolicies privacy.Policies) (*usersync.UsersyncInfo, error) { +func (s fakeSyncer) GetUsersyncInfo(privacyPolicies usersync.PrivacyPolicies) (*usersync.UsersyncInfo, error) { return nil, nil } diff --git a/exchange/utils.go b/exchange/utils.go index 8f71f32a392..c65482c5452 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -76,9 +76,20 @@ func cleanOpenRTBRequests(ctx context.Context, consent := extractConsent(orig) ampGDPRException := (labels.RType == pbsmetrics.ReqTypeAMP) && gDPR.AMPException() - var ccpaPolicy ccpa.Policy + var ccpaPolicy ccpa.ParsedPolicy if privacyConfig.CCPA.Enforce { - ccpaPolicy, _ = ccpa.ReadPolicy(orig) + policy, err := ccpa.ReadFromRequest(orig) + if err != nil { + errs = append(errs, err) + return + } + + // buuild comvinarion of aliaes and built in bidders + ccpaPolicy, err = policy.Parse(nil) + if err != nil { + errs = append(errs, err) + return + } } var lmtPolicy lmt.Policy @@ -92,7 +103,7 @@ func cleanOpenRTBRequests(ctx context.Context, LMT: lmtPolicy.ShouldEnforce(), } - privacyLabels.CCPAProvided = ccpaPolicy.Value != "" + privacyLabels.CCPAProvided = ccpaPolicy.Value != "" // todo: add Specified helepr privacyLabels.CCPAEnforced = privacyEnforcement.CCPA privacyLabels.COPPAEnforced = privacyEnforcement.COPPA privacyLabels.LMTEnforced = privacyEnforcement.LMT @@ -108,7 +119,6 @@ func cleanOpenRTBRequests(ctx context.Context, // bidder level privacy policies for bidder, bidReq := range requestsByBidder { - // CCPA privacyEnforcement.CCPA = ccpaPolicy.ShouldEnforce(bidder.String()) diff --git a/privacy/ccpa/parsedpolicy.go b/privacy/ccpa/parsedpolicy.go index fec0ef98174..1a53e0d29ae 100644 --- a/privacy/ccpa/parsedpolicy.go +++ b/privacy/ccpa/parsedpolicy.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/errortypes" ) const ( @@ -23,40 +23,43 @@ const ( const allBiddersMarker = "*" -// ParsedPolicy represents parsed and validated CCPA regulatory information from the OpenRTB bid request. +// ParsedPolicy represents parsed and validated CCPA regulatory information. Use this object +// to make enforcement decisions. type ParsedPolicy struct { - policy Policy - consentOptOut bool + consentSpecified bool + consentOptOutSale bool noSaleForAllBidders bool noSaleSpecificBidders map[string]struct{} } -// Write mutates an OpenRTB bid request with the CCPA regulatory information. -func (p ParsedPolicy) Write(req *openrtb.BidRequest) error { - return p.policy.Write(req) -} - -// Parse returns a parsed and validated ParsedPolicy which can be used for enforcement checks. -func (p Policy) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { +// Parse returns a parsed and validated ParsedPolicy intended for use in enforcement decisions. +func (p PolicyFromRequest) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { consentOptOut, err := parseConsent(p.Consent) if err != nil { - return ParsedPolicy{}, fmt.Errorf("request.regs.ext.us_privacy is invalid. %s", err.Error()) + msg := fmt.Sprintf("request.regs.ext.us_privacy is invalid: %s", err.Error()) + return ParsedPolicy{}, &errortypes.InvalidPrivacyConsent{Message: msg} } noSaleForAllBidders, noSaleSpecificBidders, err := parseNoSaleBidders(p.NoSaleBidders, validBidders) if err != nil { - return ParsedPolicy{}, fmt.Errorf("request.ext.prebid.nosale is invalid. %s", err.Error()) + return ParsedPolicy{}, fmt.Errorf("request.ext.prebid.nosale is invalid: %s", err.Error()) } return ParsedPolicy{ - policy: p.Clone(), - consentOptOut: consentOptOut, + consentSpecified: p.Consent != "", + consentOptOutSale: consentOptOut, noSaleForAllBidders: noSaleForAllBidders, noSaleSpecificBidders: noSaleSpecificBidders, }, nil } -func parseConsent(consent string) (consentOptOut bool, err error) { +// ValidateConsent returns true if the consent string is empty or valid. +func ValidateConsent(consent string) bool { + _, err := parseConsent(consent) + return err == nil +} + +func parseConsent(consent string) (consentOptOutSale bool, err error) { if consent == "" { return false, nil } @@ -90,6 +93,8 @@ func parseConsent(consent string) (consentOptOut bool, err error) { } func parseNoSaleBidders(noSaleBidders []string, validBidders map[string]struct{}) (noSaleForAllBidders bool, noSaleSpecificBidders map[string]struct{}, err error) { + noSaleSpecificBidders = make(map[string]struct{}) + if len(noSaleBidders) == 1 && noSaleBidders[0] == allBiddersMarker { noSaleForAllBidders = true return @@ -97,7 +102,7 @@ func parseNoSaleBidders(noSaleBidders []string, validBidders map[string]struct{} for _, bidder := range noSaleBidders { if bidder == allBiddersMarker { - err = errors.New("err") + err = errors.New("can only specify all bidders if no other bidders are provided") return } @@ -112,6 +117,11 @@ func parseNoSaleBidders(noSaleBidders []string, validBidders map[string]struct{} return } +// Specified returns true when consent is provided, as opposed to an empty string. +func (p ParsedPolicy) Specified() bool { + return p.consentSpecified +} + func (p ParsedPolicy) isNoSaleForBidder(bidder string) bool { if p.noSaleForAllBidders { return true @@ -123,15 +133,5 @@ func (p ParsedPolicy) isNoSaleForBidder(bidder string) bool { // ShouldEnforce returns true when the opt-out signal is explicitly detected. func (p ParsedPolicy) ShouldEnforce(bidder string) bool { - if p.isNoSaleForBidder(bidder) { - return false - } - - return p.consentOptOut + return !p.isNoSaleForBidder(bidder) && p.consentOptOutSale } - -// p := ccpa.Read(req) -// p.Clone() -// p.Write() // amp query params -> p -> write -// pp := p.Parse(bidders) -// pp.ShouldEnforce(bidder) diff --git a/privacy/ccpa/parsedpolicy_test.go b/privacy/ccpa/parsedpolicy_test.go new file mode 100644 index 00000000000..69894a822a9 --- /dev/null +++ b/privacy/ccpa/parsedpolicy_test.go @@ -0,0 +1,382 @@ +package ccpa + +import ( + "errors" + "testing" + + "github.com/mxmCherry/openrtb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestParesPolicyFromRequest(t *testing.T) { + validBidders := map[string]struct{}{"a": {}} + + testCases := []struct { + description string + consent string + noSaleBidders []string + expectedPolicy ParsedPolicy + expectedError string + }{ + { + description: "Consent Error", + consent: "malformed", + noSaleBidders: []string{}, + expectedPolicy: ParsedPolicy{}, + expectedError: "request.regs.ext.us_privacy is invalid: must contain 4 characters", + }, + { + description: "No Sale Error", + consent: "1NYN", + noSaleBidders: []string{"b"}, + expectedPolicy: ParsedPolicy{}, + expectedError: "request.ext.prebid.nosale is invalid: unrecognized bidder 'b'", + }, + { + description: "Success", + consent: "1NYN", + noSaleBidders: []string{"a"}, + expectedPolicy: ParsedPolicy{ + policyWriter: PolicyFromRequest{Consent: "1NYN", NoSaleBidders: []string{"a"}}, + consentOptOutSale: true, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{"a": {}}, + }, + }, + } + + for _, test := range testCases { + policy := PolicyFromRequest{Consent: test.consent, NoSaleBidders: test.noSaleBidders} + + result, err := policy.Parse(validBidders) + + if test.expectedError == "" { + assert.NoError(t, err, test.description) + } else { + assert.EqualError(t, err, test.expectedError, test.description) + } + + assert.Equal(t, test.expectedPolicy, result, test.description) + } +} + +func TestParesPolicyFromConsent(t *testing.T) { + testCases := []struct { + description string + consent string + expectedPolicy ParsedPolicy + expectedError string + }{ + { + description: "Success", + consent: "1NYN", + expectedPolicy: ParsedPolicy{ + policyWriter: PolicyFromConsent{Consent: "1NYN"}, + consentOptOutSale: true, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{}, + }, + }, + { + description: "Error", + consent: "malformed", + expectedPolicy: ParsedPolicy{}, + expectedError: "request.regs.ext.us_privacy is invalid: must contain 4 characters", + }, + } + + for _, test := range testCases { + policy := PolicyFromConsent{test.consent} + + result, err := policy.Parse() + + if test.expectedError == "" { + assert.NoError(t, err, test.description) + } else { + assert.EqualError(t, err, test.expectedError, test.description) + } + + assert.Equal(t, test.expectedPolicy, result, test.description) + } +} + +func TestWriteSuccess(t *testing.T) { + req := &openrtb.BidRequest{} + mockWriter := &mockPolicWriter{} + mockWriter.On("Write", req).Return(nil).Once() + parsedPolicy := &ParsedPolicy{policyWriter: mockWriter} + + resultErr := parsedPolicy.Write(req) + + mockWriter.AssertExpectations(t) + assert.NoError(t, resultErr) +} + +func TestWriteError(t *testing.T) { + req := &openrtb.BidRequest{} + mockWriter := &mockPolicWriter{} + mockWriter.On("Write", req).Return(errors.New("foo")).Once() + parsedPolicy := &ParsedPolicy{policyWriter: mockWriter} + + resultErr := parsedPolicy.Write(req) + + mockWriter.AssertExpectations(t) + assert.Error(t, resultErr, "foo") +} + +func TestParseConsent(t *testing.T) { + testCases := []struct { + description string + consent string + expectedResult bool + expectedError string + }{ + { + description: "Valid", + consent: "1NYN", + expectedResult: true, + }, + { + description: "Valid - Not Sale", + consent: "1NNN", + expectedResult: false, + }, + { + description: "Valid - Not Applicable", + consent: "1---", + expectedResult: false, + }, + { + description: "Valid - Empty", + consent: "", + expectedResult: false, + }, + { + description: "Wrong Length", + consent: "1NY", + expectedResult: false, + expectedError: "must contain 4 characters", + }, + { + description: "Wrong Version", + consent: "2---", + expectedResult: false, + expectedError: "must specify version 1", + }, + { + description: "Explicit Notice Char", + consent: "1X--", + expectedResult: false, + expectedError: "must specify 'N', 'Y', or '-' for the explicit notice", + }, + { + description: "Invalid Explicit Notice Case", + consent: "1y--", + expectedResult: false, + expectedError: "must specify 'N', 'Y', or '-' for the explicit notice", + }, + { + description: "Invalid Opt-Out Sale Char", + consent: "1-X-", + expectedResult: false, + expectedError: "must specify 'N', 'Y', or '-' for the opt-out sale", + }, + { + description: "Invalid Opt-Out Sale Case", + consent: "1-y-", + expectedResult: false, + expectedError: "must specify 'N', 'Y', or '-' for the opt-out sale", + }, + { + description: "Invalid LSPA Char", + consent: "1--X", + expectedResult: false, + expectedError: "must specify 'N', 'Y', or '-' for the limited service provider agreement", + }, + { + description: "Invalid LSPA Case", + consent: "1--y", + expectedResult: false, + expectedError: "must specify 'N', 'Y', or '-' for the limited service provider agreement", + }, + } + + for _, test := range testCases { + result, err := parseConsent(test.consent) + + if test.expectedError == "" { + assert.NoError(t, err, test.description) + } else { + assert.EqualError(t, err, test.expectedError, test.description) + } + + assert.Equal(t, test.expectedResult, result, test.description) + } +} + +func TestParseNoSaleBidders(t *testing.T) { + testCases := []struct { + description string + noSaleBidders []string + validBidders []string + expectedNoSaleForAllBidders bool + expectedNoSaleSpecificBidders map[string]struct{} + expectedError string + }{ + { + description: "Valid - No Bidders", + noSaleBidders: []string{}, + validBidders: []string{"a"}, + expectedNoSaleForAllBidders: false, + expectedNoSaleSpecificBidders: map[string]struct{}{}, + }, + { + description: "Valid - 1 Bidder", + noSaleBidders: []string{"a"}, + validBidders: []string{"a"}, + expectedNoSaleForAllBidders: false, + expectedNoSaleSpecificBidders: map[string]struct{}{"a": {}}, + }, + { + description: "Valid - 1+ Bidders", + noSaleBidders: []string{"a", "b"}, + validBidders: []string{"a", "b"}, + expectedNoSaleForAllBidders: false, + expectedNoSaleSpecificBidders: map[string]struct{}{"a": {}, "b": {}}, + }, + { + description: "Valid - All Bidders", + noSaleBidders: []string{"*"}, + validBidders: []string{"a"}, + expectedNoSaleForAllBidders: true, + expectedNoSaleSpecificBidders: map[string]struct{}{}, + }, + { + description: "Bidder Not Valid", + noSaleBidders: []string{"b"}, + validBidders: []string{"a"}, + expectedError: "unrecognized bidder 'b'", + expectedNoSaleForAllBidders: false, + expectedNoSaleSpecificBidders: map[string]struct{}{}, + }, + { + description: "All Bidder Mixed With Other Bidders Is Invalid", + noSaleBidders: []string{"*", "a"}, + validBidders: []string{"a"}, + expectedError: "can only specify all bidders if no other bidders are provided", + expectedNoSaleForAllBidders: false, + expectedNoSaleSpecificBidders: map[string]struct{}{}, + }, + { + description: "Valid Bidders Case Sensitive", + noSaleBidders: []string{"a"}, + validBidders: []string{"A"}, + expectedError: "unrecognized bidder 'a'", + expectedNoSaleForAllBidders: false, + expectedNoSaleSpecificBidders: map[string]struct{}{}, + }, + } + + for _, test := range testCases { + validBiddersMap := make(map[string]struct{}) + for _, v := range test.validBidders { + validBiddersMap[v] = struct{}{} + } + + resultNoSaleForAllBidders, resultNoSaleSpecificBidders, err := parseNoSaleBidders(test.noSaleBidders, validBiddersMap) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + } + + assert.Equal(t, test.expectedNoSaleForAllBidders, resultNoSaleForAllBidders, test.description+":allBidders") + assert.Equal(t, test.expectedNoSaleSpecificBidders, resultNoSaleSpecificBidders, test.description+":specificBidders") + } +} + +func TestShouldEnforce(t *testing.T) { + testCases := []struct { + description string + policy ParsedPolicy + bidder string + expected bool + }{ + { + description: "Not Enforced - All Bidders No Sale", + policy: ParsedPolicy{ + consentOptOutSale: true, + noSaleForAllBidders: true, + noSaleSpecificBidders: map[string]struct{}{}, + }, + bidder: "a", + expected: false, + }, + { + description: "Not Enforced - Specific Bidders No Sale", + policy: ParsedPolicy{ + consentOptOutSale: true, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{"a": {}}, + }, + bidder: "a", + expected: false, + }, + { + description: "Not Enforced - No Bidder No Sale", + policy: ParsedPolicy{ + consentOptOutSale: false, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{}, + }, + bidder: "a", + expected: false, + }, + { + description: "Not Enforced - No Sale Case Sensitive", + policy: ParsedPolicy{ + consentOptOutSale: false, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{"A": {}}, + }, + bidder: "a", + expected: false, + }, + { + description: "Enforced - No Bidder No Sale", + policy: ParsedPolicy{ + consentOptOutSale: true, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{}, + }, + bidder: "a", + expected: true, + }, + { + description: "Enforced - No Sale Case Sensitive", + policy: ParsedPolicy{ + consentOptOutSale: true, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{"A": {}}, + }, + bidder: "a", + expected: true, + }, + } + + for _, test := range testCases { + result := test.policy.ShouldEnforce(test.bidder) + assert.Equal(t, test.expected, result, test.description) + } +} + +type mockPolicWriter struct { + mock.Mock +} + +func (m *mockPolicWriter) Write(req *openrtb.BidRequest) error { + args := m.Called(req) + return args.Error(0) +} diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 1f2f8bf71cc..e6a1a718ca8 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -6,28 +6,29 @@ import ( "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/privacy" ) -// Policy represents the CCPA regulatory information from the OpenRTB bid request. +// Policy represents the CCPA regulatory information from an OpenRTB bid request. type Policy struct { Consent string NoSaleBidders []string } -// ReadPolicy extracts the CCPA regulatory information from the OpenRTB bid request. -func ReadPolicy(req *openrtb.BidRequest) (Policy, error) { +// ReadFromRequest extracts the CCPA regulatory information from an OpenRTB bid request. +func ReadFromRequest(req *openrtb.BidRequest) (Policy, error) { var consent string var noSaleBidders []string if req == nil { - return Policy{}, nil + return PolicyFromRequest{}, nil } // Read consent from request.regs.ext if req.Regs != nil && len(req.Regs.Ext) > 0 { var ext openrtb_ext.ExtRegs if err := json.Unmarshal(req.Regs.Ext, &ext); err != nil { - return Policy{}, err + return PolicyFromRequest{}, err } consent = ext.USPrivacy } @@ -36,42 +37,54 @@ func ReadPolicy(req *openrtb.BidRequest) (Policy, error) { if len(req.Ext) > 0 { var ext openrtb_ext.ExtRequest if err := json.Unmarshal(req.Ext, &ext); err != nil { - return Policy{}, err + return PolicyFromRequest{}, err } noSaleBidders = ext.Prebid.NoSale } - return Policy{consent, noSaleBidders}, nil -} - -// Clone makes a deep copy the Policy. -func (p Policy) Clone() Policy { - if p.NoSaleBidders != nil { - noSaleBiddersCopy := make([]string, len(p.NoSaleBidders)) - copy(noSaleBiddersCopy, p.NoSaleBidders) - p.NoSaleBidders = noSaleBiddersCopy - } - return p + return PolicyFromRequest{consent, noSaleBidders}, nil } // Write mutates an OpenRTB bid request with the CCPA regulatory information. -func (p Policy) Write(req *openrtb.BidRequest) (err error) { +func (p PolicyFromRequest) Write(req *openrtb.BidRequest) error { if req == nil { - return + return nil } regs, err := buildRegs(p.Consent, req.Regs) if err != nil { - return + return err } ext, err := buildExt(p.NoSaleBidders, req.Ext) if err != nil { - return + return err } req.Regs = regs req.Ext = ext - return + return nil +} + +type consentWriter struct { + consent string +} + +func (c consentWriter) Write(req *openrtb.BidRequest) error { + if req == nil { + return nil + } + + regs, err := buildRegs(c.consent, req.Regs) + if err != nil { + return err + } + req.Regs = regs + + return nil +} + +func NewConsentWriter(consent string) privacy.PolicyWriter { + return consentWriter{consent} } func buildRegs(consent string, regs *openrtb.Regs) (*openrtb.Regs, error) { @@ -123,6 +136,7 @@ func buildRegsWrite(consent string, regs *openrtb.Regs) (*openrtb.Regs, error) { if err := json.Unmarshal(regs.Ext, &extMap); err != nil { return nil, err } + extMap["us_privacy"] = consent return marshalRegsExt(*regs, extMap) } @@ -203,5 +217,17 @@ func buildExtWrite(noSaleBidders []string, ext json.RawMessage) (json.RawMessage prebidExt["nosale"] = noSaleBidders return json.Marshal(extMap) +} +// last bit, separate consent into it's own concept? +type ConsentWriter struct { + Consent string } + +// other languagds have this. but not go. + +func (ConsentWriter) Write(req) { + +} + +// all will be good if i can create a ccpa consent writer diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index 9cd009d6f4d..d7e16868b4c 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -8,41 +8,39 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRead(t *testing.T) { +func TestPolicyFromRequestRead(t *testing.T) { testCases := []struct { description string request *openrtb.BidRequest - expectedPolicy Policy + expectedPolicy PolicyFromRequest expectedError bool }{ - { - description: "Nil Request", - request: nil, - expectedPolicy: Policy{ - Consent: "", - NoSaleBidders: nil, - }, - }, { description: "Success", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"ABC"}`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "ABC", NoSaleBidders: []string{"a", "b"}, }, }, + { + description: "Nil Request", + request: nil, + expectedPolicy: PolicyFromRequest{ + Consent: "", + NoSaleBidders: nil, + }, + }, { description: "Nil Regs", request: &openrtb.BidRequest{ Regs: nil, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -53,7 +51,7 @@ func TestRead(t *testing.T) { Regs: &openrtb.Regs{}, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -61,12 +59,10 @@ func TestRead(t *testing.T) { { description: "Empty Regs.Ext", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{}`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{}`)}, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -74,12 +70,10 @@ func TestRead(t *testing.T) { { description: "Missing Regs.Ext USPrivacy Value", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"anythingElse":"42"}`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"anythingElse":"42"}`)}, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -87,32 +81,26 @@ func TestRead(t *testing.T) { { description: "Malformed Regs.Ext", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`malformed`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed`)}, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedError: true, }, { description: "Invalid Regs.Ext Type", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":123`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":123`)}, + Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, expectedError: true, }, { description: "Nil Ext", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"ABC"}`), - }, - Ext: nil, + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + Ext: nil, }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "ABC", NoSaleBidders: nil, }, @@ -120,12 +108,10 @@ func TestRead(t *testing.T) { { description: "Empty Ext", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"ABC"}`), - }, - Ext: json.RawMessage(`{}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + Ext: json.RawMessage(`{}`), }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "ABC", NoSaleBidders: nil, }, @@ -133,12 +119,10 @@ func TestRead(t *testing.T) { { description: "Missing Ext.Prebid No Sale Value", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"ABC"}`), - }, - Ext: json.RawMessage(`{"anythingElse":"42"}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + Ext: json.RawMessage(`{"anythingElse":"42"}`), }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "ABC", NoSaleBidders: nil, }, @@ -146,45 +130,39 @@ func TestRead(t *testing.T) { { description: "Malformed Ext", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"ABC"}`), - }, - Ext: json.RawMessage(`malformed`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + Ext: json.RawMessage(`malformed`), }, expectedError: true, }, { description: "Invalid Ext.Prebid.NoSale Type", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"ABC"}`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":"wrongtype"}}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, + Ext: json.RawMessage(`{"prebid":{"nosale":"wrongtype"}}`), }, expectedError: true, }, { description: "Injection Attack", request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), - }, + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`)}, }, - expectedPolicy: Policy{ + expectedPolicy: PolicyFromRequest{ Consent: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"", }, }, } for _, test := range testCases { - result, err := ReadPolicy(test.request) + result, err := ReadFromRequest(test.request) assertError(t, test.expectedError, err, test.description) assert.Equal(t, test.expectedPolicy, result, test.description) } } -func TestClone(t *testing.T) { - policy := Policy{ +func TestPolicyFromRequestClone(t *testing.T) { + policy := PolicyFromRequest{ Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}, } @@ -196,8 +174,8 @@ func TestClone(t *testing.T) { assert.ElementsMatch(t, []string{"1", "b"}, clone.NoSaleBidders, "clone") } -func TestCloneNilNoSale(t *testing.T) { - policy := Policy{ +func TestPolicyFromRequestCloneNilNoSale(t *testing.T) { + policy := PolicyFromRequest{ Consent: "anyConsent", NoSaleBidders: nil, } @@ -207,49 +185,43 @@ func TestCloneNilNoSale(t *testing.T) { assert.Nil(t, clone.NoSaleBidders) } -func TestWrite(t *testing.T) { +func TestPolicyFromRequestWrite(t *testing.T) { testCases := []struct { description string - policy Policy + policy PolicyFromRequest request *openrtb.BidRequest expected *openrtb.BidRequest expectedError bool }{ { description: "Nil Request", - policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: nil, expected: nil, }, { description: "Success", - policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{}, expected: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`), - }, - Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, + Ext: json.RawMessage(`{"prebid":{"nosale":["a","b"]}}`), }, }, { description: "Error Regs.Ext - No Partial Update To Request", - policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`malformed}`), - }, + Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, }, expectedError: true, expected: &openrtb.BidRequest{ - Regs: &openrtb.Regs{ - Ext: json.RawMessage(`malformed}`), - }, + Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, }, }, { description: "Error Ext - No Partial Update To Request", - policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: json.RawMessage(`malformed}`), }, @@ -267,6 +239,72 @@ func TestWrite(t *testing.T) { } } +func TestPolicyFromConsentRead(t *testing.T) { + testCases := []struct { + description string + consent string + expectedPolicy PolicyFromConsent + }{ + { + description: "Empty Cosent", + consent: "", + expectedPolicy: PolicyFromConsent{}, + }, + { + description: "Cosent", + consent: "anyConsent", + expectedPolicy: PolicyFromConsent{Consent: "anyConsent"}, + }, + } + + for _, test := range testCases { + result := ReadFromConsent(test.consent) + assert.Equal(t, test.expectedPolicy, result, test.description) + } +} + +func TestPolicyFromConsentWrite(t *testing.T) { + testCases := []struct { + description string + policy PolicyFromConsent + request *openrtb.BidRequest + expected *openrtb.BidRequest + expectedError bool + }{ + { + description: "Success", + policy: PolicyFromConsent{Consent: "anyConsent"}, + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{ + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, + }, + }, + { + description: "Nil Request", + policy: PolicyFromConsent{Consent: "anyConsent"}, + request: nil, + expected: nil, + }, + { + description: "Error Regs.Ext - No Update To Request", + policy: PolicyFromConsent{Consent: "anyConsent"}, + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, + }, + expectedError: true, + expected: &openrtb.BidRequest{ + Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, + }, + }, + } + + for _, test := range testCases { + err := test.policy.Write(test.request) + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, test.request, test.description) + } +} + func TestBuildRegs(t *testing.T) { testCases := []struct { description string diff --git a/privacy/policies.go b/privacy/policies.go index 23ce738dc7f..bd83cf1d035 100644 --- a/privacy/policies.go +++ b/privacy/policies.go @@ -2,28 +2,25 @@ package privacy import ( "github.com/mxmCherry/openrtb" - - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" ) +// PolicyWriter mutates an OpenRTB bid request with a policy's regulatory information. type PolicyWriter interface { Write(req *openrtb.BidRequest) error } -// ReadPolicyFromConsent inspects the consent string and returns a validated policy writer. -func ReadPolicyFromConsent(consent string) (PolicyWriter, bool) { - if len(consent) == 0 { - return nil, false - } +// NilPolicyWriter implements the PolicyWriter interface but performs no action. +type NilPolicyWriter struct{} - if err := gdpr.ValidateConsent(consent); err == nil { - return gdpr.Policy{Consent: consent}, true - } +func (NilPolicyWriter) Write(req *openrtb.BidRequest) error { + return nil +} - if p, err := ccpa.Parse(ccpa.Policy{Consent: consent}, nil); err == nil { - return p, true - } +// InvalidConsentError represents an error parsing or validating a consent string. +type InvalidConsentError struct { + Message string +} - return nil, false +func (err *InvalidConsentError) Error() string { + return err.Message } diff --git a/privacy/policies_test.go b/privacy/policies_test.go deleted file mode 100644 index 34fbe52d0e9..00000000000 --- a/privacy/policies_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package privacy - -import ( - "errors" - "testing" - - "github.com/mxmCherry/openrtb" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestWritePoliciesNone(t *testing.T) { - request := &openrtb.BidRequest{} - policyWriters := []policyWriter{} - - err := writePolicies(request, policyWriters) - - assert.NoError(t, err) -} - -func TestWritePoliciesOne(t *testing.T) { - request := &openrtb.BidRequest{} - mockWriter := new(mockPolicyWriter) - policyWriters := []policyWriter{ - mockWriter, - } - - mockWriter.On("Write", request).Return(nil).Once() - - err := writePolicies(request, policyWriters) - - assert.NoError(t, err) - mockWriter.AssertExpectations(t) -} - -func TestWritePoliciesMany(t *testing.T) { - request := &openrtb.BidRequest{} - mockWriter1 := new(mockPolicyWriter) - mockWriter2 := new(mockPolicyWriter) - policyWriters := []policyWriter{ - mockWriter1, mockWriter2, - } - - mockWriter1.On("Write", request).Return(nil).Once() - mockWriter2.On("Write", request).Return(nil).Once() - - err := writePolicies(request, policyWriters) - - assert.NoError(t, err) - mockWriter1.AssertExpectations(t) - mockWriter2.AssertExpectations(t) -} - -func TestWritePoliciesError(t *testing.T) { - request := &openrtb.BidRequest{} - mockWriter := new(mockPolicyWriter) - policyWriters := []policyWriter{ - mockWriter, - } - - expectedErr := errors.New("anyError") - mockWriter.On("Write", request).Return(expectedErr).Once() - - err := writePolicies(request, policyWriters) - - assert.Error(t, err, expectedErr) - mockWriter.AssertExpectations(t) -} - -type mockPolicyWriter struct { - mock.Mock -} - -func (m *mockPolicyWriter) Write(req *openrtb.BidRequest) error { - args := m.Called(req) - return args.Error(0) -} - -func TestReadPoliciesFromConsent(t *testing.T) { - testCases := []struct { - description string - consent string - expectedResultValue Policies - expectedResultOK bool - }{ - { - description: "Empty String", - consent: "", - expectedResultValue: Policies{}, - expectedResultOK: false, - }, - { - description: "CCPA", - consent: "1NYN", - expectedResultValue: Policies{CCPA: ccpa.Policy{Value: "1NYN"}}, - expectedResultOK: true, - }, - { - description: "GDPR TCF 1.0", - consent: "BONV8oqONXwgmADACHENAO7pqzAAppY", - expectedResultValue: Policies{GDPR: gdpr.Policy{Consent: "BONV8oqONXwgmADACHENAO7pqzAAppY"}}, - expectedResultOK: true, - }, - { - description: "Invalid", - consent: "any invalid", - expectedResultValue: Policies{}, - expectedResultOK: false, - }, - } - - for _, test := range testCases { - resultValue, resultOK := ReadPoliciesFromConsent(test.consent) - assert.Equal(t, test.expectedResultValue, resultValue, test.description+":value") - assert.Equal(t, test.expectedResultOK, resultOK, test.description+":ok") - } -} diff --git a/usersync/usersync.go b/usersync/usersync.go index 2b313a021f8..fbce052b99b 100644 --- a/usersync/usersync.go +++ b/usersync/usersync.go @@ -1,13 +1,11 @@ package usersync -import "github.com/prebid/prebid-server/privacy" - type Usersyncer interface { // GetUsersyncInfo returns basic info the browser needs in order to run a user sync. // The returned UsersyncInfo object must not be mutated by callers. // // For more information about user syncs, see http://clearcode.cc/2015/12/cookie-syncing/ - GetUsersyncInfo(privacyPolicies privacy.Policies) (*UsersyncInfo, error) + GetUsersyncInfo(privacyPolicies PrivacyPolicies) (*UsersyncInfo, error) // FamilyName should be the same as the `BidderName` for this Usersyncer. // This function only exists for legacy reasons. @@ -32,6 +30,12 @@ type UsersyncInfo struct { SupportCORS bool `json:"supportCORS,omitempty"` } +type PrivacyPolicies struct { + GDPRSignal string + GDPRConsent string + CCPAConsent string +} + type CookieSyncBidders struct { BidderCode string `json:"bidder"` NoCookie bool `json:"no_cookie,omitempty"` From 74c7765f5c235fb0b763c82172c18166417e247a Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 11:13:36 -0400 Subject: [PATCH 15/27] WIP --- config/config.go | 2 +- endpoints/cookie_sync.go | 20 +++++++++++--------- endpoints/openrtb2/amp_auction.go | 4 ++-- endpoints/openrtb2/auction.go | 7 +++---- exchange/utils.go | 21 +++++++++++++++++---- privacy/ccpa/parsedpolicy.go | 4 ++-- privacy/ccpa/policy.go | 13 ------------- privacy/gdpr/policy.go | 23 ++++++++++++++++------- privacy/policies.go | 9 --------- 9 files changed, 52 insertions(+), 51 deletions(-) diff --git a/config/config.go b/config/config.go index 8545523d238..5f7e5420f3f 100755 --- a/config/config.go +++ b/config/config.go @@ -870,7 +870,7 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("gdpr.tcf2.purpose_one_treatement.enabled", true) v.SetDefault("gdpr.tcf2.purpose_one_treatement.access_allowed", true) v.SetDefault("gdpr.amp_exception", false) - v.SetDefault("ccpa.enforce", false) + v.SetDefault("ccpa.enforce", true) v.SetDefault("lmt.enforce", true) v.SetDefault("currency_converter.fetch_url", "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json") v.SetDefault("currency_converter.fetch_interval_seconds", 1800) // fetch currency rates every 30 minutes diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index b6f8ffb1149..4c2ac339778 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -119,9 +119,8 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h parsedReq.filterForGDPR(deps.syncPermissions) - ccpaPolicy := ccpa.ReadFromConsent(privacyPolicy.CCPAConsent) - if ccpaParsedPolicy, err := ccpaPolicy.Parse(); err != nil { - parsedReq.filterForCCPA(ccpaParsedPolicy, deps.enforceCCPA) + if deps.enforceCCPA { + parsedReq.filterForCCPA() } // surviving bidders are not privacy blocked @@ -242,17 +241,20 @@ func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) { } } -func (req *cookieSyncRequest) filterForCCPA(policy ccpa.ParsedPolicy, enforceCCPA bool) { +func (req *cookieSyncRequest) filterForCCPA() { if !enforceCCPA { return } - // let's do the parsing here instead? + ccpaPolicy := &ccpa.Policy{Consent: req.USPrivacy} + ccpaParsedPolicy, err := ccpaPolicy.Parse() - for i := 0; i < len(req.Bidders); i++ { - if policy.ShouldEnforce(req.Bidders[i]) { - req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) - i-- + if err == nil { + for i := 0; i < len(req.Bidders); i++ { + if policy.ShouldEnforce(req.Bidders[i]) { + req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) + i-- + } } } } diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 628bdda54f3..5b290c4af02 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -560,8 +560,8 @@ func readConsent(url *url.URL) (privacy.PolicyWriter, error) { return privacy.NilPolicyWriter{}, nil } - if err := gdpr.ValidateConsent(consent); err == nil { - return gdpr.Policy{Consent: consent}, nil + if gdpr.ValidateConsent(consent) { + return gdpr.NewConsentWriter(consent), nil } if ccpa.ValidateConsent(consent) { diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index ae83447426d..790ed80a3c1 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -28,7 +28,6 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" "github.com/prebid/prebid-server/prebid_cache_client" - "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/privacy/ccpa" "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" @@ -326,12 +325,12 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { if ccpaPolicy, err := ccpa.ReadFromRequest(req); err != nil { return append(errL, errL...) } else if _, err := ccpaPolicy.Parse(getValidBidders(aliases)); err == nil { - if _, isInvalidConsent := err.(*privacy.InvalidConsentError); isInvalidConsent { + if _, invalidConsent := err.(*errortypes.InvalidPrivacyConsent); invalidConsent { errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) // remove invalid consent from request - ccpaPolicy.Consent = "" - if err := ccpaPolicy.Write(req); err != nil { + consentWriter := ccpa.NewConsentWriter("") + if err := consentWriter.Write(req); err != nil { return append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) } } else { diff --git a/exchange/utils.go b/exchange/utils.go index c65482c5452..3f56908c6d0 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -84,8 +84,7 @@ func cleanOpenRTBRequests(ctx context.Context, return } - // buuild comvinarion of aliaes and built in bidders - ccpaPolicy, err = policy.Parse(nil) + ccpaPolicy, err = policy.Parse(getValidBidders(aliases)) if err != nil { errs = append(errs, err) return @@ -103,8 +102,8 @@ func cleanOpenRTBRequests(ctx context.Context, LMT: lmtPolicy.ShouldEnforce(), } - privacyLabels.CCPAProvided = ccpaPolicy.Value != "" // todo: add Specified helepr - privacyLabels.CCPAEnforced = privacyEnforcement.CCPA + privacyLabels.CCPAProvided = ccpaPolicy.Specified() + privacyLabels.CCPAEnforced = ccpaPolicy.ShouldEnforce("") privacyLabels.COPPAEnforced = privacyEnforcement.COPPA privacyLabels.LMTEnforced = privacyEnforcement.LMT @@ -441,6 +440,20 @@ func parseAliases(orig *openrtb.BidRequest) (map[string]string, []error) { return aliases, nil } +func getValidBidders(aliases map[string]string) map[string]struct{} { + validBidders := make(map[string]struct{}) + + for _, v := range openrtb_ext.BidderMap { + validBidders[v.String()] = struct{}{} + } + + for k := range aliases { + validBidders[k] = struct{}{} + } + + return validBidders +} + // Quick little randomizer for a list of strings. Stuffing it in utils to keep other files clean func randomizeList(list []openrtb_ext.BidderName) { l := len(list) diff --git a/privacy/ccpa/parsedpolicy.go b/privacy/ccpa/parsedpolicy.go index 1a53e0d29ae..266093747ca 100644 --- a/privacy/ccpa/parsedpolicy.go +++ b/privacy/ccpa/parsedpolicy.go @@ -33,7 +33,7 @@ type ParsedPolicy struct { } // Parse returns a parsed and validated ParsedPolicy intended for use in enforcement decisions. -func (p PolicyFromRequest) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { +func (p Policy) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { consentOptOut, err := parseConsent(p.Consent) if err != nil { msg := fmt.Sprintf("request.regs.ext.us_privacy is invalid: %s", err.Error()) @@ -53,7 +53,7 @@ func (p PolicyFromRequest) Parse(validBidders map[string]struct{}) (ParsedPolicy }, nil } -// ValidateConsent returns true if the consent string is empty or valid. +// ValidateConsent returns true if the consent string is empty or valid per the IAB CCPA spec. func ValidateConsent(consent string) bool { _, err := parseConsent(consent) return err == nil diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index e6a1a718ca8..f0a6da8c99f 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -218,16 +218,3 @@ func buildExtWrite(noSaleBidders []string, ext json.RawMessage) (json.RawMessage prebidExt["nosale"] = noSaleBidders return json.Marshal(extMap) } - -// last bit, separate consent into it's own concept? -type ConsentWriter struct { - Consent string -} - -// other languagds have this. but not go. - -func (ConsentWriter) Write(req) { - -} - -// all will be good if i can create a ccpa consent writer diff --git a/privacy/gdpr/policy.go b/privacy/gdpr/policy.go index 4733e1edd38..50d7daa6a94 100644 --- a/privacy/gdpr/policy.go +++ b/privacy/gdpr/policy.go @@ -2,7 +2,9 @@ package gdpr import ( "encoding/json" + "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/privacy" "github.com/mxmCherry/openrtb" "github.com/prebid/go-gdpr/vendorconsent" @@ -14,9 +16,12 @@ type Policy struct { Consent string } -// Write mutates an OpenRTB bid request with the context of the GDPR policy. -func (p Policy) Write(req *openrtb.BidRequest) error { - if p.Consent == "" { +type consentWriter struct { + consent string +} + +func (c consentWriter) Write(req *openrtb.BidRequest) error { + if c.consent == "" { return nil } @@ -25,7 +30,7 @@ func (p Policy) Write(req *openrtb.BidRequest) error { } if req.User.Ext == nil { - ext, err := json.Marshal(openrtb_ext.ExtUser{Consent: p.Consent}) + ext, err := json.Marshal(openrtb_ext.ExtUser{Consent: c.consent}) if err == nil { req.User.Ext = ext } @@ -35,7 +40,7 @@ func (p Policy) Write(req *openrtb.BidRequest) error { var extMap map[string]interface{} err := json.Unmarshal(req.User.Ext, &extMap) if err == nil { - extMap["consent"] = p.Consent + extMap["consent"] = c.consent ext, err := json.Marshal(extMap) if err == nil { req.User.Ext = ext @@ -44,8 +49,12 @@ func (p Policy) Write(req *openrtb.BidRequest) error { return err } -// ValidateConsent returns an error if the GDPR consent string does not adhere to the IAB TCF spec. +func NewConsentWriter(consent string) privacy.PolicyWriter { + return consentWriter{consent} +} + +// ValidateConsent returns true if the consent string is empty or valid per the IAB TCF spec. func ValidateConsent(consent string) error { _, err := vendorconsent.ParseString(consent) - return err + return err == nil } diff --git a/privacy/policies.go b/privacy/policies.go index bd83cf1d035..bc4bf009df7 100644 --- a/privacy/policies.go +++ b/privacy/policies.go @@ -15,12 +15,3 @@ type NilPolicyWriter struct{} func (NilPolicyWriter) Write(req *openrtb.BidRequest) error { return nil } - -// InvalidConsentError represents an error parsing or validating a consent string. -type InvalidConsentError struct { - Message string -} - -func (err *InvalidConsentError) Error() string { - return err.Message -} From c1496e69cd69584cd33fc01f2142cf605b2f8012 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 15:16:44 -0400 Subject: [PATCH 16/27] Final? Pass At Privacy Package Update --- privacy/ccpa/consentwriter.go | 30 ++++++ privacy/ccpa/consentwriter_test.go | 63 +++++++++++++ privacy/ccpa/parsedpolicy.go | 14 +-- privacy/ccpa/parsedpolicy_test.go | 145 +++++++++++++++-------------- privacy/ccpa/policy.go | 33 +------ privacy/ccpa/policy_test.go | 126 ++++--------------------- privacy/gdpr/consentwriter.go | 49 ++++++++++ privacy/gdpr/consentwriter_test.go | 101 ++++++++++++++++++++ privacy/gdpr/policy.go | 45 +-------- privacy/gdpr/policy_test.go | 113 ++-------------------- privacy/lmt/policy.go | 4 +- privacy/lmt/policy_test.go | 4 +- privacy/{policies.go => writer.go} | 0 13 files changed, 365 insertions(+), 362 deletions(-) create mode 100644 privacy/ccpa/consentwriter.go create mode 100644 privacy/ccpa/consentwriter_test.go create mode 100644 privacy/gdpr/consentwriter.go create mode 100644 privacy/gdpr/consentwriter_test.go rename privacy/{policies.go => writer.go} (100%) diff --git a/privacy/ccpa/consentwriter.go b/privacy/ccpa/consentwriter.go new file mode 100644 index 00000000000..779bd8c6377 --- /dev/null +++ b/privacy/ccpa/consentwriter.go @@ -0,0 +1,30 @@ +package ccpa + +import ( + "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/privacy" +) + +type consentWriter struct { + consent string +} + +// Write mutates an OpenRTB bid request with the CCPA consent. +func (c consentWriter) Write(req *openrtb.BidRequest) error { + if req == nil { + return nil + } + + regs, err := buildRegs(c.consent, req.Regs) + if err != nil { + return err + } + req.Regs = regs + + return nil +} + +// NewConsentWriter constructs a privacy.PolicyWriter to write the CCPA consent to an OpenRTB bid request. +func NewConsentWriter(consent string) privacy.PolicyWriter { + return consentWriter{consent} +} diff --git a/privacy/ccpa/consentwriter_test.go b/privacy/ccpa/consentwriter_test.go new file mode 100644 index 00000000000..e49b68e28ff --- /dev/null +++ b/privacy/ccpa/consentwriter_test.go @@ -0,0 +1,63 @@ +package ccpa + +import ( + "encoding/json" + "testing" + + "github.com/mxmCherry/openrtb" + "github.com/stretchr/testify/assert" +) + +func TestConsentWriterWrite(t *testing.T) { + consent := "anyConsent" + testCases := []struct { + description string + request *openrtb.BidRequest + expected *openrtb.BidRequest + expectedError bool + }{ + { + description: "Success", + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{ + Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, + }, + }, + { + description: "Nil Request", + request: nil, + expected: nil, + }, + { + description: "Error With Regs.Ext - Does Not Mutate", + request: &openrtb.BidRequest{ + Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, + }, + expectedError: true, + expected: &openrtb.BidRequest{ + Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, + }, + }, + } + + for _, test := range testCases { + writer := &consentWriter{consent} + + err := writer.Write(test.request) + + assertError(t, test.expectedError, err, test.description) + assert.Equal(t, test.expected, test.request, test.description) + } +} + +func TestNewConsentWriter(t *testing.T) { + testCases := []string{ + "", + "anyConsent", + } + + for _, test := range testCases { + writer := NewConsentWriter(test).(consentWriter) + assert.Equal(t, test, writer.consent) + } +} diff --git a/privacy/ccpa/parsedpolicy.go b/privacy/ccpa/parsedpolicy.go index 266093747ca..de49560486f 100644 --- a/privacy/ccpa/parsedpolicy.go +++ b/privacy/ccpa/parsedpolicy.go @@ -23,7 +23,13 @@ const ( const allBiddersMarker = "*" -// ParsedPolicy represents parsed and validated CCPA regulatory information. Use this object +// ValidateConsent returns true if the consent string is empty or valid per the IAB CCPA spec. +func ValidateConsent(consent string) bool { + _, err := parseConsent(consent) + return err == nil +} + +// ParsedPolicy represents parsed and validated CCPA regulatory information. Use this struct // to make enforcement decisions. type ParsedPolicy struct { consentSpecified bool @@ -53,12 +59,6 @@ func (p Policy) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { }, nil } -// ValidateConsent returns true if the consent string is empty or valid per the IAB CCPA spec. -func ValidateConsent(consent string) bool { - _, err := parseConsent(consent) - return err == nil -} - func parseConsent(consent string) (consentOptOutSale bool, err error) { if consent == "" { return false, nil diff --git a/privacy/ccpa/parsedpolicy_test.go b/privacy/ccpa/parsedpolicy_test.go index 69894a822a9..00214c88f00 100644 --- a/privacy/ccpa/parsedpolicy_test.go +++ b/privacy/ccpa/parsedpolicy_test.go @@ -1,7 +1,6 @@ package ccpa import ( - "errors" "testing" "github.com/mxmCherry/openrtb" @@ -9,7 +8,41 @@ import ( "github.com/stretchr/testify/mock" ) -func TestParesPolicyFromRequest(t *testing.T) { +func TestValidateConsent(t *testing.T) { + testCases := []struct { + description string + consent string + expected bool + }{ + { + description: "Empty String", + consent: "", + expected: true, + }, + { + description: "Valid Consent With Opt Out", + consent: "1NYN", + expected: true, + }, + { + description: "Valid Consent Without Opt Out", + consent: "1NNN", + expected: true, + }, + { + description: "Invalid", + consent: "malformed", + expected: false, + }, + } + + for _, test := range testCases { + result := ValidateConsent(test.consent) + assert.Equal(t, test.expected, result, test.description) + } +} + +func TestParse(t *testing.T) { validBidders := map[string]struct{}{"a": {}} testCases := []struct { @@ -38,7 +71,7 @@ func TestParesPolicyFromRequest(t *testing.T) { consent: "1NYN", noSaleBidders: []string{"a"}, expectedPolicy: ParsedPolicy{ - policyWriter: PolicyFromRequest{Consent: "1NYN", NoSaleBidders: []string{"a"}}, + consentSpecified: true, consentOptOutSale: true, noSaleForAllBidders: false, noSaleSpecificBidders: map[string]struct{}{"a": {}}, @@ -47,7 +80,7 @@ func TestParesPolicyFromRequest(t *testing.T) { } for _, test := range testCases { - policy := PolicyFromRequest{Consent: test.consent, NoSaleBidders: test.noSaleBidders} + policy := Policy{test.consent, test.noSaleBidders} result, err := policy.Parse(validBidders) @@ -61,70 +94,6 @@ func TestParesPolicyFromRequest(t *testing.T) { } } -func TestParesPolicyFromConsent(t *testing.T) { - testCases := []struct { - description string - consent string - expectedPolicy ParsedPolicy - expectedError string - }{ - { - description: "Success", - consent: "1NYN", - expectedPolicy: ParsedPolicy{ - policyWriter: PolicyFromConsent{Consent: "1NYN"}, - consentOptOutSale: true, - noSaleForAllBidders: false, - noSaleSpecificBidders: map[string]struct{}{}, - }, - }, - { - description: "Error", - consent: "malformed", - expectedPolicy: ParsedPolicy{}, - expectedError: "request.regs.ext.us_privacy is invalid: must contain 4 characters", - }, - } - - for _, test := range testCases { - policy := PolicyFromConsent{test.consent} - - result, err := policy.Parse() - - if test.expectedError == "" { - assert.NoError(t, err, test.description) - } else { - assert.EqualError(t, err, test.expectedError, test.description) - } - - assert.Equal(t, test.expectedPolicy, result, test.description) - } -} - -func TestWriteSuccess(t *testing.T) { - req := &openrtb.BidRequest{} - mockWriter := &mockPolicWriter{} - mockWriter.On("Write", req).Return(nil).Once() - parsedPolicy := &ParsedPolicy{policyWriter: mockWriter} - - resultErr := parsedPolicy.Write(req) - - mockWriter.AssertExpectations(t) - assert.NoError(t, resultErr) -} - -func TestWriteError(t *testing.T) { - req := &openrtb.BidRequest{} - mockWriter := &mockPolicWriter{} - mockWriter.On("Write", req).Return(errors.New("foo")).Once() - parsedPolicy := &ParsedPolicy{policyWriter: mockWriter} - - resultErr := parsedPolicy.Write(req) - - mockWriter.AssertExpectations(t) - assert.Error(t, resultErr, "foo") -} - func TestParseConsent(t *testing.T) { testCases := []struct { description string @@ -297,6 +266,40 @@ func TestParseNoSaleBidders(t *testing.T) { } } +func TestSpecified(t *testing.T) { + testCases := []struct { + description string + policy ParsedPolicy + expected bool + }{ + { + description: "Specified", + policy: ParsedPolicy{ + consentSpecified: true, + consentOptOutSale: false, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{}, + }, + expected: true, + }, + { + description: "Not Specified", + policy: ParsedPolicy{ + consentSpecified: false, + consentOptOutSale: false, + noSaleForAllBidders: false, + noSaleSpecificBidders: map[string]struct{}{}, + }, + expected: false, + }, + } + + for _, test := range testCases { + result := test.policy.Specified() + assert.Equal(t, test.expected, result, test.description) + } +} + func TestShouldEnforce(t *testing.T) { testCases := []struct { description string @@ -307,6 +310,7 @@ func TestShouldEnforce(t *testing.T) { { description: "Not Enforced - All Bidders No Sale", policy: ParsedPolicy{ + consentSpecified: true, consentOptOutSale: true, noSaleForAllBidders: true, noSaleSpecificBidders: map[string]struct{}{}, @@ -317,6 +321,7 @@ func TestShouldEnforce(t *testing.T) { { description: "Not Enforced - Specific Bidders No Sale", policy: ParsedPolicy{ + consentSpecified: true, consentOptOutSale: true, noSaleForAllBidders: false, noSaleSpecificBidders: map[string]struct{}{"a": {}}, @@ -327,6 +332,7 @@ func TestShouldEnforce(t *testing.T) { { description: "Not Enforced - No Bidder No Sale", policy: ParsedPolicy{ + consentSpecified: true, consentOptOutSale: false, noSaleForAllBidders: false, noSaleSpecificBidders: map[string]struct{}{}, @@ -337,6 +343,7 @@ func TestShouldEnforce(t *testing.T) { { description: "Not Enforced - No Sale Case Sensitive", policy: ParsedPolicy{ + consentSpecified: true, consentOptOutSale: false, noSaleForAllBidders: false, noSaleSpecificBidders: map[string]struct{}{"A": {}}, @@ -347,6 +354,7 @@ func TestShouldEnforce(t *testing.T) { { description: "Enforced - No Bidder No Sale", policy: ParsedPolicy{ + consentSpecified: true, consentOptOutSale: true, noSaleForAllBidders: false, noSaleSpecificBidders: map[string]struct{}{}, @@ -357,6 +365,7 @@ func TestShouldEnforce(t *testing.T) { { description: "Enforced - No Sale Case Sensitive", policy: ParsedPolicy{ + consentSpecified: true, consentOptOutSale: true, noSaleForAllBidders: false, noSaleSpecificBidders: map[string]struct{}{"A": {}}, diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index f0a6da8c99f..1b841943137 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -6,7 +6,6 @@ import ( "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/privacy" ) // Policy represents the CCPA regulatory information from an OpenRTB bid request. @@ -21,14 +20,14 @@ func ReadFromRequest(req *openrtb.BidRequest) (Policy, error) { var noSaleBidders []string if req == nil { - return PolicyFromRequest{}, nil + return Policy{}, nil } // Read consent from request.regs.ext if req.Regs != nil && len(req.Regs.Ext) > 0 { var ext openrtb_ext.ExtRegs if err := json.Unmarshal(req.Regs.Ext, &ext); err != nil { - return PolicyFromRequest{}, err + return Policy{}, err } consent = ext.USPrivacy } @@ -37,16 +36,16 @@ func ReadFromRequest(req *openrtb.BidRequest) (Policy, error) { if len(req.Ext) > 0 { var ext openrtb_ext.ExtRequest if err := json.Unmarshal(req.Ext, &ext); err != nil { - return PolicyFromRequest{}, err + return Policy{}, err } noSaleBidders = ext.Prebid.NoSale } - return PolicyFromRequest{consent, noSaleBidders}, nil + return Policy{consent, noSaleBidders}, nil } // Write mutates an OpenRTB bid request with the CCPA regulatory information. -func (p PolicyFromRequest) Write(req *openrtb.BidRequest) error { +func (p Policy) Write(req *openrtb.BidRequest) error { if req == nil { return nil } @@ -65,28 +64,6 @@ func (p PolicyFromRequest) Write(req *openrtb.BidRequest) error { return nil } -type consentWriter struct { - consent string -} - -func (c consentWriter) Write(req *openrtb.BidRequest) error { - if req == nil { - return nil - } - - regs, err := buildRegs(c.consent, req.Regs) - if err != nil { - return err - } - req.Regs = regs - - return nil -} - -func NewConsentWriter(consent string) privacy.PolicyWriter { - return consentWriter{consent} -} - func buildRegs(consent string, regs *openrtb.Regs) (*openrtb.Regs, error) { if consent == "" { return buildRegsClear(regs) diff --git a/privacy/ccpa/policy_test.go b/privacy/ccpa/policy_test.go index d7e16868b4c..7ff896e9ebf 100644 --- a/privacy/ccpa/policy_test.go +++ b/privacy/ccpa/policy_test.go @@ -8,11 +8,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPolicyFromRequestRead(t *testing.T) { +func TestReadFromRequest(t *testing.T) { testCases := []struct { description string request *openrtb.BidRequest - expectedPolicy PolicyFromRequest + expectedPolicy Policy expectedError bool }{ { @@ -21,7 +21,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "ABC", NoSaleBidders: []string{"a", "b"}, }, @@ -29,7 +29,7 @@ func TestPolicyFromRequestRead(t *testing.T) { { description: "Nil Request", request: nil, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "", NoSaleBidders: nil, }, @@ -40,7 +40,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: nil, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -51,7 +51,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: &openrtb.Regs{}, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -62,7 +62,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: &openrtb.Regs{Ext: json.RawMessage(`{}`)}, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -73,7 +73,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: &openrtb.Regs{Ext: json.RawMessage(`{"anythingElse":"42"}`)}, Ext: json.RawMessage(`{"prebid":{"nosale":["a", "b"]}}`), }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "", NoSaleBidders: []string{"a", "b"}, }, @@ -100,7 +100,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, Ext: nil, }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "ABC", NoSaleBidders: nil, }, @@ -111,7 +111,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, Ext: json.RawMessage(`{}`), }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "ABC", NoSaleBidders: nil, }, @@ -122,7 +122,7 @@ func TestPolicyFromRequestRead(t *testing.T) { Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"ABC"}`)}, Ext: json.RawMessage(`{"anythingElse":"42"}`), }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "ABC", NoSaleBidders: nil, }, @@ -148,7 +148,7 @@ func TestPolicyFromRequestRead(t *testing.T) { request: &openrtb.BidRequest{ Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`)}, }, - expectedPolicy: PolicyFromRequest{ + expectedPolicy: Policy{ Consent: "1YYY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"", }, }, @@ -161,47 +161,23 @@ func TestPolicyFromRequestRead(t *testing.T) { } } -func TestPolicyFromRequestClone(t *testing.T) { - policy := PolicyFromRequest{ - Consent: "anyConsent", - NoSaleBidders: []string{"a", "b"}, - } - - clone := policy.Clone() - clone.NoSaleBidders[0] = "1" - - assert.ElementsMatch(t, []string{"a", "b"}, policy.NoSaleBidders, "original") - assert.ElementsMatch(t, []string{"1", "b"}, clone.NoSaleBidders, "clone") -} - -func TestPolicyFromRequestCloneNilNoSale(t *testing.T) { - policy := PolicyFromRequest{ - Consent: "anyConsent", - NoSaleBidders: nil, - } - - clone := policy.Clone() - - assert.Nil(t, clone.NoSaleBidders) -} - -func TestPolicyFromRequestWrite(t *testing.T) { +func TestWrite(t *testing.T) { testCases := []struct { description string - policy PolicyFromRequest + policy Policy request *openrtb.BidRequest expected *openrtb.BidRequest expectedError bool }{ { description: "Nil Request", - policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: nil, expected: nil, }, { description: "Success", - policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{}, expected: &openrtb.BidRequest{ Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, @@ -210,7 +186,7 @@ func TestPolicyFromRequestWrite(t *testing.T) { }, { description: "Error Regs.Ext - No Partial Update To Request", - policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, }, @@ -221,7 +197,7 @@ func TestPolicyFromRequestWrite(t *testing.T) { }, { description: "Error Ext - No Partial Update To Request", - policy: PolicyFromRequest{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, + policy: Policy{Consent: "anyConsent", NoSaleBidders: []string{"a", "b"}}, request: &openrtb.BidRequest{ Ext: json.RawMessage(`malformed}`), }, @@ -239,72 +215,6 @@ func TestPolicyFromRequestWrite(t *testing.T) { } } -func TestPolicyFromConsentRead(t *testing.T) { - testCases := []struct { - description string - consent string - expectedPolicy PolicyFromConsent - }{ - { - description: "Empty Cosent", - consent: "", - expectedPolicy: PolicyFromConsent{}, - }, - { - description: "Cosent", - consent: "anyConsent", - expectedPolicy: PolicyFromConsent{Consent: "anyConsent"}, - }, - } - - for _, test := range testCases { - result := ReadFromConsent(test.consent) - assert.Equal(t, test.expectedPolicy, result, test.description) - } -} - -func TestPolicyFromConsentWrite(t *testing.T) { - testCases := []struct { - description string - policy PolicyFromConsent - request *openrtb.BidRequest - expected *openrtb.BidRequest - expectedError bool - }{ - { - description: "Success", - policy: PolicyFromConsent{Consent: "anyConsent"}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{ - Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, - }, - }, - { - description: "Nil Request", - policy: PolicyFromConsent{Consent: "anyConsent"}, - request: nil, - expected: nil, - }, - { - description: "Error Regs.Ext - No Update To Request", - policy: PolicyFromConsent{Consent: "anyConsent"}, - request: &openrtb.BidRequest{ - Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, - }, - expectedError: true, - expected: &openrtb.BidRequest{ - Regs: &openrtb.Regs{Ext: json.RawMessage(`malformed}`)}, - }, - }, - } - - for _, test := range testCases { - err := test.policy.Write(test.request) - assertError(t, test.expectedError, err, test.description) - assert.Equal(t, test.expected, test.request, test.description) - } -} - func TestBuildRegs(t *testing.T) { testCases := []struct { description string diff --git a/privacy/gdpr/consentwriter.go b/privacy/gdpr/consentwriter.go new file mode 100644 index 00000000000..21b29693dde --- /dev/null +++ b/privacy/gdpr/consentwriter.go @@ -0,0 +1,49 @@ +package gdpr + +import ( + "encoding/json" + + "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/privacy" + + "github.com/mxmCherry/openrtb" +) + +type consentWriter struct { + consent string +} + +// Write mutates an OpenRTB bid request with the GDPR TCF consent. +func (c consentWriter) Write(req *openrtb.BidRequest) error { + if c.consent == "" { + return nil + } + + if req.User == nil { + req.User = &openrtb.User{} + } + + if req.User.Ext == nil { + ext, err := json.Marshal(openrtb_ext.ExtUser{Consent: c.consent}) + if err == nil { + req.User.Ext = ext + } + return err + } + + var extMap map[string]interface{} + err := json.Unmarshal(req.User.Ext, &extMap) + if err == nil { + extMap["consent"] = c.consent + ext, err := json.Marshal(extMap) + if err == nil { + req.User.Ext = ext + } + } + return err +} + +// NewConsentWriter constructs a privacy.PolicyWriter to write the GDPR TCF consent to an OpenRTB bid request. +func NewConsentWriter(consent string) privacy.PolicyWriter { + return consentWriter{consent} +} diff --git a/privacy/gdpr/consentwriter_test.go b/privacy/gdpr/consentwriter_test.go new file mode 100644 index 00000000000..d29c1881309 --- /dev/null +++ b/privacy/gdpr/consentwriter_test.go @@ -0,0 +1,101 @@ +package gdpr + +import ( + "encoding/json" + "testing" + + "github.com/mxmCherry/openrtb" + "github.com/stretchr/testify/assert" +) + +func TestWrite(t *testing.T) { + testCases := []struct { + description string + consent string + request *openrtb.BidRequest + expected *openrtb.BidRequest + expectedError bool + }{ + { + description: "Empty", + consent: "", + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{}, + }, + { + description: "Enabled With Nil Request User Object", + consent: "anyConsent", + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"consent":"anyConsent"}`)}}, + }, + { + description: "Enabled With Nil Request User Ext Object", + consent: "anyConsent", + request: &openrtb.BidRequest{User: &openrtb.User{}}, + expected: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"consent":"anyConsent"}`)}}, + }, + { + description: "Enabled With Existing Request User Ext Object - Doesn't Overwrite", + consent: "anyConsent", + request: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"existing":"any"}`)}}, + expected: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"consent":"anyConsent","existing":"any"}`)}}, + }, + { + description: "Enabled With Existing Request User Ext Object - Overwrites", + consent: "anyConsent", + request: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"existing":"any","consent":"toBeOverwritten"}`)}}, + expected: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"consent":"anyConsent","existing":"any"}`)}}, + }, + { + description: "Enabled With Existing Malformed Request User Ext Object", + consent: "anyConsent", + request: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`malformed`)}}, + expectedError: true, + }, + { + description: "Injection Attack With Nil Request User Object", + consent: "BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"", + request: &openrtb.BidRequest{}, + expected: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"consent":"BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), + }}, + }, + { + description: "Injection Attack With Nil Request User Ext Object", + consent: "BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"", + request: &openrtb.BidRequest{User: &openrtb.User{}}, + expected: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"consent":"BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), + }}, + }, + { + description: "Injection Attack With Existing Request User Ext Object", + consent: "BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"", + request: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"existing":"any"}`), + }}, + expected: &openrtb.BidRequest{User: &openrtb.User{ + Ext: json.RawMessage(`{"consent":"BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"","existing":"any"}`), + }}, + }, + } + + for _, test := range testCases { + writer := NewConsentWriter(test.consent) + err := writer.Write(test.request) + + if test.expectedError { + assert.Error(t, err, test.description) + } else { + assert.NoError(t, err, test.description) + assert.Equal(t, test.expected, test.request, test.description) + } + } +} diff --git a/privacy/gdpr/policy.go b/privacy/gdpr/policy.go index 50d7daa6a94..0464a9ff979 100644 --- a/privacy/gdpr/policy.go +++ b/privacy/gdpr/policy.go @@ -1,12 +1,6 @@ package gdpr import ( - "encoding/json" - - "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/privacy" - - "github.com/mxmCherry/openrtb" "github.com/prebid/go-gdpr/vendorconsent" ) @@ -16,45 +10,8 @@ type Policy struct { Consent string } -type consentWriter struct { - consent string -} - -func (c consentWriter) Write(req *openrtb.BidRequest) error { - if c.consent == "" { - return nil - } - - if req.User == nil { - req.User = &openrtb.User{} - } - - if req.User.Ext == nil { - ext, err := json.Marshal(openrtb_ext.ExtUser{Consent: c.consent}) - if err == nil { - req.User.Ext = ext - } - return err - } - - var extMap map[string]interface{} - err := json.Unmarshal(req.User.Ext, &extMap) - if err == nil { - extMap["consent"] = c.consent - ext, err := json.Marshal(extMap) - if err == nil { - req.User.Ext = ext - } - } - return err -} - -func NewConsentWriter(consent string) privacy.PolicyWriter { - return consentWriter{consent} -} - // ValidateConsent returns true if the consent string is empty or valid per the IAB TCF spec. -func ValidateConsent(consent string) error { +func ValidateConsent(consent string) bool { _, err := vendorconsent.ParseString(consent) return err == nil } diff --git a/privacy/gdpr/policy_test.go b/privacy/gdpr/policy_test.go index c9bf10cd24a..dc8f56425c5 100644 --- a/privacy/gdpr/policy_test.go +++ b/privacy/gdpr/policy_test.go @@ -1,129 +1,36 @@ package gdpr import ( - "encoding/json" "testing" - "github.com/mxmCherry/openrtb" "github.com/stretchr/testify/assert" ) -func TestWrite(t *testing.T) { - testCases := []struct { - description string - policy Policy - request *openrtb.BidRequest - expected *openrtb.BidRequest - expectedError bool - }{ - { - description: "Disabled", - policy: Policy{Consent: ""}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{}, - }, - { - description: "Enabled With Nil Request User Object", - policy: Policy{Consent: "anyConsent"}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"consent":"anyConsent"}`)}}, - }, - { - description: "Enabled With Nil Request User Ext Object", - policy: Policy{Consent: "anyConsent"}, - request: &openrtb.BidRequest{User: &openrtb.User{}}, - expected: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"consent":"anyConsent"}`)}}, - }, - { - description: "Enabled With Existing Request User Ext Object - Doesn't Overwrite", - policy: Policy{Consent: "anyConsent"}, - request: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"existing":"any"}`)}}, - expected: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"consent":"anyConsent","existing":"any"}`)}}, - }, - { - description: "Enabled With Existing Request User Ext Object - Overwrites", - policy: Policy{Consent: "anyConsent"}, - request: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"existing":"any","consent":"toBeOverwritten"}`)}}, - expected: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"consent":"anyConsent","existing":"any"}`)}}, - }, - { - description: "Enabled With Existing Malformed Request User Ext Object", - policy: Policy{Consent: "anyConsent"}, - request: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`malformed`)}}, - expectedError: true, - }, - { - description: "Injection Attack With Nil Request User Object", - policy: Policy{Consent: "BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, - request: &openrtb.BidRequest{}, - expected: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"consent":"BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), - }}, - }, - { - description: "Injection Attack With Nil Request User Ext Object", - policy: Policy{Consent: "BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, - request: &openrtb.BidRequest{User: &openrtb.User{}}, - expected: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"consent":"BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}`), - }}, - }, - { - description: "Injection Attack With Existing Request User Ext Object", - policy: Policy{Consent: "BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\""}, - request: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"existing":"any"}`), - }}, - expected: &openrtb.BidRequest{User: &openrtb.User{ - Ext: json.RawMessage(`{"consent":"BONV8oqONXwgmADACHENAO7pqzAAppY\"},\"oops\":\"malicious\",\"p\":{\"p\":\"","existing":"any"}`), - }}, - }, - } - - for _, test := range testCases { - err := test.policy.Write(test.request) - - if test.expectedError { - assert.Error(t, err, test.description) - } else { - assert.NoError(t, err, test.description) - assert.Equal(t, test.expected, test.request, test.description) - } - } -} - func TestValidateConsent(t *testing.T) { testCases := []struct { description string consent string - expectError bool + expected bool }{ { description: "Invalid", consent: "", - expectError: true, + expected: false, }, { - description: "Valid", + description: "TCF1 Valid", consent: "BONV8oqONXwgmADACHENAO7pqzAAppY", - expectError: false, + expected: true, + }, + { + description: "TCF2 Valid", + consent: "COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA", + expected: true, }, } for _, test := range testCases { result := ValidateConsent(test.consent) - - if test.expectError { - assert.Error(t, result, test.description) - } else { - assert.NoError(t, result, test.description) - } + assert.Equal(t, test.expected, result, test.description) } } diff --git a/privacy/lmt/policy.go b/privacy/lmt/policy.go index 79425bf59f7..9460fd2ecab 100644 --- a/privacy/lmt/policy.go +++ b/privacy/lmt/policy.go @@ -15,8 +15,8 @@ type Policy struct { SignalProvided bool } -// ReadPolicy extracts the LMT (Limit Ad Tracking) policy from an OpenRTB bid request. -func ReadPolicy(req *openrtb.BidRequest) Policy { +// ReadFromRequest extracts the LMT (Limit Ad Tracking) policy from an OpenRTB bid request. +func ReadFromRequest(req *openrtb.BidRequest) Policy { policy := Policy{} if req != nil && req.Device != nil && req.Device.Lmt != nil { diff --git a/privacy/lmt/policy_test.go b/privacy/lmt/policy_test.go index 45de219a9bf..356d35baf91 100644 --- a/privacy/lmt/policy_test.go +++ b/privacy/lmt/policy_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRead(t *testing.T) { +func TestReadFromRequest(t *testing.T) { var one int8 = 1 testCases := []struct { @@ -60,7 +60,7 @@ func TestRead(t *testing.T) { } for _, test := range testCases { - p := ReadPolicy(test.request) + p := ReadFromRequest(test.request) assert.Equal(t, test.expectedPolicy, p, test.description) } } diff --git a/privacy/policies.go b/privacy/writer.go similarity index 100% rename from privacy/policies.go rename to privacy/writer.go From 6cf462fabff4b689453b8f3bc2cb8d52f24f8da6 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 15:23:31 -0400 Subject: [PATCH 17/27] Updated UserSync Package + Adatper Calls --- adapters/consumable/consumable.go | 13 +++++++------ adapters/sharethrough/butler.go | 15 ++++++++------- adapters/syncer.go | 9 ++++----- usersync/usersync.go | 12 ++++++------ 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/adapters/consumable/consumable.go b/adapters/consumable/consumable.go index 243f1b8000b..73d491952b9 100644 --- a/adapters/consumable/consumable.go +++ b/adapters/consumable/consumable.go @@ -3,15 +3,16 @@ package consumable import ( "encoding/json" "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/privacy/ccpa" - "net/http" - "net/url" - "strconv" - "strings" ) type ConsumableAdapter struct { @@ -134,9 +135,9 @@ func (a *ConsumableAdapter) MakeRequests(request *openrtb.BidRequest, reqInfo *a gdpr := bidGdpr{} - ccpaPolicy, err := ccpa.ReadPolicy(request) + ccpaPolicy, err := ccpa.ReadFromRequest(request) if err == nil { - body.CCPA = ccpaPolicy.Value + body.CCPA = ccpaPolicy.Consent } // TODO: Replace with gdpr.ReadPolicy when it is available diff --git a/adapters/sharethrough/butler.go b/adapters/sharethrough/butler.go index 522bbc4967e..36af79c4534 100644 --- a/adapters/sharethrough/butler.go +++ b/adapters/sharethrough/butler.go @@ -3,16 +3,17 @@ package sharethrough import ( "encoding/json" "fmt" - "github.com/mxmCherry/openrtb" - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/errortypes" - "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/privacy/ccpa" "net/http" "net/url" "regexp" "strconv" "time" + + "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/privacy/ccpa" ) const defaultTmax = 10000 // 10 sec @@ -97,8 +98,8 @@ func (s StrOpenRTBTranslator) requestFromOpenRTB(imp openrtb.Imp, request *openr } usPolicySignal := "" - if usPolicy, err := ccpa.ReadPolicy(request); err == nil { - usPolicySignal = usPolicy.Value + if usPolicy, err := ccpa.ReadFromRequest(request); err == nil { + usPolicySignal = usPolicy.Consent } return &adapters.RequestData{ diff --git a/adapters/syncer.go b/adapters/syncer.go index c212a4366c9..944027e705b 100644 --- a/adapters/syncer.go +++ b/adapters/syncer.go @@ -5,7 +5,6 @@ import ( "github.com/prebid/prebid-server/macros" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/usersync" ) @@ -42,11 +41,11 @@ const ( SyncTypeIframe SyncType = "iframe" ) -func (s *Syncer) GetUsersyncInfo(privacyPolicies privacy.Policies) (*usersync.UsersyncInfo, error) { +func (s *Syncer) GetUsersyncInfo(privacyPolicies usersync.PrivacyPolicies) (*usersync.UsersyncInfo, error) { syncURL, err := macros.ResolveMacros(*s.urlTemplate, macros.UserSyncTemplateParams{ - GDPR: privacyPolicies.GDPR.Signal, - GDPRConsent: privacyPolicies.GDPR.Consent, - USPrivacy: privacyPolicies.CCPA.Value, + GDPR: privacyPolicies.GDPRSignal, + GDPRConsent: privacyPolicies.GDPRConsent, + USPrivacy: privacyPolicies.CCPAConsent, }) if err != nil { return nil, err diff --git a/usersync/usersync.go b/usersync/usersync.go index fbce052b99b..9e87a5d2d6f 100644 --- a/usersync/usersync.go +++ b/usersync/usersync.go @@ -30,14 +30,14 @@ type UsersyncInfo struct { SupportCORS bool `json:"supportCORS,omitempty"` } -type PrivacyPolicies struct { - GDPRSignal string - GDPRConsent string - CCPAConsent string -} - type CookieSyncBidders struct { BidderCode string `json:"bidder"` NoCookie bool `json:"no_cookie,omitempty"` UsersyncInfo *UsersyncInfo `json:"usersync,omitempty"` } + +type PrivacyPolicies struct { + GDPRSignal string + GDPRConsent string + CCPAConsent string +} From 8e9a7a8b8736ebf99cf782ef346c8636d8cc809b Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 15:49:26 -0400 Subject: [PATCH 18/27] Back To Building --- adapters/33across/usersync_test.go | 2 +- adapters/adkernel/usersync_test.go | 2 +- adapters/adkernelAdn/usersync_test.go | 2 +- adapters/adman/usersync_test.go | 2 +- adapters/admixer/usersync_test.go | 7 ++++--- adapters/adtarget/usersync_test.go | 5 +++-- adapters/aja/usersync_test.go | 5 +++-- adapters/avocet/usersync_test.go | 2 +- adapters/beachfront/usersync_test.go | 2 +- adapters/beintoo/usersync_test.go | 2 +- adapters/consumable/usersync_test.go | 2 +- adapters/datablocks/usersync_test.go | 2 +- adapters/emx_digital/usersync_test.go | 2 +- adapters/engagebdr/usersync_test.go | 2 +- adapters/gamoshi/usersync_test.go | 2 +- adapters/gumgum/usersync_test.go | 2 +- adapters/improvedigital/usersync_test.go | 2 +- adapters/marsmedia/usersync_test.go | 2 +- adapters/nanointeractive/usersync_test.go | 24 +++++++++++------------ adapters/pubmatic/usersync_test.go | 2 +- adapters/rhythmone/usersync_test.go | 2 +- adapters/smartadserver/usersync_test.go | 2 +- adapters/syncer.go | 9 +++++---- adapters/syncer_test.go | 2 +- adapters/unruly/usersync_test.go | 2 +- adapters/valueimpression/usersync_test.go | 2 +- adapters/visx/usersync_test.go | 2 +- adapters/zeroclickfraud/usersync_test.go | 2 +- endpoints/auction.go | 20 +++++++++++-------- endpoints/auction_test.go | 12 ++++++++---- endpoints/cookie_sync.go | 24 ++++++++++++----------- endpoints/openrtb2/amp_auction.go | 4 ++-- endpoints/openrtb2/auction.go | 2 +- endpoints/setuid_test.go | 3 ++- exchange/utils.go | 2 +- privacy/ccpa/consentwriter.go | 15 +++++--------- privacy/ccpa/consentwriter_test.go | 14 +------------ privacy/gdpr/consentwriter.go | 19 +++++++----------- privacy/gdpr/consentwriter_test.go | 2 +- privacy/policies.go | 14 +++++++++++++ usersync/usersync.go | 10 +++------- 41 files changed, 120 insertions(+), 117 deletions(-) create mode 100644 privacy/policies.go diff --git a/adapters/33across/usersync_test.go b/adapters/33across/usersync_test.go index a5e301b1082..a9eb4e57908 100644 --- a/adapters/33across/usersync_test.go +++ b/adapters/33across/usersync_test.go @@ -23,7 +23,7 @@ func Test33AcrossSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/adkernel/usersync_test.go b/adapters/adkernel/usersync_test.go index 0d539d11ee0..aeacf00b7f0 100644 --- a/adapters/adkernel/usersync_test.go +++ b/adapters/adkernel/usersync_test.go @@ -23,7 +23,7 @@ func TestAdkernelAdnSyncer(t *testing.T) { Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/adkernelAdn/usersync_test.go b/adapters/adkernelAdn/usersync_test.go index ecc759bdf70..92d688e6117 100644 --- a/adapters/adkernelAdn/usersync_test.go +++ b/adapters/adkernelAdn/usersync_test.go @@ -23,7 +23,7 @@ func TestAdkernelAdnSyncer(t *testing.T) { Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/adman/usersync_test.go b/adapters/adman/usersync_test.go index 55a6e2cec97..25da77db7ed 100644 --- a/adapters/adman/usersync_test.go +++ b/adapters/adman/usersync_test.go @@ -23,7 +23,7 @@ func TestAdmanSyncer(t *testing.T) { Consent: "ANDFJDS", }, CCPA: ccpa.Policy{ - Value: "1-YY", + Consent: "1-YY", }, }) diff --git a/adapters/admixer/usersync_test.go b/adapters/admixer/usersync_test.go index a5715c64a46..d31f7b10fb1 100644 --- a/adapters/admixer/usersync_test.go +++ b/adapters/admixer/usersync_test.go @@ -1,12 +1,13 @@ package admixer import ( + "testing" + "text/template" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/privacy/ccpa" "github.com/prebid/prebid-server/privacy/gdpr" "github.com/stretchr/testify/assert" - "testing" - "text/template" ) func TestAdmixerSyncer(t *testing.T) { @@ -22,7 +23,7 @@ func TestAdmixerSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/adtarget/usersync_test.go b/adapters/adtarget/usersync_test.go index 3ab2ed5b5df..ddba9e7a720 100644 --- a/adapters/adtarget/usersync_test.go +++ b/adapters/adtarget/usersync_test.go @@ -2,10 +2,11 @@ package adtarget import ( "fmt" - "github.com/prebid/prebid-server/privacy/ccpa" "testing" "text/template" + "github.com/prebid/prebid-server/privacy/ccpa" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/privacy/gdpr" "github.com/stretchr/testify/assert" @@ -25,7 +26,7 @@ func TestAdtargetSyncer(t *testing.T) { Consent: "123", }, CCPA: ccpa.Policy{ - Value: "1-YY", + Consent: "1-YY", }, }) diff --git a/adapters/aja/usersync_test.go b/adapters/aja/usersync_test.go index dbb66cc9ae2..4b6c90ef141 100644 --- a/adapters/aja/usersync_test.go +++ b/adapters/aja/usersync_test.go @@ -1,10 +1,11 @@ package aja import ( - "github.com/prebid/prebid-server/privacy/ccpa" "testing" "text/template" + "github.com/prebid/prebid-server/privacy/ccpa" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/privacy/gdpr" "github.com/stretchr/testify/assert" @@ -23,7 +24,7 @@ func TestAJASyncer(t *testing.T) { Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/avocet/usersync_test.go b/adapters/avocet/usersync_test.go index 8fba403f1b1..3df39b77fce 100644 --- a/adapters/avocet/usersync_test.go +++ b/adapters/avocet/usersync_test.go @@ -23,7 +23,7 @@ func TestAvocetSyncer(t *testing.T) { Consent: "ConsentString", }, CCPA: ccpa.Policy{ - Value: "PrivacyString", + Consent: "PrivacyString", }, }) diff --git a/adapters/beachfront/usersync_test.go b/adapters/beachfront/usersync_test.go index 0267ac05eb7..db4d825eb5a 100644 --- a/adapters/beachfront/usersync_test.go +++ b/adapters/beachfront/usersync_test.go @@ -23,7 +23,7 @@ func TestBeachfrontSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/beintoo/usersync_test.go b/adapters/beintoo/usersync_test.go index 2cfca010226..880d6a84cee 100644 --- a/adapters/beintoo/usersync_test.go +++ b/adapters/beintoo/usersync_test.go @@ -23,7 +23,7 @@ func TestBeintooSyncer(t *testing.T) { Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/consumable/usersync_test.go b/adapters/consumable/usersync_test.go index 017cb72975b..ef71c0b18c7 100644 --- a/adapters/consumable/usersync_test.go +++ b/adapters/consumable/usersync_test.go @@ -23,7 +23,7 @@ func TestConsumableSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/datablocks/usersync_test.go b/adapters/datablocks/usersync_test.go index f8500ab9b03..a7518e9b226 100644 --- a/adapters/datablocks/usersync_test.go +++ b/adapters/datablocks/usersync_test.go @@ -23,7 +23,7 @@ func TestDatablocksSyncer(t *testing.T) { Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/emx_digital/usersync_test.go b/adapters/emx_digital/usersync_test.go index 0e76936cea4..59d66d87808 100644 --- a/adapters/emx_digital/usersync_test.go +++ b/adapters/emx_digital/usersync_test.go @@ -23,7 +23,7 @@ func TestEMXDigitalSyncer(t *testing.T) { Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/engagebdr/usersync_test.go b/adapters/engagebdr/usersync_test.go index 45e1e41e196..3a6c179addf 100644 --- a/adapters/engagebdr/usersync_test.go +++ b/adapters/engagebdr/usersync_test.go @@ -23,7 +23,7 @@ func TestEngageBDRSyncer(t *testing.T) { Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/gamoshi/usersync_test.go b/adapters/gamoshi/usersync_test.go index b8e3e327e44..43dc88a4953 100644 --- a/adapters/gamoshi/usersync_test.go +++ b/adapters/gamoshi/usersync_test.go @@ -18,7 +18,7 @@ func TestGamoshiSyncer(t *testing.T) { syncer := NewGamoshiSyncer(syncURLTemplate) syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ CCPA: ccpa.Policy{ - Value: "anyValue", + Consent: "anyValue", }, }) diff --git a/adapters/gumgum/usersync_test.go b/adapters/gumgum/usersync_test.go index 3606f6ae04c..9c6dc420600 100644 --- a/adapters/gumgum/usersync_test.go +++ b/adapters/gumgum/usersync_test.go @@ -23,7 +23,7 @@ func TestGumGumSyncer(t *testing.T) { Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/improvedigital/usersync_test.go b/adapters/improvedigital/usersync_test.go index c928ebf123d..35ea89cf894 100644 --- a/adapters/improvedigital/usersync_test.go +++ b/adapters/improvedigital/usersync_test.go @@ -23,7 +23,7 @@ func TestImprovedigitalSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/marsmedia/usersync_test.go b/adapters/marsmedia/usersync_test.go index 67276a35fb6..f019c014516 100644 --- a/adapters/marsmedia/usersync_test.go +++ b/adapters/marsmedia/usersync_test.go @@ -23,7 +23,7 @@ func TestMarsmediaSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/nanointeractive/usersync_test.go b/adapters/nanointeractive/usersync_test.go index ec9787bc20d..fa78664928f 100644 --- a/adapters/nanointeractive/usersync_test.go +++ b/adapters/nanointeractive/usersync_test.go @@ -1,11 +1,12 @@ package nanointeractive import ( - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" "testing" "text/template" + "github.com/prebid/prebid-server/privacy/ccpa" + "github.com/prebid/prebid-server/privacy/gdpr" + "github.com/prebid/prebid-server/privacy" "github.com/stretchr/testify/assert" ) @@ -17,16 +18,15 @@ func TestNewNanoInteractiveSyncer(t *testing.T) { ) userSync := NewNanoInteractiveSyncer(syncURLTemplate) - syncInfo, err := userSync.GetUsersyncInfo( - privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - CCPA: ccpa.Policy{ - Value: "1NYN", - }, - }) + syncInfo, err := userSync.GetUsersyncInfo(privacy.Policies{ + GDPR: gdpr.Policy{ + Signal: "1", + Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", + }, + CCPA: ccpa.Policy{ + Consent: "1NYN", + }, + }) assert.NoError(t, err) assert.Equal(t, "https://ad.audiencemanager.de/hbs/cookie_sync?gdpr=1&consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw&us_privacy=1NYN&redirectUri=http%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dnanointeractive%26gdpr%3D1%26gdpr_consent%3DBONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw%26uid%3D%24UID", syncInfo.URL) diff --git a/adapters/pubmatic/usersync_test.go b/adapters/pubmatic/usersync_test.go index dd4a086c453..d6cd9f78af7 100644 --- a/adapters/pubmatic/usersync_test.go +++ b/adapters/pubmatic/usersync_test.go @@ -23,7 +23,7 @@ func TestPubmaticSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/rhythmone/usersync_test.go b/adapters/rhythmone/usersync_test.go index cee6e9b0259..85ecba2a8ab 100644 --- a/adapters/rhythmone/usersync_test.go +++ b/adapters/rhythmone/usersync_test.go @@ -23,7 +23,7 @@ func TestRhythmoneSyncer(t *testing.T) { Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/smartadserver/usersync_test.go b/adapters/smartadserver/usersync_test.go index e279b49e017..c4e6660693f 100644 --- a/adapters/smartadserver/usersync_test.go +++ b/adapters/smartadserver/usersync_test.go @@ -23,7 +23,7 @@ func TestSmartadserverSyncer(t *testing.T) { Consent: "COyASAoOyASAoAfAAAENAfCAAAAAAAAAAAAAAAAAAAAA", }, CCPA: ccpa.Policy{ - Value: "1YNN", + Consent: "1YNN", }, }) diff --git a/adapters/syncer.go b/adapters/syncer.go index 944027e705b..122bcc7ed38 100644 --- a/adapters/syncer.go +++ b/adapters/syncer.go @@ -5,6 +5,7 @@ import ( "github.com/prebid/prebid-server/macros" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/usersync" ) @@ -41,11 +42,11 @@ const ( SyncTypeIframe SyncType = "iframe" ) -func (s *Syncer) GetUsersyncInfo(privacyPolicies usersync.PrivacyPolicies) (*usersync.UsersyncInfo, error) { +func (s *Syncer) GetUsersyncInfo(privacyPolicies privacy.Policies) (*usersync.UsersyncInfo, error) { syncURL, err := macros.ResolveMacros(*s.urlTemplate, macros.UserSyncTemplateParams{ - GDPR: privacyPolicies.GDPRSignal, - GDPRConsent: privacyPolicies.GDPRConsent, - USPrivacy: privacyPolicies.CCPAConsent, + GDPR: privacyPolicies.GDPR.Signal, + GDPRConsent: privacyPolicies.GDPR.Consent, + USPrivacy: privacyPolicies.CCPA.Consent, }) if err != nil { return nil, err diff --git a/adapters/syncer_test.go b/adapters/syncer_test.go index 9be523091dd..ca33a9a130d 100644 --- a/adapters/syncer_test.go +++ b/adapters/syncer_test.go @@ -17,7 +17,7 @@ func TestGetUsersyncInfo(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, } diff --git a/adapters/unruly/usersync_test.go b/adapters/unruly/usersync_test.go index bdab254f370..2f0e07d813a 100644 --- a/adapters/unruly/usersync_test.go +++ b/adapters/unruly/usersync_test.go @@ -23,7 +23,7 @@ func TestUnrulySyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/valueimpression/usersync_test.go b/adapters/valueimpression/usersync_test.go index 63f123055a9..ffb3f372bd7 100644 --- a/adapters/valueimpression/usersync_test.go +++ b/adapters/valueimpression/usersync_test.go @@ -23,7 +23,7 @@ func TestValueImpressionSyncer(t *testing.T) { Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/adapters/visx/usersync_test.go b/adapters/visx/usersync_test.go index a77136c9240..b410cda6061 100644 --- a/adapters/visx/usersync_test.go +++ b/adapters/visx/usersync_test.go @@ -23,7 +23,7 @@ func TestVisxSyncer(t *testing.T) { Consent: "B", }, CCPA: ccpa.Policy{ - Value: "C", + Consent: "C", }, }) diff --git a/adapters/zeroclickfraud/usersync_test.go b/adapters/zeroclickfraud/usersync_test.go index 30ade771a4c..5e8f8fdf111 100644 --- a/adapters/zeroclickfraud/usersync_test.go +++ b/adapters/zeroclickfraud/usersync_test.go @@ -23,7 +23,7 @@ func TestZeroClickFraudSyncer(t *testing.T) { Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", }, CCPA: ccpa.Policy{ - Value: "1NYN", + Consent: "1NYN", }, }) diff --git a/endpoints/auction.go b/endpoints/auction.go index ca6711d31c5..cf8657e9685 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -23,6 +23,8 @@ import ( "github.com/prebid/prebid-server/pbs" "github.com/prebid/prebid-server/pbsmetrics" pbc "github.com/prebid/prebid-server/prebid_cache_client" + "github.com/prebid/prebid-server/privacy" + gdprPrivacy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync" ) @@ -188,20 +190,20 @@ func (a *auction) recoverSafely(inner func(*pbs.PBSBidder, pbsmetrics.AdapterLab } } -func (a *auction) shouldUsersync(ctx context.Context, bidder openrtb_ext.BidderName, privacyPolicies usersync.PrivacyPolicies) bool { - switch privacyPolicies.GDPRSignal { +func (a *auction) shouldUsersync(ctx context.Context, bidder openrtb_ext.BidderName, privacyPolicies privacy.Policies) bool { + switch privacyPolicies.GDPR.Signal { case "0": return true case "1": - if privacyPolicies.GDPRConsent == "" { + if privacyPolicies.GDPR.Consent == "" { return false } fallthrough default: - if canSync, err := a.gdprPerms.HostCookiesAllowed(ctx, privacyPolicies.GDPRConsent); !canSync || err != nil { + if canSync, err := a.gdprPerms.HostCookiesAllowed(ctx, privacyPolicies.GDPR.Consent); !canSync || err != nil { return false } - canSync, err := a.gdprPerms.BidderSyncAllowed(ctx, bidder, privacyPolicies.GDPRConsent) + canSync, err := a.gdprPerms.BidderSyncAllowed(ctx, bidder, privacyPolicies.GDPR.Consent) return canSync && err == nil } } @@ -508,9 +510,11 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl uid, _, _ := req.Cookie.GetUID(syncer.FamilyName()) if uid == "" { bidder.NoCookie = true - privacyPolicies := usersync.PrivacyPolicies{ - GDPRSignal: req.ParseGDPR(), - GDPRConsent: req.ParseConsent(), + privacyPolicies := privacy.Policies{ + GDPR: gdprPrivacy.Policy{ + Signal: req.ParseGDPR(), + Consent: req.ParseConsent(), + }, } if a.shouldUsersync(*ctx, openrtb_ext.BidderName(syncerCode), privacyPolicies) { syncInfo, err := syncer.GetUsersyncInfo(privacyPolicies) diff --git a/endpoints/auction_test.go b/endpoints/auction_test.go index 5e9e9639a9c..e6db0c35d36 100644 --- a/endpoints/auction_test.go +++ b/endpoints/auction_test.go @@ -19,6 +19,7 @@ import ( "github.com/prebid/prebid-server/pbsmetrics" metricsConf "github.com/prebid/prebid-server/pbsmetrics/config" "github.com/prebid/prebid-server/prebid_cache_client" + "github.com/prebid/prebid-server/privacy" gdprPolicy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync/usersyncers" "github.com/spf13/viper" @@ -387,11 +388,14 @@ func TestShouldUsersync(t *testing.T) { }, metricsEngine: nil, } - privacyPolicy := gdprPolicy.Policy{ - Signal: gdprApplies, - Consent: consent, + privacyPolicies := privacy.Policies{ + GDPR: gdprPolicy.Policy{ + Signal: gdprApplies, + Consent: consent, + }, } - allowSyncs := deps.shouldUsersync(context.Background(), openrtb_ext.BidderAdform, privacyPolicy) + + allowSyncs := deps.shouldUsersync(context.Background(), openrtb_ext.BidderAdform, privacyPolicies) if allowSyncs != expectAllow { t.Errorf("Expected syncs: %t, allowed syncs: %t", expectAllow, allowSyncs) } diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 4c2ac339778..9f64e6ceb54 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -18,7 +18,9 @@ import ( "github.com/prebid/prebid-server/gdpr" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/privacy/ccpa" + gdprPrivacy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync" ) @@ -111,10 +113,14 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h adapterSyncs[openrtb_ext.BidderName(b)] = true } - privacyPolicy := usersync.PrivacyPolicies{ - GDPRSignal: gdprToString(parsedReq.GDPR), - GDPRConsent: parsedReq.Consent, - CCPAConsent: parsedReq.USPrivacy, + privacyPolicy := privacy.Policies{ + GDPR: gdprPrivacy.Policy{ + Signal: gdprToString(parsedReq.GDPR), + Consent: parsedReq.Consent, + }, + CCPA: ccpa.Policy{ + Consent: parsedReq.USPrivacy, + }, } parsedReq.filterForGDPR(deps.syncPermissions) @@ -138,7 +144,7 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h } for i := 0; i < len(parsedReq.Bidders); i++ { bidder := parsedReq.Bidders[i] - syncInfo, err := deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(privacyPolicy) //PrivacyPolicies + syncInfo, err := deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(privacyPolicy) if err == nil { newSync := &usersync.CookieSyncBidders{ BidderCode: bidder, @@ -242,16 +248,12 @@ func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) { } func (req *cookieSyncRequest) filterForCCPA() { - if !enforceCCPA { - return - } - ccpaPolicy := &ccpa.Policy{Consent: req.USPrivacy} - ccpaParsedPolicy, err := ccpaPolicy.Parse() + ccpaParsedPolicy, err := ccpaPolicy.Parse(nil) // need valid bidders if err == nil { for i := 0; i < len(req.Bidders); i++ { - if policy.ShouldEnforce(req.Bidders[i]) { + if ccpaParsedPolicy.ShouldEnforce(req.Bidders[i]) { req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) i-- } diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 5b290c4af02..5216b8ac63e 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -561,11 +561,11 @@ func readConsent(url *url.URL) (privacy.PolicyWriter, error) { } if gdpr.ValidateConsent(consent) { - return gdpr.NewConsentWriter(consent), nil + return gdpr.ConsentWriter{consent}, nil } if ccpa.ValidateConsent(consent) { - return ccpa.NewConsentWriter(consent), nil + return ccpa.ConsentWriter{consent}, nil } return privacy.NilPolicyWriter{}, &errortypes.InvalidPrivacyConsent{ diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 790ed80a3c1..93619d99905 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -329,7 +329,7 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) // remove invalid consent from request - consentWriter := ccpa.NewConsentWriter("") + consentWriter := ccpa.ConsentWriter{""} if err := consentWriter.Write(req); err != nil { return append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) } diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 2ad1cc8b2bb..3f47b257d2e 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -12,6 +12,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/pbsmetrics" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" @@ -460,7 +461,7 @@ func (s fakeSyncer) FamilyName() string { } // GetUsersyncInfo implements the Usersyncer interface with a no-op. -func (s fakeSyncer) GetUsersyncInfo(privacyPolicies usersync.PrivacyPolicies) (*usersync.UsersyncInfo, error) { +func (s fakeSyncer) GetUsersyncInfo(privacyPolicies privacy.Policies) (*usersync.UsersyncInfo, error) { return nil, nil } diff --git a/exchange/utils.go b/exchange/utils.go index 3f56908c6d0..80f04ab80d0 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -93,7 +93,7 @@ func cleanOpenRTBRequests(ctx context.Context, var lmtPolicy lmt.Policy if privacyConfig.LMT.Enforce { - lmtPolicy = lmt.ReadPolicy(orig) + lmtPolicy = lmt.ReadFromRequest(orig) } // request level privacy policies diff --git a/privacy/ccpa/consentwriter.go b/privacy/ccpa/consentwriter.go index 779bd8c6377..bfbecdf4bf0 100644 --- a/privacy/ccpa/consentwriter.go +++ b/privacy/ccpa/consentwriter.go @@ -2,20 +2,20 @@ package ccpa import ( "github.com/mxmCherry/openrtb" - "github.com/prebid/prebid-server/privacy" ) -type consentWriter struct { - consent string +// ConsentWriter implements the PolicyWriter interface for CCPA. +type ConsentWriter struct { + Consent string } // Write mutates an OpenRTB bid request with the CCPA consent. -func (c consentWriter) Write(req *openrtb.BidRequest) error { +func (c ConsentWriter) Write(req *openrtb.BidRequest) error { if req == nil { return nil } - regs, err := buildRegs(c.consent, req.Regs) + regs, err := buildRegs(c.Consent, req.Regs) if err != nil { return err } @@ -23,8 +23,3 @@ func (c consentWriter) Write(req *openrtb.BidRequest) error { return nil } - -// NewConsentWriter constructs a privacy.PolicyWriter to write the CCPA consent to an OpenRTB bid request. -func NewConsentWriter(consent string) privacy.PolicyWriter { - return consentWriter{consent} -} diff --git a/privacy/ccpa/consentwriter_test.go b/privacy/ccpa/consentwriter_test.go index e49b68e28ff..465ed7a9620 100644 --- a/privacy/ccpa/consentwriter_test.go +++ b/privacy/ccpa/consentwriter_test.go @@ -41,7 +41,7 @@ func TestConsentWriterWrite(t *testing.T) { } for _, test := range testCases { - writer := &consentWriter{consent} + writer := ConsentWriter{consent} err := writer.Write(test.request) @@ -49,15 +49,3 @@ func TestConsentWriterWrite(t *testing.T) { assert.Equal(t, test.expected, test.request, test.description) } } - -func TestNewConsentWriter(t *testing.T) { - testCases := []string{ - "", - "anyConsent", - } - - for _, test := range testCases { - writer := NewConsentWriter(test).(consentWriter) - assert.Equal(t, test, writer.consent) - } -} diff --git a/privacy/gdpr/consentwriter.go b/privacy/gdpr/consentwriter.go index 21b29693dde..040bbd6c94b 100644 --- a/privacy/gdpr/consentwriter.go +++ b/privacy/gdpr/consentwriter.go @@ -4,18 +4,18 @@ import ( "encoding/json" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/privacy" "github.com/mxmCherry/openrtb" ) -type consentWriter struct { - consent string +// ConsentWriter implements the PolicyWriter interface for GDPR TCF. +type ConsentWriter struct { + Consent string } // Write mutates an OpenRTB bid request with the GDPR TCF consent. -func (c consentWriter) Write(req *openrtb.BidRequest) error { - if c.consent == "" { +func (c ConsentWriter) Write(req *openrtb.BidRequest) error { + if c.Consent == "" { return nil } @@ -24,7 +24,7 @@ func (c consentWriter) Write(req *openrtb.BidRequest) error { } if req.User.Ext == nil { - ext, err := json.Marshal(openrtb_ext.ExtUser{Consent: c.consent}) + ext, err := json.Marshal(openrtb_ext.ExtUser{Consent: c.Consent}) if err == nil { req.User.Ext = ext } @@ -34,7 +34,7 @@ func (c consentWriter) Write(req *openrtb.BidRequest) error { var extMap map[string]interface{} err := json.Unmarshal(req.User.Ext, &extMap) if err == nil { - extMap["consent"] = c.consent + extMap["consent"] = c.Consent ext, err := json.Marshal(extMap) if err == nil { req.User.Ext = ext @@ -42,8 +42,3 @@ func (c consentWriter) Write(req *openrtb.BidRequest) error { } return err } - -// NewConsentWriter constructs a privacy.PolicyWriter to write the GDPR TCF consent to an OpenRTB bid request. -func NewConsentWriter(consent string) privacy.PolicyWriter { - return consentWriter{consent} -} diff --git a/privacy/gdpr/consentwriter_test.go b/privacy/gdpr/consentwriter_test.go index d29c1881309..a4d17d2465c 100644 --- a/privacy/gdpr/consentwriter_test.go +++ b/privacy/gdpr/consentwriter_test.go @@ -88,7 +88,7 @@ func TestWrite(t *testing.T) { } for _, test := range testCases { - writer := NewConsentWriter(test.consent) + writer := ConsentWriter{test.consent} err := writer.Write(test.request) if test.expectedError { diff --git a/privacy/policies.go b/privacy/policies.go new file mode 100644 index 00000000000..bc844a4e463 --- /dev/null +++ b/privacy/policies.go @@ -0,0 +1,14 @@ +package privacy + +import ( + "github.com/prebid/prebid-server/privacy/ccpa" + "github.com/prebid/prebid-server/privacy/gdpr" + "github.com/prebid/prebid-server/privacy/lmt" +) + +// Policies represents the privacy regulations for an OpenRTB bid request. +type Policies struct { + CCPA ccpa.Policy + GDPR gdpr.Policy + LMT lmt.Policy +} diff --git a/usersync/usersync.go b/usersync/usersync.go index 9e87a5d2d6f..2b313a021f8 100644 --- a/usersync/usersync.go +++ b/usersync/usersync.go @@ -1,11 +1,13 @@ package usersync +import "github.com/prebid/prebid-server/privacy" + type Usersyncer interface { // GetUsersyncInfo returns basic info the browser needs in order to run a user sync. // The returned UsersyncInfo object must not be mutated by callers. // // For more information about user syncs, see http://clearcode.cc/2015/12/cookie-syncing/ - GetUsersyncInfo(privacyPolicies PrivacyPolicies) (*UsersyncInfo, error) + GetUsersyncInfo(privacyPolicies privacy.Policies) (*UsersyncInfo, error) // FamilyName should be the same as the `BidderName` for this Usersyncer. // This function only exists for legacy reasons. @@ -35,9 +37,3 @@ type CookieSyncBidders struct { NoCookie bool `json:"no_cookie,omitempty"` UsersyncInfo *UsersyncInfo `json:"usersync,omitempty"` } - -type PrivacyPolicies struct { - GDPRSignal string - GDPRConsent string - CCPAConsent string -} From 4e36e02359a85d0762543dc9c2bfcd4a47a303ba Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 15:54:21 -0400 Subject: [PATCH 19/27] Always Pass Valid Bidders To CCPA Parse --- endpoints/cookie_sync.go | 7 ++++++- endpoints/openrtb2/auction.go | 16 +--------------- exchange/utils.go | 4 ++-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 9f64e6ceb54..60da4f0bd16 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -248,8 +248,13 @@ func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) { } func (req *cookieSyncRequest) filterForCCPA() { + validBidders := make(map[string]struct{}) + for _, v := range openrtb_ext.BidderMap { + validBidders[v.String()] = struct{}{} + } + ccpaPolicy := &ccpa.Policy{Consent: req.USPrivacy} - ccpaParsedPolicy, err := ccpaPolicy.Parse(nil) // need valid bidders + ccpaParsedPolicy, err := ccpaPolicy.Parse(validBidders) if err == nil { for i := 0; i < len(req.Bidders); i++ { diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 93619d99905..b83c7b7ade2 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -324,7 +324,7 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { if ccpaPolicy, err := ccpa.ReadFromRequest(req); err != nil { return append(errL, errL...) - } else if _, err := ccpaPolicy.Parse(getValidBidders(aliases)); err == nil { + } else if _, err := ccpaPolicy.Parse(exchange.GetValidBidders(aliases)); err == nil { if _, invalidConsent := err.(*errortypes.InvalidPrivacyConsent); invalidConsent { errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) @@ -1292,17 +1292,3 @@ func validateAccount(cfg *config.Configuration, pubID string) error { } return err } - -func getValidBidders(aliases map[string]string) map[string]struct{} { - validBidders := make(map[string]struct{}) - - for _, v := range openrtb_ext.BidderMap { - validBidders[v.String()] = struct{}{} - } - - for k := range aliases { - validBidders[k] = struct{}{} - } - - return validBidders -} diff --git a/exchange/utils.go b/exchange/utils.go index 80f04ab80d0..72cd08d4f43 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -84,7 +84,7 @@ func cleanOpenRTBRequests(ctx context.Context, return } - ccpaPolicy, err = policy.Parse(getValidBidders(aliases)) + ccpaPolicy, err = policy.Parse(GetValidBidders(aliases)) if err != nil { errs = append(errs, err) return @@ -440,7 +440,7 @@ func parseAliases(orig *openrtb.BidRequest) (map[string]string, []error) { return aliases, nil } -func getValidBidders(aliases map[string]string) map[string]struct{} { +func GetValidBidders(aliases map[string]string) map[string]struct{} { validBidders := make(map[string]struct{}) for _, v := range openrtb_ext.BidderMap { From 986670ad0d1caaefc4420a81067edb5575a7984f Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 16:25:03 -0400 Subject: [PATCH 20/27] Reverted Unnecessary Changes + Fix Error Check Bug --- endpoints/auction.go | 12 ++++++------ endpoints/auction_test.go | 11 ++++------- endpoints/openrtb2/auction.go | 4 +--- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/endpoints/auction.go b/endpoints/auction.go index cf8657e9685..c6fd57123c7 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -190,20 +190,20 @@ func (a *auction) recoverSafely(inner func(*pbs.PBSBidder, pbsmetrics.AdapterLab } } -func (a *auction) shouldUsersync(ctx context.Context, bidder openrtb_ext.BidderName, privacyPolicies privacy.Policies) bool { - switch privacyPolicies.GDPR.Signal { +func (a *auction) shouldUsersync(ctx context.Context, bidder openrtb_ext.BidderName, gdprPrivacyPolicy gdprPrivacy.Policy) bool { + switch gdprPrivacyPolicy.Signal { case "0": return true case "1": - if privacyPolicies.GDPR.Consent == "" { + if gdprPrivacyPolicy.Consent == "" { return false } fallthrough default: - if canSync, err := a.gdprPerms.HostCookiesAllowed(ctx, privacyPolicies.GDPR.Consent); !canSync || err != nil { + if canSync, err := a.gdprPerms.HostCookiesAllowed(ctx, gdprPrivacyPolicy.Consent); !canSync || err != nil { return false } - canSync, err := a.gdprPerms.BidderSyncAllowed(ctx, bidder, privacyPolicies.GDPR.Consent) + canSync, err := a.gdprPerms.BidderSyncAllowed(ctx, bidder, gdprPrivacyPolicy.Consent) return canSync && err == nil } } @@ -516,7 +516,7 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl Consent: req.ParseConsent(), }, } - if a.shouldUsersync(*ctx, openrtb_ext.BidderName(syncerCode), privacyPolicies) { + if a.shouldUsersync(*ctx, openrtb_ext.BidderName(syncerCode), privacyPolicies.GDPR) { syncInfo, err := syncer.GetUsersyncInfo(privacyPolicies) if err == nil { bidder.UsersyncInfo = syncInfo diff --git a/endpoints/auction_test.go b/endpoints/auction_test.go index e6db0c35d36..c7e5e3143ab 100644 --- a/endpoints/auction_test.go +++ b/endpoints/auction_test.go @@ -19,7 +19,6 @@ import ( "github.com/prebid/prebid-server/pbsmetrics" metricsConf "github.com/prebid/prebid-server/pbsmetrics/config" "github.com/prebid/prebid-server/prebid_cache_client" - "github.com/prebid/prebid-server/privacy" gdprPolicy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync/usersyncers" "github.com/spf13/viper" @@ -388,14 +387,12 @@ func TestShouldUsersync(t *testing.T) { }, metricsEngine: nil, } - privacyPolicies := privacy.Policies{ - GDPR: gdprPolicy.Policy{ - Signal: gdprApplies, - Consent: consent, - }, + gdprPrivacyPolicy := gdprPolicy.Policy{ + Signal: gdprApplies, + Consent: consent, } - allowSyncs := deps.shouldUsersync(context.Background(), openrtb_ext.BidderAdform, privacyPolicies) + allowSyncs := deps.shouldUsersync(context.Background(), openrtb_ext.BidderAdform, gdprPrivacyPolicy) if allowSyncs != expectAllow { t.Errorf("Expected syncs: %t, allowed syncs: %t", expectAllow, allowSyncs) } diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index b83c7b7ade2..e04af12baad 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -324,11 +324,9 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { if ccpaPolicy, err := ccpa.ReadFromRequest(req); err != nil { return append(errL, errL...) - } else if _, err := ccpaPolicy.Parse(exchange.GetValidBidders(aliases)); err == nil { + } else if _, err := ccpaPolicy.Parse(exchange.GetValidBidders(aliases)); err != nil { if _, invalidConsent := err.(*errortypes.InvalidPrivacyConsent); invalidConsent { errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) - - // remove invalid consent from request consentWriter := ccpa.ConsentWriter{""} if err := consentWriter.Write(req); err != nil { return append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) From 1dcf17ac6f03ff6ac3e9404ade3ece2fcf77f2ca Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 16:48:56 -0400 Subject: [PATCH 21/27] Corrected Test Expecations --- exchange/utils_test.go | 16 ++++++++++++++++ privacy/ccpa/parsedpolicy.go | 2 +- privacy/ccpa/parsedpolicy_test.go | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 6051a54ddce..bd58c9ef3f5 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -121,20 +121,35 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { { description: "Feature Flag Enabled - No Sale Star - Doesn't Scrub", reqExt: json.RawMessage(`{"prebid":{"nosale":["*"]}}`), + ccpaConsent: "1-Y-", enforceCCPA: true, expectDataScrub: false, + expectPrivacyLabels: pbsmetrics.PrivacyLabels{ + CCPAProvided: true, + CCPAEnforced: false, + }, }, { description: "Feature Flag Enabled - No Sale Specific Bidder - Doesn't Scrub", reqExt: json.RawMessage(`{"prebid":{"nosale":["appnexus"]}}`), + ccpaConsent: "1-Y-", enforceCCPA: true, expectDataScrub: false, + expectPrivacyLabels: pbsmetrics.PrivacyLabels{ + CCPAProvided: true, + CCPAEnforced: true, + }, }, { description: "Feature Flag Enabled - No Sale Different Bidder - Scrubs", reqExt: json.RawMessage(`{"prebid":{"nosale":["rubicon"]}}`), + ccpaConsent: "1-Y-", enforceCCPA: true, expectDataScrub: true, + expectPrivacyLabels: pbsmetrics.PrivacyLabels{ + CCPAProvided: true, + CCPAEnforced: true, + }, }, { description: "Feature Flag Disabled", @@ -150,6 +165,7 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { for _, test := range testCases { req := newBidRequest(t) + req.Ext = test.reqExt req.Regs = &openrtb.Regs{ Ext: json.RawMessage(`{"us_privacy":"` + test.ccpaConsent + `"}`), } diff --git a/privacy/ccpa/parsedpolicy.go b/privacy/ccpa/parsedpolicy.go index de49560486f..022b1708554 100644 --- a/privacy/ccpa/parsedpolicy.go +++ b/privacy/ccpa/parsedpolicy.go @@ -42,7 +42,7 @@ type ParsedPolicy struct { func (p Policy) Parse(validBidders map[string]struct{}) (ParsedPolicy, error) { consentOptOut, err := parseConsent(p.Consent) if err != nil { - msg := fmt.Sprintf("request.regs.ext.us_privacy is invalid: %s", err.Error()) + msg := fmt.Sprintf("request.regs.ext.us_privacy %s", err.Error()) return ParsedPolicy{}, &errortypes.InvalidPrivacyConsent{Message: msg} } diff --git a/privacy/ccpa/parsedpolicy_test.go b/privacy/ccpa/parsedpolicy_test.go index 00214c88f00..42014fa43b6 100644 --- a/privacy/ccpa/parsedpolicy_test.go +++ b/privacy/ccpa/parsedpolicy_test.go @@ -57,7 +57,7 @@ func TestParse(t *testing.T) { consent: "malformed", noSaleBidders: []string{}, expectedPolicy: ParsedPolicy{}, - expectedError: "request.regs.ext.us_privacy is invalid: must contain 4 characters", + expectedError: "request.regs.ext.us_privacy must contain 4 characters", }, { description: "No Sale Error", From e06c13373b1369b4492b83069c2c283ecfcf16f3 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 17:23:56 -0400 Subject: [PATCH 22/27] Expand Test Coverage --- endpoints/openrtb2/auction.go | 4 +-- endpoints/openrtb2/auction_test.go | 47 ++++++++++++++++++++++++++++++ exchange/utils_test.go | 40 +++++++++++++++++++++++++ privacy/ccpa/policy.go | 5 ++-- 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index e04af12baad..a05e1d60f08 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -323,7 +323,7 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { } if ccpaPolicy, err := ccpa.ReadFromRequest(req); err != nil { - return append(errL, errL...) + return append(errL, err) } else if _, err := ccpaPolicy.Parse(exchange.GetValidBidders(aliases)); err != nil { if _, invalidConsent := err.(*errortypes.InvalidPrivacyConsent); invalidConsent { errL = append(errL, &errortypes.InvalidPrivacyConsent{Message: fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err)}) @@ -332,7 +332,7 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error { return append(errL, fmt.Errorf("Unable to remove invalid CCPA consent from the request. (%v)", err)) } } else { - return append(errL, errL...) + return append(errL, err) } } diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 957760c61c9..aad8a31d6e7 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -1222,6 +1222,53 @@ func TestCCPAInvalid(t *testing.T) { assert.Empty(t, req.Regs.Ext, "Invalid Consent Removed From Request") } +func TestNoSaleInvalid(t *testing.T) { + deps := &endpointDeps{ + &nobidExchange{}, + newParamsValidator(t), + &mockStoredReqFetcher{}, + empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, + &config.Configuration{}, + pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), + analyticsConf.NewPBSAnalytics(&config.Analytics{}), + map[string]string{}, + false, + []byte{}, + openrtb_ext.BidderMap, + nil, + nil, + hardcodedResponseIPValidator{response: true}, + } + + ui := uint64(1) + req := openrtb.BidRequest{ + ID: "someID", + Imp: []openrtb.Imp{ + { + ID: "imp-ID", + Banner: &openrtb.Banner{ + W: &ui, + H: &ui, + }, + Ext: json.RawMessage(`{"appnexus": {"placementId": 5667}}`), + }, + }, + Site: &openrtb.Site{ + ID: "myID", + }, + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"1NYN"}`), + }, + Ext: json.RawMessage(`{"prebid":{"nosale":["*", "appnexus"]}}`), + } + + errL := deps.validateRequest(&req) + + expectedError := errors.New("request.ext.prebid.nosale is invalid: can only specify all bidders if no other bidders are provided") + assert.ElementsMatch(t, errL, []error{expectedError}) +} + func TestSChainInvalid(t *testing.T) { deps := &endpointDeps{ &nobidExchange{}, diff --git a/exchange/utils_test.go b/exchange/utils_test.go index bd58c9ef3f5..2ecf82ff2d2 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -3,10 +3,12 @@ package exchange import ( "context" "encoding/json" + "errors" "testing" "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" "github.com/stretchr/testify/assert" @@ -191,6 +193,44 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { } } +func TestCleanOpenRTBRequestsCCPAErrors(t *testing.T) { + testCases := []struct { + description string + reqExt json.RawMessage + reqRegsExt json.RawMessage + expectError error + }{ + { + description: "Invalid Consent", + reqExt: json.RawMessage(`{"prebid":{"nosale":["*"]}}`), + reqRegsExt: json.RawMessage(`{"us_privacy":"malformed"}`), + expectError: &errortypes.InvalidPrivacyConsent{"request.regs.ext.us_privacy must contain 4 characters"}, + }, + { + description: "Invalid No Sale Bidders", + reqExt: json.RawMessage(`{"prebid":{"nosale":["*", "another"]}}`), + reqRegsExt: json.RawMessage(`{"us_privacy":"1NYN"}`), + expectError: errors.New("request.ext.prebid.nosale is invalid: can only specify all bidders if no other bidders are provided"), + }, + } + + for _, test := range testCases { + req := newBidRequest(t) + req.Ext = test.reqExt + req.Regs = &openrtb.Regs{Ext: test.reqRegsExt} + + privacyConfig := config.Privacy{ + CCPA: config.CCPA{ + Enforce: true, + }, + } + + _, _, _, errs := cleanOpenRTBRequests(context.Background(), req, &emptyUsersync{}, map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels{}, pbsmetrics.Labels{}, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) + + assert.ElementsMatch(t, []error{test.expectError}, errs, test.description) + } +} + func TestCleanOpenRTBRequestsCOPPA(t *testing.T) { testCases := []struct { description string diff --git a/privacy/ccpa/policy.go b/privacy/ccpa/policy.go index 1b841943137..a9f1c49e47d 100644 --- a/privacy/ccpa/policy.go +++ b/privacy/ccpa/policy.go @@ -3,6 +3,7 @@ package ccpa import ( "encoding/json" "errors" + "fmt" "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/openrtb_ext" @@ -27,7 +28,7 @@ func ReadFromRequest(req *openrtb.BidRequest) (Policy, error) { if req.Regs != nil && len(req.Regs.Ext) > 0 { var ext openrtb_ext.ExtRegs if err := json.Unmarshal(req.Regs.Ext, &ext); err != nil { - return Policy{}, err + return Policy{}, fmt.Errorf("error reading request.regs.ext: %s", err) } consent = ext.USPrivacy } @@ -36,7 +37,7 @@ func ReadFromRequest(req *openrtb.BidRequest) (Policy, error) { if len(req.Ext) > 0 { var ext openrtb_ext.ExtRequest if err := json.Unmarshal(req.Ext, &ext); err != nil { - return Policy{}, err + return Policy{}, fmt.Errorf("error reading request.ext.prebid: %s", err) } noSaleBidders = ext.Prebid.NoSale } From 74441c864403b2e11574e5f64871ffda1611601d Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 17:34:13 -0400 Subject: [PATCH 23/27] Fix Merge Conflict --- exchange/utils_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 5d5882a3d30..b8b677df16a 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -220,13 +220,16 @@ func TestCleanOpenRTBRequestsCCPAErrors(t *testing.T) { req.Ext = test.reqExt req.Regs = &openrtb.Regs{Ext: test.reqRegsExt} + var reqExtStruct openrtb_ext.ExtRequest + err := json.Unmarshal(req.Ext, &reqExtStruct) + assert.NoError(t, err, test.description+":marshal_ext") + privacyConfig := config.Privacy{ CCPA: config.CCPA{ Enforce: true, }, } - - _, _, _, errs := cleanOpenRTBRequests(context.Background(), req, &emptyUsersync{}, map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels{}, pbsmetrics.Labels{}, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) + _, _, _, errs := cleanOpenRTBRequests(context.Background(), req, &reqExtStruct, &emptyUsersync{}, map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels{}, pbsmetrics.Labels{}, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) assert.ElementsMatch(t, []error{test.expectError}, errs, test.description) } From 7160c6696cfeb80dd885d9700a9eb38921c8b900 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 27 Aug 2020 17:38:08 -0400 Subject: [PATCH 24/27] Revert Behavior Change --- exchange/utils.go | 22 +++++++++++----------- exchange/utils_test.go | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/exchange/utils.go b/exchange/utils.go index 59795f73110..f01f7d1c3ef 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -73,15 +73,15 @@ func cleanOpenRTBRequests(ctx context.Context, consent := extractConsent(orig) ampGDPRException := (labels.RType == pbsmetrics.ReqTypeAMP) && gDPR.AMPException() - var ccpaPolicy ccpa.ParsedPolicy - if privacyConfig.CCPA.Enforce { - policy, err := ccpa.ReadFromRequest(orig) - if err != nil { - errs = append(errs, err) - return - } + ccpaPolicy, err := ccpa.ReadFromRequest(orig) + if err != nil { + errs = append(errs, err) + return + } - ccpaPolicy, err = policy.Parse(GetValidBidders(aliases)) + var ccpaParsedPolicy ccpa.ParsedPolicy + if privacyConfig.CCPA.Enforce { + ccpaParsedPolicy, err = ccpaPolicy.Parse(GetValidBidders(aliases)) if err != nil { errs = append(errs, err) return @@ -99,8 +99,8 @@ func cleanOpenRTBRequests(ctx context.Context, LMT: lmtPolicy.ShouldEnforce(), } - privacyLabels.CCPAProvided = ccpaPolicy.Specified() - privacyLabels.CCPAEnforced = ccpaPolicy.ShouldEnforce("") + privacyLabels.CCPAProvided = ccpaPolicy.Consent != "" + privacyLabels.CCPAEnforced = ccpaParsedPolicy.ShouldEnforce("") privacyLabels.COPPAEnforced = privacyEnforcement.COPPA privacyLabels.LMTEnforced = privacyEnforcement.LMT @@ -116,7 +116,7 @@ func cleanOpenRTBRequests(ctx context.Context, // bidder level privacy policies for bidder, bidReq := range requestsByBidder { // CCPA - privacyEnforcement.CCPA = ccpaPolicy.ShouldEnforce(bidder.String()) + privacyEnforcement.CCPA = ccpaParsedPolicy.ShouldEnforce(bidder.String()) // GDPR if gdpr == 1 { diff --git a/exchange/utils_test.go b/exchange/utils_test.go index b8b677df16a..0dd6c0311ab 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -160,7 +160,7 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { enforceCCPA: false, expectDataScrub: false, expectPrivacyLabels: pbsmetrics.PrivacyLabels{ - CCPAProvided: false, + CCPAProvided: true, CCPAEnforced: false, }, }, From 12ae72414766dc1fcb5ccfaac7871db178e273ea Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Fri, 28 Aug 2020 14:12:17 -0400 Subject: [PATCH 25/27] Introduced Privacy Enforcer --- exchange/utils.go | 54 ++++++++++++++++--------- privacy/ccpa/consentwriter.go | 2 +- privacy/ccpa/consentwriter_test.go | 12 +++--- privacy/ccpa/parsedpolicy.go | 4 +- privacy/ccpa/parsedpolicy_test.go | 4 +- privacy/enforcer.go | 43 ++++++++++++++++++++ privacy/enforcer_test.go | 18 +++++++++ privacy/gdpr/consentwriter_test.go | 2 +- privacy/lmt/policy.go | 12 +++--- privacy/lmt/policy_test.go | 64 +++++++++++++++++++++++++++++- privacy/writer.go | 1 + privacy/writer_test.go | 25 ++++++++++++ 12 files changed, 204 insertions(+), 37 deletions(-) create mode 100644 privacy/enforcer.go create mode 100644 privacy/enforcer_test.go create mode 100644 privacy/writer_test.go diff --git a/exchange/utils.go b/exchange/utils.go index f01f7d1c3ef..7c4926137e6 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -19,6 +19,8 @@ import ( "github.com/prebid/prebid-server/privacy/lmt" ) +const unknownBidder string = "" + func BidderToPrebidSChains(req *openrtb_ext.ExtRequest) (map[string]*openrtb_ext.ExtRequestPrebidSChainSChain, error) { bidderToSChains := make(map[string]*openrtb_ext.ExtRequestPrebidSChainSChain) @@ -73,36 +75,24 @@ func cleanOpenRTBRequests(ctx context.Context, consent := extractConsent(orig) ampGDPRException := (labels.RType == pbsmetrics.ReqTypeAMP) && gDPR.AMPException() - ccpaPolicy, err := ccpa.ReadFromRequest(orig) + ccpaEnforcer, err := extractCCPA(orig, privacyConfig, aliases) if err != nil { errs = append(errs, err) return } - var ccpaParsedPolicy ccpa.ParsedPolicy - if privacyConfig.CCPA.Enforce { - ccpaParsedPolicy, err = ccpaPolicy.Parse(GetValidBidders(aliases)) - if err != nil { - errs = append(errs, err) - return - } - } - - var lmtPolicy lmt.Policy - if privacyConfig.LMT.Enforce { - lmtPolicy = lmt.ReadFromRequest(orig) - } + lmtEnforcer := extractLMT(orig, privacyConfig) // request level privacy policies privacyEnforcement := privacy.Enforcement{ COPPA: orig.Regs != nil && orig.Regs.COPPA == 1, - LMT: lmtPolicy.ShouldEnforce(), + LMT: lmtEnforcer.ShouldEnforce(unknownBidder), } - privacyLabels.CCPAProvided = ccpaPolicy.Consent != "" - privacyLabels.CCPAEnforced = ccpaParsedPolicy.ShouldEnforce("") + privacyLabels.CCPAProvided = ccpaEnforcer.CanEnforce() + privacyLabels.CCPAEnforced = ccpaEnforcer.ShouldEnforce(unknownBidder) privacyLabels.COPPAEnforced = privacyEnforcement.COPPA - privacyLabels.LMTEnforced = privacyEnforcement.LMT + privacyLabels.LMTEnforced = lmtEnforcer.ShouldEnforce(unknownBidder) if gdpr == 1 { privacyLabels.GDPREnforced = true @@ -116,7 +106,7 @@ func cleanOpenRTBRequests(ctx context.Context, // bidder level privacy policies for bidder, bidReq := range requestsByBidder { // CCPA - privacyEnforcement.CCPA = ccpaParsedPolicy.ShouldEnforce(bidder.String()) + privacyEnforcement.CCPA = ccpaEnforcer.ShouldEnforce(bidder.String()) // GDPR if gdpr == 1 { @@ -139,6 +129,32 @@ func cleanOpenRTBRequests(ctx context.Context, return } +func extractCCPA(orig *openrtb.BidRequest, privacyConfig config.Privacy, aliases map[string]string) (privacy.PolicyEnforcer, error) { + ccpaPolicy, err := ccpa.ReadFromRequest(orig) + if err != nil { + return privacy.NilPolicyEnforcer{}, err + } + + validBidders := GetValidBidders(aliases) + ccpaParsedPolicy, err := ccpaPolicy.Parse(validBidders) + if err != nil { + return privacy.NilPolicyEnforcer{}, err + } + + ccpaEnforcer := privacy.EnabledPolicyEnforcer{ + Enabled: privacyConfig.CCPA.Enforce, + PolicyEnforcer: ccpaParsedPolicy, + } + return ccpaEnforcer, nil +} + +func extractLMT(orig *openrtb.BidRequest, privacyConfig config.Privacy) privacy.PolicyEnforcer { + return privacy.EnabledPolicyEnforcer{ + Enabled: privacyConfig.LMT.Enforce, + PolicyEnforcer: lmt.ReadFromRequest(orig), + } +} + func splitBidRequest(req *openrtb.BidRequest, requestExt *openrtb_ext.ExtRequest, impsByBidder map[string][]openrtb.Imp, diff --git a/privacy/ccpa/consentwriter.go b/privacy/ccpa/consentwriter.go index bfbecdf4bf0..4856655402b 100644 --- a/privacy/ccpa/consentwriter.go +++ b/privacy/ccpa/consentwriter.go @@ -9,7 +9,7 @@ type ConsentWriter struct { Consent string } -// Write mutates an OpenRTB bid request with the CCPA consent. +// Write mutates an OpenRTB bid request with the CCPA consent string. func (c ConsentWriter) Write(req *openrtb.BidRequest) error { if req == nil { return nil diff --git a/privacy/ccpa/consentwriter_test.go b/privacy/ccpa/consentwriter_test.go index 465ed7a9620..57a7f8f4ddf 100644 --- a/privacy/ccpa/consentwriter_test.go +++ b/privacy/ccpa/consentwriter_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestConsentWriterWrite(t *testing.T) { +func TestConsentWriter(t *testing.T) { consent := "anyConsent" testCases := []struct { description string @@ -16,6 +16,11 @@ func TestConsentWriterWrite(t *testing.T) { expected *openrtb.BidRequest expectedError bool }{ + { + description: "Nil Request", + request: nil, + expected: nil, + }, { description: "Success", request: &openrtb.BidRequest{}, @@ -23,11 +28,6 @@ func TestConsentWriterWrite(t *testing.T) { Regs: &openrtb.Regs{Ext: json.RawMessage(`{"us_privacy":"anyConsent"}`)}, }, }, - { - description: "Nil Request", - request: nil, - expected: nil, - }, { description: "Error With Regs.Ext - Does Not Mutate", request: &openrtb.BidRequest{ diff --git a/privacy/ccpa/parsedpolicy.go b/privacy/ccpa/parsedpolicy.go index 022b1708554..3c934e67822 100644 --- a/privacy/ccpa/parsedpolicy.go +++ b/privacy/ccpa/parsedpolicy.go @@ -117,8 +117,8 @@ func parseNoSaleBidders(noSaleBidders []string, validBidders map[string]struct{} return } -// Specified returns true when consent is provided, as opposed to an empty string. -func (p ParsedPolicy) Specified() bool { +// CanEnforce returns true when consent is specifically provided by the publisher, as opposed to an empty string. +func (p ParsedPolicy) CanEnforce() bool { return p.consentSpecified } diff --git a/privacy/ccpa/parsedpolicy_test.go b/privacy/ccpa/parsedpolicy_test.go index 42014fa43b6..2f7e8bfd683 100644 --- a/privacy/ccpa/parsedpolicy_test.go +++ b/privacy/ccpa/parsedpolicy_test.go @@ -266,7 +266,7 @@ func TestParseNoSaleBidders(t *testing.T) { } } -func TestSpecified(t *testing.T) { +func TestCanEnforce(t *testing.T) { testCases := []struct { description string policy ParsedPolicy @@ -295,7 +295,7 @@ func TestSpecified(t *testing.T) { } for _, test := range testCases { - result := test.policy.Specified() + result := test.policy.CanEnforce() assert.Equal(t, test.expected, result, test.description) } } diff --git a/privacy/enforcer.go b/privacy/enforcer.go new file mode 100644 index 00000000000..0d5ecad5309 --- /dev/null +++ b/privacy/enforcer.go @@ -0,0 +1,43 @@ +package privacy + +// PolicyEnforcer determines if personally identifiable information (PII) should be removed or anonymized per the policy. +type PolicyEnforcer interface { + // CanEnforce returns true when policy information is specifically provided by the publisher. + CanEnforce() bool + + // ShouldEnforce returns true when the OpenRTB request should have personally identifiable + // information (PII) removed or anonymized per the policy. + ShouldEnforce(bidder string) bool +} + +// NilPolicyEnforcer implements the PolicyEnforcer interface but will always return false. +type NilPolicyEnforcer struct{} + +// CanEnforce is hardcoded to always return false. +func (NilPolicyEnforcer) CanEnforce() bool { + return false +} + +// ShouldEnforce is hardcoded to always return false. +func (NilPolicyEnforcer) ShouldEnforce(bidder string) bool { + return false +} + +// EnabledPolicyEnforcer decorates a PolicyEnforcer with an enabled flag. +type EnabledPolicyEnforcer struct { + Enabled bool + PolicyEnforcer PolicyEnforcer +} + +// CanEnforce returns true when the PolicyEnforcer can enforce. +func (p EnabledPolicyEnforcer) CanEnforce() bool { + return p.PolicyEnforcer.CanEnforce() +} + +// ShouldEnforce returns true when the enforcer is enabled the PolicyEnforcer allows enforcement. +func (p EnabledPolicyEnforcer) ShouldEnforce(bidder string) bool { + if p.Enabled { + return p.PolicyEnforcer.ShouldEnforce(bidder) + } + return false +} diff --git a/privacy/enforcer_test.go b/privacy/enforcer_test.go new file mode 100644 index 00000000000..b0c4032c714 --- /dev/null +++ b/privacy/enforcer_test.go @@ -0,0 +1,18 @@ +package privacy + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNilEnforcerCanEnforce(t *testing.T) { + nilEnforcer := &NilPolicyEnforcer{} + assert.False(t, nilEnforcer.CanEnforce()) +} + +func TestNilEnforcerShouldEnforce(t *testing.T) { + nilEnforcer := &NilPolicyEnforcer{} + assert.False(t, nilEnforcer.ShouldEnforce("")) + assert.False(t, nilEnforcer.ShouldEnforce("anyBidder")) +} diff --git a/privacy/gdpr/consentwriter_test.go b/privacy/gdpr/consentwriter_test.go index a4d17d2465c..77fbdf88d92 100644 --- a/privacy/gdpr/consentwriter_test.go +++ b/privacy/gdpr/consentwriter_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestWrite(t *testing.T) { +func TestConsentWriter(t *testing.T) { testCases := []struct { description string consent string diff --git a/privacy/lmt/policy.go b/privacy/lmt/policy.go index 9460fd2ecab..295dcc46469 100644 --- a/privacy/lmt/policy.go +++ b/privacy/lmt/policy.go @@ -16,18 +16,20 @@ type Policy struct { } // ReadFromRequest extracts the LMT (Limit Ad Tracking) policy from an OpenRTB bid request. -func ReadFromRequest(req *openrtb.BidRequest) Policy { - policy := Policy{} - +func ReadFromRequest(req *openrtb.BidRequest) (policy Policy) { if req != nil && req.Device != nil && req.Device.Lmt != nil { policy.Signal = int(*req.Device.Lmt) policy.SignalProvided = true } + return +} - return policy +// CanEnforce returns true the LMT (Limit Ad Tracking) signal is provided by the publisher. +func (p Policy) CanEnforce() bool { + return p.SignalProvided } // ShouldEnforce returns true when the LMT (Limit Ad Tracking) policy is in effect. -func (p Policy) ShouldEnforce() bool { +func (p Policy) ShouldEnforce(bidder string) bool { return p.SignalProvided && p.Signal == trackingRestricted } diff --git a/privacy/lmt/policy_test.go b/privacy/lmt/policy_test.go index 356d35baf91..3027414fd02 100644 --- a/privacy/lmt/policy_test.go +++ b/privacy/lmt/policy_test.go @@ -65,6 +65,68 @@ func TestReadFromRequest(t *testing.T) { } } +func TestCanEnforce(t *testing.T) { + testCases := []struct { + description string + policy Policy + expected bool + }{ + { + description: "Signal Not Provided - Zero", + policy: Policy{ + Signal: 0, + SignalProvided: false, + }, + expected: false, + }, + { + description: "Signal Not Provided - One", + policy: Policy{ + Signal: 1, + SignalProvided: false, + }, + expected: false, + }, + { + description: "Signal Not Provided - Other", + policy: Policy{ + Signal: 42, + SignalProvided: false, + }, + expected: false, + }, + { + description: "Signal Provided - Zero", + policy: Policy{ + Signal: 0, + SignalProvided: true, + }, + expected: true, + }, + { + description: "Signal Provided - One", + policy: Policy{ + Signal: 1, + SignalProvided: true, + }, + expected: true, + }, + { + description: "Signal Provided - Other", + policy: Policy{ + Signal: 42, + SignalProvided: true, + }, + expected: true, + }, + } + + for _, test := range testCases { + result := test.policy.CanEnforce() + assert.Equal(t, test.expected, result, test.description) + } +} + func TestShouldEnforce(t *testing.T) { testCases := []struct { description string @@ -122,7 +184,7 @@ func TestShouldEnforce(t *testing.T) { } for _, test := range testCases { - result := test.policy.ShouldEnforce() + result := test.policy.ShouldEnforce("") assert.Equal(t, test.expected, result, test.description) } } diff --git a/privacy/writer.go b/privacy/writer.go index bc4bf009df7..c61767f16c8 100644 --- a/privacy/writer.go +++ b/privacy/writer.go @@ -12,6 +12,7 @@ type PolicyWriter interface { // NilPolicyWriter implements the PolicyWriter interface but performs no action. type NilPolicyWriter struct{} +// Write is hardcoded to perform no action with the OpenRTB bid request. func (NilPolicyWriter) Write(req *openrtb.BidRequest) error { return nil } diff --git a/privacy/writer_test.go b/privacy/writer_test.go new file mode 100644 index 00000000000..79170cfc451 --- /dev/null +++ b/privacy/writer_test.go @@ -0,0 +1,25 @@ +package privacy + +import ( + "encoding/json" + "testing" + + "github.com/mxmCherry/openrtb" + "github.com/stretchr/testify/assert" +) + +func TestNilWriter(t *testing.T) { + request := &openrtb.BidRequest{ + ID: "anyID", + Ext: json.RawMessage(`{"anyJson":"anyValue"}`), + } + expectedRequest := &openrtb.BidRequest{ + ID: "anyID", + Ext: json.RawMessage(`{"anyJson":"anyValue"}`), + } + + nilWriter := &NilPolicyWriter{} + nilWriter.Write(request) + + assert.Equal(t, expectedRequest, request) +} From 923e5249a8d089cd50f82d8473f639915391116f Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 3 Sep 2020 16:51:22 -0400 Subject: [PATCH 26/27] Fixed Merge Conflict --- endpoints/openrtb2/auction_test.go | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 659c511bea8..14294fc4e1c 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -1303,6 +1303,55 @@ func TestNoSaleInvalid(t *testing.T) { assert.ElementsMatch(t, errL, []error{expectedError}) } +func TestValidateSourceTID(t *testing.T) { + cfg := &config.Configuration{ + AutoGenSourceTID: true, + } + + deps := &endpointDeps{ + &nobidExchange{}, + newParamsValidator(t), + &mockStoredReqFetcher{}, + empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, + cfg, + pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), + analyticsConf.NewPBSAnalytics(&config.Analytics{}), + map[string]string{}, + false, + []byte{}, + openrtb_ext.BidderMap, + nil, + nil, + hardcodedResponseIPValidator{response: true}, + } + + ui := uint64(1) + req := openrtb.BidRequest{ + ID: "someID", + Imp: []openrtb.Imp{ + { + ID: "imp-ID", + Banner: &openrtb.Banner{ + W: &ui, + H: &ui, + }, + Ext: json.RawMessage(`{"appnexus": {"placementId": 5667}}`), + }, + }, + Site: &openrtb.Site{ + ID: "myID", + }, + Regs: &openrtb.Regs{ + Ext: json.RawMessage(`{"us_privacy":"invalid by length"}`), + }, + } + + deps.validateRequest(&req) + assert.NotEmpty(t, req.Source.TID, "Expected req.Source.TID to be filled with a randomly generated UID") +} + func TestSChainInvalid(t *testing.T) { deps := &endpointDeps{ &nobidExchange{}, From 293e163913900a7faa8d30909e272c26c6a31ef8 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Fri, 4 Sep 2020 15:27:36 -0400 Subject: [PATCH 27/27] Rename Method To Improve Clarity --- endpoints/openrtb2/amp_auction.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 6a2d714af90..78b198920d3 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -404,7 +404,7 @@ func (deps *endpointDeps) overrideWithParams(httpRequest *http.Request, req *ope req.Imp[0].TagID = slot } - policyWriter, policyWriterErr := readConsent(httpRequest.URL) + policyWriter, policyWriterErr := readPolicyFromUrl(httpRequest.URL) if policyWriterErr != nil { return []error{policyWriterErr} } @@ -554,7 +554,7 @@ func setAmpExt(site *openrtb.Site, value string) { } } -func readConsent(url *url.URL) (privacy.PolicyWriter, error) { +func readPolicyFromUrl(url *url.URL) (privacy.PolicyWriter, error) { consent := readConsentFromURL(url) if len(consent) == 0 {