Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename GDPR UserSyncIfAmbiguous to DefaultValue #1858

Merged
merged 4 commits into from
Jun 15, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ type Privacy struct {
type GDPR struct {
Enabled bool `mapstructure:"enabled"`
HostVendorID int `mapstructure:"host_vendor_id"`
UsersyncIfAmbiguous bool `mapstructure:"usersync_if_ambiguous"`
DefaultValue string `mapstructure:"default_value"`
Timeouts GDPRTimeouts `mapstructure:"timeouts_ms"`
NonStandardPublishers []string `mapstructure:"non_standard_publishers,flow"`
NonStandardPublisherMap map[string]struct{}
Expand All @@ -202,12 +202,15 @@ type GDPR struct {
// EEACountries (EEA = European Economic Area) are a list of countries where we should assume GDPR applies.
// If the gdpr flag is unset in a request, but geo.country is set, we will assume GDPR applies if and only
// if the country matches one on this list. If both the GDPR flag and country are not set, we default
// to UsersyncIfAmbiguous
// to DefaultValue
EEACountries []string `mapstructure:"eea_countries"`
EEACountriesMap map[string]struct{}
}

func (cfg *GDPR) validate(errs []error) []error {
if cfg.DefaultValue != "0" && cfg.DefaultValue != "1" {
errs = append(errs, fmt.Errorf("gdpr.default_value must be 0 or 1"))
}
if cfg.HostVendorID < 0 || cfg.HostVendorID > 0xffff {
errs = append(errs, fmt.Errorf("gdpr.host_vendor_id must be in the range [0, %d]. Got %d", 0xffff, cfg.HostVendorID))
}
Expand Down Expand Up @@ -937,7 +940,7 @@ func SetupViper(v *viper.Viper, filename string) {
v.SetDefault("amp_timeout_adjustment_ms", 0)
v.SetDefault("gdpr.enabled", true)
v.SetDefault("gdpr.host_vendor_id", 0)
v.SetDefault("gdpr.usersync_if_ambiguous", false)
v.SetDefault("gdpr.default_value", "1")
v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0)
v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0)
v.SetDefault("gdpr.non_standard_publishers", []string{""})
Expand Down
13 changes: 11 additions & 2 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func TestDefaults(t *testing.T) {
var fullConfig = []byte(`
gdpr:
host_vendor_id: 15
usersync_if_ambiguous: true
default_value: "0"
non_standard_publishers: ["siteID","fake-site-id","appID","agltb3B1Yi1pbmNyDAsSA0FwcBiJkfIUDA"]
ccpa:
enforce: true
Expand Down Expand Up @@ -352,7 +352,7 @@ func TestFullConfig(t *testing.T) {
cmpInts(t, "http_client_cache.max_idle_connections_per_host", cfg.CacheClient.MaxIdleConnsPerHost, 2)
cmpInts(t, "http_client_cache.idle_connection_timeout_seconds", cfg.CacheClient.IdleConnTimeout, 3)
cmpInts(t, "gdpr.host_vendor_id", cfg.GDPR.HostVendorID, 15)
cmpBools(t, "gdpr.usersync_if_ambiguous", cfg.GDPR.UsersyncIfAmbiguous, true)
cmpStrings(t, "gdpr.default_value", cfg.GDPR.DefaultValue, "0")

//Assert the NonStandardPublishers was correctly unmarshalled
cmpStrings(t, "gdpr.non_standard_publishers", cfg.GDPR.NonStandardPublishers[0], "siteID")
Expand Down Expand Up @@ -460,6 +460,9 @@ func TestUnmarshalAdapterExtraInfo(t *testing.T) {

func TestValidConfig(t *testing.T) {
cfg := Configuration{
GDPR: GDPR{
DefaultValue: "1",
},
StoredRequests: StoredRequests{
Files: FileFetcherConfig{Enabled: true},
InMemoryCache: InMemoryCache{
Expand Down Expand Up @@ -650,6 +653,12 @@ func TestInvalidAMPException(t *testing.T) {
assertOneError(t, cfg.validate(), "gdpr.amp_exception has been discontinued and must be removed from your config. If you need to disable GDPR for AMP, you may do so per-account (gdpr.integration_enabled.amp) or at the host level for the default account (account_defaults.gdpr.integration_enabled.amp)")
}

func TestInvalidGDPRDefaultValue(t *testing.T) {
cfg := newDefaultConfig(t)
cfg.GDPR.DefaultValue = "2"
assertOneError(t, cfg.validate(), "gdpr.default_value must be 0 or 1")
}

func TestNegativeCurrencyConverterFetchInterval(t *testing.T) {
cfg := Configuration{
CurrencyConverter: CurrencyConverter{
Expand Down
6 changes: 3 additions & 3 deletions endpoints/cookie_sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h
}

parsedReq := &cookieSyncRequest{}
if err := parseRequest(parsedReq, bodyBytes, deps.gDPR.UsersyncIfAmbiguous); err != nil {
if err := parseRequest(parsedReq, bodyBytes, deps.gDPR.DefaultValue); err != nil {
co.Status = http.StatusBadRequest
co.Errors = append(co.Errors, err)
http.Error(w, co.Errors[len(co.Errors)-1].Error(), co.Status)
Expand Down Expand Up @@ -181,7 +181,7 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h
enc.Encode(csResp)
}

func parseRequest(parsedReq *cookieSyncRequest, bodyBytes []byte, usersyncIfAmbiguous bool) error {
func parseRequest(parsedReq *cookieSyncRequest, bodyBytes []byte, gdprDefaultValue string) error {
if err := json.Unmarshal(bodyBytes, parsedReq); err != nil {
return fmt.Errorf("JSON parsing failed: %s", err.Error())
}
Expand All @@ -193,7 +193,7 @@ func parseRequest(parsedReq *cookieSyncRequest, bodyBytes []byte, usersyncIfAmbi
if parsedReq.GDPR == nil {
var gdpr = new(int)
*gdpr = 1
if usersyncIfAmbiguous {
if gdprDefaultValue == "0" {
*gdpr = 0
}
parsedReq.GDPR = gdpr
Expand Down
4 changes: 2 additions & 2 deletions endpoints/cookie_sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func TestCCPA(t *testing.T) {
}

for _, test := range testCases {
gdpr := config.GDPR{UsersyncIfAmbiguous: true}
gdpr := config.GDPR{DefaultValue: "0"}
ccpa := config.CCPA{Enforce: test.enforceCCPA}
rr := doConfigurablePost(test.requestBody, nil, true, syncersForTest(), gdpr, ccpa)
assert.Equal(t, http.StatusOK, rr.Code, test.description+":httpResponseCode")
Expand Down Expand Up @@ -149,7 +149,7 @@ func TestCookieSyncNoBidders(t *testing.T) {
}

func TestCookieSyncNoCookiesBrokenGDPR(t *testing.T) {
rr := doConfigurablePost(`{"bidders":["appnexus", "audienceNetwork", "random"],"gdpr_consent":"GLKHGKGKKGK"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{}, config.GDPR{UsersyncIfAmbiguous: true}, config.CCPA{})
rr := doConfigurablePost(`{"bidders":["appnexus", "audienceNetwork", "random"],"gdpr_consent":"GLKHGKGKKGK"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{}, config.GDPR{DefaultValue: "0"}, config.CCPA{})
assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8")
assert.Equal(t, http.StatusOK, rr.Code)
assert.ElementsMatch(t, []string{"appnexus", "audienceNetwork"}, parseSyncs(t, rr.Body.Bytes()))
Expand Down
58 changes: 29 additions & 29 deletions exchange/exchange.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,18 @@ type IdFetcher interface {
}

type exchange struct {
adapterMap map[openrtb_ext.BidderName]adaptedBidder
bidderInfo config.BidderInfos
me metrics.MetricsEngine
cache prebid_cache_client.Client
cacheTime time.Duration
gDPR gdpr.Permissions
currencyConverter *currency.RateConverter
externalURL string
UsersyncIfAmbiguous bool
privacyConfig config.Privacy
categoriesFetcher stored_requests.CategoryFetcher
bidIDGenerator BidIDGenerator
adapterMap map[openrtb_ext.BidderName]adaptedBidder
bidderInfo config.BidderInfos
me metrics.MetricsEngine
cache prebid_cache_client.Client
cacheTime time.Duration
gDPR gdpr.Permissions
currencyConverter *currency.RateConverter
externalURL string
gdprDefaultValue string
privacyConfig config.Privacy
categoriesFetcher stored_requests.CategoryFetcher
bidIDGenerator BidIDGenerator
}

// Container to pass out response ext data from the GetAllBids goroutines back into the main thread
Expand Down Expand Up @@ -111,16 +111,16 @@ func (randomDeduplicateBidBooleanGenerator) Generate() bool {

func NewExchange(adapters map[openrtb_ext.BidderName]adaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gDPR gdpr.Permissions, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher) Exchange {
return &exchange{
adapterMap: adapters,
bidderInfo: infos,
cache: cache,
cacheTime: time.Duration(cfg.CacheURL.ExpectedTimeMillis) * time.Millisecond,
categoriesFetcher: categoriesFetcher,
currencyConverter: currencyConverter,
externalURL: cfg.ExternalURL,
gDPR: gDPR,
me: metricsEngine,
UsersyncIfAmbiguous: cfg.GDPR.UsersyncIfAmbiguous,
adapterMap: adapters,
bidderInfo: infos,
cache: cache,
cacheTime: time.Duration(cfg.CacheURL.ExpectedTimeMillis) * time.Millisecond,
categoriesFetcher: categoriesFetcher,
currencyConverter: currencyConverter,
externalURL: cfg.ExternalURL,
gDPR: gDPR,
me: metricsEngine,
gdprDefaultValue: cfg.GDPR.DefaultValue,
privacyConfig: config.Privacy{
CCPA: cfg.CCPA,
GDPR: cfg.GDPR,
Expand Down Expand Up @@ -186,10 +186,10 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog *
recordImpMetrics(r.BidRequest, e.me)

// Make our best guess if GDPR applies
usersyncIfAmbiguous := e.parseUsersyncIfAmbiguous(r.BidRequest)
gdprDefaultValue := e.parseGDPRDefaultValue(r.BidRequest)

// Slice of BidRequests, each a copy of the original cleaned to only contain bidder data for the named bidder
bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(ctx, r, requestExt, e.gDPR, e.me, usersyncIfAmbiguous, e.privacyConfig, &r.Account)
bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(ctx, r, requestExt, e.gDPR, e.me, gdprDefaultValue, e.privacyConfig, &r.Account)

e.me.RecordRequestPrivacy(privacyLabels)

Expand Down Expand Up @@ -301,8 +301,8 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog *
return e.buildBidResponse(ctx, liveAdapters, adapterBids, r.BidRequest, adapterExtra, auc, bidResponseExt, cacheInstructions.returnCreative, errs)
}

func (e *exchange) parseUsersyncIfAmbiguous(bidRequest *openrtb2.BidRequest) bool {
usersyncIfAmbiguous := e.UsersyncIfAmbiguous
func (e *exchange) parseGDPRDefaultValue(bidRequest *openrtb2.BidRequest) string {
gdprDefaultValue := e.gdprDefaultValue
var geo *openrtb2.Geo = nil

if bidRequest.User != nil && bidRequest.User.Geo != nil {
Expand All @@ -314,14 +314,14 @@ func (e *exchange) parseUsersyncIfAmbiguous(bidRequest *openrtb2.BidRequest) boo
// If we have a country set, and it is on the list, we assume GDPR applies if not set on the request.
// Otherwise we assume it does not apply as long as it appears "valid" (is 3 characters long).
if _, found := e.privacyConfig.GDPR.EEACountriesMap[strings.ToUpper(geo.Country)]; found {
usersyncIfAmbiguous = false
gdprDefaultValue = "1"
} else if len(geo.Country) == 3 {
// The country field is formatted properly as a three character country code
usersyncIfAmbiguous = true
gdprDefaultValue = "0"
}
}

return usersyncIfAmbiguous
return gdprDefaultValue
}

func recordImpMetrics(bidRequest *openrtb2.BidRequest, metricsEngine metrics.MetricsEngine) {
Expand Down
37 changes: 22 additions & 15 deletions exchange/exchange_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2004,6 +2004,13 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) {
eeac[c] = s
}

var gdprDefaultValue string
if spec.AssumeGDPRApplies {
gdprDefaultValue = "1"
} else {
gdprDefaultValue = "0"
}

privacyConfig := config.Privacy{
CCPA: config.CCPA{
Enforce: spec.EnforceCCPA,
Expand All @@ -2012,9 +2019,9 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) {
Enforce: spec.EnforceLMT,
},
GDPR: config.GDPR{
Enabled: spec.GDPREnabled,
UsersyncIfAmbiguous: !spec.AssumeGDPRApplies,
EEACountriesMap: eeac,
Enabled: spec.GDPREnabled,
DefaultValue: gdprDefaultValue,
EEACountriesMap: eeac,
},
}
bidIdGenerator := &mockBidIDGenerator{}
Expand Down Expand Up @@ -2156,18 +2163,18 @@ func newExchangeForTests(t *testing.T, filename string, expectations map[string]
}

return &exchange{
adapterMap: bidderAdapters,
me: metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.CoreBidderNames()),
cache: &wellBehavedCache{},
cacheTime: 0,
gDPR: &permissionsMock{allowAllBidders: true},
currencyConverter: currency.NewRateConverter(&http.Client{}, "", time.Duration(0)),
UsersyncIfAmbiguous: privacyConfig.GDPR.UsersyncIfAmbiguous,
privacyConfig: privacyConfig,
categoriesFetcher: categoriesFetcher,
bidderInfo: bidderInfos,
externalURL: "http://localhost",
bidIDGenerator: bidIDGenerator,
adapterMap: bidderAdapters,
me: metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.CoreBidderNames()),
cache: &wellBehavedCache{},
cacheTime: 0,
gDPR: &permissionsMock{allowAllBidders: true},
currencyConverter: currency.NewRateConverter(&http.Client{}, "", time.Duration(0)),
gdprDefaultValue: privacyConfig.GDPR.DefaultValue,
privacyConfig: privacyConfig,
categoriesFetcher: categoriesFetcher,
bidderInfo: bidderInfos,
externalURL: "http://localhost",
bidIDGenerator: bidIDGenerator,
}
}

Expand Down
18 changes: 9 additions & 9 deletions exchange/targeting_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,15 +87,15 @@ func runTargetingAuction(t *testing.T, mockBids map[openrtb_ext.BidderName][]*op
}

ex := &exchange{
adapterMap: buildAdapterMap(mockBids, server.URL, server.Client()),
me: &metricsConf.DummyMetricsEngine{},
cache: &wellBehavedCache{},
cacheTime: time.Duration(0),
gDPR: gdpr.AlwaysAllow{},
currencyConverter: currency.NewRateConverter(&http.Client{}, "", time.Duration(0)),
UsersyncIfAmbiguous: false,
categoriesFetcher: categoriesFetcher,
bidIDGenerator: &mockBidIDGenerator{false, false},
adapterMap: buildAdapterMap(mockBids, server.URL, server.Client()),
me: &metricsConf.DummyMetricsEngine{},
cache: &wellBehavedCache{},
cacheTime: time.Duration(0),
gDPR: gdpr.AlwaysAllow{},
currencyConverter: currency.NewRateConverter(&http.Client{}, "", time.Duration(0)),
gdprDefaultValue: "1",
categoriesFetcher: categoriesFetcher,
bidIDGenerator: &mockBidIDGenerator{false, false},
}

imps := buildImps(t, mockBids)
Expand Down
4 changes: 2 additions & 2 deletions exchange/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func cleanOpenRTBRequests(ctx context.Context,
requestExt *openrtb_ext.ExtRequest,
gDPR gdpr.Permissions,
metricsEngine metrics.MetricsEngine,
usersyncIfAmbiguous bool,
gdprDefaultValue string,
privacyConfig config.Privacy,
account *config.Account) (allowedBidderRequests []BidderRequest, privacyLabels metrics.PrivacyLabels, errs []error) {

Expand Down Expand Up @@ -87,7 +87,7 @@ func cleanOpenRTBRequests(ctx context.Context,
if err != nil {
errs = append(errs, err)
}
gdprEnforced := gdprSignal == gdpr.SignalYes || (gdprSignal == gdpr.SignalAmbiguous && !usersyncIfAmbiguous)
gdprEnforced := gdprSignal == gdpr.SignalYes || (gdprSignal == gdpr.SignalAmbiguous && gdprDefaultValue == "1")

ccpaEnforcer, err := extractCCPA(req.BidRequest, privacyConfig, &req.Account, aliases, integrationTypeMap[req.LegacyLabels.RType])
if err != nil {
Expand Down
Loading