Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove GDPR TCF1 #1854

Merged
merged 6 commits into from
Jun 11, 2021
Merged

Remove GDPR TCF1 #1854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7088319
Remove TCF v1
bsardo May 17, 2021
d060186
Fix misspelled purpose one treatment in config
bsardo May 25, 2021
00e7692
Remove TCF2 references in tests
bsardo May 26, 2021
239f969
Remove TCF1 references in metrics
bsardo May 26, 2021
274c354
Maintain purpose_one_treatment config backwards compatibility
bsardo May 27, 2021
7683071
Add TCF1 consent string unit test
bsardo Jun 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 23 additions & 21 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,6 @@ type GDPR struct {
Timeouts GDPRTimeouts `mapstructure:"timeouts_ms"`
NonStandardPublishers []string `mapstructure:"non_standard_publishers,flow"`
NonStandardPublisherMap map[string]struct{}
TCF1 TCF1 `mapstructure:"tcf1"`
TCF2 TCF2 `mapstructure:"tcf2"`
AMPException bool `mapstructure:"amp_exception"` // Deprecated: Use account-level GDPR settings (gdpr.integration_enabled.amp) instead
// EEACountries (EEA = European Economic Area) are a list of countries where we should assume GDPR applies.
Expand All @@ -218,9 +217,6 @@ func (cfg *GDPR) validate(errs []error) []error {
if cfg.AMPException == true {
errs = append(errs, fmt.Errorf("gdpr.amp_exception has been discontinued and must be removed from your config. If you need to disable GDPR for AMP, you may do so per-account (gdpr.integration_enabled.amp) or at the host level for the default account (account_defaults.gdpr.integration_enabled.amp)"))
}
if cfg.TCF1.FetchGVL == true {
errs = append(errs, fmt.Errorf("gdpr.tcf1.fetch_gvl has been discontinued and must be removed from your config. TCF1 will always use the fallback GVL going forward"))
}
return errs
}

Expand All @@ -237,28 +233,22 @@ func (t *GDPRTimeouts) ActiveTimeout() time.Duration {
return time.Duration(t.ActiveVendorlistFetch) * time.Millisecond
}

// TCF1 defines the TCF1 specific configurations for GDPR
type TCF1 struct {
FetchGVL bool `mapstructure:"fetch_gvl"` // Deprecated: In a future version TCF1 will always use the fallback GVL
FallbackGVLPath string `mapstructure:"fallback_gvl_path"`
}

// TCF2 defines the TCF2 specific configurations for GDPR
type TCF2 struct {
Enabled bool `mapstructure:"enabled"`
Purpose1 PurposeDetail `mapstructure:"purpose1"`
Purpose2 PurposeDetail `mapstructure:"purpose2"`
Purpose7 PurposeDetail `mapstructure:"purpose7"`
SpecialPurpose1 PurposeDetail `mapstructure:"special_purpose1"`
PurposeOneTreatment PurposeOneTreatement `mapstructure:"purpose_one_treatement"`
Enabled bool `mapstructure:"enabled"`
Purpose1 PurposeDetail `mapstructure:"purpose1"`
Purpose2 PurposeDetail `mapstructure:"purpose2"`
Purpose7 PurposeDetail `mapstructure:"purpose7"`
SpecialPurpose1 PurposeDetail `mapstructure:"special_purpose1"`
PurposeOneTreatment PurposeOneTreatment `mapstructure:"purpose_one_treatment"`
}

// Making a purpose struct so purpose specific details can be added later.
type PurposeDetail struct {
Enabled bool `mapstructure:"enabled"`
}

type PurposeOneTreatement struct {
type PurposeOneTreatment struct {
Enabled bool `mapstructure:"enabled"`
AccessAllowed bool `mapstructure:"access_allowed"`
}
Expand Down Expand Up @@ -947,16 +937,12 @@ func SetupViper(v *viper.Viper, filename string) {
v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0)
v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0)
v.SetDefault("gdpr.non_standard_publishers", []string{""})
v.SetDefault("gdpr.tcf1.fetch_gvl", false)
v.SetDefault("gdpr.tcf1.fallback_gvl_path", "./static/tcf1/fallback_gvl.json")
v.SetDefault("gdpr.tcf2.enabled", true)
v.SetDefault("gdpr.tcf2.purpose1.enabled", true)
v.SetDefault("gdpr.tcf2.purpose2.enabled", true)
v.SetDefault("gdpr.tcf2.purpose4.enabled", true)
v.SetDefault("gdpr.tcf2.purpose7.enabled", true)
v.SetDefault("gdpr.tcf2.special_purpose1.enabled", true)
v.SetDefault("gdpr.tcf2.purpose_one_treatement.enabled", true)
v.SetDefault("gdpr.tcf2.purpose_one_treatement.access_allowed", true)
v.SetDefault("gdpr.amp_exception", false)
v.SetDefault("gdpr.eea_countries", []string{"ALA", "AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK", "EST",
"FIN", "FRA", "GUF", "DEU", "GIB", "GRC", "GLP", "GGY", "HUN", "ISL", "IRL", "IMN", "ITA", "JEY", "LVA",
Expand Down Expand Up @@ -1010,6 +996,10 @@ func SetupViper(v *viper.Viper, filename string) {

// Migrate config settings to maintain compatibility with old configs
migrateConfig(v)
migrateConfigPurposeOneTreatment(v)

v.SetDefault("gdpr.tcf2.purpose_one_treatment.enabled", true)
v.SetDefault("gdpr.tcf2.purpose_one_treatment.access_allowed", true)
}

func migrateConfig(v *viper.Viper) {
Expand All @@ -1025,6 +1015,18 @@ func migrateConfig(v *viper.Viper) {
}
}

func migrateConfigPurposeOneTreatment(v *viper.Viper) {
if oldConfig, ok := v.Get("gdpr.tcf2.purpose_one_treatement").(map[string]interface{}); ok {
if v.IsSet("gdpr.tcf2.purpose_one_treatment") {
glog.Warning("using gdpr.tcf2.purpose_one_treatment and ignoring deprecated gdpr.tcf2.purpose_one_treatement")
} else {
glog.Warning("gdpr.tcf2.purpose_one_treatement.enabled should be changed to gdpr.tcf2.purpose_one_treatment.enabled")
glog.Warning("gdpr.tcf2.purpose_one_treatement.access_allowed should be changed to gdpr.tcf2.purpose_one_treatment.access_allowed")
v.Set("gdpr.tcf2.purpose_one_treatment", oldConfig)
}
}
}

func setBidderDefaults(v *viper.Viper, bidder string) {
adapterCfgPrefix := "adapters."
v.SetDefault(adapterCfgPrefix+bidder+".endpoint", "")
Expand Down
81 changes: 75 additions & 6 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,8 @@ func TestDefaults(t *testing.T) {
cmpStrings(t, "stored_requests.filesystem.directorypath", "./stored_requests/data/by_id", cfg.StoredRequests.Files.Path)
cmpBools(t, "auto_gen_source_tid", cfg.AutoGenSourceTID, true)
cmpBools(t, "generate_bid_id", cfg.GenerateBidID, false)
cmpBools(t, "gdpr.tcf2.purpose_one_treatment.enabled", true, cfg.GDPR.TCF2.PurposeOneTreatment.Enabled)
cmpBools(t, "gdpr.tcf2.purpose_one_treatment.access_allowed", true, cfg.GDPR.TCF2.PurposeOneTreatment.AccessAllowed)
}

var fullConfig = []byte(`
Expand Down Expand Up @@ -509,6 +511,79 @@ func TestMigrateConfigFromEnv(t *testing.T) {
cmpBools(t, "stored_requests.filesystem.enabled", true, cfg.StoredRequests.Files.Enabled)
}

func TestMigrateConfigPurposeOneTreatment(t *testing.T) {
oldPurposeOneTreatmentConfig := []byte(`
gdpr:
tcf2:
purpose_one_treatement:
enabled: true
access_allowed: true
`)
newPurposeOneTreatmentConfig := []byte(`
gdpr:
tcf2:
purpose_one_treatment:
enabled: true
access_allowed: true
`)
oldAndNewPurposeOneTreatmentConfig := []byte(`
gdpr:
tcf2:
purpose_one_treatement:
enabled: false
access_allowed: true
purpose_one_treatment:
enabled: true
access_allowed: false
`)

tests := []struct {
description string
config []byte
wantPurpose1TreatmentEnabled bool
wantPurpose1TreatmentAccessAllowed bool
}{
{
description: "New config and old config not set",
config: []byte{},
},
{
description: "New config not set, old config set",
config: oldPurposeOneTreatmentConfig,
wantPurpose1TreatmentEnabled: true,
wantPurpose1TreatmentAccessAllowed: true,
},
{
description: "New config set, old config not set",
config: newPurposeOneTreatmentConfig,
wantPurpose1TreatmentEnabled: true,
wantPurpose1TreatmentAccessAllowed: true,
},
{
description: "New config and old config set",
config: oldAndNewPurposeOneTreatmentConfig,
wantPurpose1TreatmentEnabled: true,
wantPurpose1TreatmentAccessAllowed: false,
},
}

for _, tt := range tests {
v := viper.New()
v.SetConfigType("yaml")
v.ReadConfig(bytes.NewBuffer(tt.config))

migrateConfigPurposeOneTreatment(v)

if len(tt.config) > 0 {
assert.Equal(t, tt.wantPurpose1TreatmentEnabled, v.Get("gdpr.tcf2.purpose_one_treatment.enabled").(bool), tt.description)
assert.Equal(t, tt.wantPurpose1TreatmentAccessAllowed, v.Get("gdpr.tcf2.purpose_one_treatment.access_allowed").(bool), tt.description)
} else {
assert.Nil(t, v.Get("gdpr.tcf2.purpose_one_treatment.enabled"), tt.description)
assert.Nil(t, v.Get("gdpr.tcf2.purpose_one_treatment.access_allowed"), tt.description)
}
}
}

func TestInvalidAdapterEndpointConfig(t *testing.T) {
v := viper.New()
SetupViper(v, "")
Expand Down Expand Up @@ -568,12 +643,6 @@ func TestInvalidHostVendorID(t *testing.T) {
}
}

func TestInvalidFetchGVL(t *testing.T) {
cfg := newDefaultConfig(t)
cfg.GDPR.TCF1.FetchGVL = true
assertOneError(t, cfg.validate(), "gdpr.tcf1.fetch_gvl has been discontinued and must be removed from your config. TCF1 will always use the fallback GVL going forward")
}

func TestInvalidAMPException(t *testing.T) {
cfg := newDefaultConfig(t)
cfg.GDPR.AMPException = true
Expand Down
57 changes: 22 additions & 35 deletions exchange/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1447,8 +1447,7 @@ func TestCleanOpenRTBRequestsLMT(t *testing.T) {
}
}

func TestCleanOpenRTBRequestsGDPRScrub(t *testing.T) {
tcf1Consent := "BONV8oqONXwgmADACHENAO7pqzAAppY"
func TestCleanOpenRTBRequestsGDPR(t *testing.T) {
tcf2Consent := "COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA"
trueValue, falseValue := true, false

Expand Down Expand Up @@ -1477,19 +1476,7 @@ func TestCleanOpenRTBRequestsGDPRScrub(t *testing.T) {
},
},
{
description: "Enforce - TCF 1",
gdprAccountEnabled: &trueValue,
gdprHostEnabled: true,
gdpr: "1",
gdprConsent: tcf1Consent,
gdprScrub: true,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: true,
GDPRTCFVersion: metrics.TCFVersionV1,
},
},
{
description: "Enforce - TCF 2",
description: "Enforce",
gdprAccountEnabled: &trueValue,
gdprHostEnabled: true,
gdpr: "1",
Expand All @@ -1501,72 +1488,72 @@ func TestCleanOpenRTBRequestsGDPRScrub(t *testing.T) {
},
},
{
description: "Not Enforce - TCF 1",
description: "Not Enforce",
gdprAccountEnabled: &trueValue,
gdprHostEnabled: true,
gdpr: "0",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: false,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: false,
GDPRTCFVersion: "",
},
},
{
description: "Enforce - TCF 1; GDPR signal extraction error",
description: "Enforce; GDPR signal extraction error",
gdprAccountEnabled: &trueValue,
gdprHostEnabled: true,
gdpr: "0{",
gdprConsent: "BONV8oqONXwgmADACHENAO7pqzAAppY",
gdprConsent: tcf2Consent,
gdprScrub: true,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: true,
GDPRTCFVersion: metrics.TCFVersionV1,
GDPRTCFVersion: metrics.TCFVersionV2,
},
expectError: true,
},
{
description: "Enforce - TCF 1; account GDPR enabled, host GDPR setting disregarded",
description: "Enforce; account GDPR enabled, host GDPR setting disregarded",
gdprAccountEnabled: &trueValue,
gdprHostEnabled: false,
gdpr: "1",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: true,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: true,
GDPRTCFVersion: metrics.TCFVersionV1,
GDPRTCFVersion: metrics.TCFVersionV2,
},
},
{
description: "Not Enforce - TCF 1; account GDPR disabled, host GDPR setting disregarded",
description: "Not Enforce; account GDPR disabled, host GDPR setting disregarded",
gdprAccountEnabled: &falseValue,
gdprHostEnabled: true,
gdpr: "1",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: false,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: false,
GDPRTCFVersion: "",
},
},
{
description: "Enforce - TCF 1; account GDPR not specified, host GDPR enabled",
description: "Enforce; account GDPR not specified, host GDPR enabled",
gdprAccountEnabled: nil,
gdprHostEnabled: true,
gdpr: "1",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: true,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: true,
GDPRTCFVersion: metrics.TCFVersionV1,
GDPRTCFVersion: metrics.TCFVersionV2,
},
},
{
description: "Not Enforce - TCF 1; account GDPR not specified, host GDPR disabled",
description: "Not Enforce; account GDPR not specified, host GDPR disabled",
gdprAccountEnabled: nil,
gdprHostEnabled: false,
gdpr: "1",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: false,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: false,
Expand All @@ -1578,20 +1565,20 @@ func TestCleanOpenRTBRequestsGDPRScrub(t *testing.T) {
gdprAccountEnabled: nil,
gdprHostEnabled: true,
gdpr: "null",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: true,
userSyncIfAmbiguous: false,
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: true,
GDPRTCFVersion: metrics.TCFVersionV1,
GDPRTCFVersion: metrics.TCFVersionV2,
},
},
{
description: "Not Enforce - Ambiguous signal, sync user if ambiguous",
gdprAccountEnabled: nil,
gdprHostEnabled: true,
gdpr: "null",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: false,
userSyncIfAmbiguous: true,
expectPrivacyLabels: metrics.PrivacyLabels{
Expand All @@ -1604,12 +1591,12 @@ func TestCleanOpenRTBRequestsGDPRScrub(t *testing.T) {
gdprAccountEnabled: nil,
gdprHostEnabled: true,
gdpr: "1",
gdprConsent: tcf1Consent,
gdprConsent: tcf2Consent,
gdprScrub: true,
permissionsError: errors.New("Some error"),
expectPrivacyLabels: metrics.PrivacyLabels{
GDPREnforced: true,
GDPRTCFVersion: metrics.TCFVersionV1,
GDPRTCFVersion: metrics.TCFVersionV2,
},
},
}
Expand Down
4 changes: 1 addition & 3 deletions gdpr/gdpr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ type Permissions interface {

// Versions of the GDPR TCF technical specification.
const (
tcf1SpecVersion uint8 = 1
tcf2SpecVersion uint8 = 2
)

Expand All @@ -44,8 +43,7 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
cfg: cfg,
vendorIDs: vendorIDs,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tcf1SpecVersion: newVendorListFetcherTCF1(cfg),
tcf2SpecVersion: newVendorListFetcherTCF2(ctx, cfg, client, vendorListURLMaker)},
tcf2SpecVersion: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker)},
}

if cfg.HostVendorID == 0 {
Expand Down
Loading