Fetching GDPR Vendor Lists (prebid#511)

* Added a basic gdpr module for consent management. No tests yet, and probably still lots of bugs

* Updated gopkg.lock.

* Moved the URL creation into a function.

* Made the URL-maker function a method arg, so it could be mocked out during testing.

* Started some test helper code.

* Added lots of tests.

Gopkg.toml

@@ -81,6 +81,10 @@
  name = "github.com/chasex/log"
  branch = "analytics"
 
+[[constraint]]
+ name = "github.com/prebid/go-gdpr"
+ version = "0.3.0"
+
 [prune]
  go-tests = true
  unused-packages = true

gdpr/gdpr.go

@@ -0,0 +1,31 @@
+package gdpr
+
+import (
+ "context"
+ "net/http"
+
+ "github.com/prebid/prebid-server/config"
+ "github.com/prebid/prebid-server/openrtb_ext"
+)
+
+type Permissions interface {
+ // Determines whether or not the host company is allowed to read/write cookies.
+ HostCookiesAllowed(ctx context.Context, consent string) (bool, error)
+
+ // Determines whether or not the given bidder is allowed to user personal info for ad targeting.
+ BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error)
+}
+
+// NewPermissions gets an instance of the Permissions for use elsewhere in the project.
+func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_ext.BidderName]uint16, client *http.Client) Permissions {
+ // If the host doesn't buy into the IAB GDPR consent framework, then save some cycles and let all syncs happen.
+ if cfg.HostVendorID == 0 {
+ return alwaysAllow{}
+ }
+
+ return &permissionsImpl{
+ cfg: cfg,
+ vendorIDs: vendorIDs,
+ fetchVendorList: newVendorListFetcher(ctx, client, vendorListURLMaker),
+ }
+}

gdpr/impl.go

@@ -0,0 +1,104 @@
+package gdpr
+
+import (
+ "context"
+ "encoding/base64"
+
+ "github.com/prebid/go-gdpr/consentconstants"
+ "github.com/prebid/go-gdpr/vendorconsent"
+ "github.com/prebid/go-gdpr/vendorlist"
+ "github.com/prebid/prebid-server/config"
+ "github.com/prebid/prebid-server/openrtb_ext"
+)
+
+// This file implements GDPR permissions for the app.
+// For more info, see https://github.com/prebid/prebid-server/issues/501
+//
+// Nothing in this file is exported. Public APIs can be found in gdpr.go
+
+type permissionsImpl struct {
+ cfg config.GDPR
+ vendorIDs map[openrtb_ext.BidderName]uint16
+ fetchVendorList func(ctx context.Context, id uint16) (vendorlist.VendorList, error)
+}
+
+func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
+ // If we're not given a consent string, respect the preferences in the app config.
+ if consent == "" {
+ return p.cfg.UsersyncIfAmbiguous, nil
+ }
+
+ data, err := base64.RawURLEncoding.DecodeString(consent)
+ if err != nil {
+ return false, err
+ }
+
+ parsedConsent, err := vendorconsent.Parse([]byte(data))
+ if err != nil {
+ return false, err
+ }
+
+ vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
+ if err != nil {
+ return false, err
+ }
+
+ // Config validation makes uint16 conversion safe here
+ return hasPermissions(parsedConsent, vendorList, uint16(p.cfg.HostVendorID), consentconstants.InfoStorageAccess), nil
+}
+
+func (p *permissionsImpl) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error) {
+ // If we're not given a consent string, respect the preferences in the app config.
+ if consent == "" {
+ return p.cfg.UsersyncIfAmbiguous, nil
+ }
+
+ id, ok := p.vendorIDs[bidder]
+ if !ok {
+ return false, nil
+ }
+
+ data, err := base64.RawURLEncoding.DecodeString(consent)
+ if err != nil {
+ return false, err
+ }
+
+ parsedConsent, err := vendorconsent.Parse([]byte(data))
+ if err != nil {
+ return false, err
+ }
+
+ vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
+ if err != nil {
+ return false, err
+ }
+
+ return hasPermissions(parsedConsent, vendorList, id, consentconstants.AdSelectionDeliveryReporting), nil
+}
+
+func hasPermissions(consent vendorconsent.VendorConsents, vendorList vendorlist.VendorList, vendorID uint16, purpose consentconstants.Purpose) bool {
+ vendor := vendorList.Vendor(vendorID)
+ if vendor == nil {
+ return false
+ }
+ if vendor.LegitimateInterest(purpose) {
+ return true
+ }
+
+ // If the host declared writing cookies to be a "normal" purpose, only do the sync if the user consented to it.
+ if vendor.Purpose(purpose) && consent.PurposeAllowed(purpose) && consent.VendorConsent(vendorID) {
+ return true
+ }
+
+ return false
+}
+
+type alwaysAllow struct{}
+
+func (a alwaysAllow) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
+ return true, nil
+}
+
+func (a alwaysAllow) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error) {
+ return true, nil
+}

gdpr/impl_test.go

@@ -0,0 +1,185 @@
+package gdpr
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "testing"
+
+ "github.com/prebid/prebid-server/config"
+ "github.com/prebid/prebid-server/openrtb_ext"
+
+ "github.com/prebid/go-gdpr/vendorlist"
+)
+
+func TestNoConsentButAllowByDefault(t *testing.T) {
+ perms := permissionsImpl{
+ cfg: config.GDPR{
+ HostVendorID: 3,
+ UsersyncIfAmbiguous: true,
+ },
+ vendorIDs: nil,
+ fetchVendorList: failedListFetcher,
+ }
+ allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
+ assertBoolsEqual(t, true, allowSync)
+ assertNilErr(t, err)
+ allowSync, err = perms.HostCookiesAllowed(context.Background(), "")
+ assertBoolsEqual(t, true, allowSync)
+ assertNilErr(t, err)
+}
+
+func TestNoConsentAndRejectByDefault(t *testing.T) {
+ perms := permissionsImpl{
+ cfg: config.GDPR{
+ HostVendorID: 3,
+ UsersyncIfAmbiguous: false,
+ },
+ vendorIDs: nil,
+ fetchVendorList: failedListFetcher,
+ }
+ allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
+ assertBoolsEqual(t, false, allowSync)
+ assertNilErr(t, err)
+ allowSync, err = perms.HostCookiesAllowed(context.Background(), "")
+ assertBoolsEqual(t, false, allowSync)
+ assertNilErr(t, err)
+}
+
+func TestAllowedSyncs(t *testing.T) {
+ vendorListData := mockVendorListData(t, 1, map[uint16]*purposes{
+ 2: &purposes{
+ purposes: []uint8{1}, // cookie reads/writes
+ },
+ 3: &purposes{
+ purposes: []uint8{3}, // ad personalization
+ },
+ })
+ perms := permissionsImpl{
+ cfg: config.GDPR{
+ HostVendorID: 2,
+ },
+ vendorIDs: map[openrtb_ext.BidderName]uint16{
+ openrtb_ext.BidderAppnexus: 2,
+ openrtb_ext.BidderPubmatic: 3,
+ },
+ fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
+ 1: parseVendorListData(t, vendorListData),
+ }),
+ }
+
+ allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABoAAAAAMw")
+ assertNilErr(t, err)
+ assertBoolsEqual(t, true, allowSync)
+
+ allowSync, err = perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderPubmatic, "BON3PCUON3PCUABABBAAABoAAAAAMw")
+ assertNilErr(t, err)
+ assertBoolsEqual(t, true, allowSync)
+}
+
+func TestProhibitedPurposes(t *testing.T) {
+ vendorListData := mockVendorListData(t, 1, map[uint16]*purposes{
+ 2: &purposes{
+ purposes: []uint8{1}, // cookie reads/writes
+ },
+ 3: &purposes{
+ purposes: []uint8{3}, // ad personalization
+ },
+ })
+ perms := permissionsImpl{
+ cfg: config.GDPR{
+ HostVendorID: 2,
+ },
+ vendorIDs: map[openrtb_ext.BidderName]uint16{
+ openrtb_ext.BidderAppnexus: 2,
+ openrtb_ext.BidderPubmatic: 3,
+ },
+ fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
+ 1: parseVendorListData(t, vendorListData),
+ }),
+ }
+
+ allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABAAAAAAMw")
+ assertNilErr(t, err)
+ assertBoolsEqual(t, false, allowSync)
+
+ allowSync, err = perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderPubmatic, "BON3PCUON3PCUABABBAAABAAAAAAMw")
+ assertNilErr(t, err)
+ assertBoolsEqual(t, false, allowSync)
+}
+
+func TestProhibitedVendors(t *testing.T) {
+ vendorListData := mockVendorListData(t, 1, map[uint16]*purposes{
+ 2: &purposes{
+ purposes: []uint8{1}, // cookie reads/writes
+ },
+ 3: &purposes{
+ purposes: []uint8{3}, // ad personalization
+ },
+ })
+ perms := permissionsImpl{
+ cfg: config.GDPR{
+ HostVendorID: 2,
+ },
+ vendorIDs: map[openrtb_ext.BidderName]uint16{
+ openrtb_ext.BidderAppnexus: 2,
+ openrtb_ext.BidderPubmatic: 3,
+ },
+ fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
+ 1: parseVendorListData(t, vendorListData),
+ }),
+ }
+
+ allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABoAAAAANA")
+ assertNilErr(t, err)
+ assertBoolsEqual(t, false, allowSync)
+
+ allowSync, err = perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderPubmatic, "BON3PCUON3PCUABABBAAABoAAAAANA")
+ assertNilErr(t, err)
+ assertBoolsEqual(t, false, allowSync)
+}
+
+func parseVendorListData(t *testing.T, data string) vendorlist.VendorList {
+ t.Helper()
+ parsed, err := vendorlist.ParseEagerly([]byte(data))
+ if err != nil {
+ t.Fatalf("Failed to parse vendor list data. %v", err)
+ }
+ return parsed
+}
+
+func listFetcher(lists map[uint16]vendorlist.VendorList) func(context.Context, uint16) (vendorlist.VendorList, error) {
+ return func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
+ data, ok := lists[id]
+ if ok {
+ return data, nil
+ } else {
+ return nil, fmt.Errorf("vendorlist id=%d not found", id)
+ }
+ }
+}
+
+func failedListFetcher(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
+ return nil, errors.New("vendor list can't be fetched")
+}
+
+func assertNilErr(t *testing.T, err error) {
+ t.Helper()
+ if err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ }
+}
+
+func assertErr(t *testing.T, err error) {
+ t.Helper()
+ if err == nil {
+ t.Errorf("Expected error did not occur.")
+ }
+}
+
+func assertBoolsEqual(t *testing.T, expected bool, actual bool) {
+ t.Helper()
+ if expected != actual {
+ t.Errorf("Expected %t, got %t", expected, actual)
+ }
+}