Skip to content

Commit

Permalink
Stricter Privacy Scrubbing (#1286)
Browse files Browse the repository at this point in the history 
* Stricter Privacy Scrubbing

* Update Unit Test Style

* Fixed Whitespace
  • Loading branch information
@SyntaxNode
SyntaxNode authored May 6, 2020
1 parent cd90991 commit cc3d2da
Show file tree
Hide file tree
Showing 6 changed files with 244 additions and 159 deletions.
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ require (
github.com/spf13/jwalterweatherman v0.0.0-20180814060501-14d3d4c51834 // indirect
github.com/spf13/pflag v1.0.2 // indirect
github.com/spf13/viper v1.1.0
github.com/stretchr/testify v1.3.0
github.com/stretchr/testify v1.5.1
github.com/valyala/fasthttp v1.9.0
github.com/vrischmann/go-metrics-influxdb v0.0.0-20160917065939-43af8332c303
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
Expand All @@ -70,5 +70,5 @@ require (
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
golang.org/x/text v0.3.0
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
gopkg.in/yaml.v2 v2.2.1
gopkg.in/yaml.v2 v2.2.2
)
4 changes: 4 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw=
Expand Down Expand Up @@ -196,3 +198,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
18 changes: 5 additions & 13 deletions privacy/enforcement.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,11 @@ func (e Enforcement) Apply(bidRequest *openrtb.BidRequest) {

func (e Enforcement) apply(bidRequest *openrtb.BidRequest, scrubber Scrubber) {
if bidRequest != nil && e.Any() {
bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getDeviceMacAndIFA(), e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy())
bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getUserScrubStrategy(), e.getGeoScrubStrategy())
bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy())
bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getDemographicScrubStrategy(), e.getGeoScrubStrategy())
}
}

func (e Enforcement) getDeviceMacAndIFA() bool {
return e.COPPA
}

func (e Enforcement) getIPv6ScrubStrategy() ScrubStrategyIPV6 {
if e.COPPA {
return ScrubStrategyIPV6Lowest32
Expand All @@ -56,14 +52,10 @@ func (e Enforcement) getGeoScrubStrategy() ScrubStrategyGeo {
return ScrubStrategyGeoNone
}

func (e Enforcement) getUserScrubStrategy() ScrubStrategyUser {
func (e Enforcement) getDemographicScrubStrategy() ScrubStrategyDemographic {
if e.COPPA {
return ScrubStrategyUserFull
}

if e.GDPR || e.CCPA {
return ScrubStrategyUserBuyerIDOnly
return ScrubStrategyDemographicAgeAndGender
}

return ScrubStrategyUserNone
return ScrubStrategyDemographicNone
}
106 changes: 54 additions & 52 deletions privacy/enforcement_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,31 +15,31 @@ func TestAny(t *testing.T) {
description string
}{
{
description: "All False",
enforcement: Enforcement{
CCPA: false,
COPPA: false,
GDPR: false,
},
expected: false,
description: "All False",
expected: false,
},
{
description: "All True",
enforcement: Enforcement{
CCPA: true,
COPPA: true,
GDPR: true,
},
expected: true,
description: "All True",
expected: true,
},
{
description: "Mixed",
enforcement: Enforcement{
CCPA: false,
COPPA: true,
GDPR: false,
},
expected: true,
description: "Mixed",
expected: true,
},
}

Expand All @@ -51,117 +51,119 @@ func TestAny(t *testing.T) {

func TestApply(t *testing.T) {
testCases := []struct {
description string
enforcement Enforcement
expectedDeviceMacAndIFA bool
expectedDeviceIPv6 ScrubStrategyIPV6
expectedDeviceGeo ScrubStrategyGeo
expectedUser ScrubStrategyUser
expectedUserDemographic ScrubStrategyDemographic
expectedUserGeo ScrubStrategyGeo
description string
}{
{
description: "All Enforced",
enforcement: Enforcement{
CCPA: true,
COPPA: true,
GDPR: true,
},
expectedDeviceMacAndIFA: true,
expectedDeviceIPv6: ScrubStrategyIPV6Lowest32,
expectedDeviceGeo: ScrubStrategyGeoFull,
expectedUser: ScrubStrategyUserFull,
expectedUserDemographic: ScrubStrategyDemographicAgeAndGender,
expectedUserGeo: ScrubStrategyGeoFull,
description: "All Enforced - Most Strict",
},
{
description: "CCPA Only",
enforcement: Enforcement{
CCPA: true,
COPPA: false,
GDPR: false,
},
expectedDeviceIPv6: ScrubStrategyIPV6Lowest16,
expectedDeviceGeo: ScrubStrategyGeoReducedPrecision,
expectedUserDemographic: ScrubStrategyDemographicNone,
expectedUserGeo: ScrubStrategyGeoReducedPrecision,
},
{
description: "COPPA Only",
enforcement: Enforcement{
CCPA: false,
COPPA: true,
GDPR: false,
},
expectedDeviceMacAndIFA: true,
expectedDeviceIPv6: ScrubStrategyIPV6Lowest32,
expectedDeviceGeo: ScrubStrategyGeoFull,
expectedUser: ScrubStrategyUserFull,
expectedUserDemographic: ScrubStrategyDemographicAgeAndGender,
expectedUserGeo: ScrubStrategyGeoFull,
description: "COPPA",
},
{
description: "GDPR Only",
enforcement: Enforcement{
CCPA: false,
COPPA: false,
GDPR: true,
},
expectedDeviceMacAndIFA: false,
expectedDeviceIPv6: ScrubStrategyIPV6Lowest16,
expectedDeviceGeo: ScrubStrategyGeoReducedPrecision,
expectedUser: ScrubStrategyUserBuyerIDOnly,
expectedUserDemographic: ScrubStrategyDemographicNone,
expectedUserGeo: ScrubStrategyGeoReducedPrecision,
description: "GDPR",
},
{
enforcement: Enforcement{
CCPA: true,
COPPA: false,
GDPR: false,
},
expectedDeviceMacAndIFA: false,
expectedDeviceIPv6: ScrubStrategyIPV6Lowest16,
expectedDeviceGeo: ScrubStrategyGeoReducedPrecision,
expectedUser: ScrubStrategyUserBuyerIDOnly,
expectedUserGeo: ScrubStrategyGeoReducedPrecision,
description: "CCPA",
},
}

for _, test := range testCases {
req := &openrtb.BidRequest{
Device: &openrtb.Device{DIDSHA1: "before"},
User: &openrtb.User{ID: "before"},
Device: &openrtb.Device{},
User: &openrtb.User{},
}
device := &openrtb.Device{DIDSHA1: "after"}
user := &openrtb.User{ID: "after"}
replacedDevice := &openrtb.Device{}
replacedUser := &openrtb.User{}

m := &mockScrubber{}
m.On("ScrubDevice", req.Device, test.expectedDeviceMacAndIFA, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(device).Once()
m.On("ScrubUser", req.User, test.expectedUser, test.expectedUserGeo).Return(user).Once()
m.On("ScrubDevice", req.Device, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(replacedDevice).Once()
m.On("ScrubUser", req.User, test.expectedUserDemographic, test.expectedUserGeo).Return(replacedUser).Once()

test.enforcement.apply(req, m)

m.AssertExpectations(t)
assert.Equal(t, device, req.Device, "Device Set Correctly")
assert.Equal(t, user, req.User, "User Set Correctly")
assert.Same(t, replacedDevice, req.Device, "Device")
assert.Same(t, replacedUser, req.User, "User")
}
}

func TestApplyNoneApplicable(t *testing.T) {
enforcement := Enforcement{}
device := &openrtb.Device{DIDSHA1: "original"}
user := &openrtb.User{ID: "original"}
req := &openrtb.BidRequest{
Device: device,
User: user,
}
req := &openrtb.BidRequest{}

m := &mockScrubber{}

enforcement := Enforcement{
CCPA: false,
COPPA: false,
GDPR: false,
}
enforcement.apply(req, m)

m.AssertNotCalled(t, "ScrubDevice")
m.AssertNotCalled(t, "ScrubUser")
assert.Equal(t, device, req.Device, "Device Set Correctly")
assert.Equal(t, user, req.User, "User Set Correctly")
}

func TestApplyNil(t *testing.T) {
m := &mockScrubber{}

enforcement := Enforcement{}
enforcement.apply(nil, m)

m.AssertNotCalled(t, "ScrubDevice")
m.AssertNotCalled(t, "ScrubUser")
}

type mockScrubber struct {
mock.Mock
}

func (m *mockScrubber) ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
args := m.Called(device, macAndIFA, ipv6, geo)
func (m *mockScrubber) ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
args := m.Called(device, ipv6, geo)
return args.Get(0).(*openrtb.Device)
}

func (m *mockScrubber) ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User {
args := m.Called(user, strategy, geo)
func (m *mockScrubber) ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User {
args := m.Called(user, demographic, geo)
return args.Get(0).(*openrtb.User)
}
51 changes: 18 additions & 33 deletions privacy/scrubber.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,24 +34,21 @@ const (
ScrubStrategyGeoReducedPrecision
)

// ScrubStrategyUser defines the approach to scrub PII from user data.
type ScrubStrategyUser int
// ScrubStrategyDemographic defines the approach to non-location demographic data.
type ScrubStrategyDemographic int

const (
// ScrubStrategyUserNone does not remove user data.
ScrubStrategyUserNone ScrubStrategyUser = iota
// ScrubStrategyDemographicNone does not remove non-location demographic data.
ScrubStrategyDemographicNone ScrubStrategyDemographic = iota

// ScrubStrategyUserFull removes the user's buyer id, exchange id year of birth, and gender.
ScrubStrategyUserFull

// ScrubStrategyUserBuyerIDOnly removes the user's buyer id.
ScrubStrategyUserBuyerIDOnly
// ScrubStrategyDemographicAgeAndGender removes age and gender data.
ScrubStrategyDemographicAgeAndGender
)

// Scrubber removes PII from parts of an OpenRTB request.
type Scrubber interface {
ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device
ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User
ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device
ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User
}

type scrubber struct{}
Expand All @@ -61,25 +58,21 @@ func NewScrubber() Scrubber {
return scrubber{}
}

func (scrubber) ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
func (scrubber) ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
if device == nil {
return nil
}

deviceCopy := *device

deviceCopy.DIDMD5 = ""
deviceCopy.DIDSHA1 = ""
deviceCopy.DPIDMD5 = ""
deviceCopy.DPIDSHA1 = ""
deviceCopy.IFA = ""
deviceCopy.MACMD5 = ""
deviceCopy.MACSHA1 = ""
deviceCopy.IP = scrubIPV4(device.IP)

if macAndIFA {
deviceCopy.MACSHA1 = ""
deviceCopy.MACMD5 = ""
deviceCopy.IFA = ""
}

switch ipv6 {
case ScrubStrategyIPV6Lowest16:
deviceCopy.IPv6 = scrubIPV6Lowest16Bits(device.IPv6)
Expand All @@ -97,21 +90,19 @@ func (scrubber) ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubSt
return &deviceCopy
}

func (scrubber) ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User {
func (scrubber) ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User {
if user == nil {
return nil
}

userCopy := *user
userCopy.BuyerUID = ""
userCopy.ID = ""

switch strategy {
case ScrubStrategyUserFull:
userCopy.BuyerUID = ""
userCopy.ID = ""
switch demographic {
case ScrubStrategyDemographicAgeAndGender:
userCopy.Yob = 0
userCopy.Gender = ""
case ScrubStrategyUserBuyerIDOnly:
userCopy.BuyerUID = ""
}

switch geo {
Expand Down Expand Up @@ -169,13 +160,7 @@ func scrubGeoFull(geo *openrtb.Geo) *openrtb.Geo {
return nil
}

geoCopy := *geo
geoCopy.Lat = 0
geoCopy.Lon = 0
geoCopy.Metro = ""
geoCopy.City = ""
geoCopy.ZIP = ""
return &geoCopy
return &openrtb.Geo{}
}

func scrubGeoPrecision(geo *openrtb.Geo) *openrtb.Geo {
Expand Down
Loading

0 comments on commit cc3d2da

Please sign in to comment.