Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add required information to DBP NA<->FE API for removal timeline support #3268

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Add required information to DBP NA<->FE API for removal timeline support #3268

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
1c82d0f
Add new fields to DBP->FE API, and associated constructors
THISISDINOSAUR Sep 12, 2024
6388873
Make UIMapper use new constructors
THISISDINOSAUR Sep 12, 2024
24d3fc9
Remove unused data field
THISISDINOSAUR Sep 12, 2024
19ba1c8
Merge branch 'main' into elle/add-removal-timeline-support
THISISDINOSAUR Sep 13, 2024
2f45004
Refactor DBPUIDateBrokerProfileMatch inits to reduce duplication
THISISDINOSAUR Sep 16, 2024
9e326ef
Fix DBPUIDataBrokerProfileMatch creation when created date is default
THISISDINOSAUR Sep 16, 2024
fa5f675
Add unit tests for DBPUIDataBrokerProfileMatch creation
THISISDINOSAUR Sep 16, 2024
59988dd
Change estimatedRemovalDate to be calculated from optOutSubmittedDate…
THISISDINOSAUR Sep 18, 2024
0865a78
Add match testing function to Extracted profile
THISISDINOSAUR Sep 18, 2024
9aa5b41
Add unit tests for matching function
THISISDINOSAUR Sep 18, 2024
1af1ebe
Add parent matching to DBPUIDataBrokerProfileMatch
THISISDINOSAUR Sep 18, 2024
29c8bb8
Add UIMapper tests for profile matching
THISISDINOSAUR Sep 18, 2024
c2d4d9f
Add parent URL field to UIDataBroker
THISISDINOSAUR Sep 18, 2024
06b1020
Fix incorrect data being given when looking for parent matches
THISISDINOSAUR Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...ages/DataBrokerProtection/Sources/DataBrokerProtection/Model/BrokerProfileQueryData.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ public struct BrokerProfileQueryData: Sendable {
optOutJobData.map { $0.extractedProfile }
}

var extractedProfilesWithOptOutJobData: [(ExtractedProfile, OptOutJobData)] {
optOutJobData.map { ($0.extractedProfile, $0) }
}

var events: [HistoryEvent] {
operationsData.flatMap { $0.historyEvents }.sorted { $0.date < $1.date }
}
Expand Down
70 changes: 69 additions & 1 deletion ...ges/DataBrokerProtection/Sources/DataBrokerProtection/Model/DBPUICommunicationModel.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,15 @@ struct DBPUIUserProfileAddress: Codable {
let zipCode: String?
}

extension DBPUIUserProfileAddress {
init(addressCityState: AddressCityState) {
self.init(street: addressCityState.fullAddress,
city: addressCityState.city,
state: addressCityState.state,
zipCode: nil)
}
}

/// Message Object representing a user profile containing one or more names and addresses
/// also contains the user profile's birth year
struct DBPUIUserProfile: Codable {
Expand Down Expand Up @@ -112,6 +121,10 @@ struct DBPUIDataBroker: Codable, Hashable {
self.date = date
}

init(dataBroker: DataBroker) {
self.init(name: dataBroker.name, url: dataBroker.url)
}

func hash(into hasher: inout Hasher) {
hasher.combine(name)
}
Expand All @@ -135,7 +148,41 @@ struct DBPUIDataBrokerProfileMatch: Codable {
let addresses: [DBPUIUserProfileAddress]
let alternativeNames: [String]
let relatives: [String]
let date: Double? // Used in some methods to set the removedDate or found date
let foundDate: Double
let optOutSubmittedDate: Double?
let estimatedRemovalDate: Double?
let removedDate: Double?
}

extension DBPUIDataBrokerProfileMatch {
init(extractedProfile: ExtractedProfile, optOutJobData: OptOutJobData, dataBroker: DataBroker) {
let estimatedRemovalDate = Calendar.current.date(byAdding: .day, value: 14, to: optOutJobData.createdDate)
self.init(dataBroker: DBPUIDataBroker(dataBroker: dataBroker),
name: extractedProfile.fullName ?? "No name",
addresses: extractedProfile.addresses?.map {DBPUIUserProfileAddress(addressCityState: $0) } ?? [],
alternativeNames: extractedProfile.alternativeNames ?? [String](),
relatives: extractedProfile.relatives ?? [String](),
foundDate: optOutJobData.createdDate.timeIntervalSince1970,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't love that this information is gained from the optOutJobData, but what you can't see from looking just at this struct is if you see further above is that all these extracted profiles are taken from the opt out jobs the first place.
So I don't think there's anything I can do about it outside of making more serious challenges to our data architecture (which I'd love to do. In making this PR I really noticed how my issues with the DB scheme trickle all the way up to cause a lot of the things I don't like in the UI mapping stuff)

optOutSubmittedDate: optOutJobData.submittedSuccessfullyDate?.timeIntervalSince1970,
estimatedRemovalDate: estimatedRemovalDate?.timeIntervalSince1970,
removedDate: extractedProfile.removedDate?.timeIntervalSince1970)
}

init(extractedProfile: ExtractedProfile,
optOutJobData: OptOutJobData,
dataBrokerName: String,
databrokerURL: String) {
let estimatedRemovalDate = Calendar.current.date(byAdding: .day, value: 14, to: optOutJobData.createdDate)
self.init(dataBroker: DBPUIDataBroker(name: dataBrokerName, url: databrokerURL),
name: extractedProfile.fullName ?? "No name",
addresses: extractedProfile.addresses?.map {DBPUIUserProfileAddress(addressCityState: $0) } ?? [],
alternativeNames: extractedProfile.alternativeNames ?? [String](),
relatives: extractedProfile.relatives ?? [String](),
foundDate: optOutJobData.createdDate.timeIntervalSince1970,
optOutSubmittedDate: optOutJobData.submittedSuccessfullyDate?.timeIntervalSince1970,
estimatedRemovalDate: estimatedRemovalDate?.timeIntervalSince1970,
removedDate: extractedProfile.removedDate?.timeIntervalSince1970)
}
}

/// Protocol to represent a message that can be passed from the host to the UI
Expand All @@ -156,6 +203,27 @@ struct DBPUIOptOutMatch: DBPUISendableMessage {
let alternativeNames: [String]
let addresses: [DBPUIUserProfileAddress]
let date: Double
let foundDate: Double
let optOutSubmittedDate: Double?
let estimatedRemovalDate: Double?
let removedDate: Double?
}

extension DBPUIOptOutMatch {
init?(profileMatch: DBPUIDataBrokerProfileMatch, matches: Int) {
guard let removedDate = profileMatch.removedDate else { return nil }
let dataBroker = profileMatch.dataBroker
self.init(dataBroker: dataBroker,
matches: matches,
name: profileMatch.name,
alternativeNames: profileMatch.alternativeNames,
addresses: profileMatch.addresses,
date: removedDate,
foundDate: profileMatch.foundDate,
optOutSubmittedDate: profileMatch.optOutSubmittedDate,
estimatedRemovalDate: nil,
removedDate: removedDate)
}
}

/// Data representing the initial scan progress
Expand Down
60 changes: 17 additions & 43 deletions LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,36 +22,6 @@ import os.log

struct MapperToUI {

func mapToUI(_ dataBroker: DataBroker, extractedProfile: ExtractedProfile) -> DBPUIDataBrokerProfileMatch {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All this has now moved into constructors on the individual UI objects

DBPUIDataBrokerProfileMatch(
dataBroker: mapToUI(dataBroker),
name: extractedProfile.fullName ?? "No name",
addresses: extractedProfile.addresses?.map(mapToUI) ?? [],
alternativeNames: extractedProfile.alternativeNames ?? [String](),
relatives: extractedProfile.relatives ?? [String](),
date: extractedProfile.removedDate?.timeIntervalSince1970
)
}

func mapToUI(_ dataBrokerName: String, databrokerURL: String, extractedProfile: ExtractedProfile) -> DBPUIDataBrokerProfileMatch {
DBPUIDataBrokerProfileMatch(
dataBroker: DBPUIDataBroker(name: dataBrokerName, url: databrokerURL),
name: extractedProfile.fullName ?? "No name",
addresses: extractedProfile.addresses?.map(mapToUI) ?? [],
alternativeNames: extractedProfile.alternativeNames ?? [String](),
relatives: extractedProfile.relatives ?? [String](),
date: extractedProfile.removedDate?.timeIntervalSince1970
)
}

func mapToUI(_ dataBroker: DataBroker) -> DBPUIDataBroker {
DBPUIDataBroker(name: dataBroker.name, url: dataBroker.url)
}

func mapToUI(_ address: AddressCityState) -> DBPUIUserProfileAddress {
DBPUIUserProfileAddress(street: address.fullAddress, city: address.city, state: address.state, zipCode: nil)
}

func initialScanState(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> DBPUIInitialScanState {

let withoutDeprecated = brokerProfileQueryData.filter { !$0.profileQuery.deprecated }
Expand Down Expand Up @@ -82,13 +52,18 @@ struct MapperToUI {
private func mapMatchesToUI(_ brokerProfileQueryData: [BrokerProfileQueryData]) -> [DBPUIDataBrokerProfileMatch] {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't like how much this and the maintenance scans function duplicate each other. However I'm also afraid to touch them since there seem to be some subtle differences which I assume are deliberate

return brokerProfileQueryData.flatMap {
var profiles = [DBPUIDataBrokerProfileMatch]()
for extractedProfile in $0.extractedProfiles where !$0.profileQuery.deprecated {
profiles.append(mapToUI($0.dataBroker, extractedProfile: extractedProfile))
for (extractedProfile, optOutJobData) in $0.extractedProfilesWithOptOutJobData where !$0.profileQuery.deprecated {
profiles.append(DBPUIDataBrokerProfileMatch(extractedProfile: extractedProfile,
optOutJobData: optOutJobData,
dataBroker: $0.dataBroker))

if !$0.dataBroker.mirrorSites.isEmpty {
let mirrorSitesMatches = $0.dataBroker.mirrorSites.compactMap { mirrorSite in
if mirrorSite.shouldWeIncludeMirrorSite() {
return mapToUI(mirrorSite.name, databrokerURL: mirrorSite.url, extractedProfile: extractedProfile)
return DBPUIDataBrokerProfileMatch(extractedProfile: extractedProfile,
optOutJobData: optOutJobData,
dataBrokerName: mirrorSite.name,
databrokerURL: mirrorSite.url)
}

return nil
Expand All @@ -113,7 +88,9 @@ struct MapperToUI {
let scanJob = $0.scanJobData
for optOutJob in $0.optOutJobData {
let extractedProfile = optOutJob.extractedProfile
let profileMatch = mapToUI(dataBroker, extractedProfile: extractedProfile)
let profileMatch = DBPUIDataBrokerProfileMatch(extractedProfile: extractedProfile,
optOutJobData: optOutJob,
dataBroker: dataBroker)

if extractedProfile.removedDate == nil {
inProgressOptOuts.append(profileMatch)
Expand All @@ -123,7 +100,10 @@ struct MapperToUI {

if let closestMatchesFoundEvent = scanJob.closestMatchesFoundEvent() {
for mirrorSite in dataBroker.mirrorSites where mirrorSite.shouldWeIncludeMirrorSite(for: closestMatchesFoundEvent.date) {
let mirrorSiteMatch = mapToUI(mirrorSite.name, databrokerURL: mirrorSite.url, extractedProfile: extractedProfile)
let mirrorSiteMatch = DBPUIDataBrokerProfileMatch(extractedProfile: extractedProfile,
optOutJobData: optOutJob,
dataBrokerName: mirrorSite.name,
databrokerURL: mirrorSite.url)

if let extractedProfileRemovedDate = extractedProfile.removedDate,
mirrorSite.shouldWeIncludeMirrorSite(for: extractedProfileRemovedDate) {
Expand All @@ -137,15 +117,9 @@ struct MapperToUI {
}

let completedOptOutsDictionary = Dictionary(grouping: removedProfiles, by: { $0.dataBroker })
let completedOptOuts: [DBPUIOptOutMatch] = completedOptOutsDictionary.compactMap { (key: DBPUIDataBroker, value: [DBPUIDataBrokerProfileMatch]) in
let completedOptOuts: [DBPUIOptOutMatch] = completedOptOutsDictionary.compactMap { (_, value: [DBPUIDataBrokerProfileMatch]) in
value.compactMap { match in
guard let removedDate = match.date else { return nil }
return DBPUIOptOutMatch(dataBroker: key,
matches: value.count,
name: match.name,
alternativeNames: match.alternativeNames,
addresses: match.addresses,
date: removedDate)
return DBPUIOptOutMatch(profileMatch: match, matches: value.count)
}
}.flatMap { $0 }

Expand Down
Loading