From 4d89dfe57eed7ce32bc92be416b1a7a365ca53db Mon Sep 17 00:00:00 2001 From: rory Date: Mon, 19 Jun 2023 12:25:03 -0700 Subject: [PATCH 1/6] Write down details for data that needs to be migrated --- .../migrations/PersonalDetailsByAccountID.js | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/libs/migrations/PersonalDetailsByAccountID.js diff --git a/src/libs/migrations/PersonalDetailsByAccountID.js b/src/libs/migrations/PersonalDetailsByAccountID.js new file mode 100644 index 000000000000..a9ac88082771 --- /dev/null +++ b/src/libs/migrations/PersonalDetailsByAccountID.js @@ -0,0 +1,32 @@ +/** + * Migrate Onyx data to hide emails where necessary. + * + * - personalDetails -> personalDetailsList + * - Key by accountID instead of email + * - Must check if users are "known" or not, and include their contact info iff they are "known" + * - policyMemberList_ -> policyMember_ + * - Key by accountID instead of email + * - reportAction_ + * - originalMessage.oldLogin -> originalMessage.oldAccountID + * - originalMessage.newLogin -> originalMessage.newAccountID + * - originalMessage.participants -> originalMessage.participantAccountIDs + * - actorEmail -> actorAccountID + * - childManagerEmail -> childManagerAccountID + * - whisperedTo -> whisperedToAccountID + * - childOldestFourEmails -> childOldestFourAccountIDs + * - accountEmail -> accountID + * - report_ + * - ownerEmail -> ownerAccountID + * - managerEmail -> managerID + * - lastActorEmail -> lastActorAccountID + * - participants -> participantAccountID + * + * - User A "knows" user B if any of the following are true: + * - They share at least one non-public workspace room or domain room + * - User B has ever sent user A a message in a DM or group DM. + * + * @returns {Promise} + */ +export default function () { + return Promise.resolve(); +} From 5af1081c59619cc88c35da164a8c70091f928e8a Mon Sep 17 00:00:00 2001 From: rory Date: Mon, 19 Jun 2023 14:55:46 -0700 Subject: [PATCH 2/6] Create migration for email => accountID --- .../migrations/PersonalDetailsByAccountID.js | 319 +++++++++++++++++- 1 file changed, 310 insertions(+), 9 deletions(-) diff --git a/src/libs/migrations/PersonalDetailsByAccountID.js b/src/libs/migrations/PersonalDetailsByAccountID.js index a9ac88082771..9bc3dc03affe 100644 --- a/src/libs/migrations/PersonalDetailsByAccountID.js +++ b/src/libs/migrations/PersonalDetailsByAccountID.js @@ -1,14 +1,79 @@ +import _ from 'underscore'; +import Onyx from 'react-native-onyx'; +import ONYXKEYS from '../../ONYXKEYS'; +import * as ReportUtils from '../ReportUtils'; + +const DEPRECATED_ONYX_KEYS = { + // Deprecated personal details object which was keyed by login instead of accountID. + PERSONAL_DETAILS: 'personalDetails', + + COLLECTION: { + POLICY_MEMBER_LIST: 'policyMemberList_', + }, +}; + +function getReportsFromOnyx() { + return new Promise((resolve) => { + const connectionID = Onyx.connect({ + key: ONYXKEYS.COLLECTION.REPORT, + waitForCollectionCallback: true, + callback: (allReports) => { + Onyx.disconnect(connectionID); + return resolve(allReports); + }, + }); + }); +} + +function getReportActionsFromOnyx() { + return new Promise((resolve) => { + const connectionID = Onyx.connect({ + key: ONYXKEYS.COLLECTION.REPORT_ACTIONS, + waitForCollectionCallback: true, + callback: (allReportActions) => { + Onyx.disconnect(connectionID); + return resolve(allReportActions); + }, + }); + }); +} + +function getDeprecatedPersonalDetailsFromOnyx() { + return new Promise((resolve) => { + const connectionID = Onyx.connect({ + key: DEPRECATED_ONYX_KEYS.PERSONAL_DETAILS, + callback: (allPersonalDetails) => { + Onyx.disconnect(connectionID); + return resolve(allPersonalDetails); + }, + }); + }); +} + +function getDeprecatedPolicyMemberListFromOnyx() { + return new Promise((resolve) => { + const connectionID = Onyx.connect({ + key: DEPRECATED_ONYX_KEYS.COLLECTION.POLICY_MEMBER_LIST, + waitForCollectionCallback: true, + callback: (allPolicyMembers) => { + Onyx.disconnect(connectionID); + return resolve(allPolicyMembers); + }, + }); + }); +} + /** * Migrate Onyx data to hide emails where necessary. * * - personalDetails -> personalDetailsList * - Key by accountID instead of email - * - Must check if users are "known" or not, and include their contact info iff they are "known" + * - Must check if users are "known" or not, and include their contact info if and only if they are "known" * - policyMemberList_ -> policyMember_ * - Key by accountID instead of email * - reportAction_ - * - originalMessage.oldLogin -> originalMessage.oldAccountID - * - originalMessage.newLogin -> originalMessage.newAccountID + * - oldLogin -> oldAccountID + * - newLogin -> newAccountID * - originalMessage.participants -> originalMessage.participantAccountIDs * - actorEmail -> actorAccountID * - childManagerEmail -> childManagerAccountID @@ -19,14 +84,250 @@ * - ownerEmail -> ownerAccountID * - managerEmail -> managerID * - lastActorEmail -> lastActorAccountID - * - participants -> participantAccountID - * - * - User A "knows" user B if any of the following are true: - * - They share at least one non-public workspace room or domain room - * - User B has ever sent user A a message in a DM or group DM. + * - participants -> participantAccountIDs * * @returns {Promise} */ export default function () { - return Promise.resolve(); + return Promise.all([getReportsFromOnyx(), getReportActionsFromOnyx(), getDeprecatedPersonalDetailsFromOnyx(), getDeprecatedPolicyMemberListFromOnyx()]).then( + ([oldReports, oldReportActions, oldPersonalDetails, oldPolicyMemberLists]) => { + /** + * User A "knows" user B if any of the following are true: + * - They share at least one non-public workspace room or domain room + * - User B has ever sent user A a message in a DM or group DM. + * + * @param {String} login + * @returns {Boolean} + */ + function isLoginOfKnownUser(login) { + return _.some(oldReports, (report) => { + if (!report) { + return false; + } + + if (ReportUtils.isPublicRoom(report)) { + return false; + } + + const participants = report.participants; + if (!_.has(participants, login)) { + return false; + } + + // If this is true, it means that the user shares a non-public room, which means they're on the same workspace. + if (ReportUtils.isChatRoom(report)) { + return true; + } + + // It might be possible that the user is a member of the same policy, but are not in any overlapping private rooms. + // So we check the policy member list as well. + const isMemberOfSamePolicy = _.some(oldPolicyMemberLists, (memberList) => _.has(memberList, login)); + if (isMemberOfSamePolicy) { + return true; + } + + const reportActionsForReport = oldReportActions[`${ONYXKEYS.COLLECTION.REPORT_ACTIONS}${report.reportID}`]; + return _.some(reportActionsForReport, (reportAction) => reportAction.actorEmail === login); + }); + } + + // Migrate personalDetails => personalDetailsList + const newPersonalDetailsList = {}; + _.each(oldPersonalDetails, (value, login) => { + if (!value) { + return; + } + + let newPersonalDetailsForUser; + const isKnownUser = isLoginOfKnownUser(login); + if (!isKnownUser) { + newPersonalDetailsForUser = _.omit(value, ['login', 'phoneNumber']); + newPersonalDetailsForUser.displayName = ''; + } else { + newPersonalDetailsForUser = value; + } + + newPersonalDetailsList[value.accountID] = newPersonalDetailsForUser; + }); + + // Migrate policyMemberList_ to policyMember_ + const newPolicyMemberLists = {}; + _.each(oldPolicyMemberLists, (value, policyKey) => { + const policyID = policyKey.substr(DEPRECATED_ONYX_KEYS.COLLECTION.POLICY_MEMBER_LIST.length); + const newMemberList = {}; + _.each(value, (member, login) => { + const accountID = _.get(oldPersonalDetails, [login, 'accountID']); + if (!accountID) { + return; + } + newMemberList[accountID] = member; + }); + newPolicyMemberLists[`${ONYXKEYS.COLLECTION.POLICY_MEMBERS}${policyID}`] = newMemberList; + }); + + // Migrate reportActions + const newReportActions = {}; + _.each(oldReportActions, (reportActionsForReport, onyxKey) => { + if (!reportActionsForReport) { + return; + } + + const newReportActionsForReport = {}; + _.each(reportActionsForReport, (reportAction) => { + if (!reportAction) { + return; + } + + const newReportAction = _.omit(reportAction, [ + 'oldLogin', + 'newLogin', + 'actorEmail', + 'accountEmail', + 'childManagerEmail', + 'originalMessage', + 'whisperedTo', + 'childOldestFourEmails', + ]); + + if (reportAction.oldLogin) { + const oldAccountID = _.get(oldPersonalDetails, [reportAction.oldLogin, 'accountID']); + if (oldAccountID) { + newReportAction.oldAccountID = oldAccountID; + } + } + + if (reportAction.newLogin) { + const newAccountID = _.get(oldPersonalDetails, [reportAction.newLogin, 'accountID']); + if (newAccountID) { + newReportAction.newAccountID = newAccountID; + } + } + + if (reportAction.actorEmail) { + const newAccountID = _.get(oldPersonalDetails, [reportAction.actorEmail, 'accountID']); + if (newAccountID) { + newReportAction.actorAccountID = newAccountID; + } + } + + if (reportAction.accountEmail) { + const accountID = _.get(oldPersonalDetails, [reportAction.accountEmail, 'accountID']); + if (accountID) { + newReportAction.accountID = accountID; + } + } + + if (reportAction.childManagerEmail) { + const childManagerAccountID = _.get(oldPersonalDetails, [reportAction.childManagerEmail, 'accountID']); + if (childManagerAccountID) { + newReportAction.childManagerAccountID = childManagerAccountID; + } + } + + const newOriginalMessage = _.omit(reportAction.originalMessage, ['participants']); + const oldParticipants = _.get(reportAction, ['originalMessage', 'participants'], []); + const newParticipants = []; + _.each(oldParticipants, (login) => { + const accountID = _.get(oldPersonalDetails, [login, 'accountID']); + newParticipants.push(accountID); + }); + newOriginalMessage.participantAccountIDs = newParticipants; + newReportAction.originalMessage = newOriginalMessage; + + if (reportAction.whisperedTo) { + const whisperedToAccountIDs = []; + _.each(reportAction.whisperedTo, (whisperedToLogin) => { + const whisperedToAccountID = _.get(oldPersonalDetails, [whisperedToLogin, 'accountID']); + if (whisperedToAccountID) { + whisperedToAccountIDs.push(whisperedToAccountID); + } + }); + newReportAction.whisperedToAccountIDs = whisperedToAccountIDs; + } + + if (reportAction.childOldestFourEmails) { + const childOldestFourEmails = reportAction.childOldestFourEmails.split(','); + const childOldestFourAccountIDs = []; + _.each(childOldestFourEmails, (login) => { + const accountID = _.get(oldPersonalDetails, [login, 'accountID']); + if (accountID) { + childOldestFourAccountIDs.push(accountID); + } + }); + newReportAction.childOldestFourAccountIDs = childOldestFourAccountIDs.join(','); + } + + newReportActionsForReport[newReportAction.reportActionID] = newReportAction; + }); + + newReportActions[onyxKey] = newReportActionsForReport; + }); + + // Migrate reports + const newReports = {}; + _.each(oldReports, (report, onyxKey) => { + const newReport = _.omit(report, ['ownerEmail', 'managerEmail', 'lastActorEmail', 'participants']); + + if (report.ownerEmail) { + const ownerAccountID = _.get(oldPersonalDetails, [report.ownerEmail, 'accountID']); + if (ownerAccountID) { + newReport.ownerAccountId = ownerAccountID; + } + } + + if (report.managerEmail) { + const managerAccountID = _.get(oldPersonalDetails, [report.managerEmail, 'accountID']); + if (managerAccountID) { + newReport.managerAccountID = managerAccountID; + } + } + + if (report.lastActorEmail) { + const lastActorAccountID = _.get(oldPersonalDetails, [report.lastActorEmail, 'accountID']); + if (lastActorAccountID) { + newReport.lastActorAccountID = lastActorAccountID; + } + } + + if (report.participants) { + const participantAccountIDs = []; + _.each(report.participants, (login) => { + const accountID = _.get(oldPersonalDetails, [login, 'accountID']); + participantAccountIDs.push(accountID); + }); + newReport.participantAccountIDs = participantAccountIDs; + } + + newReports[onyxKey] = newReport; + }); + + const onyxData = { + [DEPRECATED_ONYX_KEYS.PERSONAL_DETAILS]: null, + [ONYXKEYS.PERSONAL_DETAILS_LIST]: newPersonalDetailsList, + }; + + _.each(oldPolicyMemberLists, (value, key) => { + onyxData[key] = null; + }); + _.each(newPolicyMemberLists, (value, key) => { + onyxData[key] = value; + }); + + _.each(oldReports, (value, key) => { + onyxData[key] = null; + }); + _.each(newReports, (value, key) => { + onyxData[key] = value; + }); + + _.each(oldReportActions, (value, key) => { + onyxData[key] = null; + }); + _.each(newReportActions, (value, key) => { + onyxData[key] = value; + }); + + return Onyx.multiSet(onyxData); + }, + ); } From 98f80a304a23e1bb46e80c63565917fb3539e276 Mon Sep 17 00:00:00 2001 From: rory Date: Mon, 19 Jun 2023 14:57:40 -0700 Subject: [PATCH 3/6] Add missing JSDocs --- src/libs/migrations/PersonalDetailsByAccountID.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libs/migrations/PersonalDetailsByAccountID.js b/src/libs/migrations/PersonalDetailsByAccountID.js index 9bc3dc03affe..fdc866ac76f6 100644 --- a/src/libs/migrations/PersonalDetailsByAccountID.js +++ b/src/libs/migrations/PersonalDetailsByAccountID.js @@ -12,6 +12,9 @@ const DEPRECATED_ONYX_KEYS = { }, }; +/** + * @returns {Promise} + */ function getReportsFromOnyx() { return new Promise((resolve) => { const connectionID = Onyx.connect({ @@ -25,6 +28,9 @@ function getReportsFromOnyx() { }); } +/** + * @returns {Promise} + */ function getReportActionsFromOnyx() { return new Promise((resolve) => { const connectionID = Onyx.connect({ @@ -38,6 +44,9 @@ function getReportActionsFromOnyx() { }); } +/** + * @returns {Promise} + */ function getDeprecatedPersonalDetailsFromOnyx() { return new Promise((resolve) => { const connectionID = Onyx.connect({ @@ -50,6 +59,9 @@ function getDeprecatedPersonalDetailsFromOnyx() { }); } +/** + * @returns {Promise} + */ function getDeprecatedPolicyMemberListFromOnyx() { return new Promise((resolve) => { const connectionID = Onyx.connect({ From 5785ac52a0cfcd849ebde173b5a0c92c30fa1a8b Mon Sep 17 00:00:00 2001 From: rory Date: Mon, 19 Jun 2023 15:07:20 -0700 Subject: [PATCH 4/6] Use set for knownUsers instead of inner function --- .../migrations/PersonalDetailsByAccountID.js | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/src/libs/migrations/PersonalDetailsByAccountID.js b/src/libs/migrations/PersonalDetailsByAccountID.js index fdc866ac76f6..4e12cec08f46 100644 --- a/src/libs/migrations/PersonalDetailsByAccountID.js +++ b/src/libs/migrations/PersonalDetailsByAccountID.js @@ -103,45 +103,42 @@ function getDeprecatedPolicyMemberListFromOnyx() { export default function () { return Promise.all([getReportsFromOnyx(), getReportActionsFromOnyx(), getDeprecatedPersonalDetailsFromOnyx(), getDeprecatedPolicyMemberListFromOnyx()]).then( ([oldReports, oldReportActions, oldPersonalDetails, oldPolicyMemberLists]) => { - /** - * User A "knows" user B if any of the following are true: - * - They share at least one non-public workspace room or domain room - * - User B has ever sent user A a message in a DM or group DM. - * - * @param {String} login - * @returns {Boolean} - */ - function isLoginOfKnownUser(login) { - return _.some(oldReports, (report) => { - if (!report) { - return false; - } + const knownUsers = new Set(); + _.each(oldReports, (report) => { + if (!report) { + return; + } - if (ReportUtils.isPublicRoom(report)) { - return false; - } + if (ReportUtils.isPublicRoom(report)) { + return; + } + if (ReportUtils.isChatRoom(report)) { const participants = report.participants; - if (!_.has(participants, login)) { - return false; - } + _.each(participants, (login) => knownUsers.add(login)); + return; + } - // If this is true, it means that the user shares a non-public room, which means they're on the same workspace. - if (ReportUtils.isChatRoom(report)) { - return true; + const reportActionsForReport = oldReportActions[`${ONYXKEYS.COLLECTION.REPORT_ACTIONS}${report.reportID}`]; + _.each(reportActionsForReport, (reportAction) => { + if (!reportAction) { + return; } - // It might be possible that the user is a member of the same policy, but are not in any overlapping private rooms. - // So we check the policy member list as well. - const isMemberOfSamePolicy = _.some(oldPolicyMemberLists, (memberList) => _.has(memberList, login)); - if (isMemberOfSamePolicy) { - return true; + const actorEmail = reportAction.actorEmail; + if (!actorEmail) { + return; } - const reportActionsForReport = oldReportActions[`${ONYXKEYS.COLLECTION.REPORT_ACTIONS}${report.reportID}`]; - return _.some(reportActionsForReport, (reportAction) => reportAction.actorEmail === login); + knownUsers.add(actorEmail); }); - } + }); + + _.each(oldPolicyMemberLists, (memberList) => { + _.each(memberList, (login) => { + knownUsers.add(login); + }); + }); // Migrate personalDetails => personalDetailsList const newPersonalDetailsList = {}; @@ -151,8 +148,7 @@ export default function () { } let newPersonalDetailsForUser; - const isKnownUser = isLoginOfKnownUser(login); - if (!isKnownUser) { + if (!knownUsers.has(login)) { newPersonalDetailsForUser = _.omit(value, ['login', 'phoneNumber']); newPersonalDetailsForUser.displayName = ''; } else { From 6ac8a6d073dd1de63eaafcb54ba2a8be702ab7aa Mon Sep 17 00:00:00 2001 From: rory Date: Mon, 19 Jun 2023 15:13:36 -0700 Subject: [PATCH 5/6] Eliminate unnecessary extra loops --- .../migrations/PersonalDetailsByAccountID.js | 40 ++++--------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/src/libs/migrations/PersonalDetailsByAccountID.js b/src/libs/migrations/PersonalDetailsByAccountID.js index 4e12cec08f46..b2ab45b8819c 100644 --- a/src/libs/migrations/PersonalDetailsByAccountID.js +++ b/src/libs/migrations/PersonalDetailsByAccountID.js @@ -103,6 +103,8 @@ function getDeprecatedPolicyMemberListFromOnyx() { export default function () { return Promise.all([getReportsFromOnyx(), getReportActionsFromOnyx(), getDeprecatedPersonalDetailsFromOnyx(), getDeprecatedPolicyMemberListFromOnyx()]).then( ([oldReports, oldReportActions, oldPersonalDetails, oldPolicyMemberLists]) => { + const onyxData = {}; + const knownUsers = new Set(); _.each(oldReports, (report) => { if (!report) { @@ -157,9 +159,10 @@ export default function () { newPersonalDetailsList[value.accountID] = newPersonalDetailsForUser; }); + onyxData[DEPRECATED_ONYX_KEYS.PERSONAL_DETAILS] = null; + onyxData[ONYXKEYS.PERSONAL_DETAILS_LIST] = newPersonalDetailsList; // Migrate policyMemberList_ to policyMember_ - const newPolicyMemberLists = {}; _.each(oldPolicyMemberLists, (value, policyKey) => { const policyID = policyKey.substr(DEPRECATED_ONYX_KEYS.COLLECTION.POLICY_MEMBER_LIST.length); const newMemberList = {}; @@ -170,11 +173,11 @@ export default function () { } newMemberList[accountID] = member; }); - newPolicyMemberLists[`${ONYXKEYS.COLLECTION.POLICY_MEMBERS}${policyID}`] = newMemberList; + onyxData[policyKey] = null; + onyxData[`${ONYXKEYS.COLLECTION.POLICY_MEMBERS}${policyID}`] = newMemberList; }); // Migrate reportActions - const newReportActions = {}; _.each(oldReportActions, (reportActionsForReport, onyxKey) => { if (!reportActionsForReport) { return; @@ -268,11 +271,10 @@ export default function () { newReportActionsForReport[newReportAction.reportActionID] = newReportAction; }); - newReportActions[onyxKey] = newReportActionsForReport; + onyxData[onyxKey] = newReportActionsForReport; }); // Migrate reports - const newReports = {}; _.each(oldReports, (report, onyxKey) => { const newReport = _.omit(report, ['ownerEmail', 'managerEmail', 'lastActorEmail', 'participants']); @@ -306,33 +308,7 @@ export default function () { newReport.participantAccountIDs = participantAccountIDs; } - newReports[onyxKey] = newReport; - }); - - const onyxData = { - [DEPRECATED_ONYX_KEYS.PERSONAL_DETAILS]: null, - [ONYXKEYS.PERSONAL_DETAILS_LIST]: newPersonalDetailsList, - }; - - _.each(oldPolicyMemberLists, (value, key) => { - onyxData[key] = null; - }); - _.each(newPolicyMemberLists, (value, key) => { - onyxData[key] = value; - }); - - _.each(oldReports, (value, key) => { - onyxData[key] = null; - }); - _.each(newReports, (value, key) => { - onyxData[key] = value; - }); - - _.each(oldReportActions, (value, key) => { - onyxData[key] = null; - }); - _.each(newReportActions, (value, key) => { - onyxData[key] = value; + onyxData[onyxKey] = newReport; }); return Onyx.multiSet(onyxData); From 05d949a44826938b01e1a2388d910c2ab2792511 Mon Sep 17 00:00:00 2001 From: rory Date: Mon, 19 Jun 2023 15:14:11 -0700 Subject: [PATCH 6/6] Add migration to migrateOnyx --- src/libs/migrateOnyx.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libs/migrateOnyx.js b/src/libs/migrateOnyx.js index fd48ad1fa6eb..9389a9b66fbc 100644 --- a/src/libs/migrateOnyx.js +++ b/src/libs/migrateOnyx.js @@ -7,6 +7,7 @@ import MoveToIndexedDB from './migrations/MoveToIndexedDB'; import RenameExpensifyNewsStatus from './migrations/RenameExpensifyNewsStatus'; import AddLastVisibleActionCreated from './migrations/AddLastVisibleActionCreated'; import KeyReportActionsByReportActionID from './migrations/KeyReportActionsByReportActionID'; +import PersonalDetailsByAccountID from './migrations/PersonalDetailsByAccountID'; export default function () { const startTime = Date.now(); @@ -22,6 +23,7 @@ export default function () { RenameExpensifyNewsStatus, AddLastVisibleActionCreated, KeyReportActionsByReportActionID, + PersonalDetailsByAccountID, ]; // Reduce all promises down to a single promise. All promises run in a linear fashion, waiting for the