From 89c2ff64b98602abe83c2fa992c7978f27171cdb Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Thu, 17 Aug 2023 20:52:11 -0600 Subject: [PATCH 01/79] Add skeleton application methods --- src/CONST.js | 4 ++ src/libs/Middleware/SaveResponseInOnyx.js | 50 ++++++++++++++--------- src/libs/actions/App.js | 32 ++++++++++++++- src/libs/actions/OnyxUpdates.js | 10 ++++- src/libs/actions/User.js | 21 +++++++--- 5 files changed, 88 insertions(+), 29 deletions(-) diff --git a/src/CONST.js b/src/CONST.js index 6920f8fbcc64..dc31da3f37b4 100755 --- a/src/CONST.js +++ b/src/CONST.js @@ -2552,6 +2552,10 @@ const CONST = { NAVIGATE: 'NAVIGATE', }, }, + ONYX_UPDATE_TYPES: { + HTTPS: 'https', + PUSHER: 'pusher', + }, }; export default CONST; diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index b347e45ab61c..895a94f5f9fb 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -16,29 +16,39 @@ function SaveResponseInOnyx(response, request) { } // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server - OnyxUpdates.saveUpdateIDs(Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0)); + OnyxUpdates.saveUpdateIDs( + { + updateType: CONST.ONYX_UPDATE_TYPES.HTTPS, + data: { + request, + response, + }, + }, + Number(responseData.lastUpdateID || 0), + Number(responseData.previousUpdateID || 0), + ); - // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in - // the UI. See https://github.com/Expensify/App/issues/12775 for more info. - const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; + // // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in + // // the UI. See https://github.com/Expensify/App/issues/12775 for more info. + // const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; - // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then - // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained - // in successData/failureData until after the component has received and API data. - const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); + // // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then + // // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained + // // in successData/failureData until after the component has received and API data. + // const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); - return onyxDataUpdatePromise - .then(() => { - // Handle the request's success/failure data (client-side data) - if (responseData.jsonCode === 200 && request.successData) { - return updateHandler(request.successData); - } - if (responseData.jsonCode !== 200 && request.failureData) { - return updateHandler(request.failureData); - } - return Promise.resolve(); - }) - .then(() => responseData); + // return onyxDataUpdatePromise + // .then(() => { + // // Handle the request's success/failure data (client-side data) + // if (responseData.jsonCode === 200 && request.successData) { + // return updateHandler(request.successData); + // } + // if (responseData.jsonCode !== 200 && request.failureData) { + // return updateHandler(request.failureData); + // } + // return Promise.resolve(); + // }) + // .then(() => responseData); }); } diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index d82c1e78fe0c..b33a90ab3fc0 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -234,6 +234,36 @@ function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) { // The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has. // When a gap is detected, the missing updates are fetched from the API. +/** + * @param {Object} data + * @param {Object} data.request + * @param {Object} data.response + */ +function applyHTTPSOnyxUpdates({request, response}) {} + +/** + * @param {Object} data + * @param {Object} data.onyxData + */ +function applyPusherOnyxUpdates({onyxData}) {} + +/** + * @param {Object[]} updateParams + * @param {String} updateParams.type + * @param {Object} updateParams.data + * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.onyxData] Exists if updateParams.type === 'pusher' + */ +function applyOnyxUpdates({type, data}) { + if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { + applyHTTPSOnyxUpdates(data); + } + if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { + applyPusherOnyxUpdates(data); + } +} + // These key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated let lastUpdateIDAppliedToClient = 0; Onyx.connect({ @@ -248,7 +278,7 @@ Onyx.connect({ return; } - const {lastUpdateIDFromServer, previousUpdateIDFromServer} = val; + const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index e582016f0109..96cee3a228bc 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -2,11 +2,16 @@ import Onyx from 'react-native-onyx'; import ONYXKEYS from '../../ONYXKEYS'; /** - * + * @param {Object[]} updateParams + * @param {String} updateParams.type + * @param {Object} updateParams.data + * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.onyxData] Exists if updateParams.type === 'pusher' * @param {Number} [lastUpdateID] * @param {Number} [previousUpdateID] */ -function saveUpdateIDs(lastUpdateID = 0, previousUpdateID = 0) { +function saveUpdateIDs(updateParams, lastUpdateID = 0, previousUpdateID = 0) { // Return early if there were no updateIDs if (!lastUpdateID) { return; @@ -15,6 +20,7 @@ function saveUpdateIDs(lastUpdateID = 0, previousUpdateID = 0) { Onyx.merge(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, { lastUpdateIDFromServer: lastUpdateID, previousUpdateIDFromServer: previousUpdateID, + updateParams, }); } diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index 9648b77220ac..92c0e805017d 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -552,13 +552,22 @@ function subscribeToUserEvents() { // lastUpdateID, previousUpdateID and updates if (_.isArray(pushJSON)) { updates = pushJSON; - } else { - updates = pushJSON.updates; - OnyxUpdates.saveUpdateIDs(Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0)); + _.each(updates, (multipleEvent) => { + PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); + }); + return; } - _.each(updates, (multipleEvent) => { - PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); - }); + + OnyxUpdates.saveUpdateIDs( + { + updateType: CONST.ONYX_UPDATE_TYPES.PUSHER, + data: { + onyxUpdates: pushJSON.updates, + }, + }, + Number(pushJSON.lastUpdateID || 0), + Number(pushJSON.previousUpdateID || 0), + ); }); // Handles Onyx updates coming from Pusher through the mega multipleEvents. From 424f9e9dc2cff25644a929dbaed88bf63397442f Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Thu, 17 Aug 2023 21:07:18 -0600 Subject: [PATCH 02/79] Build update application methods --- src/libs/Middleware/SaveResponseInOnyx.js | 26 +------------ src/libs/actions/App.js | 45 ++++++++++++++++++++--- src/libs/actions/OnyxUpdates.js | 4 +- src/libs/actions/User.js | 2 +- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 895a94f5f9fb..695b050610e6 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -1,6 +1,4 @@ -import Onyx from 'react-native-onyx'; import CONST from '../../CONST'; -import * as QueuedOnyxUpdates from '../actions/QueuedOnyxUpdates'; import * as OnyxUpdates from '../actions/OnyxUpdates'; /** @@ -21,34 +19,14 @@ function SaveResponseInOnyx(response, request) { updateType: CONST.ONYX_UPDATE_TYPES.HTTPS, data: { request, - response, + responseData, }, }, Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0), ); - // // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in - // // the UI. See https://github.com/Expensify/App/issues/12775 for more info. - // const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; - - // // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then - // // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained - // // in successData/failureData until after the component has received and API data. - // const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); - - // return onyxDataUpdatePromise - // .then(() => { - // // Handle the request's success/failure data (client-side data) - // if (responseData.jsonCode === 200 && request.successData) { - // return updateHandler(request.successData); - // } - // if (responseData.jsonCode !== 200 && request.failureData) { - // return updateHandler(request.failureData); - // } - // return Promise.resolve(); - // }) - // .then(() => responseData); + return responseData; }); } diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index b33a90ab3fc0..c4114e05d100 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -19,6 +19,8 @@ import * as ReportActionsUtils from '../ReportActionsUtils'; import Timing from './Timing'; import * as Browser from '../Browser'; import * as SequentialQueue from '../Network/SequentialQueue'; +import PusherUtils from '../PusherUtils'; +import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; let currentUserAccountID; let currentUserEmail; @@ -237,15 +239,41 @@ function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) { /** * @param {Object} data * @param {Object} data.request - * @param {Object} data.response + * @param {Object} data.responseData */ -function applyHTTPSOnyxUpdates({request, response}) {} +function applyHTTPSOnyxUpdates({request, responseData}) { + // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in + // the UI. See https://github.com/Expensify/App/issues/12775 for more info. + const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; + + // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then + // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained + // in successData/failureData until after the component has received and API data. + const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); + + onyxDataUpdatePromise + .then(() => { + // Handle the request's success/failure data (client-side data) + if (responseData.jsonCode === 200 && request.successData) { + return updateHandler(request.successData); + } + if (responseData.jsonCode !== 200 && request.failureData) { + return updateHandler(request.failureData); + } + return Promise.resolve(); + }) + .then(() => responseData); +} /** * @param {Object} data - * @param {Object} data.onyxData + * @param {Object} data.multipleEvents */ -function applyPusherOnyxUpdates({onyxData}) {} +function applyPusherOnyxUpdates({multipleEvents}) { + _.each(multipleEvents, (multipleEvent) => { + PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); + }); +} /** * @param {Object[]} updateParams @@ -253,7 +281,7 @@ function applyPusherOnyxUpdates({onyxData}) {} * @param {Object} updateParams.data * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.onyxData] Exists if updateParams.type === 'pusher' + * @param {Object} [updateParams.data.multipleEvents] Exists if updateParams.type === 'pusher' */ function applyOnyxUpdates({type, data}) { if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { @@ -293,7 +321,12 @@ Onyx.connect({ lastUpdateIDAppliedToClient, }); SequentialQueue.pause(); - getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(SequentialQueue.unpause); + getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { + applyOnyxUpdates(updateParams); + SequentialQueue.unpause(); + }); + } else { + applyOnyxUpdates(updateParams); } if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 96cee3a228bc..6e8dffc1b047 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -6,8 +6,8 @@ import ONYXKEYS from '../../ONYXKEYS'; * @param {String} updateParams.type * @param {Object} updateParams.data * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.onyxData] Exists if updateParams.type === 'pusher' + * @param {Object} [updateParams.data.responseData] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.multipleEvents] Exists if updateParams.type === 'pusher' * @param {Number} [lastUpdateID] * @param {Number} [previousUpdateID] */ diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index 92c0e805017d..30d5af52decd 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -562,7 +562,7 @@ function subscribeToUserEvents() { { updateType: CONST.ONYX_UPDATE_TYPES.PUSHER, data: { - onyxUpdates: pushJSON.updates, + multipleEvents: pushJSON.updates, }, }, Number(pushJSON.lastUpdateID || 0), From 59fd99f7f57bcecdde9968b03b381168c7d36cd9 Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Thu, 17 Aug 2023 21:32:29 -0600 Subject: [PATCH 03/79] Use a promise to know when updates are done --- src/libs/actions/App.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index c4114e05d100..3ae191390014 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -240,6 +240,7 @@ function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) { * @param {Object} data * @param {Object} data.request * @param {Object} data.responseData + * @returns {Promise} */ function applyHTTPSOnyxUpdates({request, responseData}) { // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in @@ -251,7 +252,7 @@ function applyHTTPSOnyxUpdates({request, responseData}) { // in successData/failureData until after the component has received and API data. const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); - onyxDataUpdatePromise + return onyxDataUpdatePromise .then(() => { // Handle the request's success/failure data (client-side data) if (responseData.jsonCode === 200 && request.successData) { @@ -270,6 +271,7 @@ function applyHTTPSOnyxUpdates({request, responseData}) { * @param {Object} data.multipleEvents */ function applyPusherOnyxUpdates({multipleEvents}) { + console.log('timxxx', 1, multipleEvents); _.each(multipleEvents, (multipleEvent) => { PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); }); @@ -282,13 +284,15 @@ function applyPusherOnyxUpdates({multipleEvents}) { * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.multipleEvents] Exists if updateParams.type === 'pusher' + * @returns {Promise} */ function applyOnyxUpdates({type, data}) { if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { - applyHTTPSOnyxUpdates(data); + return applyHTTPSOnyxUpdates(data); } if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { applyPusherOnyxUpdates(data); + return new Promise().resolve(); } } @@ -322,8 +326,7 @@ Onyx.connect({ }); SequentialQueue.pause(); getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { - applyOnyxUpdates(updateParams); - SequentialQueue.unpause(); + applyOnyxUpdates(updateParams).then(SequentialQueue.unpause); }); } else { applyOnyxUpdates(updateParams); From ea1ebe33913ea621f75a85d407354bc0cbe15d34 Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Thu, 17 Aug 2023 21:48:20 -0600 Subject: [PATCH 04/79] Remove debug --- src/libs/actions/App.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index 3ae191390014..a0e4289ddb11 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -271,7 +271,6 @@ function applyHTTPSOnyxUpdates({request, responseData}) { * @param {Object} data.multipleEvents */ function applyPusherOnyxUpdates({multipleEvents}) { - console.log('timxxx', 1, multipleEvents); _.each(multipleEvents, (multipleEvent) => { PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); }); From b9ecfc78f2497c8c10abfc0a25a11a83134e48bb Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Sat, 19 Aug 2023 13:31:10 -0600 Subject: [PATCH 05/79] Refactor the location of code --- src/libs/actions/App.js | 109 +---------------------- src/libs/actions/OnyxUpdatesManager.js | 116 +++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 108 deletions(-) create mode 100644 src/libs/actions/OnyxUpdatesManager.js diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index 098e99063fc7..56a313e81c0b 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -18,9 +18,6 @@ import * as Session from './Session'; import * as ReportActionsUtils from '../ReportActionsUtils'; import Timing from './Timing'; import * as Browser from '../Browser'; -import * as SequentialQueue from '../Network/SequentialQueue'; -import PusherUtils from '../PusherUtils'; -import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; let currentUserAccountID; let currentUserEmail; @@ -233,111 +230,6 @@ function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) { ); } -// The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has. -// When a gap is detected, the missing updates are fetched from the API. - -/** - * @param {Object} data - * @param {Object} data.request - * @param {Object} data.responseData - * @returns {Promise} - */ -function applyHTTPSOnyxUpdates({request, responseData}) { - // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in - // the UI. See https://github.com/Expensify/App/issues/12775 for more info. - const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; - - // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then - // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained - // in successData/failureData until after the component has received and API data. - const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); - - return onyxDataUpdatePromise - .then(() => { - // Handle the request's success/failure data (client-side data) - if (responseData.jsonCode === 200 && request.successData) { - return updateHandler(request.successData); - } - if (responseData.jsonCode !== 200 && request.failureData) { - return updateHandler(request.failureData); - } - return Promise.resolve(); - }) - .then(() => responseData); -} - -/** - * @param {Object} data - * @param {Object} data.multipleEvents - */ -function applyPusherOnyxUpdates({multipleEvents}) { - _.each(multipleEvents, (multipleEvent) => { - PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); - }); -} - -/** - * @param {Object[]} updateParams - * @param {String} updateParams.type - * @param {Object} updateParams.data - * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.multipleEvents] Exists if updateParams.type === 'pusher' - * @returns {Promise} - */ -function applyOnyxUpdates({type, data}) { - if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { - return applyHTTPSOnyxUpdates(data); - } - if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { - applyPusherOnyxUpdates(data); - return new Promise().resolve(); - } -} - -// These key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated -let lastUpdateIDAppliedToClient = 0; -Onyx.connect({ - key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, - callback: (val) => (lastUpdateIDAppliedToClient = val), -}); - -Onyx.connect({ - key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, - callback: (val) => { - if (!val) { - return; - } - - const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; - console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); - console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); - console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); - - // If the previous update from the server does not match the last update the client got, then the client is missing some updates. - // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. - if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { - console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates'); - Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { - lastUpdateIDFromServer, - previousUpdateIDFromServer, - lastUpdateIDAppliedToClient, - }); - SequentialQueue.pause(); - getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { - applyOnyxUpdates(updateParams).then(SequentialQueue.unpause); - }); - } else { - applyOnyxUpdates(updateParams); - } - - if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { - // Update this value so that it matches what was just received from the server - Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); - } - }, -}); - /** * This promise is used so that deeplink component know when a transition is end. * This is necessary because we want to begin deeplink redirection after the transition is end. @@ -542,4 +434,5 @@ export { beginDeepLinkRedirect, beginDeepLinkRedirectAfterTransition, createWorkspaceAndNavigateToIt, + getMissingOnyxUpdates, }; diff --git a/src/libs/actions/OnyxUpdatesManager.js b/src/libs/actions/OnyxUpdatesManager.js new file mode 100644 index 000000000000..cdfa610cd146 --- /dev/null +++ b/src/libs/actions/OnyxUpdatesManager.js @@ -0,0 +1,116 @@ +import _ from 'underscore'; +import Onyx from 'react-native-onyx'; +import ONYXKEYS from '../../ONYXKEYS'; +import CONST from '../../CONST'; +import Log from '../Log'; +import * as SequentialQueue from '../Network/SequentialQueue'; +import PusherUtils from '../PusherUtils'; +import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; +import * as App from './App'; + +// The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has. +// When a gap is detected, the missing updates are fetched from the API. + +/** + * @param {Object} data + * @param {Object} data.request + * @param {Object} data.responseData + * @returns {Promise} + */ +function applyHTTPSOnyxUpdates({request, responseData}) { + // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in + // the UI. See https://github.com/Expensify/App/issues/12775 for more info. + const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; + + // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then + // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained + // in successData/failureData until after the component has received and API data. + const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); + + return onyxDataUpdatePromise + .then(() => { + // Handle the request's success/failure data (client-side data) + if (responseData.jsonCode === 200 && request.successData) { + return updateHandler(request.successData); + } + if (responseData.jsonCode !== 200 && request.failureData) { + return updateHandler(request.failureData); + } + return Promise.resolve(); + }) + .then(() => responseData); +} + +/** + * @param {Object} data + * @param {Object} data.multipleEvents + */ +function applyPusherOnyxUpdates({multipleEvents}) { + _.each(multipleEvents, (multipleEvent) => { + PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); + }); +} + +/** + * @param {Object[]} updateParams + * @param {String} updateParams.type + * @param {Object} updateParams.data + * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.multipleEvents] Exists if updateParams.type === 'pusher' + * @returns {Promise} + */ +function applyOnyxUpdates({type, data}) { + if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { + return applyHTTPSOnyxUpdates(data); + } + if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { + applyPusherOnyxUpdates(data); + return new Promise().resolve(); + } +} + +// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated +let lastUpdateIDAppliedToClient = 0; +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, + callback: (val) => (lastUpdateIDAppliedToClient = val), +}); + +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, + callback: (val) => { + if (!val) { + return; + } + + const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; + console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); + console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); + console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); + + // If the previous update from the server does not match the last update the client got, then the client is missing some updates. + // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. + if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates'); + Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { + lastUpdateIDFromServer, + previousUpdateIDFromServer, + lastUpdateIDAppliedToClient, + }); + + SequentialQueue.pause(); + + App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { + applyOnyxUpdates(updateParams).then(SequentialQueue.unpause); + }); + } else { + applyOnyxUpdates(updateParams); + } + + if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { + // Update this value so that it matches what was just received from the server + Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); + } + }, +}); From 842ec64cc4167f68d0d679dbb809875b59428989 Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Sat, 19 Aug 2023 13:32:57 -0600 Subject: [PATCH 06/79] Start the update manager from authscreens --- .../Navigation/AppNavigator/AuthScreens.js | 2 + src/libs/actions/OnyxUpdatesManager.js | 66 ++++++++++--------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/src/libs/Navigation/AppNavigator/AuthScreens.js b/src/libs/Navigation/AppNavigator/AuthScreens.js index 7317306cdbe6..8078d1d8db84 100644 --- a/src/libs/Navigation/AppNavigator/AuthScreens.js +++ b/src/libs/Navigation/AppNavigator/AuthScreens.js @@ -35,6 +35,7 @@ import DesktopSignInRedirectPage from '../../../pages/signin/DesktopSignInRedire import styles from '../../../styles/styles'; import * as SessionUtils from '../../SessionUtils'; import getNavigationModalCardStyle from '../../../styles/getNavigationModalCardStyles'; +import OnyxUpdatesManager from '../../actions/OnyxUpdatesManager'; let timezone; let currentAccountID; @@ -125,6 +126,7 @@ class AuthScreens extends React.Component { } componentDidMount() { + OnyxUpdatesManager(); NetworkConnection.listenForReconnect(); NetworkConnection.onReconnect(() => App.reconnectApp(this.props.lastUpdateIDAppliedToClient)); PusherConnectionManager.init(); diff --git a/src/libs/actions/OnyxUpdatesManager.js b/src/libs/actions/OnyxUpdatesManager.js index cdfa610cd146..38a021ae4540 100644 --- a/src/libs/actions/OnyxUpdatesManager.js +++ b/src/libs/actions/OnyxUpdatesManager.js @@ -77,40 +77,42 @@ Onyx.connect({ callback: (val) => (lastUpdateIDAppliedToClient = val), }); -Onyx.connect({ - key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, - callback: (val) => { - if (!val) { - return; - } +export default () => { + Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, + callback: (val) => { + if (!val) { + return; + } - const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; - console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); - console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); - console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); + const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; + console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); + console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); + console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); - // If the previous update from the server does not match the last update the client got, then the client is missing some updates. - // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. - if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { - console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates'); - Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { - lastUpdateIDFromServer, - previousUpdateIDFromServer, - lastUpdateIDAppliedToClient, - }); + // If the previous update from the server does not match the last update the client got, then the client is missing some updates. + // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. + if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates'); + Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { + lastUpdateIDFromServer, + previousUpdateIDFromServer, + lastUpdateIDAppliedToClient, + }); - SequentialQueue.pause(); + SequentialQueue.pause(); - App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { - applyOnyxUpdates(updateParams).then(SequentialQueue.unpause); - }); - } else { - applyOnyxUpdates(updateParams); - } + App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { + applyOnyxUpdates(updateParams).then(SequentialQueue.unpause); + }); + } else { + applyOnyxUpdates(updateParams); + } - if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { - // Update this value so that it matches what was just received from the server - Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); - } - }, -}); + if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { + // Update this value so that it matches what was just received from the server + Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); + } + }, + }); +}; From ba59617ccd7575fcab247daf2c4b6d0f405cdee4 Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Sat, 19 Aug 2023 13:34:13 -0600 Subject: [PATCH 07/79] Improve logging --- src/libs/actions/OnyxUpdatesManager.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libs/actions/OnyxUpdatesManager.js b/src/libs/actions/OnyxUpdatesManager.js index 38a021ae4540..85120cdbd83b 100644 --- a/src/libs/actions/OnyxUpdatesManager.js +++ b/src/libs/actions/OnyxUpdatesManager.js @@ -78,6 +78,7 @@ Onyx.connect({ }); export default () => { + console.debug('[OnyxUpdateManager] Listening for updates from the server'); Onyx.connect({ key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, callback: (val) => { @@ -86,14 +87,14 @@ export default () => { } const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; - console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); - console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); - console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); + console.debug('[OnyxUpdateManager] Received lastUpdateID from server', lastUpdateIDFromServer); + console.debug('[OnyxUpdateManager] Received previousUpdateID from server', previousUpdateIDFromServer); + console.debug('[OnyxUpdateManager] Last update ID applied to the client', lastUpdateIDAppliedToClient); // If the previous update from the server does not match the last update the client got, then the client is missing some updates. // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { - console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates'); + console.debug('[OnyxUpdateManager] Gap detected in update IDs so fetching incremental updates'); Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { lastUpdateIDFromServer, previousUpdateIDFromServer, From 427043eb2e6fdb2ce7b7aa959d7d551937d626fb Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Sat, 19 Aug 2023 13:35:20 -0600 Subject: [PATCH 08/79] Rename method --- src/libs/Middleware/SaveResponseInOnyx.js | 2 +- src/libs/actions/OnyxUpdates.js | 4 ++-- src/libs/actions/User.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index db978b990751..c3bc66e86fbb 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -38,7 +38,7 @@ function SaveResponseInOnyx(response, request) { }); // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server - OnyxUpdates.saveUpdateIDs( + OnyxUpdates.saveUpdateInformation( { updateType: CONST.ONYX_UPDATE_TYPES.HTTPS, data: { diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 6e8dffc1b047..48ec66280fdd 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -11,7 +11,7 @@ import ONYXKEYS from '../../ONYXKEYS'; * @param {Number} [lastUpdateID] * @param {Number} [previousUpdateID] */ -function saveUpdateIDs(updateParams, lastUpdateID = 0, previousUpdateID = 0) { +function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID = 0) { // Return early if there were no updateIDs if (!lastUpdateID) { return; @@ -25,4 +25,4 @@ function saveUpdateIDs(updateParams, lastUpdateID = 0, previousUpdateID = 0) { } // eslint-disable-next-line import/prefer-default-export -export {saveUpdateIDs}; +export {saveUpdateInformation}; diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index bd018f1d3e15..c83bd3f37891 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -563,7 +563,7 @@ function subscribeToUserEvents() { return; } - OnyxUpdates.saveUpdateIDs( + OnyxUpdates.saveUpdateInformation( { updateType: CONST.ONYX_UPDATE_TYPES.PUSHER, data: { From ebe24bf6e1895536d27cd830c3951bd4e164f4fc Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Sat, 19 Aug 2023 13:55:26 -0600 Subject: [PATCH 09/79] Rename file and add promises to pusher events --- .../Navigation/AppNavigator/AuthScreens.js | 4 +- src/libs/PusherUtils.js | 5 ++- ...UpdatesManager.js => OnyxUpdateManager.js} | 45 ++++++++++++++++--- src/libs/actions/User.js | 12 +++-- 4 files changed, 52 insertions(+), 14 deletions(-) rename src/libs/actions/{OnyxUpdatesManager.js => OnyxUpdateManager.js} (69%) diff --git a/src/libs/Navigation/AppNavigator/AuthScreens.js b/src/libs/Navigation/AppNavigator/AuthScreens.js index 8078d1d8db84..dce177fe5d97 100644 --- a/src/libs/Navigation/AppNavigator/AuthScreens.js +++ b/src/libs/Navigation/AppNavigator/AuthScreens.js @@ -35,7 +35,7 @@ import DesktopSignInRedirectPage from '../../../pages/signin/DesktopSignInRedire import styles from '../../../styles/styles'; import * as SessionUtils from '../../SessionUtils'; import getNavigationModalCardStyle from '../../../styles/getNavigationModalCardStyles'; -import OnyxUpdatesManager from '../../actions/OnyxUpdatesManager'; +import OnyxUpdateManager from '../../actions/OnyxUpdateManager'; let timezone; let currentAccountID; @@ -126,7 +126,7 @@ class AuthScreens extends React.Component { } componentDidMount() { - OnyxUpdatesManager(); + OnyxUpdateManager(); NetworkConnection.listenForReconnect(); NetworkConnection.onReconnect(() => App.reconnectApp(this.props.lastUpdateIDAppliedToClient)); PusherConnectionManager.init(); diff --git a/src/libs/PusherUtils.js b/src/libs/PusherUtils.js index 9d84bd4012fe..7fdd12cf6296 100644 --- a/src/libs/PusherUtils.js +++ b/src/libs/PusherUtils.js @@ -18,12 +18,13 @@ function subscribeToMultiEvent(eventType, callback) { /** * @param {String} eventType * @param {Mixed} data + * @returns {Promise} */ function triggerMultiEventHandler(eventType, data) { if (!multiEventCallbackMapping[eventType]) { - return; + return new Promise().resolve(); } - multiEventCallbackMapping[eventType](data); + return multiEventCallbackMapping[eventType](data); } /** diff --git a/src/libs/actions/OnyxUpdatesManager.js b/src/libs/actions/OnyxUpdateManager.js similarity index 69% rename from src/libs/actions/OnyxUpdatesManager.js rename to src/libs/actions/OnyxUpdateManager.js index 85120cdbd83b..de9a928f6ee0 100644 --- a/src/libs/actions/OnyxUpdatesManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -38,16 +38,27 @@ function applyHTTPSOnyxUpdates({request, responseData}) { } return Promise.resolve(); }) - .then(() => responseData); + .then(() => { + console.debug('[OnyxUpdateManager] Done applying HTTPS update'); + }); } /** * @param {Object} data * @param {Object} data.multipleEvents + * @returns {Promise} */ function applyPusherOnyxUpdates({multipleEvents}) { - _.each(multipleEvents, (multipleEvent) => { - PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); + const pusherEventPromises = _.reduce( + multipleEvents, + (result, multipleEvent) => { + result.push(PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data)); + return result; + }, + [], + ); + return Promise.all(pusherEventPromises).then(() => { + console.debug('[OnyxUpdateManager] Done applying Pusher update'); }); } @@ -65,8 +76,7 @@ function applyOnyxUpdates({type, data}) { return applyHTTPSOnyxUpdates(data); } if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { - applyPusherOnyxUpdates(data); - return new Promise().resolve(); + return applyPusherOnyxUpdates(data); } } @@ -91,6 +101,14 @@ export default () => { console.debug('[OnyxUpdateManager] Received previousUpdateID from server', previousUpdateIDFromServer); console.debug('[OnyxUpdateManager] Last update ID applied to the client', lastUpdateIDAppliedToClient); + // This can happen when a user has just started getting reliable updates from the server but they haven't + // had an OpenApp or ReconnectApp call yet. This can result in never getting reliable updates because + // lastUpdateIDAppliedToClient will always be null. For this case, reconnectApp() will be triggered for them + // to kick start the reliable updates. + if (!lastUpdateIDAppliedToClient && previousUpdateIDFromServer > 0) { + App.reconnectApp(); + } + // If the previous update from the server does not match the last update the client got, then the client is missing some updates. // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { @@ -101,10 +119,25 @@ export default () => { lastUpdateIDAppliedToClient, }); + // Pause the sequential queue while the missing Onyx updates are fetched from the server. This is important + // so that the updates are applied in their correct and specific order. If this queue was not paused, then + // there would be a lot of onyx data being applied while we are fetching the missing updates and that would + // put them all out of order. SequentialQueue.pause(); App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { - applyOnyxUpdates(updateParams).then(SequentialQueue.unpause); + console.debug('[OnyxUpdateManager] Done Getting missing onyx updates'); + + // The onyx update from the initial request could have been either from HTTPS or Pusher. + // Now that the missing onyx updates have been applied, we can apply the original onyxUpdates from + // the API request. + applyOnyxUpdates(updateParams).then(() => { + console.debug('[OnyxUpdateManager] Done applying all updates'); + + // Finally, the missing updates were applied, the original update was applied, and now the + // sequential queue is free to continue. + SequentialQueue.unpause(); + }); }); } else { applyOnyxUpdates(updateParams); diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index c83bd3f37891..8822afd9ec08 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -576,17 +576,21 @@ function subscribeToUserEvents() { }); // Handles Onyx updates coming from Pusher through the mega multipleEvents. - PusherUtils.subscribeToMultiEvent(Pusher.TYPE.MULTIPLE_EVENT_TYPE.ONYX_API_UPDATE, (pushJSON) => { + PusherUtils.subscribeToMultiEvent(Pusher.TYPE.MULTIPLE_EVENT_TYPE.ONYX_API_UPDATE, (pushJSON) => SequentialQueue.getCurrentRequest().then(() => { // If we don't have the currentUserAccountID (user is logged out) we don't want to update Onyx with data from Pusher if (!currentUserAccountID) { return; } - Onyx.update(pushJSON); + const onyxUpdatePromise = Onyx.update(pushJSON); triggerNotifications(pushJSON); - }); - }); + + // Return a promise when Onyx is done updating so that the OnyxUpdatesManager can properly apply all + // the onyx updates in order + return onyxUpdatePromise; + }), + ); } /** From ff9a773ae32ff1b19530f714280fadf9b0525d69 Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Sat, 19 Aug 2023 13:57:53 -0600 Subject: [PATCH 10/79] Improve logs --- src/libs/actions/OnyxUpdateManager.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index de9a928f6ee0..ecd81ffbddcf 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -106,13 +106,14 @@ export default () => { // lastUpdateIDAppliedToClient will always be null. For this case, reconnectApp() will be triggered for them // to kick start the reliable updates. if (!lastUpdateIDAppliedToClient && previousUpdateIDFromServer > 0) { + console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); App.reconnectApp(); } // If the previous update from the server does not match the last update the client got, then the client is missing some updates. // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { - console.debug('[OnyxUpdateManager] Gap detected in update IDs so fetching incremental updates'); + console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { lastUpdateIDFromServer, previousUpdateIDFromServer, @@ -140,6 +141,7 @@ export default () => { }); }); } else { + console.debug(`[OnyxUpdateManager] Client is in sync with the server`); applyOnyxUpdates(updateParams); } From 528062b1c45cc94531039e63dccd010d84bcc494 Mon Sep 17 00:00:00 2001 From: Tim Golen Date: Sat, 19 Aug 2023 14:31:40 -0600 Subject: [PATCH 11/79] Fix the updating and improve comments --- src/App.js | 2 ++ src/libs/Middleware/SaveResponseInOnyx.js | 2 +- src/libs/Navigation/AppNavigator/AuthScreens.js | 2 -- src/libs/actions/OnyxUpdateManager.js | 16 +++++++++++----- src/libs/actions/OnyxUpdates.js | 3 ++- src/libs/actions/User.js | 2 +- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/App.js b/src/App.js index d8faa911f86b..f072f46961ef 100644 --- a/src/App.js +++ b/src/App.js @@ -23,6 +23,7 @@ import ThemeStylesProvider from './styles/ThemeStylesProvider'; import {CurrentReportIDContextProvider} from './components/withCurrentReportID'; import {EnvironmentProvider} from './components/withEnvironment'; import * as Session from './libs/actions/Session'; +import OnyxUpdateManager from './libs/actions/OnyxUpdateManager'; // For easier debugging and development, when we are in web we expose Onyx to the window, so you can more easily set data into Onyx if (window && Environment.isDevelopment()) { @@ -40,6 +41,7 @@ LogBox.ignoreLogs([ const fill = {flex: 1}; function App() { + OnyxUpdateManager(); return ( App.reconnectApp(this.props.lastUpdateIDAppliedToClient)); PusherConnectionManager.init(); diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index ecd81ffbddcf..d08feb8504c7 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -8,8 +8,14 @@ import PusherUtils from '../PusherUtils'; import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; import * as App from './App'; -// The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has. -// When a gap is detected, the missing updates are fetched from the API. +// This file is in charge of looking at the updateIDs coming from the server and comparing them to the last updateID that the client has. +// If the client is behind the server, then we need to pause everything, get the missing updates from the server, apply those updates, +// then restart everything. This will ensure that the client is up-to-date with the server and all the updates have been applied +// in the correct order. +// It's important that this file is separate and not imported by OnyxUpdates.js, so that there are no circular dependencies. Onyx +// is used as a pub/sub mechanism to break out of the circular dependencies. +// The circular dependency happen because this file calls the API GetMissingOnyxUpdates which uses the SaveResponseInOnyx.js file +// (as a middleware). Therefore, SaveResponseInOnyx.js can't import and use this file directly. /** * @param {Object} data @@ -18,6 +24,7 @@ import * as App from './App'; * @returns {Promise} */ function applyHTTPSOnyxUpdates({request, responseData}) { + console.debug('[OnyxUpdateManager] Applying https update'); // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; @@ -49,6 +56,7 @@ function applyHTTPSOnyxUpdates({request, responseData}) { * @returns {Promise} */ function applyPusherOnyxUpdates({multipleEvents}) { + console.debug('[OnyxUpdateManager] Applying pusher update'); const pusherEventPromises = _.reduce( multipleEvents, (result, multipleEvent) => { @@ -72,6 +80,7 @@ function applyPusherOnyxUpdates({multipleEvents}) { * @returns {Promise} */ function applyOnyxUpdates({type, data}) { + console.debug(`[OnyxUpdateManager] Applying update type: ${type}`, data); if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { return applyHTTPSOnyxUpdates(data); } @@ -97,9 +106,6 @@ export default () => { } const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; - console.debug('[OnyxUpdateManager] Received lastUpdateID from server', lastUpdateIDFromServer); - console.debug('[OnyxUpdateManager] Received previousUpdateID from server', previousUpdateIDFromServer); - console.debug('[OnyxUpdateManager] Last update ID applied to the client', lastUpdateIDAppliedToClient); // This can happen when a user has just started getting reliable updates from the server but they haven't // had an OpenApp or ReconnectApp call yet. This can result in never getting reliable updates because diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 48ec66280fdd..bb09015393f4 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -17,7 +17,8 @@ function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID return; } - Onyx.merge(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, { + // Always use set() here so that the updateParams are never merged and always unique to the request that came in + Onyx.set(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, { lastUpdateIDFromServer: lastUpdateID, previousUpdateIDFromServer: previousUpdateID, updateParams, diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index 8822afd9ec08..b5429ae0893d 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -565,7 +565,7 @@ function subscribeToUserEvents() { OnyxUpdates.saveUpdateInformation( { - updateType: CONST.ONYX_UPDATE_TYPES.PUSHER, + type: CONST.ONYX_UPDATE_TYPES.PUSHER, data: { multipleEvents: pushJSON.updates, }, From 7b421695dad00701c1b166386d04333b60085a9f Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 23 Aug 2023 21:21:20 +0200 Subject: [PATCH 12/79] added ANY to format on updates since that's still undefined --- src/ONYXKEYS.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ONYXKEYS.ts b/src/ONYXKEYS.ts index d4d2ab1f90a6..d4cb591c704d 100755 --- a/src/ONYXKEYS.ts +++ b/src/ONYXKEYS.ts @@ -350,7 +350,7 @@ type OnyxValues = { [ONYXKEYS.SELECTED_TAB]: string; [ONYXKEYS.RECEIPT_MODAL]: OnyxTypes.ReceiptModal; [ONYXKEYS.MAPBOX_ACCESS_TOKEN]: OnyxTypes.MapboxAccessToken; - [ONYXKEYS.ONYX_UPDATES_FROM_SERVER]: number; + [ONYXKEYS.ONYX_UPDATES_FROM_SERVER]: any; [ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT]: number; // Collections From 3002c64da3ef4811ce2685af3223bd30f1e994c3 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 23 Aug 2023 21:21:54 +0200 Subject: [PATCH 13/79] adding early return in case of responses without onyxData --- src/libs/Middleware/SaveResponseInOnyx.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index efb38ef19018..24bf60e5cdaf 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -27,6 +27,13 @@ function SaveResponseInOnyx(response, request) { // Supports both the old format and the new format const onyxUpdates = _.isArray(responseData) ? responseData : responseData.onyxData; + + // Sometimes we call requests that are successfull but they don't have any response. Let's return early since + // we don't need to store anything here. + if(!onyxUpdates){ + return; + } + // If there is an OnyxUpdate for using memory only keys, enable them _.find(onyxUpdates, ({key, value}) => { if (key !== ONYXKEYS.IS_USING_MEMORY_ONLY_KEYS || !value) { From c4d40949c3ad2de650a5561956a874de640a1d09 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 23 Aug 2023 21:22:45 +0200 Subject: [PATCH 14/79] updating save function to not return early if missing lastUpdateID --- src/libs/actions/OnyxUpdates.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index bb09015393f4..f5b7adeab23c 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -12,11 +12,6 @@ import ONYXKEYS from '../../ONYXKEYS'; * @param {Number} [previousUpdateID] */ function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID = 0) { - // Return early if there were no updateIDs - if (!lastUpdateID) { - return; - } - // Always use set() here so that the updateParams are never merged and always unique to the request that came in Onyx.set(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, { lastUpdateIDFromServer: lastUpdateID, From fe50fd0ebfa76ea8556bf7a6edf8beff0e556adb Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 23 Aug 2023 21:23:40 +0200 Subject: [PATCH 15/79] adding check on reconnect app after switching to updates beta to not do the request if this is open app --- src/libs/actions/OnyxUpdateManager.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index d08feb8504c7..8003fd09bc95 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -104,6 +104,7 @@ export default () => { if (!val) { return; } + debugger; const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; @@ -111,7 +112,12 @@ export default () => { // had an OpenApp or ReconnectApp call yet. This can result in never getting reliable updates because // lastUpdateIDAppliedToClient will always be null. For this case, reconnectApp() will be triggered for them // to kick start the reliable updates. - if (!lastUpdateIDAppliedToClient && previousUpdateIDFromServer > 0) { + if ( + !lastUpdateIDAppliedToClient + && previousUpdateIDFromServer > 0 + && updateParams.type == CONST.ONYX_UPDATE_TYPES.HTTPS + && updateParams.request.command !== "OpenApp" + ) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); App.reconnectApp(); } From a5e812751a1b87ebee06a9832b7ce6c966f77c79 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 23 Aug 2023 21:25:05 +0200 Subject: [PATCH 16/79] adding reconnect app to the list of methods we shouldn't check --- src/libs/actions/OnyxUpdateManager.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 8003fd09bc95..71b6f0607899 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -104,7 +104,6 @@ export default () => { if (!val) { return; } - debugger; const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; @@ -116,8 +115,10 @@ export default () => { !lastUpdateIDAppliedToClient && previousUpdateIDFromServer > 0 && updateParams.type == CONST.ONYX_UPDATE_TYPES.HTTPS - && updateParams.request.command !== "OpenApp" - ) { + && ( + updateParams.request.command !== "OpenApp" + || updateParams.request.command !== "ReconnectApp" + )) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); App.reconnectApp(); } From ba47492e63aefa135d49954fd43c94fbc51b84d5 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 23 Aug 2023 22:27:53 +0200 Subject: [PATCH 17/79] adding check to prevent us from running both ReconnectApp and GetOnyxUpdates on the same request --- src/libs/actions/OnyxUpdateManager.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 71b6f0607899..3cdcfeda40d0 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -105,27 +105,37 @@ export default () => { return; } + // Since people will have migrations hapening in their accounts while they use the app, we don't want to trigger + // a full ReconnectApp and then trigger a GetOnyxUpdates. Let's use this as a control variable until we enable + // the beta to all users. + let updateEnablesBeta = false; + const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; // This can happen when a user has just started getting reliable updates from the server but they haven't // had an OpenApp or ReconnectApp call yet. This can result in never getting reliable updates because // lastUpdateIDAppliedToClient will always be null. For this case, reconnectApp() will be triggered for them - // to kick start the reliable updates. + // to kick start the reliable updates. We also filter OpenApp and ReconnectApp so we don't create a loop. if ( !lastUpdateIDAppliedToClient && previousUpdateIDFromServer > 0 - && updateParams.type == CONST.ONYX_UPDATE_TYPES.HTTPS && ( - updateParams.request.command !== "OpenApp" - || updateParams.request.command !== "ReconnectApp" + updateParams.type == CONST.ONYX_UPDATE_TYPES.PUSHER + || ( + updateParams.type == CONST.ONYX_UPDATE_TYPES.HTTPS + && ( + updateParams.data.request.command !== "OpenApp" + || updateParams.data.request.command !== "ReconnectApp" + )) )) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); App.reconnectApp(); + updateEnablesBeta = true; } // If the previous update from the server does not match the last update the client got, then the client is missing some updates. // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. - if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + if (!updateEnablesBeta && previousUpdateIDFromServer && (lastUpdateIDAppliedToClient < previousUpdateIDFromServer)) { console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { lastUpdateIDFromServer, From dde0814580ebab0af4e384ffdb705512abeeaf75 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 23 Aug 2023 22:28:51 +0200 Subject: [PATCH 18/79] prettier --- src/libs/Middleware/SaveResponseInOnyx.js | 2 +- src/libs/actions/OnyxUpdateManager.js | 20 +++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 24bf60e5cdaf..13244dd64e36 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -30,7 +30,7 @@ function SaveResponseInOnyx(response, request) { // Sometimes we call requests that are successfull but they don't have any response. Let's return early since // we don't need to store anything here. - if(!onyxUpdates){ + if (!onyxUpdates) { return; } diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 3cdcfeda40d0..039f89e146c3 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -109,7 +109,7 @@ export default () => { // a full ReconnectApp and then trigger a GetOnyxUpdates. Let's use this as a control variable until we enable // the beta to all users. let updateEnablesBeta = false; - + const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; // This can happen when a user has just started getting reliable updates from the server but they haven't @@ -117,17 +117,11 @@ export default () => { // lastUpdateIDAppliedToClient will always be null. For this case, reconnectApp() will be triggered for them // to kick start the reliable updates. We also filter OpenApp and ReconnectApp so we don't create a loop. if ( - !lastUpdateIDAppliedToClient - && previousUpdateIDFromServer > 0 - && ( - updateParams.type == CONST.ONYX_UPDATE_TYPES.PUSHER - || ( - updateParams.type == CONST.ONYX_UPDATE_TYPES.HTTPS - && ( - updateParams.data.request.command !== "OpenApp" - || updateParams.data.request.command !== "ReconnectApp" - )) - )) { + !lastUpdateIDAppliedToClient && + previousUpdateIDFromServer > 0 && + (updateParams.type == CONST.ONYX_UPDATE_TYPES.PUSHER || + (updateParams.type == CONST.ONYX_UPDATE_TYPES.HTTPS && (updateParams.data.request.command !== 'OpenApp' || updateParams.data.request.command !== 'ReconnectApp'))) + ) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); App.reconnectApp(); updateEnablesBeta = true; @@ -135,7 +129,7 @@ export default () => { // If the previous update from the server does not match the last update the client got, then the client is missing some updates. // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. - if (!updateEnablesBeta && previousUpdateIDFromServer && (lastUpdateIDAppliedToClient < previousUpdateIDFromServer)) { + if (!updateEnablesBeta && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { lastUpdateIDFromServer, From 321041a4b27eb80b8402421591a68d2b2e6735d3 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Thu, 24 Aug 2023 18:33:11 +0200 Subject: [PATCH 19/79] adding fields that were missing in the request object --- src/types/onyx/Request.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/types/onyx/Request.ts b/src/types/onyx/Request.ts index e730dfd807fb..cc57c9410e30 100644 --- a/src/types/onyx/Request.ts +++ b/src/types/onyx/Request.ts @@ -1,8 +1,12 @@ +import { OnyxUpdate } from "react-native-onyx"; + type Request = { command?: string; data?: Record; type?: string; shouldUseSecure?: boolean; + successData?: OnyxUpdate[]; + failureData?: OnyxUpdate[]; }; export default Request; From 85bd8ba018fa2b8df89b2cd2d6e68c8b317138eb Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Thu, 24 Aug 2023 18:33:30 +0200 Subject: [PATCH 20/79] adding response object --- src/types/onyx/Response.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/types/onyx/Response.ts diff --git a/src/types/onyx/Response.ts b/src/types/onyx/Response.ts new file mode 100644 index 000000000000..01956adffc87 --- /dev/null +++ b/src/types/onyx/Response.ts @@ -0,0 +1,11 @@ +import { OnyxUpdate } from "react-native-onyx"; + +type Request = { + previousUpdateID?: number; + lastUpdateID?: number; + jsonCode?: number; + onyxData?: OnyxUpdate[]; + requestID?: string; +}; + +export default Response; From 659de586ca7bdf3c1bf7ed414bfc09fadfaa8e45 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Thu, 24 Aug 2023 18:33:45 +0200 Subject: [PATCH 21/79] adding OnyxUpdatesFromServer object --- src/types/onyx/OnyxUpdatesFromServer.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/types/onyx/OnyxUpdatesFromServer.ts diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts new file mode 100644 index 000000000000..e510c3d75c4a --- /dev/null +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -0,0 +1,20 @@ +import { OnyxUpdate } from "react-native-onyx"; +import Request from "./Request"; +import Response from "./Response"; + + +type OnyxUpdateFromServerData = { + request?: Request; + response?: Response; + updates?: OnyxUpdate[]; +} + + +type OnyxUpdateFromServer = { + lastUpdateID: number; + previousUpdateID: number; + type: 'HTTPS' | 'PUSHER' + data: OnyxUpdateFromServerData +}; + +export default OnyxUpdateFromServer; \ No newline at end of file From 85b4ef07ea5f628fae4075991c4903a507951c0a Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Thu, 24 Aug 2023 18:36:24 +0200 Subject: [PATCH 22/79] renaming and changing object in onyxKeys --- src/ONYXKEYS.ts | 2 +- src/types/onyx/OnyxUpdatesFromServer.ts | 8 ++++---- src/types/onyx/index.ts | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ONYXKEYS.ts b/src/ONYXKEYS.ts index d4cb591c704d..716e3288ebcc 100755 --- a/src/ONYXKEYS.ts +++ b/src/ONYXKEYS.ts @@ -350,7 +350,7 @@ type OnyxValues = { [ONYXKEYS.SELECTED_TAB]: string; [ONYXKEYS.RECEIPT_MODAL]: OnyxTypes.ReceiptModal; [ONYXKEYS.MAPBOX_ACCESS_TOKEN]: OnyxTypes.MapboxAccessToken; - [ONYXKEYS.ONYX_UPDATES_FROM_SERVER]: any; + [ONYXKEYS.ONYX_UPDATES_FROM_SERVER]: OnyxTypes.OnyxUpdatesFromServer; [ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT]: number; // Collections diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index e510c3d75c4a..22bf42b38706 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -3,18 +3,18 @@ import Request from "./Request"; import Response from "./Response"; -type OnyxUpdateFromServerData = { +type OnyxUpdatesFromServerData = { request?: Request; response?: Response; updates?: OnyxUpdate[]; } -type OnyxUpdateFromServer = { +type OnyxUpdatesFromServer = { lastUpdateID: number; previousUpdateID: number; type: 'HTTPS' | 'PUSHER' - data: OnyxUpdateFromServerData + data: OnyxUpdatesFromServerData }; -export default OnyxUpdateFromServer; \ No newline at end of file +export default OnyxUpdatesFromServer; \ No newline at end of file diff --git a/src/types/onyx/index.ts b/src/types/onyx/index.ts index e7ab360551c5..7733b27138b0 100644 --- a/src/types/onyx/index.ts +++ b/src/types/onyx/index.ts @@ -32,6 +32,7 @@ import ReimbursementAccountDraft from './ReimbursementAccountDraft'; import WalletTransfer from './WalletTransfer'; import ReceiptModal from './ReceiptModal'; import MapboxAccessToken from './MapboxAccessToken'; +import OnyxUpdatesFromServer from './OnyxUpdatesFromServer'; import Download from './Download'; import PolicyMember from './PolicyMember'; @@ -89,4 +90,5 @@ export type { Transaction, Form, AddDebitCardForm, + OnyxUpdatesFromServer, }; From 42555c1051e0655870f4dee06dca307b6b004cab Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Thu, 24 Aug 2023 18:37:41 +0200 Subject: [PATCH 23/79] renaming property to updates --- src/libs/actions/OnyxUpdateManager.js | 12 ++++++------ src/types/onyx/OnyxUpdatesFromServer.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 039f89e146c3..1ae1343dc79f 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -52,15 +52,15 @@ function applyHTTPSOnyxUpdates({request, responseData}) { /** * @param {Object} data - * @param {Object} data.multipleEvents + * @param {Object} data.updates * @returns {Promise} */ -function applyPusherOnyxUpdates({multipleEvents}) { +function applyPusherOnyxUpdates({updates}) { console.debug('[OnyxUpdateManager] Applying pusher update'); const pusherEventPromises = _.reduce( - multipleEvents, - (result, multipleEvent) => { - result.push(PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data)); + updates, + (result, update) => { + result.push(PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); return result; }, [], @@ -76,7 +76,7 @@ function applyPusherOnyxUpdates({multipleEvents}) { * @param {Object} updateParams.data * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.multipleEvents] Exists if updateParams.type === 'pusher' + * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' * @returns {Promise} */ function applyOnyxUpdates({type, data}) { diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index 22bf42b38706..2e55d40b9c87 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -13,7 +13,7 @@ type OnyxUpdatesFromServerData = { type OnyxUpdatesFromServer = { lastUpdateID: number; previousUpdateID: number; - type: 'HTTPS' | 'PUSHER' + type: 'https' | 'pusher' data: OnyxUpdatesFromServerData }; From 4d02eb5b37e46801d0f6c49fc06a2ec6a3c69c81 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Thu, 24 Aug 2023 18:38:42 +0200 Subject: [PATCH 24/79] renaming multipleEvents to updates --- src/libs/actions/OnyxUpdates.js | 2 +- src/libs/actions/User.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index f5b7adeab23c..8834fc4e34f7 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -7,7 +7,7 @@ import ONYXKEYS from '../../ONYXKEYS'; * @param {Object} updateParams.data * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.responseData] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.multipleEvents] Exists if updateParams.type === 'pusher' + * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' * @param {Number} [lastUpdateID] * @param {Number} [previousUpdateID] */ diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index b5429ae0893d..940ed780f655 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -567,7 +567,7 @@ function subscribeToUserEvents() { { type: CONST.ONYX_UPDATE_TYPES.PUSHER, data: { - multipleEvents: pushJSON.updates, + updates: pushJSON.updates, }, }, Number(pushJSON.lastUpdateID || 0), From c5d726476955438a0b8340ce11723537031614ee Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 29 Aug 2023 10:42:43 +0200 Subject: [PATCH 25/79] prettier --- src/types/onyx/OnyxUpdatesFromServer.ts | 20 +++++++++----------- src/types/onyx/Request.ts | 2 +- src/types/onyx/Response.ts | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index 2e55d40b9c87..c2c76afa0aa8 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -1,20 +1,18 @@ -import { OnyxUpdate } from "react-native-onyx"; -import Request from "./Request"; -import Response from "./Response"; - +import {OnyxUpdate} from 'react-native-onyx'; +import Request from './Request'; +import Response from './Response'; type OnyxUpdatesFromServerData = { request?: Request; response?: Response; updates?: OnyxUpdate[]; -} - +}; type OnyxUpdatesFromServer = { - lastUpdateID: number; - previousUpdateID: number; - type: 'https' | 'pusher' - data: OnyxUpdatesFromServerData + lastUpdateID: number | string; + previousUpdateID: number | string; + type: 'https' | 'pusher'; + data: OnyxUpdatesFromServerData; }; -export default OnyxUpdatesFromServer; \ No newline at end of file +export default OnyxUpdatesFromServer; diff --git a/src/types/onyx/Request.ts b/src/types/onyx/Request.ts index cc57c9410e30..1df20cfb28fe 100644 --- a/src/types/onyx/Request.ts +++ b/src/types/onyx/Request.ts @@ -1,4 +1,4 @@ -import { OnyxUpdate } from "react-native-onyx"; +import {OnyxUpdate} from 'react-native-onyx'; type Request = { command?: string; diff --git a/src/types/onyx/Response.ts b/src/types/onyx/Response.ts index 01956adffc87..27077cc9bc74 100644 --- a/src/types/onyx/Response.ts +++ b/src/types/onyx/Response.ts @@ -1,4 +1,4 @@ -import { OnyxUpdate } from "react-native-onyx"; +import {OnyxUpdate} from 'react-native-onyx'; type Request = { previousUpdateID?: number; From 473d5df1dbdd6c3019c07bc25ae8511ae046342b Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 29 Aug 2023 10:43:43 +0200 Subject: [PATCH 26/79] renaming property --- ios/NewExpensify.xcodeproj/project.pbxproj | 8 ++++++-- src/libs/actions/OnyxUpdateManager.js | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ios/NewExpensify.xcodeproj/project.pbxproj b/ios/NewExpensify.xcodeproj/project.pbxproj index d87226269a8b..6aeb6cd1cf7d 100644 --- a/ios/NewExpensify.xcodeproj/project.pbxproj +++ b/ios/NewExpensify.xcodeproj/project.pbxproj @@ -29,7 +29,7 @@ 70CF6E82262E297300711ADC /* BootSplash.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 70CF6E81262E297300711ADC /* BootSplash.storyboard */; }; BDB853621F354EBB84E619C2 /* ExpensifyNewKansas-MediumItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = D2AFB39EC1D44BF9B91D3227 /* ExpensifyNewKansas-MediumItalic.otf */; }; DD79042B2792E76D004484B4 /* RCTBootSplash.m in Sources */ = {isa = PBXBuildFile; fileRef = DD79042A2792E76D004484B4 /* RCTBootSplash.m */; }; - E51DC681C7DEE40AEBDDFBFE /* BuildFile in Frameworks */ = {isa = PBXBuildFile; }; + E51DC681C7DEE40AEBDDFBFE /* (null) in Frameworks */ = {isa = PBXBuildFile; }; E9DF872D2525201700607FDC /* AirshipConfig.plist in Resources */ = {isa = PBXBuildFile; fileRef = E9DF872C2525201700607FDC /* AirshipConfig.plist */; }; ED222ED90E074A5481A854FA /* ExpensifyNeue-BoldItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = 8B28D84EF339436DBD42A203 /* ExpensifyNeue-BoldItalic.otf */; }; F0C450EA2705020500FD2970 /* colors.json in Resources */ = {isa = PBXBuildFile; fileRef = F0C450E92705020500FD2970 /* colors.json */; }; @@ -112,7 +112,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - E51DC681C7DEE40AEBDDFBFE /* BuildFile in Frameworks */, + E51DC681C7DEE40AEBDDFBFE /* (null) in Frameworks */, 5A464BC8112CDB1DE1E38F1C /* libPods-NewExpensify.a in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -701,6 +701,7 @@ CODE_SIGN_STYLE = Manual; CURRENT_PROJECT_VERSION = 3; DEVELOPMENT_TEAM = 368M544MTT; + "DEVELOPMENT_TEAM[sdk=iphoneos*]" = 368M544MTT; ENABLE_BITCODE = NO; INFOPLIST_FILE = "$(SRCROOT)/NewExpensify/Info.plist"; IPHONEOS_DEPLOYMENT_TARGET = 13.0; @@ -714,6 +715,7 @@ PRODUCT_BUNDLE_IDENTIFIER = com.expensify.chat.dev; PRODUCT_NAME = "New Expensify Dev"; PROVISIONING_PROFILE_SPECIFIER = expensify_chat_dev; + "PROVISIONING_PROFILE_SPECIFIER[sdk=iphoneos*]" = expensify_chat_dev; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; @@ -733,6 +735,7 @@ CODE_SIGN_STYLE = Manual; CURRENT_PROJECT_VERSION = 3; DEVELOPMENT_TEAM = 368M544MTT; + "DEVELOPMENT_TEAM[sdk=iphoneos*]" = 368M544MTT; INFOPLIST_FILE = NewExpensify/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; @@ -745,6 +748,7 @@ PRODUCT_BUNDLE_IDENTIFIER = com.expensify.chat.dev; PRODUCT_NAME = "New Expensify Dev"; PROVISIONING_PROFILE_SPECIFIER = expensify_chat_dev; + "PROVISIONING_PROFILE_SPECIFIER[sdk=iphoneos*]" = expensify_chat_dev; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; VERSIONING_SYSTEM = "apple-generic"; diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 1ae1343dc79f..358be3c07690 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -108,7 +108,7 @@ export default () => { // Since people will have migrations hapening in their accounts while they use the app, we don't want to trigger // a full ReconnectApp and then trigger a GetOnyxUpdates. Let's use this as a control variable until we enable // the beta to all users. - let updateEnablesBeta = false; + let isUserGettingReliableUpdatesForTheVeryFirstTime = false; const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; @@ -124,12 +124,12 @@ export default () => { ) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); App.reconnectApp(); - updateEnablesBeta = true; + isUserGettingReliableUpdatesForTheVeryFirstTime = true; } // If the previous update from the server does not match the last update the client got, then the client is missing some updates. // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. - if (!updateEnablesBeta && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + if (!isUserGettingReliableUpdatesForTheVeryFirstTime && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { lastUpdateIDFromServer, From 6a51999af2786b55aa202adb380f0314e274df33 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 29 Aug 2023 12:22:12 +0200 Subject: [PATCH 27/79] lint errors --- src/libs/actions/OnyxUpdateManager.js | 4 ++-- src/types/onyx/Response.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 358be3c07690..97456d8d704f 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -119,8 +119,8 @@ export default () => { if ( !lastUpdateIDAppliedToClient && previousUpdateIDFromServer > 0 && - (updateParams.type == CONST.ONYX_UPDATE_TYPES.PUSHER || - (updateParams.type == CONST.ONYX_UPDATE_TYPES.HTTPS && (updateParams.data.request.command !== 'OpenApp' || updateParams.data.request.command !== 'ReconnectApp'))) + (updateParams.type === CONST.ONYX_UPDATE_TYPES.PUSHER || + (updateParams.type === CONST.ONYX_UPDATE_TYPES.HTTPS && (updateParams.data.request.command !== 'OpenApp' || updateParams.data.request.command !== 'ReconnectApp'))) ) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); App.reconnectApp(); diff --git a/src/types/onyx/Response.ts b/src/types/onyx/Response.ts index 27077cc9bc74..d9395b6e8dab 100644 --- a/src/types/onyx/Response.ts +++ b/src/types/onyx/Response.ts @@ -1,6 +1,6 @@ import {OnyxUpdate} from 'react-native-onyx'; -type Request = { +type Response = { previousUpdateID?: number; lastUpdateID?: number; jsonCode?: number; From 72629ebb6d701d1b73a85d4b61bc9d2444dd8249 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 29 Aug 2023 23:39:22 +0200 Subject: [PATCH 28/79] reverting file to main --- ios/NewExpensify.xcodeproj/project.pbxproj | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ios/NewExpensify.xcodeproj/project.pbxproj b/ios/NewExpensify.xcodeproj/project.pbxproj index 6aeb6cd1cf7d..d87226269a8b 100644 --- a/ios/NewExpensify.xcodeproj/project.pbxproj +++ b/ios/NewExpensify.xcodeproj/project.pbxproj @@ -29,7 +29,7 @@ 70CF6E82262E297300711ADC /* BootSplash.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 70CF6E81262E297300711ADC /* BootSplash.storyboard */; }; BDB853621F354EBB84E619C2 /* ExpensifyNewKansas-MediumItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = D2AFB39EC1D44BF9B91D3227 /* ExpensifyNewKansas-MediumItalic.otf */; }; DD79042B2792E76D004484B4 /* RCTBootSplash.m in Sources */ = {isa = PBXBuildFile; fileRef = DD79042A2792E76D004484B4 /* RCTBootSplash.m */; }; - E51DC681C7DEE40AEBDDFBFE /* (null) in Frameworks */ = {isa = PBXBuildFile; }; + E51DC681C7DEE40AEBDDFBFE /* BuildFile in Frameworks */ = {isa = PBXBuildFile; }; E9DF872D2525201700607FDC /* AirshipConfig.plist in Resources */ = {isa = PBXBuildFile; fileRef = E9DF872C2525201700607FDC /* AirshipConfig.plist */; }; ED222ED90E074A5481A854FA /* ExpensifyNeue-BoldItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = 8B28D84EF339436DBD42A203 /* ExpensifyNeue-BoldItalic.otf */; }; F0C450EA2705020500FD2970 /* colors.json in Resources */ = {isa = PBXBuildFile; fileRef = F0C450E92705020500FD2970 /* colors.json */; }; @@ -112,7 +112,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - E51DC681C7DEE40AEBDDFBFE /* (null) in Frameworks */, + E51DC681C7DEE40AEBDDFBFE /* BuildFile in Frameworks */, 5A464BC8112CDB1DE1E38F1C /* libPods-NewExpensify.a in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -701,7 +701,6 @@ CODE_SIGN_STYLE = Manual; CURRENT_PROJECT_VERSION = 3; DEVELOPMENT_TEAM = 368M544MTT; - "DEVELOPMENT_TEAM[sdk=iphoneos*]" = 368M544MTT; ENABLE_BITCODE = NO; INFOPLIST_FILE = "$(SRCROOT)/NewExpensify/Info.plist"; IPHONEOS_DEPLOYMENT_TARGET = 13.0; @@ -715,7 +714,6 @@ PRODUCT_BUNDLE_IDENTIFIER = com.expensify.chat.dev; PRODUCT_NAME = "New Expensify Dev"; PROVISIONING_PROFILE_SPECIFIER = expensify_chat_dev; - "PROVISIONING_PROFILE_SPECIFIER[sdk=iphoneos*]" = expensify_chat_dev; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; @@ -735,7 +733,6 @@ CODE_SIGN_STYLE = Manual; CURRENT_PROJECT_VERSION = 3; DEVELOPMENT_TEAM = 368M544MTT; - "DEVELOPMENT_TEAM[sdk=iphoneos*]" = 368M544MTT; INFOPLIST_FILE = NewExpensify/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; @@ -748,7 +745,6 @@ PRODUCT_BUNDLE_IDENTIFIER = com.expensify.chat.dev; PRODUCT_NAME = "New Expensify Dev"; PROVISIONING_PROFILE_SPECIFIER = expensify_chat_dev; - "PROVISIONING_PROFILE_SPECIFIER[sdk=iphoneos*]" = expensify_chat_dev; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; VERSIONING_SYSTEM = "apple-generic"; From c2f3097941439015ebca802c3939ee7b44077a4f Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 30 Aug 2023 16:57:53 +0200 Subject: [PATCH 29/79] add onyxupdatemanager on tests --- tests/actions/IOUTest.js | 2 ++ tests/actions/ReportTest.js | 2 ++ tests/actions/SessionTest.js | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tests/actions/IOUTest.js b/tests/actions/IOUTest.js index 6fbbe19cec8e..afb06cdb6fb3 100644 --- a/tests/actions/IOUTest.js +++ b/tests/actions/IOUTest.js @@ -9,6 +9,7 @@ import DateUtils from '../../src/libs/DateUtils'; import * as NumberUtils from '../../src/libs/NumberUtils'; import * as ReportActions from '../../src/libs/actions/ReportActions'; import * as Report from '../../src/libs/actions/Report'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; const CARLOS_EMAIL = 'cmartins@expensifail.com'; const CARLOS_ACCOUNT_ID = 1; @@ -19,6 +20,7 @@ const RORY_ACCOUNT_ID = 3; const VIT_EMAIL = 'vit@expensifail.com'; const VIT_ACCOUNT_ID = 4; +OnyxUpdateManager(); describe('actions/IOU', () => { beforeAll(() => { Onyx.init({ diff --git a/tests/actions/ReportTest.js b/tests/actions/ReportTest.js index c06d3bc83766..978186fcf9c4 100644 --- a/tests/actions/ReportTest.js +++ b/tests/actions/ReportTest.js @@ -14,6 +14,7 @@ import * as PersistedRequests from '../../src/libs/actions/PersistedRequests'; import * as User from '../../src/libs/actions/User'; import * as ReportUtils from '../../src/libs/ReportUtils'; import DateUtils from '../../src/libs/DateUtils'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; jest.mock('../../src/libs/actions/Report', () => { const originalModule = jest.requireActual('../../src/libs/actions/Report'); @@ -24,6 +25,7 @@ jest.mock('../../src/libs/actions/Report', () => { }; }); +OnyxUpdateManager(); describe('actions/Report', () => { beforeAll(() => { PusherHelper.setup(); diff --git a/tests/actions/SessionTest.js b/tests/actions/SessionTest.js index d8bfa144e358..59a7441679ea 100644 --- a/tests/actions/SessionTest.js +++ b/tests/actions/SessionTest.js @@ -7,6 +7,7 @@ import * as TestHelper from '../utils/TestHelper'; import CONST from '../../src/CONST'; import PushNotification from '../../src/libs/Notification/PushNotification'; import * as App from '../../src/libs/actions/App'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; // This lib needs to be imported, but it has nothing to export since all it contains is an Onyx connection // eslint-disable-next-line no-unused-vars @@ -24,6 +25,7 @@ Onyx.init({ registerStorageEventListener: () => {}, }); +OnyxUpdateManager(); beforeEach(() => Onyx.clear().then(waitForPromisesToResolve)); describe('Session', () => { From 57874c757bc92154e4147ddfb5ad4d4ad196ad91 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 30 Aug 2023 17:00:07 +0200 Subject: [PATCH 30/79] adding additional checks for early return --- src/libs/Middleware/SaveResponseInOnyx.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 13244dd64e36..448574af8101 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -28,9 +28,9 @@ function SaveResponseInOnyx(response, request) { // Supports both the old format and the new format const onyxUpdates = _.isArray(responseData) ? responseData : responseData.onyxData; - // Sometimes we call requests that are successfull but they don't have any response. Let's return early since + // Sometimes we call requests that are successfull but they don't have any response or any success/failure data to set. Let's return early since // we don't need to store anything here. - if (!onyxUpdates) { + if (!onyxUpdates && !request.successData && !request.failureData) { return; } From 4cf1934c1d00cd2fc2032f517f4b31ec8ff2df9b Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 30 Aug 2023 19:18:58 +0200 Subject: [PATCH 31/79] adding onyx update manager to Network tests --- tests/unit/NetworkTest.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/NetworkTest.js b/tests/unit/NetworkTest.js index c8dcda0e2af5..0b577dc82469 100644 --- a/tests/unit/NetworkTest.js +++ b/tests/unit/NetworkTest.js @@ -14,6 +14,7 @@ import Log from '../../src/libs/Log'; import * as MainQueue from '../../src/libs/Network/MainQueue'; import * as App from '../../src/libs/actions/App'; import NetworkConnection from '../../src/libs/NetworkConnection'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; jest.mock('../../src/libs/Log'); jest.useFakeTimers(); @@ -21,7 +22,7 @@ jest.useFakeTimers(); Onyx.init({ keys: ONYXKEYS, }); - +OnyxUpdateManager(); const originalXHR = HttpUtils.xhr; beforeEach(() => { From 64e006c0cf7c4f7eff875111ed518bb3fc033429 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 30 Aug 2023 19:21:31 +0200 Subject: [PATCH 32/79] adding space for style --- tests/unit/NetworkTest.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/NetworkTest.js b/tests/unit/NetworkTest.js index 0b577dc82469..7d8c4f23197c 100644 --- a/tests/unit/NetworkTest.js +++ b/tests/unit/NetworkTest.js @@ -22,6 +22,7 @@ jest.useFakeTimers(); Onyx.init({ keys: ONYXKEYS, }); + OnyxUpdateManager(); const originalXHR = HttpUtils.xhr; From 3ed8dcee4df0fe6adb789af11a04682ce890af0f Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 03:49:25 +0200 Subject: [PATCH 33/79] adding new method to return promise on reconnectApp --- src/libs/actions/App.js | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index 56a313e81c0b..2e3185ea4d1b 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -207,6 +207,49 @@ function reconnectApp(updateIDFrom = 0) { }); } +/** + * Fetches data when the app will call reconnectApp without params for the last time. This is a separate function + * because it will follow patterns that are not recommended so we can be sure we're not putting the app in a unusable + * state because of race conditions between reconnectApp and other pusher updates being applied at the same time. + * @return {Promise} + */ +function lastReconnectAppAfterActivatingReliableUpdates() { + console.debug(`[OnyxUpdates] Executing last reconnect app with promise`); + return getPolicyParamsForOpenOrReconnect().then((policyParams) => { + const params = {...policyParams}; + + // When the app reconnects we do a fast "sync" of the LHN and only return chats that have new messages. We achieve this by sending the most recent reportActionID. + // we have locally. And then only update the user about chats with messages that have occurred after that reportActionID. + // + // - Look through the local report actions and reports to find the most recently modified report action or report. + // - We send this to the server so that it can compute which new chats the user needs to see and return only those as an optimization. + Timing.start(CONST.TIMING.CALCULATE_MOST_RECENT_LAST_MODIFIED_ACTION); + params.mostRecentReportActionLastModified = ReportActionsUtils.getMostRecentReportActionLastModified(); + Timing.end(CONST.TIMING.CALCULATE_MOST_RECENT_LAST_MODIFIED_ACTION, '', 500); + + // Include the update IDs when reconnecting so that the server can send incremental updates if they are available. + // Otherwise, a full set of app data will be returned. + if (updateIDFrom) { + params.updateIDFrom = updateIDFrom; + } + + // It is SUPER BAD FORM to return promises from action methods. + // DO NOT FOLLOW THIS PATTERN!!!!! + // It was absolutely necessary in order to not break the app while migrating to the new reliable updates pattern. This method will be removed + // as soon as we have everyone migrated to it + // eslint-disable-next-line rulesdir/no-api-side-effects-method + return API.makeRequestWithSideEffects( + 'ReconnectApp', + { + updateIDFrom, + updateIDTo, + }, + getOnyxDataForOpenOrReconnect(), + ); + }); + +} + /** * Fetches data when the client has discovered it missed some Onyx updates from the server * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from @@ -435,4 +478,5 @@ export { beginDeepLinkRedirectAfterTransition, createWorkspaceAndNavigateToIt, getMissingOnyxUpdates, + lastReconnectAppAfterActivatingReliableUpdates, }; From d5cc741047ab671767c6b1967685a083ed3fad70 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 03:49:46 +0200 Subject: [PATCH 34/79] adding usage to new method to reconnect app --- src/libs/actions/OnyxUpdateManager.js | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 97456d8d704f..c29c83b3885b 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -27,6 +27,7 @@ function applyHTTPSOnyxUpdates({request, responseData}) { console.debug('[OnyxUpdateManager] Applying https update'); // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. + // 2023-08-31 - const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then @@ -105,11 +106,6 @@ export default () => { return; } - // Since people will have migrations hapening in their accounts while they use the app, we don't want to trigger - // a full ReconnectApp and then trigger a GetOnyxUpdates. Let's use this as a control variable until we enable - // the beta to all users. - let isUserGettingReliableUpdatesForTheVeryFirstTime = false; - const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; // This can happen when a user has just started getting reliable updates from the server but they haven't @@ -120,16 +116,13 @@ export default () => { !lastUpdateIDAppliedToClient && previousUpdateIDFromServer > 0 && (updateParams.type === CONST.ONYX_UPDATE_TYPES.PUSHER || - (updateParams.type === CONST.ONYX_UPDATE_TYPES.HTTPS && (updateParams.data.request.command !== 'OpenApp' || updateParams.data.request.command !== 'ReconnectApp'))) + (updateParams.type === CONST.ONYX_UPDATE_TYPES.HTTPS && updateParams.data.request.command !== 'OpenApp' && updateParams.data.request.command !== 'ReconnectApp')) ) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); - App.reconnectApp(); - isUserGettingReliableUpdatesForTheVeryFirstTime = true; - } - - // If the previous update from the server does not match the last update the client got, then the client is missing some updates. - // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. - if (!isUserGettingReliableUpdatesForTheVeryFirstTime && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + App.lastReconnectAppAfterActivatingReliableUpdates().finally(() => { + applyOnyxUpdates(updateParams); + }); + } else if (previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { lastUpdateIDFromServer, From c613170dfffd9afa6ef657740eb79a7e79560c6b Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 04:14:13 +0200 Subject: [PATCH 35/79] refactor the code a little bit --- src/libs/actions/OnyxUpdateManager.js | 80 +++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index c29c83b3885b..46130467c86d 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -108,51 +108,51 @@ export default () => { const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; - // This can happen when a user has just started getting reliable updates from the server but they haven't - // had an OpenApp or ReconnectApp call yet. This can result in never getting reliable updates because - // lastUpdateIDAppliedToClient will always be null. For this case, reconnectApp() will be triggered for them - // to kick start the reliable updates. We also filter OpenApp and ReconnectApp so we don't create a loop. - if ( - !lastUpdateIDAppliedToClient && - previousUpdateIDFromServer > 0 && - (updateParams.type === CONST.ONYX_UPDATE_TYPES.PUSHER || - (updateParams.type === CONST.ONYX_UPDATE_TYPES.HTTPS && updateParams.data.request.command !== 'OpenApp' && updateParams.data.request.command !== 'ReconnectApp')) - ) { - console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); - App.lastReconnectAppAfterActivatingReliableUpdates().finally(() => { - applyOnyxUpdates(updateParams); - }); - } else if (previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { - console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); - Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { - lastUpdateIDFromServer, - previousUpdateIDFromServer, - lastUpdateIDAppliedToClient, - }); + // If we don't have a previousUpdateID from the request, or if we if it's the same as we currently have stored + // we can simply apply the updates and move on. + if (!previousUpdateIDFromServer || lastUpdateIDAppliedToClient === previousUpdateIDFromServer) { + console.debug(`[OnyxUpdateManager] Client is in sync with the server, applying updates`); + applyOnyxUpdates(updateParams); + } else { - // Pause the sequential queue while the missing Onyx updates are fetched from the server. This is important - // so that the updates are applied in their correct and specific order. If this queue was not paused, then - // there would be a lot of onyx data being applied while we are fetching the missing updates and that would - // put them all out of order. + // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient + // we need to perform one of the 2 possible cases: + // + // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before + // fully migrating to the reliable updates mode; + // 2. This this client already has the reliable updates mode enabled, but it's missing some updates and it + // needs to fech those. + // + // To to both of those, we need to pause the sequential queue. This is important so that the updates are + // applied in their correct and specific order. If this queue was not paused, then there would be a lot of + // onyx data being applied while we are fetching the missing updates and that would put them all out of order. SequentialQueue.pause(); - App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(() => { - console.debug('[OnyxUpdateManager] Done Getting missing onyx updates'); - - // The onyx update from the initial request could have been either from HTTPS or Pusher. - // Now that the missing onyx updates have been applied, we can apply the original onyxUpdates from - // the API request. - applyOnyxUpdates(updateParams).then(() => { - console.debug('[OnyxUpdateManager] Done applying all updates'); - - // Finally, the missing updates were applied, the original update was applied, and now the - // sequential queue is free to continue. - SequentialQueue.unpause(); + let promise; + + // The flow below is setting the promise to a reconnect app to address flow (1) explained above. + if (!lastUpdateIDAppliedToClient && + (updateParams.type === CONST.ONYX_UPDATE_TYPES.PUSHER || updateParams.data.request.command !== 'OpenApp' && updateParams.data.request.command !== 'ReconnectApp') + ) { + console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); + Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); + promise = App.lastReconnectAppAfterActivatingReliableUpdates(); + } else if (previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above. + console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); + Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { + lastUpdateIDFromServer, + previousUpdateIDFromServer, + lastUpdateIDAppliedToClient, }); + promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); + } + + promise.finally(() => { + console.debug('[OnyxUpdateManager] Done applying all updates'); + applyOnyxUpdates(updateParams); + SequentialQueue.unpause(); }); - } else { - console.debug(`[OnyxUpdateManager] Client is in sync with the server`); - applyOnyxUpdates(updateParams); } if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { From 377cb9353cac64e9e714b516ca08b84cb233c50c Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 04:14:30 +0200 Subject: [PATCH 36/79] adding a few more checks and refactoring --- src/libs/actions/OnyxUpdateManager.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 46130467c86d..0b952cad9ef5 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -108,13 +108,14 @@ export default () => { const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; - // If we don't have a previousUpdateID from the request, or if we if it's the same as we currently have stored + // If we don't have a previousUpdateID from the request, or if we if it's the less or equal w we currently have stored // we can simply apply the updates and move on. if (!previousUpdateIDFromServer || lastUpdateIDAppliedToClient === previousUpdateIDFromServer) { console.debug(`[OnyxUpdateManager] Client is in sync with the server, applying updates`); applyOnyxUpdates(updateParams); + } else if (lastUpdateIDFromServer < lastUpdateIDAppliedToClient) { + console.debug(`[OnyxUpdateManager] Client is more up to date than the update received, skipping processing`); } else { - // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient // we need to perform one of the 2 possible cases: // @@ -127,7 +128,6 @@ export default () => { // applied in their correct and specific order. If this queue was not paused, then there would be a lot of // onyx data being applied while we are fetching the missing updates and that would put them all out of order. SequentialQueue.pause(); - let promise; // The flow below is setting the promise to a reconnect app to address flow (1) explained above. @@ -137,7 +137,7 @@ export default () => { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); promise = App.lastReconnectAppAfterActivatingReliableUpdates(); - } else if (previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + } else { // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above. console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { From df82f2efdb04e0e32215791f2a06173bdfbb65ae Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 04:21:09 +0200 Subject: [PATCH 37/79] adding check if queue is paused before moving on with applying queued onyx updates --- src/libs/Network/SequentialQueue.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index f8ea396663a5..4894456fa789 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -94,7 +94,7 @@ function flush() { isSequentialQueueRunning = false; resolveIsReadyPromise(); currentRequest = null; - Onyx.update(QueuedOnyxUpdates.getQueuedUpdates()).then(QueuedOnyxUpdates.clear); + flushAndClearOnyxQueue(); }); }, }); @@ -149,6 +149,17 @@ function waitForIdle() { return isReadyPromise; } + +/** + * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. + */ +function flushAndClearOnyxQueue() { + if (isQueuePaused) { + return; + } + Onyx.update(QueuedOnyxUpdates.getQueuedUpdates()).then(QueuedOnyxUpdates.clear); +} + /** * Puts the queue into a paused state so that no requests will be processed */ @@ -172,6 +183,10 @@ function unpause() { const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); isQueuePaused = false; + + // Since the writes may happen async to the queue, in case we had any writes happen while the queue was paused + // (because of race conditions), let's also apply the queued updates and clear them before continuing the queue. + flushAndClearOnyxQueue(); flush(); } From 181c54a317baa79b47bc815a5e715e6a38043e06 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 04:21:39 +0200 Subject: [PATCH 38/79] prettier --- src/libs/Network/SequentialQueue.js | 3 +-- src/libs/actions/App.js | 3 +-- src/libs/actions/OnyxUpdateManager.js | 17 +++++++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 4894456fa789..10c1329cca28 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -149,7 +149,6 @@ function waitForIdle() { return isReadyPromise; } - /** * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. */ @@ -184,7 +183,7 @@ function unpause() { console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); isQueuePaused = false; - // Since the writes may happen async to the queue, in case we had any writes happen while the queue was paused + // Since the writes may happen async to the queue, in case we had any writes happen while the queue was paused // (because of race conditions), let's also apply the queued updates and clear them before continuing the queue. flushAndClearOnyxQueue(); flush(); diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index 2e3185ea4d1b..820173a7be8e 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -232,7 +232,7 @@ function lastReconnectAppAfterActivatingReliableUpdates() { if (updateIDFrom) { params.updateIDFrom = updateIDFrom; } - + // It is SUPER BAD FORM to return promises from action methods. // DO NOT FOLLOW THIS PATTERN!!!!! // It was absolutely necessary in order to not break the app while migrating to the new reliable updates pattern. This method will be removed @@ -247,7 +247,6 @@ function lastReconnectAppAfterActivatingReliableUpdates() { getOnyxDataForOpenOrReconnect(), ); }); - } /** diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 0b952cad9ef5..83fda3234557 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -27,7 +27,7 @@ function applyHTTPSOnyxUpdates({request, responseData}) { console.debug('[OnyxUpdateManager] Applying https update'); // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. - // 2023-08-31 - + // 2023-08-31 - const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then @@ -113,7 +113,7 @@ export default () => { if (!previousUpdateIDFromServer || lastUpdateIDAppliedToClient === previousUpdateIDFromServer) { console.debug(`[OnyxUpdateManager] Client is in sync with the server, applying updates`); applyOnyxUpdates(updateParams); - } else if (lastUpdateIDFromServer < lastUpdateIDAppliedToClient) { + } else if (lastUpdateIDFromServer < lastUpdateIDAppliedToClient) { console.debug(`[OnyxUpdateManager] Client is more up to date than the update received, skipping processing`); } else { // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient @@ -121,18 +121,19 @@ export default () => { // // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before // fully migrating to the reliable updates mode; - // 2. This this client already has the reliable updates mode enabled, but it's missing some updates and it + // 2. This this client already has the reliable updates mode enabled, but it's missing some updates and it // needs to fech those. // - // To to both of those, we need to pause the sequential queue. This is important so that the updates are - // applied in their correct and specific order. If this queue was not paused, then there would be a lot of + // To to both of those, we need to pause the sequential queue. This is important so that the updates are + // applied in their correct and specific order. If this queue was not paused, then there would be a lot of // onyx data being applied while we are fetching the missing updates and that would put them all out of order. SequentialQueue.pause(); let promise; - + // The flow below is setting the promise to a reconnect app to address flow (1) explained above. - if (!lastUpdateIDAppliedToClient && - (updateParams.type === CONST.ONYX_UPDATE_TYPES.PUSHER || updateParams.data.request.command !== 'OpenApp' && updateParams.data.request.command !== 'ReconnectApp') + if ( + !lastUpdateIDAppliedToClient && + (updateParams.type === CONST.ONYX_UPDATE_TYPES.PUSHER || (updateParams.data.request.command !== 'OpenApp' && updateParams.data.request.command !== 'ReconnectApp')) ) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); From 97ef06e729b189ef0bab59ef6bd0a8972008aaee Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 04:43:36 +0200 Subject: [PATCH 39/79] prettier, some refactoring and adding the unpause in the right place --- src/libs/Network/SequentialQueue.js | 6 +++--- src/libs/actions/App.js | 15 +-------------- src/libs/actions/OnyxUpdateManager.js | 22 +++++++++++++++------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 10c1329cca28..27dd0755c966 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -94,7 +94,7 @@ function flush() { isSequentialQueueRunning = false; resolveIsReadyPromise(); currentRequest = null; - flushAndClearOnyxQueue(); + flushAndClearOnyxUpdatesQueue(); }); }, }); @@ -152,7 +152,7 @@ function waitForIdle() { /** * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. */ -function flushAndClearOnyxQueue() { +function flushAndClearOnyxUpdatesQueue() { if (isQueuePaused) { return; } @@ -185,7 +185,7 @@ function unpause() { // Since the writes may happen async to the queue, in case we had any writes happen while the queue was paused // (because of race conditions), let's also apply the queued updates and clear them before continuing the queue. - flushAndClearOnyxQueue(); + flushAndClearOnyxUpdatesQueue(); flush(); } diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index 820173a7be8e..0bba4249bf11 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -227,25 +227,12 @@ function lastReconnectAppAfterActivatingReliableUpdates() { params.mostRecentReportActionLastModified = ReportActionsUtils.getMostRecentReportActionLastModified(); Timing.end(CONST.TIMING.CALCULATE_MOST_RECENT_LAST_MODIFIED_ACTION, '', 500); - // Include the update IDs when reconnecting so that the server can send incremental updates if they are available. - // Otherwise, a full set of app data will be returned. - if (updateIDFrom) { - params.updateIDFrom = updateIDFrom; - } - // It is SUPER BAD FORM to return promises from action methods. // DO NOT FOLLOW THIS PATTERN!!!!! // It was absolutely necessary in order to not break the app while migrating to the new reliable updates pattern. This method will be removed // as soon as we have everyone migrated to it // eslint-disable-next-line rulesdir/no-api-side-effects-method - return API.makeRequestWithSideEffects( - 'ReconnectApp', - { - updateIDFrom, - updateIDTo, - }, - getOnyxDataForOpenOrReconnect(), - ); + return API.makeRequestWithSideEffects('ReconnectApp', params, getOnyxDataForOpenOrReconnect()); }); } diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 83fda3234557..28c74874a564 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -21,9 +21,10 @@ import * as App from './App'; * @param {Object} data * @param {Object} data.request * @param {Object} data.responseData + * @param {Boolean} data.unpauseQueue * @returns {Promise} */ -function applyHTTPSOnyxUpdates({request, responseData}) { +function applyHTTPSOnyxUpdates({request, responseData, unpauseQueue}) { console.debug('[OnyxUpdateManager] Applying https update'); // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. @@ -47,6 +48,9 @@ function applyHTTPSOnyxUpdates({request, responseData}) { return Promise.resolve(); }) .then(() => { + if (unpauseQueue) { + SequentialQueue.unpause(); + } console.debug('[OnyxUpdateManager] Done applying HTTPS update'); }); } @@ -54,9 +58,10 @@ function applyHTTPSOnyxUpdates({request, responseData}) { /** * @param {Object} data * @param {Object} data.updates + * @param {Boolean} data.unpauseQueue * @returns {Promise} */ -function applyPusherOnyxUpdates({updates}) { +function applyPusherOnyxUpdates({updates, unpauseQueue}) { console.debug('[OnyxUpdateManager] Applying pusher update'); const pusherEventPromises = _.reduce( updates, @@ -67,6 +72,9 @@ function applyPusherOnyxUpdates({updates}) { [], ); return Promise.all(pusherEventPromises).then(() => { + if (unpauseQueue) { + SequentialQueue.unpause(); + } console.debug('[OnyxUpdateManager] Done applying Pusher update'); }); } @@ -74,19 +82,20 @@ function applyPusherOnyxUpdates({updates}) { /** * @param {Object[]} updateParams * @param {String} updateParams.type + * @param {Boolean} updateParams.unpauseQueue * @param {Object} updateParams.data * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' * @returns {Promise} */ -function applyOnyxUpdates({type, data}) { +function applyOnyxUpdates({type, data, unpauseQueue = false}) { console.debug(`[OnyxUpdateManager] Applying update type: ${type}`, data); if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { - return applyHTTPSOnyxUpdates(data); + return applyHTTPSOnyxUpdates({...data, unpauseQueue}); } if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { - return applyPusherOnyxUpdates(data); + return applyPusherOnyxUpdates({...data, unpauseQueue}); } } @@ -151,8 +160,7 @@ export default () => { promise.finally(() => { console.debug('[OnyxUpdateManager] Done applying all updates'); - applyOnyxUpdates(updateParams); - SequentialQueue.unpause(); + applyOnyxUpdates({...updateParams, unpauseQueue: true}); }); } From 03da9725ba4ba8604a3a34c2b54b25c67dfb1930 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 04:45:30 +0200 Subject: [PATCH 40/79] using the promise instead of passing variable down --- src/libs/actions/OnyxUpdateManager.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 28c74874a564..24bceda6e181 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -21,10 +21,9 @@ import * as App from './App'; * @param {Object} data * @param {Object} data.request * @param {Object} data.responseData - * @param {Boolean} data.unpauseQueue * @returns {Promise} */ -function applyHTTPSOnyxUpdates({request, responseData, unpauseQueue}) { +function applyHTTPSOnyxUpdates({request, responseData}) { console.debug('[OnyxUpdateManager] Applying https update'); // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. @@ -48,9 +47,6 @@ function applyHTTPSOnyxUpdates({request, responseData, unpauseQueue}) { return Promise.resolve(); }) .then(() => { - if (unpauseQueue) { - SequentialQueue.unpause(); - } console.debug('[OnyxUpdateManager] Done applying HTTPS update'); }); } @@ -58,10 +54,9 @@ function applyHTTPSOnyxUpdates({request, responseData, unpauseQueue}) { /** * @param {Object} data * @param {Object} data.updates - * @param {Boolean} data.unpauseQueue * @returns {Promise} */ -function applyPusherOnyxUpdates({updates, unpauseQueue}) { +function applyPusherOnyxUpdates({updates}) { console.debug('[OnyxUpdateManager] Applying pusher update'); const pusherEventPromises = _.reduce( updates, @@ -72,9 +67,6 @@ function applyPusherOnyxUpdates({updates, unpauseQueue}) { [], ); return Promise.all(pusherEventPromises).then(() => { - if (unpauseQueue) { - SequentialQueue.unpause(); - } console.debug('[OnyxUpdateManager] Done applying Pusher update'); }); } @@ -160,7 +152,9 @@ export default () => { promise.finally(() => { console.debug('[OnyxUpdateManager] Done applying all updates'); - applyOnyxUpdates({...updateParams, unpauseQueue: true}); + applyOnyxUpdates(updateParams).finally(() => { + SequentialQueue.unpause(); + }); }); } From d5fd7edcc7c9d4a019036321343bdb4c222a9f38 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 05:35:12 +0200 Subject: [PATCH 41/79] linting files and adjusting checks --- src/libs/Network/SequentialQueue.js | 21 +++++++++++---------- src/libs/actions/OnyxUpdateManager.js | 7 ++----- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 27dd0755c966..2bbc766d868f 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -62,6 +62,17 @@ function process() { return currentRequest; } + +/** + * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. + */ +function flushAndClearOnyxUpdatesQueue() { + if (isQueuePaused) { + return; + } + Onyx.update(QueuedOnyxUpdates.getQueuedUpdates()).then(QueuedOnyxUpdates.clear); +} + function flush() { // When the queue is paused, return early. This will keep an requests in the queue and they will get flushed again when the queue is unpaused if (isQueuePaused) { @@ -149,16 +160,6 @@ function waitForIdle() { return isReadyPromise; } -/** - * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. - */ -function flushAndClearOnyxUpdatesQueue() { - if (isQueuePaused) { - return; - } - Onyx.update(QueuedOnyxUpdates.getQueuedUpdates()).then(QueuedOnyxUpdates.clear); -} - /** * Puts the queue into a paused state so that no requests will be processed */ diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 24bceda6e181..f1a9973ade4c 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -111,7 +111,7 @@ export default () => { // If we don't have a previousUpdateID from the request, or if we if it's the less or equal w we currently have stored // we can simply apply the updates and move on. - if (!previousUpdateIDFromServer || lastUpdateIDAppliedToClient === previousUpdateIDFromServer) { + if (!previousUpdateIDFromServer || lastUpdateIDAppliedToClient === previousUpdateIDFromServer || (updateParams.type === CONST.ONYX_UPDATE_TYPES.HTTPS && (updateParams.data.request.command === 'OpenApp' || updateParams.data.request.command === 'ReconnectApp'))) { console.debug(`[OnyxUpdateManager] Client is in sync with the server, applying updates`); applyOnyxUpdates(updateParams); } else if (lastUpdateIDFromServer < lastUpdateIDAppliedToClient) { @@ -132,10 +132,7 @@ export default () => { let promise; // The flow below is setting the promise to a reconnect app to address flow (1) explained above. - if ( - !lastUpdateIDAppliedToClient && - (updateParams.type === CONST.ONYX_UPDATE_TYPES.PUSHER || (updateParams.data.request.command !== 'OpenApp' && updateParams.data.request.command !== 'ReconnectApp')) - ) { + if (!lastUpdateIDAppliedToClient) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); promise = App.lastReconnectAppAfterActivatingReliableUpdates(); From b8abf43c92492b8892836974f3f6ac5b0378fd51 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 1 Sep 2023 06:10:51 +0200 Subject: [PATCH 42/79] adding the right check in lastUpdateAppliedToClient --- src/libs/actions/OnyxUpdateManager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index f1a9973ade4c..0028ee9776ec 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -114,7 +114,7 @@ export default () => { if (!previousUpdateIDFromServer || lastUpdateIDAppliedToClient === previousUpdateIDFromServer || (updateParams.type === CONST.ONYX_UPDATE_TYPES.HTTPS && (updateParams.data.request.command === 'OpenApp' || updateParams.data.request.command === 'ReconnectApp'))) { console.debug(`[OnyxUpdateManager] Client is in sync with the server, applying updates`); applyOnyxUpdates(updateParams); - } else if (lastUpdateIDFromServer < lastUpdateIDAppliedToClient) { + } else if (lastUpdateIDFromServer <= lastUpdateIDAppliedToClient) { console.debug(`[OnyxUpdateManager] Client is more up to date than the update received, skipping processing`); } else { // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient From ee3cddf663a081ffa054c26905bb859c29b16bdd Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 20:44:20 +0200 Subject: [PATCH 43/79] refactoring in progress --- src/libs/actions/OnyxUpdateManager.js | 150 ++++++-------------------- src/libs/actions/OnyxUpdates.js | 99 ++++++++++++++++- 2 files changed, 132 insertions(+), 117 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 0028ee9776ec..25c98b0ce8e0 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -4,9 +4,8 @@ import ONYXKEYS from '../../ONYXKEYS'; import CONST from '../../CONST'; import Log from '../Log'; import * as SequentialQueue from '../Network/SequentialQueue'; -import PusherUtils from '../PusherUtils'; -import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; import * as App from './App'; +import * as OnyxUpdates from './OnyxUpdates' // This file is in charge of looking at the updateIDs coming from the server and comparing them to the last updateID that the client has. // If the client is behind the server, then we need to pause everything, get the missing updates from the server, apply those updates, @@ -17,79 +16,7 @@ import * as App from './App'; // The circular dependency happen because this file calls the API GetMissingOnyxUpdates which uses the SaveResponseInOnyx.js file // (as a middleware). Therefore, SaveResponseInOnyx.js can't import and use this file directly. -/** - * @param {Object} data - * @param {Object} data.request - * @param {Object} data.responseData - * @returns {Promise} - */ -function applyHTTPSOnyxUpdates({request, responseData}) { - console.debug('[OnyxUpdateManager] Applying https update'); - // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in - // the UI. See https://github.com/Expensify/App/issues/12775 for more info. - // 2023-08-31 - - const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; - // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then - // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained - // in successData/failureData until after the component has received and API data. - const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); - - return onyxDataUpdatePromise - .then(() => { - // Handle the request's success/failure data (client-side data) - if (responseData.jsonCode === 200 && request.successData) { - return updateHandler(request.successData); - } - if (responseData.jsonCode !== 200 && request.failureData) { - return updateHandler(request.failureData); - } - return Promise.resolve(); - }) - .then(() => { - console.debug('[OnyxUpdateManager] Done applying HTTPS update'); - }); -} - -/** - * @param {Object} data - * @param {Object} data.updates - * @returns {Promise} - */ -function applyPusherOnyxUpdates({updates}) { - console.debug('[OnyxUpdateManager] Applying pusher update'); - const pusherEventPromises = _.reduce( - updates, - (result, update) => { - result.push(PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); - return result; - }, - [], - ); - return Promise.all(pusherEventPromises).then(() => { - console.debug('[OnyxUpdateManager] Done applying Pusher update'); - }); -} - -/** - * @param {Object[]} updateParams - * @param {String} updateParams.type - * @param {Boolean} updateParams.unpauseQueue - * @param {Object} updateParams.data - * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' - * @returns {Promise} - */ -function applyOnyxUpdates({type, data, unpauseQueue = false}) { - console.debug(`[OnyxUpdateManager] Applying update type: ${type}`, data); - if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { - return applyHTTPSOnyxUpdates({...data, unpauseQueue}); - } - if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { - return applyPusherOnyxUpdates({...data, unpauseQueue}); - } -} // This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated let lastUpdateIDAppliedToClient = 0; @@ -109,52 +36,43 @@ export default () => { const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; - // If we don't have a previousUpdateID from the request, or if we if it's the less or equal w we currently have stored - // we can simply apply the updates and move on. - if (!previousUpdateIDFromServer || lastUpdateIDAppliedToClient === previousUpdateIDFromServer || (updateParams.type === CONST.ONYX_UPDATE_TYPES.HTTPS && (updateParams.data.request.command === 'OpenApp' || updateParams.data.request.command === 'ReconnectApp'))) { - console.debug(`[OnyxUpdateManager] Client is in sync with the server, applying updates`); - applyOnyxUpdates(updateParams); - } else if (lastUpdateIDFromServer <= lastUpdateIDAppliedToClient) { - console.debug(`[OnyxUpdateManager] Client is more up to date than the update received, skipping processing`); - } else { - // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient - // we need to perform one of the 2 possible cases: - // - // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before - // fully migrating to the reliable updates mode; - // 2. This this client already has the reliable updates mode enabled, but it's missing some updates and it - // needs to fech those. - // - // To to both of those, we need to pause the sequential queue. This is important so that the updates are - // applied in their correct and specific order. If this queue was not paused, then there would be a lot of - // onyx data being applied while we are fetching the missing updates and that would put them all out of order. - SequentialQueue.pause(); - let promise; - - // The flow below is setting the promise to a reconnect app to address flow (1) explained above. - if (!lastUpdateIDAppliedToClient) { - console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); - Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); - promise = App.lastReconnectAppAfterActivatingReliableUpdates(); - } else { - // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above. - console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); - Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { - lastUpdateIDFromServer, - previousUpdateIDFromServer, - lastUpdateIDAppliedToClient, - }); - promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); - } + // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient + // we need to perform one of the 2 possible cases: + // + // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before + // fully migrating to the reliable updates mode; + // 2. This this client already has the reliable updates mode enabled, but it's missing some updates and it + // needs to fech those. + // + // To to both of those, we need to pause the sequential queue. This is important so that the updates are + // applied in their correct and specific order. If this queue was not paused, then there would be a lot of + // onyx data being applied while we are fetching the missing updates and that would put them all out of order. + SequentialQueue.pause(); + let promise; - promise.finally(() => { - console.debug('[OnyxUpdateManager] Done applying all updates'); - applyOnyxUpdates(updateParams).finally(() => { - SequentialQueue.unpause(); - }); + // The flow below is setting the promise to a reconnect app to address flow (1) explained above. + if (!lastUpdateIDAppliedToClient) { + console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); + Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); + promise = App.lastReconnectAppAfterActivatingReliableUpdates(); + } else { + // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above. + console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); + Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { + lastUpdateIDFromServer, + previousUpdateIDFromServer, + lastUpdateIDAppliedToClient, }); + promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); } + promise.finally(() => { + console.debug('[OnyxUpdateManager] Done applying all updates'); + OnyxUpdates.applyOnyxUpdates(updateParams).finally(() => { + SequentialQueue.unpause(); + }); + }); + if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { // Update this value so that it matches what was just received from the server Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 8834fc4e34f7..f3caead988d4 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -1,5 +1,79 @@ import Onyx from 'react-native-onyx'; import ONYXKEYS from '../../ONYXKEYS'; +import * as QueuedOnyxUpdates from './QueuedOnyxUpdates' + + +/** + * @param {Object} data + * @param {Object} data.request + * @param {Object} data.responseData + * @returns {Promise} + */ +function applyHTTPSOnyxUpdates({request, responseData}) { + console.debug('[OnyxUpdateManager] Applying https update'); + // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in + // the UI. See https://github.com/Expensify/App/issues/12775 for more info. + const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; + + // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then + // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained + // in successData/failureData until after the component has received and API data. + const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); + + return onyxDataUpdatePromise + .then(() => { + // Handle the request's success/failure data (client-side data) + if (responseData.jsonCode === 200 && request.successData) { + return updateHandler(request.successData); + } + if (responseData.jsonCode !== 200 && request.failureData) { + return updateHandler(request.failureData); + } + return Promise.resolve(); + }) + .then(() => { + console.debug('[OnyxUpdateManager] Done applying HTTPS update'); + }); +} + +/** + * @param {Object} data + * @param {Object} data.updates + * @returns {Promise} + */ +function applyPusherOnyxUpdates({updates}) { + console.debug('[OnyxUpdateManager] Applying pusher update'); + const pusherEventPromises = _.reduce( + updates, + (result, update) => { + result.push(PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); + return result; + }, + [], + ); + return Promise.all(pusherEventPromises).then(() => { + console.debug('[OnyxUpdateManager] Done applying Pusher update'); + }); +} + +/** + * @param {Object[]} updateParams + * @param {String} updateParams.type + * @param {Object} updateParams.data + * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' + * @returns {Promise} + */ +function applyOnyxUpdates({type, data}) { + console.debug(`[OnyxUpdateManager] Applying update type: ${type}`, data); + if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { + return applyHTTPSOnyxUpdates(data); + } + if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { + return applyPusherOnyxUpdates(data); + } +} /** * @param {Object[]} updateParams @@ -20,5 +94,28 @@ function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID }); } +// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated +let lastUpdateIDAppliedToClient = 0; +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, + callback: (val) => (lastUpdateIDAppliedToClient = val), +}); + + +function needsToUpdateClient(previousUpdateID = 0) { + + // If no previousUpdateID is sent, this is not a WRITE request so we don't need to update our current state + if(!previousUpdateID) { + return false; + } + + // If we don't have any value in lastUpdateIDAppliedToClient, this is the first time we're receiving anything, so we need to do a last reconnectApp + if(!lastUpdateIDAppliedToClient) { + return true; + } + + return lastUpdateIDAppliedToClient < previousUpdateID; +} + // eslint-disable-next-line import/prefer-default-export -export {saveUpdateInformation}; +export {saveUpdateInformation, needsToUpdateClient, applyOnyxUpdates}; From db642ebb41409868391dd300bac1dfd075e347bd Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 20:57:12 +0200 Subject: [PATCH 44/79] adding clarifying comment --- src/libs/API.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libs/API.js b/src/libs/API.js index 9405fb8f3a51..4eade86a68bf 100644 --- a/src/libs/API.js +++ b/src/libs/API.js @@ -21,7 +21,8 @@ Request.use(Middleware.RecheckConnection); // Reauthentication - Handles jsonCode 407 which indicates an expired authToken. We need to reauthenticate and get a new authToken with our stored credentials. Request.use(Middleware.Reauthentication); -// SaveResponseInOnyx - Merges either the successData or failureData into Onyx depending on if the call was successful or not +// SaveResponseInOnyx - Merges either the successData or failureData into Onyx depending on if the call was successful or not. This needs to be the LAST middleware we use, don't add any +// middlewares after this. Request.use(Middleware.SaveResponseInOnyx); /** From bd72f2a514e66f0babd0b059a63b21ac6f3722b1 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 20:59:57 +0200 Subject: [PATCH 45/79] adding syncronous processing in case there's no need to fetch updates from the server --- src/libs/Middleware/SaveResponseInOnyx.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 448574af8101..e164a2bdb644 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -4,6 +4,9 @@ import ONYXKEYS from '../../ONYXKEYS'; import * as MemoryOnlyKeys from '../actions/MemoryOnlyKeys/MemoryOnlyKeys'; import * as OnyxUpdates from '../actions/OnyxUpdates'; +// If we're executing any of these requests, we don't need to trigger our OnyxUpdates flow to update the current data even if our current value is out of date. +const requestsToIgnoreLastUpdateID = ['OpenApp', 'ReconnectApp', 'GetMissingOnyxMessages']; + /** * @param {Promise} response * @param {Object} request @@ -31,7 +34,7 @@ function SaveResponseInOnyx(response, request) { // Sometimes we call requests that are successfull but they don't have any response or any success/failure data to set. Let's return early since // we don't need to store anything here. if (!onyxUpdates && !request.successData && !request.failureData) { - return; + return Promise.resolve(responseData); } // If there is an OnyxUpdate for using memory only keys, enable them @@ -44,6 +47,14 @@ function SaveResponseInOnyx(response, request) { return true; }); + if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.needsToUpdateClient(Number(responseData.previousUpdateID || 0))) { + return OnyxUpdates.applyOnyxUpdates({ + type: CONST.ONYX_UPDATE_TYPES.HTTPS, + request: request, + responseData: responseData + }); + } + // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server OnyxUpdates.saveUpdateInformation( { @@ -57,7 +68,7 @@ function SaveResponseInOnyx(response, request) { Number(responseData.previousUpdateID || 0), ); - return responseData; + return Promise.resolve(responseData); }); } From b758fd4b52aa5a5c72508b05fef6e29e65627a04 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 21:01:19 +0200 Subject: [PATCH 46/79] fixing parameters --- src/libs/Middleware/SaveResponseInOnyx.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index e164a2bdb644..6775a927c6d2 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -50,8 +50,10 @@ function SaveResponseInOnyx(response, request) { if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.needsToUpdateClient(Number(responseData.previousUpdateID || 0))) { return OnyxUpdates.applyOnyxUpdates({ type: CONST.ONYX_UPDATE_TYPES.HTTPS, - request: request, - responseData: responseData + data: { + request, + responseData, + }, }); } From fb9e4831d9435b4926c5a0027490737bcb6e051e Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 21:01:49 +0200 Subject: [PATCH 47/79] removing unecessary check for APIs --- src/libs/Middleware/SaveResponseInOnyx.js | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 6775a927c6d2..0ffc77c00e6a 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -18,18 +18,7 @@ function SaveResponseInOnyx(response, request) { if (!responseData) { return; } - - // The data for this response comes in two different formats: - // 1. Original format - this is what was sent before the RELIABLE_UPDATES project and will go away once RELIABLE_UPDATES is fully complete - // - The data is an array of objects, where each object is an onyx update - // Example: [{onyxMethod: 'whatever', key: 'foo', value: 'bar'}] - // 1. Reliable updates format - this is what was sent with the RELIABLE_UPDATES project and will be the format from now on - // - The data is an object, containing updateIDs from the server and an array of onyx updates (this array is the same format as the original format above) - // Example: {lastUpdateID: 1, previousUpdateID: 0, onyxData: [{onyxMethod: 'whatever', key: 'foo', value: 'bar'}]} - // NOTE: This is slightly different than the format of the pusher event data, where pusher has "updates" and HTTPS responses have "onyxData" (long story) - - // Supports both the old format and the new format - const onyxUpdates = _.isArray(responseData) ? responseData : responseData.onyxData; + const onyxUpdates = responseData.onyxData; // Sometimes we call requests that are successfull but they don't have any response or any success/failure data to set. Let's return early since // we don't need to store anything here. From 06d72505900c42041f298c0812ea06580dd89785 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 21:03:04 +0200 Subject: [PATCH 48/79] renaming redundant name --- src/libs/Middleware/SaveResponseInOnyx.js | 2 +- src/libs/actions/OnyxUpdateManager.js | 2 +- src/libs/actions/OnyxUpdates.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 0ffc77c00e6a..b1971ec3560a 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -37,7 +37,7 @@ function SaveResponseInOnyx(response, request) { }); if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.needsToUpdateClient(Number(responseData.previousUpdateID || 0))) { - return OnyxUpdates.applyOnyxUpdates({ + return OnyxUpdates.apply({ type: CONST.ONYX_UPDATE_TYPES.HTTPS, data: { request, diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 25c98b0ce8e0..bf378e683a34 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -68,7 +68,7 @@ export default () => { promise.finally(() => { console.debug('[OnyxUpdateManager] Done applying all updates'); - OnyxUpdates.applyOnyxUpdates(updateParams).finally(() => { + OnyxUpdates.apply(updateParams).finally(() => { SequentialQueue.unpause(); }); }); diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index f3caead988d4..730293c36907 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -65,7 +65,7 @@ function applyPusherOnyxUpdates({updates}) { * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' * @returns {Promise} */ -function applyOnyxUpdates({type, data}) { +function apply({type, data}) { console.debug(`[OnyxUpdateManager] Applying update type: ${type}`, data); if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { return applyHTTPSOnyxUpdates(data); @@ -118,4 +118,4 @@ function needsToUpdateClient(previousUpdateID = 0) { } // eslint-disable-next-line import/prefer-default-export -export {saveUpdateInformation, needsToUpdateClient, applyOnyxUpdates}; +export {saveUpdateInformation, needsToUpdateClient, apply}; From a516500ea1329d913a4ba06b7f1e20fb7d2a5345 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 21:40:48 +0200 Subject: [PATCH 49/79] adding check on network if we should pause after a request is done --- src/libs/Network/SequentialQueue.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 2bbc766d868f..febf51ab94fb 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -44,7 +44,12 @@ function process() { // Set the current request to a promise awaiting its processing so that getCurrentRequest can be used to take some action after the current request has processed. currentRequest = Request.processWithMiddleware(requestToProcess, true) - .then(() => { + .then((responseData) => { + // While processing the request, we might return the property to pause the queue if we notice that we're out of date. We're doing this here so we avoid + // circular dependencies. + if(responseData.pauseQueue) { + pause(); + } PersistedRequests.remove(requestToProcess); RequestThrottle.clear(); return process(); From 440eaf9208deb4ea359dde20fbb38490e09521d6 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 21:41:39 +0200 Subject: [PATCH 50/79] adding new property on middleware response so we can block the queue if we need to hold the queue --- src/libs/Middleware/SaveResponseInOnyx.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index b1971ec3560a..4a7cca1dd6eb 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -59,7 +59,10 @@ function SaveResponseInOnyx(response, request) { Number(responseData.previousUpdateID || 0), ); - return Promise.resolve(responseData); + return Promise.resolve({ + ...responseData, + pauseQueue: true + }); }); } From 204cfd11d05c764c7fc26ed63625e8f111dd4a33 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 21:43:05 +0200 Subject: [PATCH 51/79] working on onyx updates queue so we don't lose updates when flushing --- src/libs/Network/SequentialQueue.js | 8 ++++---- src/libs/actions/QueuedOnyxUpdates.js | 24 ++++++++++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index febf51ab94fb..0e3db0663ce0 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -71,11 +71,11 @@ function process() { /** * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. */ -function flushAndClearOnyxUpdatesQueue() { +function flushOnyxUpdatesQueue() { if (isQueuePaused) { return; } - Onyx.update(QueuedOnyxUpdates.getQueuedUpdates()).then(QueuedOnyxUpdates.clear); + QueuedOnyxUpdates.flushQueue(); } function flush() { @@ -110,7 +110,7 @@ function flush() { isSequentialQueueRunning = false; resolveIsReadyPromise(); currentRequest = null; - flushAndClearOnyxUpdatesQueue(); + flushOnyxUpdatesQueue(); }); }, }); @@ -191,7 +191,7 @@ function unpause() { // Since the writes may happen async to the queue, in case we had any writes happen while the queue was paused // (because of race conditions), let's also apply the queued updates and clear them before continuing the queue. - flushAndClearOnyxUpdatesQueue(); + flushOnyxUpdatesQueue(); flush(); } diff --git a/src/libs/actions/QueuedOnyxUpdates.js b/src/libs/actions/QueuedOnyxUpdates.js index 486108dd56cf..30d6ee04cc9c 100644 --- a/src/libs/actions/QueuedOnyxUpdates.js +++ b/src/libs/actions/QueuedOnyxUpdates.js @@ -3,6 +3,8 @@ import ONYXKEYS from '../../ONYXKEYS'; // In this file we manage a queue of Onyx updates while the SequentialQueue is processing. There are functions to get the updates and clear the queue after saving the updates in Onyx. +let isFlushing = false; +let flushPromise; let queuedOnyxUpdates = []; Onyx.connect({ key: ONYXKEYS.QUEUED_ONYX_UPDATES, @@ -21,11 +23,25 @@ function clear() { Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, null); } +function internalFlush() { + const currentFlush = queuedOnyxUpdates; + queuedOnyxUpdates = []; + + Onyx.update(queuedOnyxUpdates) + return Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, null).then(() => isFlushing = false); +} + /** - * @returns {Array} + * @returns {Promise} */ -function getQueuedUpdates() { - return queuedOnyxUpdates; +function flushQueue() { + + if(isFlushing) { + flushPromise.then(internalFlush); + } + isFlushing = true; + flushPromise = internalFlush(); + return flushPromise; } -export {queueOnyxUpdates, clear, getQueuedUpdates}; +export {queueOnyxUpdates, clear, flushQueue}; From 798dde27cda1117620b921cb263203647ba0b919 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 22:16:58 +0200 Subject: [PATCH 52/79] a little bit of refactor --- src/libs/Network/SequentialQueue.js | 3 --- src/libs/actions/QueuedOnyxUpdates.js | 18 ++---------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 0e3db0663ce0..8d937c16f7d7 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -189,9 +189,6 @@ function unpause() { console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); isQueuePaused = false; - // Since the writes may happen async to the queue, in case we had any writes happen while the queue was paused - // (because of race conditions), let's also apply the queued updates and clear them before continuing the queue. - flushOnyxUpdatesQueue(); flush(); } diff --git a/src/libs/actions/QueuedOnyxUpdates.js b/src/libs/actions/QueuedOnyxUpdates.js index 30d6ee04cc9c..af98f406f8fb 100644 --- a/src/libs/actions/QueuedOnyxUpdates.js +++ b/src/libs/actions/QueuedOnyxUpdates.js @@ -23,25 +23,11 @@ function clear() { Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, null); } -function internalFlush() { - const currentFlush = queuedOnyxUpdates; - queuedOnyxUpdates = []; - - Onyx.update(queuedOnyxUpdates) - return Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, null).then(() => isFlushing = false); -} - /** * @returns {Promise} */ function flushQueue() { - - if(isFlushing) { - flushPromise.then(internalFlush); - } - isFlushing = true; - flushPromise = internalFlush(); - return flushPromise; + return Onyx.update(queuedOnyxUpdates).then(clear); } -export {queueOnyxUpdates, clear, flushQueue}; +export {flushQueue}; From 4de4fb70793ddb8850d2ca087a3260cff50028f0 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 22:33:31 +0200 Subject: [PATCH 53/79] more refactor --- src/libs/Middleware/SaveResponseInOnyx.js | 25 ++++++++++------------- src/libs/actions/OnyxUpdateManager.js | 19 ++++++++--------- src/libs/actions/OnyxUpdates.js | 23 ++++++++++++--------- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 4a7cca1dd6eb..a5e31462bae0 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -36,25 +36,22 @@ function SaveResponseInOnyx(response, request) { return true; }); + const responseToApply = { + type: CONST.ONYX_UPDATE_TYPES.HTTPS, + lastUpdateID: Number(responseData.lastUpdateID || 0), + data: { + request, + responseData, + }, + }; + if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.needsToUpdateClient(Number(responseData.previousUpdateID || 0))) { - return OnyxUpdates.apply({ - type: CONST.ONYX_UPDATE_TYPES.HTTPS, - data: { - request, - responseData, - }, - }); + return OnyxUpdates.apply(responseToApply); } // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server OnyxUpdates.saveUpdateInformation( - { - type: CONST.ONYX_UPDATE_TYPES.HTTPS, - data: { - request, - responseData, - }, - }, + responseToApply, Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0), ); diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index bf378e683a34..b10fefd0924c 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -42,7 +42,7 @@ export default () => { // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before // fully migrating to the reliable updates mode; // 2. This this client already has the reliable updates mode enabled, but it's missing some updates and it - // needs to fech those. + // needs to fetch those. // // To to both of those, we need to pause the sequential queue. This is important so that the updates are // applied in their correct and specific order. If this queue was not paused, then there would be a lot of @@ -54,6 +54,8 @@ export default () => { if (!lastUpdateIDAppliedToClient) { console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); + + // Since this is a full reconnectApp, we'll not apply the updates we received - those will come in the reconnect app request. promise = App.lastReconnectAppAfterActivatingReliableUpdates(); } else { // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above. @@ -63,20 +65,17 @@ export default () => { previousUpdateIDFromServer, lastUpdateIDAppliedToClient, }); - promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); + promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).then(() => { + OnyxUpdates.apply({...updateParams, lastUpdateID: lastUpdateIDFromServer}).finally(() => { + console.debug('[OnyxUpdateManager] Done applying all updates'); + }); + }); } promise.finally(() => { - console.debug('[OnyxUpdateManager] Done applying all updates'); - OnyxUpdates.apply(updateParams).finally(() => { - SequentialQueue.unpause(); - }); + SequentialQueue.unpause(); }); - if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { - // Update this value so that it matches what was just received from the server - Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); - } }, }); }; diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 730293c36907..7a1211907529 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -3,6 +3,13 @@ import ONYXKEYS from '../../ONYXKEYS'; import * as QueuedOnyxUpdates from './QueuedOnyxUpdates' +// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated +let lastUpdateIDAppliedToClient = 0; +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, + callback: (val) => (lastUpdateIDAppliedToClient = val), +}); + /** * @param {Object} data * @param {Object} data.request @@ -59,14 +66,18 @@ function applyPusherOnyxUpdates({updates}) { /** * @param {Object[]} updateParams * @param {String} updateParams.type + * @param {Number} updateParams.lastUpdateID * @param {Object} updateParams.data * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' * @returns {Promise} */ -function apply({type, data}) { - console.debug(`[OnyxUpdateManager] Applying update type: ${type}`, data); +function apply({lastUpdateID, type, data}) { + console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, data); + if (lastUpdateID && lastUpdateID > lastUpdateIDAppliedToClient) { + Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateID); + } if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { return applyHTTPSOnyxUpdates(data); } @@ -94,14 +105,6 @@ function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID }); } -// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated -let lastUpdateIDAppliedToClient = 0; -Onyx.connect({ - key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, - callback: (val) => (lastUpdateIDAppliedToClient = val), -}); - - function needsToUpdateClient(previousUpdateID = 0) { // If no previousUpdateID is sent, this is not a WRITE request so we don't need to update our current state From b3da51b6fa1e59f6b5728d519f7de62ade6bd907 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 22:39:06 +0200 Subject: [PATCH 54/79] fixing User.js calls --- src/libs/actions/User.js | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index f723032f0431..04c4a51c8677 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -546,8 +546,6 @@ function subscribeToUserEvents() { // Handles the mega multipleEvents from Pusher which contains an array of single events. // Each single event is passed to PusherUtils in order to trigger the callbacks for that event PusherUtils.subscribeToPrivateUserChannelEvent(Pusher.TYPE.MULTIPLE_EVENTS, currentUserAccountID, (pushJSON) => { - let updates; - // The data for this push event comes in two different formats: // 1. Original format - this is what was sent before the RELIABLE_UPDATES project and will go away once RELIABLE_UPDATES is fully complete // - The data is an array of objects, where each object is an onyx update @@ -556,20 +554,29 @@ function subscribeToUserEvents() { // - The data is an object, containing updateIDs from the server and an array of onyx updates (this array is the same format as the original format above) // Example: {lastUpdateID: 1, previousUpdateID: 0, updates: [{onyxMethod: 'whatever', key: 'foo', value: 'bar'}]} if (_.isArray(pushJSON)) { - updates = pushJSON; - _.each(updates, (multipleEvent) => { + _.each(pushJSON, (multipleEvent) => { PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); }); return; } - OnyxUpdates.saveUpdateInformation( - { - type: CONST.ONYX_UPDATE_TYPES.PUSHER, - data: { - updates: pushJSON.updates, - }, + const updates = { + type: CONST.ONYX_UPDATE_TYPES.PUSHER, + lastUpdateID: Number(pushJSON.lastUpdateID || 0), + data: { + updates: pushJSON.updates, }, + }; + if (!OnyxUpdates.needsToUpdateClient(Number(responseData.previousUpdateID || 0))) { + OnyxUpdates.apply(responseToApply); + return; + } + + // If we reached this point, we need to pause the queue while we prepare to fetch older OnyxUpdates. This needs to happen here since adding it on OnyxUpdates + // would cause a circular reference issue. + SequentialQueue.pause(); + OnyxUpdates.saveUpdateInformation( + updates, Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0), ); From a07073b80e8ec3eca14bb24eb24a5acd68779de3 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 22:42:08 +0200 Subject: [PATCH 55/79] adding property --- src/types/onyx/OnyxUpdatesFromServer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index c2c76afa0aa8..781a6b8564aa 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -3,6 +3,7 @@ import Request from './Request'; import Response from './Response'; type OnyxUpdatesFromServerData = { + lastUpdateID: number | string; request?: Request; response?: Response; updates?: OnyxUpdate[]; From 48934f8778591f2909e794ccbe55aba5daa0a3b4 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 22:43:44 +0200 Subject: [PATCH 56/79] returning responseData on applyHTTPSOnyxUpdates --- src/libs/actions/OnyxUpdates.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 7a1211907529..fb6cf018f8ba 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -40,6 +40,7 @@ function applyHTTPSOnyxUpdates({request, responseData}) { }) .then(() => { console.debug('[OnyxUpdateManager] Done applying HTTPS update'); + return responseData; }); } From d30a4eb9c958f8dc3aee242f6e9001f7b84f4c13 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 22:45:38 +0200 Subject: [PATCH 57/79] adding comment for clarity --- src/libs/Middleware/SaveResponseInOnyx.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index a5e31462bae0..c1890f6eb3b8 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -55,7 +55,9 @@ function SaveResponseInOnyx(response, request) { Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0), ); - + + // We'll add the pauseQueue property here to guarantee we're pausing the queue before we execute the next request. If we do it only on the trigger in + // OnyxUpdateManager, then we might have already started another request while we wait. return Promise.resolve({ ...responseData, pauseQueue: true From ec719bf5a6944cbd95eafe3c382610c9d5dd7440 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 22:48:11 +0200 Subject: [PATCH 58/79] clearing unused method --- src/libs/actions/QueuedOnyxUpdates.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libs/actions/QueuedOnyxUpdates.js b/src/libs/actions/QueuedOnyxUpdates.js index af98f406f8fb..5aa8159624d7 100644 --- a/src/libs/actions/QueuedOnyxUpdates.js +++ b/src/libs/actions/QueuedOnyxUpdates.js @@ -11,14 +11,6 @@ Onyx.connect({ callback: (val) => (queuedOnyxUpdates = val || []), }); -/** - * @param {Array} updates Onyx updates to queue for later - * @returns {Promise} - */ -function queueOnyxUpdates(updates) { - return Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, [...queuedOnyxUpdates, ...updates]); -} - function clear() { Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, null); } From 7aab96e5980c721661aa65178dfea3524e91876b Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 23:00:19 +0200 Subject: [PATCH 59/79] prettier, fixing const reference --- src/libs/Middleware/SaveResponseInOnyx.js | 14 +++++--------- src/libs/Network/SequentialQueue.js | 3 +-- src/libs/actions/OnyxUpdateManager.js | 7 ++----- src/libs/actions/OnyxUpdates.js | 13 ++++++------- src/libs/actions/User.js | 6 +----- 5 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index c1890f6eb3b8..2cdab0630eb0 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -36,7 +36,7 @@ function SaveResponseInOnyx(response, request) { return true; }); - const responseToApply = { + const responseToApply = { type: CONST.ONYX_UPDATE_TYPES.HTTPS, lastUpdateID: Number(responseData.lastUpdateID || 0), data: { @@ -50,17 +50,13 @@ function SaveResponseInOnyx(response, request) { } // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server - OnyxUpdates.saveUpdateInformation( - responseToApply, - Number(responseData.lastUpdateID || 0), - Number(responseData.previousUpdateID || 0), - ); - - // We'll add the pauseQueue property here to guarantee we're pausing the queue before we execute the next request. If we do it only on the trigger in + OnyxUpdates.saveUpdateInformation(responseToApply, Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0)); + + // We'll add the pauseQueue property here to guarantee we're pausing the queue before we execute the next request. If we do it only on the trigger in // OnyxUpdateManager, then we might have already started another request while we wait. return Promise.resolve({ ...responseData, - pauseQueue: true + pauseQueue: true, }); }); } diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 8d937c16f7d7..dbb9edf18499 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -47,7 +47,7 @@ function process() { .then((responseData) => { // While processing the request, we might return the property to pause the queue if we notice that we're out of date. We're doing this here so we avoid // circular dependencies. - if(responseData.pauseQueue) { + if (responseData.pauseQueue) { pause(); } PersistedRequests.remove(requestToProcess); @@ -67,7 +67,6 @@ function process() { return currentRequest; } - /** * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. */ diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index b10fefd0924c..967b1f6515af 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -5,7 +5,7 @@ import CONST from '../../CONST'; import Log from '../Log'; import * as SequentialQueue from '../Network/SequentialQueue'; import * as App from './App'; -import * as OnyxUpdates from './OnyxUpdates' +import * as OnyxUpdates from './OnyxUpdates'; // This file is in charge of looking at the updateIDs coming from the server and comparing them to the last updateID that the client has. // If the client is behind the server, then we need to pause everything, get the missing updates from the server, apply those updates, @@ -16,8 +16,6 @@ import * as OnyxUpdates from './OnyxUpdates' // The circular dependency happen because this file calls the API GetMissingOnyxUpdates which uses the SaveResponseInOnyx.js file // (as a middleware). Therefore, SaveResponseInOnyx.js can't import and use this file directly. - - // This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated let lastUpdateIDAppliedToClient = 0; Onyx.connect({ @@ -67,7 +65,7 @@ export default () => { }); promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).then(() => { OnyxUpdates.apply({...updateParams, lastUpdateID: lastUpdateIDFromServer}).finally(() => { - console.debug('[OnyxUpdateManager] Done applying all updates'); + console.debug('[OnyxUpdateManager] Done applying all updates'); }); }); } @@ -75,7 +73,6 @@ export default () => { promise.finally(() => { SequentialQueue.unpause(); }); - }, }); }; diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index fb6cf018f8ba..6caa6c351a1e 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -1,7 +1,7 @@ import Onyx from 'react-native-onyx'; import ONYXKEYS from '../../ONYXKEYS'; -import * as QueuedOnyxUpdates from './QueuedOnyxUpdates' - +import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; +import CONST from './CONST'; // This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated let lastUpdateIDAppliedToClient = 0; @@ -76,7 +76,7 @@ function applyPusherOnyxUpdates({updates}) { */ function apply({lastUpdateID, type, data}) { console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, data); - if (lastUpdateID && lastUpdateID > lastUpdateIDAppliedToClient) { + if (lastUpdateID && lastUpdateID > lastUpdateIDAppliedToClient) { Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateID); } if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { @@ -107,14 +107,13 @@ function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID } function needsToUpdateClient(previousUpdateID = 0) { - // If no previousUpdateID is sent, this is not a WRITE request so we don't need to update our current state - if(!previousUpdateID) { + if (!previousUpdateID) { return false; } - + // If we don't have any value in lastUpdateIDAppliedToClient, this is the first time we're receiving anything, so we need to do a last reconnectApp - if(!lastUpdateIDAppliedToClient) { + if (!lastUpdateIDAppliedToClient) { return true; } diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index 425caebc6096..b2cba5b9adf4 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -575,11 +575,7 @@ function subscribeToUserEvents() { // If we reached this point, we need to pause the queue while we prepare to fetch older OnyxUpdates. This needs to happen here since adding it on OnyxUpdates // would cause a circular reference issue. SequentialQueue.pause(); - OnyxUpdates.saveUpdateInformation( - updates, - Number(pushJSON.lastUpdateID || 0), - Number(pushJSON.previousUpdateID || 0), - ); + OnyxUpdates.saveUpdateInformation(updates, Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0)); }); // Handles Onyx updates coming from Pusher through the mega multipleEvents. From 9ede8ad1ba3512361bf8b18ecc353a681c51d5fc Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Tue, 5 Sep 2023 23:10:21 +0200 Subject: [PATCH 60/79] fixing import to the right path --- src/libs/actions/OnyxUpdates.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 6caa6c351a1e..01b7ff7ec1bb 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -1,7 +1,7 @@ import Onyx from 'react-native-onyx'; import ONYXKEYS from '../../ONYXKEYS'; import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; -import CONST from './CONST'; +import CONST from '../../CONST'; // This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated let lastUpdateIDAppliedToClient = 0; From 8a73c0a5726056cde611399002ea935fda378c52 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 00:28:43 +0200 Subject: [PATCH 61/79] adding back a method that is used --- src/libs/actions/QueuedOnyxUpdates.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libs/actions/QueuedOnyxUpdates.js b/src/libs/actions/QueuedOnyxUpdates.js index 5aa8159624d7..dbda87ea5530 100644 --- a/src/libs/actions/QueuedOnyxUpdates.js +++ b/src/libs/actions/QueuedOnyxUpdates.js @@ -11,6 +11,14 @@ Onyx.connect({ callback: (val) => (queuedOnyxUpdates = val || []), }); +/** + * @param {Array} updates Onyx updates to queue for later + * @returns {Promise} + */ +function queueOnyxUpdates(updates) { + return Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, [...queuedOnyxUpdates, ...updates]); +} + function clear() { Onyx.set(ONYXKEYS.QUEUED_ONYX_UPDATES, null); } @@ -22,4 +30,4 @@ function flushQueue() { return Onyx.update(queuedOnyxUpdates).then(clear); } -export {flushQueue}; +export {queueOnyxUpdates, flushQueue}; From a8697eed212e7c3366e116533ced66dc14600403 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 00:29:22 +0200 Subject: [PATCH 62/79] adding back the flush call when restarting the queue --- src/libs/Network/SequentialQueue.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index dbb9edf18499..2ff77a207c53 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -187,7 +187,7 @@ function unpause() { const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); isQueuePaused = false; - + flushOnyxUpdatesQueue(); flush(); } From 2cda55b87caa1c91b9a1d627bf4eb4aad1e4c567 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 00:30:03 +0200 Subject: [PATCH 63/79] adding sequentialqueue.unpause on promise cal --- src/libs/actions/OnyxUpdateManager.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 967b1f6515af..b8e58458b61c 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -63,15 +63,14 @@ export default () => { previousUpdateIDFromServer, lastUpdateIDAppliedToClient, }); - promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).then(() => { - OnyxUpdates.apply({...updateParams, lastUpdateID: lastUpdateIDFromServer}).finally(() => { - console.debug('[OnyxUpdateManager] Done applying all updates'); - }); - }); + promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); } promise.finally(() => { - SequentialQueue.unpause(); + OnyxUpdates.apply({...updateParams, lastUpdateID: lastUpdateIDFromServer}).finally(() => { + console.debug('[OnyxUpdateManager] Done applying all updates'); + SequentialQueue.unpause(); + }); }); }, }); From b333592c44990a4ffd8911586cc57905868a3cde Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 00:31:20 +0200 Subject: [PATCH 64/79] prettier and adding a check to ignore updates if they are older than current state --- src/libs/actions/OnyxUpdates.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 01b7ff7ec1bb..8d34213ac3ad 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -76,6 +76,11 @@ function applyPusherOnyxUpdates({updates}) { */ function apply({lastUpdateID, type, data}) { console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, data); + + if (lastUpdateID < lastUpdateIDAppliedToClient) { + console.debug('[OnyxUpdateManager] Update received was older than current state, returning without applying the updates'); + return new Promise().resolve(); + } if (lastUpdateID && lastUpdateID > lastUpdateIDAppliedToClient) { Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateID); } From 45d3a6df9dd91227bd1addd240fec1ec79f96dee Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 00:50:40 +0200 Subject: [PATCH 65/79] addressing comments, fixing lint issues and prettier --- src/libs/API.js | 2 +- src/libs/Middleware/SaveResponseInOnyx.js | 10 ++-- src/libs/Network/SequentialQueue.js | 62 ++++++++++++----------- src/libs/actions/App.js | 6 +-- src/libs/actions/OnyxUpdateManager.js | 34 +++++++------ src/libs/actions/OnyxUpdates.js | 13 +++-- src/libs/actions/QueuedOnyxUpdates.js | 2 - src/libs/actions/User.js | 4 +- 8 files changed, 71 insertions(+), 62 deletions(-) diff --git a/src/libs/API.js b/src/libs/API.js index 4eade86a68bf..491503f07381 100644 --- a/src/libs/API.js +++ b/src/libs/API.js @@ -22,7 +22,7 @@ Request.use(Middleware.RecheckConnection); Request.use(Middleware.Reauthentication); // SaveResponseInOnyx - Merges either the successData or failureData into Onyx depending on if the call was successful or not. This needs to be the LAST middleware we use, don't add any -// middlewares after this. +// middlewares after this, because the SequentialQueue depends on the result of this middleware to pause the queue (if needed) to bring the app to an up-to-date state. Request.use(Middleware.SaveResponseInOnyx); /** diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 2cdab0630eb0..ce178a2b2e16 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -4,7 +4,8 @@ import ONYXKEYS from '../../ONYXKEYS'; import * as MemoryOnlyKeys from '../actions/MemoryOnlyKeys/MemoryOnlyKeys'; import * as OnyxUpdates from '../actions/OnyxUpdates'; -// If we're executing any of these requests, we don't need to trigger our OnyxUpdates flow to update the current data even if our current value is out of date. +// If we're executing any of these requests, we don't need to trigger our OnyxUpdates flow to update the current data even if our current value is out of +// date because all these requests are updating the app to the most current state. const requestsToIgnoreLastUpdateID = ['OpenApp', 'ReconnectApp', 'GetMissingOnyxMessages']; /** @@ -45,18 +46,17 @@ function SaveResponseInOnyx(response, request) { }, }; - if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.needsToUpdateClient(Number(responseData.previousUpdateID || 0))) { + if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.doesClientNeedToBeUpdated(Number(responseData.previousUpdateID || 0))) { return OnyxUpdates.apply(responseToApply); } // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server OnyxUpdates.saveUpdateInformation(responseToApply, Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0)); - // We'll add the pauseQueue property here to guarantee we're pausing the queue before we execute the next request. If we do it only on the trigger in - // OnyxUpdateManager, then we might have already started another request while we wait. + // Ensure the queue is paused while the client resolves the gap in onyx updates so that updates are guaranteed to happen in a specific order. return Promise.resolve({ ...responseData, - pauseQueue: true, + shouldPauseQueue: true, }); }); } diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 2ff77a207c53..e3e6f4b3f780 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -21,6 +21,33 @@ let isSequentialQueueRunning = false; let currentRequest = null; let isQueuePaused = false; +/** + * Puts the queue into a paused state so that no requests will be processed + */ +function pause() { + if (isQueuePaused) { + return; + } + + console.debug('[SequentialQueue] Pausing the queue'); + isQueuePaused = true; +} + +/** + * Unpauses the queue and flushes all the requests that were in it or were added to it while paused + */ +function unpause() { + if (!isQueuePaused) { + return; + } + + const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; + console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); + isQueuePaused = false; + flushOnyxUpdatesQueue(); + flush(); +} + /** * Process any persisted requests, when online, one at a time until the queue is empty. * @@ -45,9 +72,9 @@ function process() { // Set the current request to a promise awaiting its processing so that getCurrentRequest can be used to take some action after the current request has processed. currentRequest = Request.processWithMiddleware(requestToProcess, true) .then((responseData) => { - // While processing the request, we might return the property to pause the queue if we notice that we're out of date. We're doing this here so we avoid - // circular dependencies. - if (responseData.pauseQueue) { + // A response might indicate that the queue should be paused. This happens when a gap in onyx updates is detected between the client and the server and + // that gap needs resolved before the queue can continue. + if (responseData.shouldPauseQueue) { pause(); } PersistedRequests.remove(requestToProcess); @@ -71,6 +98,8 @@ function process() { * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. */ function flushOnyxUpdatesQueue() { + // The only situation where the queue is paused is if we found a gap between the app current data state and our server's. If that happens, + // we'll trigger async calls to make the client updated again. While we do that, we don't want to insert anything in Onyx. if (isQueuePaused) { return; } @@ -164,31 +193,4 @@ function waitForIdle() { return isReadyPromise; } -/** - * Puts the queue into a paused state so that no requests will be processed - */ -function pause() { - if (isQueuePaused) { - return; - } - - console.debug('[SequentialQueue] Pausing the queue'); - isQueuePaused = true; -} - -/** - * Unpauses the queue and flushes all the requests that were in it or were added to it while paused - */ -function unpause() { - if (!isQueuePaused) { - return; - } - - const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; - console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); - isQueuePaused = false; - flushOnyxUpdatesQueue(); - flush(); -} - export {flush, getCurrentRequest, isRunning, push, waitForIdle, pause, unpause}; diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index 076a89917d7e..90c2a9ec4f16 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -213,7 +213,7 @@ function reconnectApp(updateIDFrom = 0) { * state because of race conditions between reconnectApp and other pusher updates being applied at the same time. * @return {Promise} */ -function lastReconnectAppAfterActivatingReliableUpdates() { +function finalReconnectAppAfterActivatingReliableUpdates() { console.debug(`[OnyxUpdates] Executing last reconnect app with promise`); return getPolicyParamsForOpenOrReconnect().then((policyParams) => { const params = {...policyParams}; @@ -230,7 +230,7 @@ function lastReconnectAppAfterActivatingReliableUpdates() { // It is SUPER BAD FORM to return promises from action methods. // DO NOT FOLLOW THIS PATTERN!!!!! // It was absolutely necessary in order to not break the app while migrating to the new reliable updates pattern. This method will be removed - // as soon as we have everyone migrated to it + // as soon as we have everyone migrated to the reliableUpdate beta. // eslint-disable-next-line rulesdir/no-api-side-effects-method return API.makeRequestWithSideEffects('ReconnectApp', params, getOnyxDataForOpenOrReconnect()); }); @@ -471,5 +471,5 @@ export { beginDeepLinkRedirectAfterTransition, createWorkspaceAndNavigateToIt, getMissingOnyxUpdates, - lastReconnectAppAfterActivatingReliableUpdates, + finalReconnectAppAfterActivatingReliableUpdates, }; diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index b8e58458b61c..8d0335ed03b7 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -1,22 +1,25 @@ -import _ from 'underscore'; import Onyx from 'react-native-onyx'; import ONYXKEYS from '../../ONYXKEYS'; -import CONST from '../../CONST'; import Log from '../Log'; import * as SequentialQueue from '../Network/SequentialQueue'; import * as App from './App'; import * as OnyxUpdates from './OnyxUpdates'; // This file is in charge of looking at the updateIDs coming from the server and comparing them to the last updateID that the client has. -// If the client is behind the server, then we need to pause everything, get the missing updates from the server, apply those updates, -// then restart everything. This will ensure that the client is up-to-date with the server and all the updates have been applied -// in the correct order. +// If the client is behind the server, then we need to +// 1. Pause all sequential queue requests +// 2. Pause all Onyx updates from Pusher +// 3. Get the missing updates from the server +// 4. Apply those updates +// 5. Apply the original update that triggered this request (it could have come from either HTTPS or Pusher) +// 6. Restart the sequential queue +// 7. Restart the Onyx updates from Pusher +// This will ensure that the client is up-to-date with the server and all the updates have been applied in the correct order. // It's important that this file is separate and not imported by OnyxUpdates.js, so that there are no circular dependencies. Onyx -// is used as a pub/sub mechanism to break out of the circular dependencies. -// The circular dependency happen because this file calls the API GetMissingOnyxUpdates which uses the SaveResponseInOnyx.js file +// is used as a pub/sub mechanism to break out of the circular dependency. +// The circular dependency happens because this file calls API.GetMissingOnyxUpdates() which uses the SaveResponseInOnyx.js file // (as a middleware). Therefore, SaveResponseInOnyx.js can't import and use this file directly. -// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated let lastUpdateIDAppliedToClient = 0; Onyx.connect({ key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, @@ -38,23 +41,22 @@ export default () => { // we need to perform one of the 2 possible cases: // // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before - // fully migrating to the reliable updates mode; - // 2. This this client already has the reliable updates mode enabled, but it's missing some updates and it + // fully migrating to the reliable updates mode. + // 2. This client already has the reliable updates mode enabled, but it's missing some updates and it // needs to fetch those. // - // To to both of those, we need to pause the sequential queue. This is important so that the updates are + // For both of those, we need to pause the sequential queue. This is important so that the updates are // applied in their correct and specific order. If this queue was not paused, then there would be a lot of // onyx data being applied while we are fetching the missing updates and that would put them all out of order. SequentialQueue.pause(); - let promise; + let canUnpauseQueuePromise; // The flow below is setting the promise to a reconnect app to address flow (1) explained above. if (!lastUpdateIDAppliedToClient) { - console.debug('[OnyxUpdateManager] Client has not gotten reliable updates before so reconnecting the app to start the process'); Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); // Since this is a full reconnectApp, we'll not apply the updates we received - those will come in the reconnect app request. - promise = App.lastReconnectAppAfterActivatingReliableUpdates(); + canUnpauseQueuePromise = App.finalReconnectAppAfterActivatingReliableUpdates(); } else { // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above. console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); @@ -63,10 +65,10 @@ export default () => { previousUpdateIDFromServer, lastUpdateIDAppliedToClient, }); - promise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); + canUnpauseQueuePromise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); } - promise.finally(() => { + canUnpauseQueuePromise.finally(() => { OnyxUpdates.apply({...updateParams, lastUpdateID: lastUpdateIDFromServer}).finally(() => { console.debug('[OnyxUpdateManager] Done applying all updates'); SequentialQueue.unpause(); diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 8d34213ac3ad..939c2d4a50d2 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -1,9 +1,12 @@ +import _ from 'underscore'; +import PusherUtils from '../PusherUtils'; import Onyx from 'react-native-onyx'; import ONYXKEYS from '../../ONYXKEYS'; import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; import CONST from '../../CONST'; -// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated +// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated. If that +// callback were triggered it would lead to duplicate processing of server updates. let lastUpdateIDAppliedToClient = 0; Onyx.connect({ key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, @@ -111,7 +114,11 @@ function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID }); } -function needsToUpdateClient(previousUpdateID = 0) { +/** + * This function will receive the previousUpdateID from any request/pusher update that has it, compare to our current app state + * and return if an update is needed + */ +function doesClientNeedToBeUpdated(previousUpdateID = 0) { // If no previousUpdateID is sent, this is not a WRITE request so we don't need to update our current state if (!previousUpdateID) { return false; @@ -126,4 +133,4 @@ function needsToUpdateClient(previousUpdateID = 0) { } // eslint-disable-next-line import/prefer-default-export -export {saveUpdateInformation, needsToUpdateClient, apply}; +export {saveUpdateInformation, doesClientNeedToBeUpdated, apply}; diff --git a/src/libs/actions/QueuedOnyxUpdates.js b/src/libs/actions/QueuedOnyxUpdates.js index dbda87ea5530..06f15be1340f 100644 --- a/src/libs/actions/QueuedOnyxUpdates.js +++ b/src/libs/actions/QueuedOnyxUpdates.js @@ -3,8 +3,6 @@ import ONYXKEYS from '../../ONYXKEYS'; // In this file we manage a queue of Onyx updates while the SequentialQueue is processing. There are functions to get the updates and clear the queue after saving the updates in Onyx. -let isFlushing = false; -let flushPromise; let queuedOnyxUpdates = []; Onyx.connect({ key: ONYXKEYS.QUEUED_ONYX_UPDATES, diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index b2cba5b9adf4..a54568463c70 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -567,8 +567,8 @@ function subscribeToUserEvents() { updates: pushJSON.updates, }, }; - if (!OnyxUpdates.needsToUpdateClient(Number(responseData.previousUpdateID || 0))) { - OnyxUpdates.apply(responseToApply); + if (!OnyxUpdates.doesClientNeedToBeUpdated(Number(pushJSON.previousUpdateID || 0))) { + OnyxUpdates.apply(updates); return; } From 7e0e60a0811fcdf3f2e5e8297db96f6f7b4d34e2 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 00:58:03 +0200 Subject: [PATCH 66/79] lint --- src/libs/Network/SequentialQueue.js | 92 ++++++++++++++--------------- src/libs/actions/OnyxUpdates.js | 4 +- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index e3e6f4b3f780..c4f6100dfb0f 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -48,52 +48,6 @@ function unpause() { flush(); } -/** - * Process any persisted requests, when online, one at a time until the queue is empty. - * - * If a request fails due to some kind of network error, such as a request being throttled or when our backend is down, then we retry it with an exponential back off process until a response - * is successfully returned. The first time a request fails we set a random, small, initial wait time. After waiting, we retry the request. If there are subsequent failures the request wait - * time is doubled creating an exponential back off in the frequency of requests hitting the server. Since the initial wait time is random and it increases exponentially, the load of - * requests to our backend is evenly distributed and it gradually decreases with time, which helps the servers catch up. - * @returns {Promise} - */ -function process() { - // When the queue is paused, return early. This prevents any new requests from happening. The queue will be flushed again when the queue is unpaused. - if (isQueuePaused) { - return Promise.resolve(); - } - - const persistedRequests = PersistedRequests.getAll(); - if (_.isEmpty(persistedRequests) || NetworkStore.isOffline()) { - return Promise.resolve(); - } - const requestToProcess = persistedRequests[0]; - - // Set the current request to a promise awaiting its processing so that getCurrentRequest can be used to take some action after the current request has processed. - currentRequest = Request.processWithMiddleware(requestToProcess, true) - .then((responseData) => { - // A response might indicate that the queue should be paused. This happens when a gap in onyx updates is detected between the client and the server and - // that gap needs resolved before the queue can continue. - if (responseData.shouldPauseQueue) { - pause(); - } - PersistedRequests.remove(requestToProcess); - RequestThrottle.clear(); - return process(); - }) - .catch((error) => { - // On sign out we cancel any in flight requests from the user. Since that user is no longer signed in their requests should not be retried. - // Duplicate records don't need to be retried as they just mean the record already exists on the server - if (error.name === CONST.ERROR.REQUEST_CANCELLED || error.message === CONST.ERROR.DUPLICATE_RECORD) { - PersistedRequests.remove(requestToProcess); - RequestThrottle.clear(); - return process(); - } - return RequestThrottle.sleep().then(process); - }); - return currentRequest; -} - /** * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. */ @@ -144,6 +98,52 @@ function flush() { }); } +/** + * Process any persisted requests, when online, one at a time until the queue is empty. + * + * If a request fails due to some kind of network error, such as a request being throttled or when our backend is down, then we retry it with an exponential back off process until a response + * is successfully returned. The first time a request fails we set a random, small, initial wait time. After waiting, we retry the request. If there are subsequent failures the request wait + * time is doubled creating an exponential back off in the frequency of requests hitting the server. Since the initial wait time is random and it increases exponentially, the load of + * requests to our backend is evenly distributed and it gradually decreases with time, which helps the servers catch up. + * @returns {Promise} + */ +function process() { + // When the queue is paused, return early. This prevents any new requests from happening. The queue will be flushed again when the queue is unpaused. + if (isQueuePaused) { + return Promise.resolve(); + } + + const persistedRequests = PersistedRequests.getAll(); + if (_.isEmpty(persistedRequests) || NetworkStore.isOffline()) { + return Promise.resolve(); + } + const requestToProcess = persistedRequests[0]; + + // Set the current request to a promise awaiting its processing so that getCurrentRequest can be used to take some action after the current request has processed. + currentRequest = Request.processWithMiddleware(requestToProcess, true) + .then((responseData) => { + // A response might indicate that the queue should be paused. This happens when a gap in onyx updates is detected between the client and the server and + // that gap needs resolved before the queue can continue. + if (responseData.shouldPauseQueue) { + pause(); + } + PersistedRequests.remove(requestToProcess); + RequestThrottle.clear(); + return process(); + }) + .catch((error) => { + // On sign out we cancel any in flight requests from the user. Since that user is no longer signed in their requests should not be retried. + // Duplicate records don't need to be retried as they just mean the record already exists on the server + if (error.name === CONST.ERROR.REQUEST_CANCELLED || error.message === CONST.ERROR.DUPLICATE_RECORD) { + PersistedRequests.remove(requestToProcess); + RequestThrottle.clear(); + return process(); + } + return RequestThrottle.sleep().then(process); + }); + return currentRequest; +} + /** * @returns {Boolean} */ diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 939c2d4a50d2..29c3f39d16dd 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -1,6 +1,6 @@ +import Onyx from 'react-native-onyx'; import _ from 'underscore'; import PusherUtils from '../PusherUtils'; -import Onyx from 'react-native-onyx'; import ONYXKEYS from '../../ONYXKEYS'; import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; import CONST from '../../CONST'; @@ -117,6 +117,8 @@ function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID /** * This function will receive the previousUpdateID from any request/pusher update that has it, compare to our current app state * and return if an update is needed + * @param {Number} previousUpdateID The previousUpdateID contained in the response object + * @returns {Boolean} */ function doesClientNeedToBeUpdated(previousUpdateID = 0) { // If no previousUpdateID is sent, this is not a WRITE request so we don't need to update our current state From 1d91a4def7cb0c8349ac43d5e474e5b7b246b26b Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 01:06:38 +0200 Subject: [PATCH 67/79] fixing order hopefully for the last time for linter --- src/libs/Network/SequentialQueue.js | 106 ++++++++++++++-------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index c4f6100dfb0f..56a11b492442 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -33,21 +33,6 @@ function pause() { isQueuePaused = true; } -/** - * Unpauses the queue and flushes all the requests that were in it or were added to it while paused - */ -function unpause() { - if (!isQueuePaused) { - return; - } - - const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; - console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); - isQueuePaused = false; - flushOnyxUpdatesQueue(); - flush(); -} - /** * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. */ @@ -60,44 +45,6 @@ function flushOnyxUpdatesQueue() { QueuedOnyxUpdates.flushQueue(); } -function flush() { - // When the queue is paused, return early. This will keep an requests in the queue and they will get flushed again when the queue is unpaused - if (isQueuePaused) { - return; - } - - if (isSequentialQueueRunning || _.isEmpty(PersistedRequests.getAll())) { - return; - } - - // ONYXKEYS.PERSISTED_REQUESTS is shared across clients, thus every client/tab will have a copy - // It is very important to only process the queue from leader client otherwise requests will be duplicated. - if (!ActiveClientManager.isClientTheLeader()) { - return; - } - - isSequentialQueueRunning = true; - - // Reset the isReadyPromise so that the queue will be flushed as soon as the request is finished - isReadyPromise = new Promise((resolve) => { - resolveIsReadyPromise = resolve; - }); - - // Ensure persistedRequests are read from storage before proceeding with the queue - const connectionID = Onyx.connect({ - key: ONYXKEYS.PERSISTED_REQUESTS, - callback: () => { - Onyx.disconnect(connectionID); - process().finally(() => { - isSequentialQueueRunning = false; - resolveIsReadyPromise(); - currentRequest = null; - flushOnyxUpdatesQueue(); - }); - }, - }); -} - /** * Process any persisted requests, when online, one at a time until the queue is empty. * @@ -144,6 +91,59 @@ function process() { return currentRequest; } +function flush() { + // When the queue is paused, return early. This will keep an requests in the queue and they will get flushed again when the queue is unpaused + if (isQueuePaused) { + return; + } + + if (isSequentialQueueRunning || _.isEmpty(PersistedRequests.getAll())) { + return; + } + + // ONYXKEYS.PERSISTED_REQUESTS is shared across clients, thus every client/tab will have a copy + // It is very important to only process the queue from leader client otherwise requests will be duplicated. + if (!ActiveClientManager.isClientTheLeader()) { + return; + } + + isSequentialQueueRunning = true; + + // Reset the isReadyPromise so that the queue will be flushed as soon as the request is finished + isReadyPromise = new Promise((resolve) => { + resolveIsReadyPromise = resolve; + }); + + // Ensure persistedRequests are read from storage before proceeding with the queue + const connectionID = Onyx.connect({ + key: ONYXKEYS.PERSISTED_REQUESTS, + callback: () => { + Onyx.disconnect(connectionID); + process().finally(() => { + isSequentialQueueRunning = false; + resolveIsReadyPromise(); + currentRequest = null; + flushOnyxUpdatesQueue(); + }); + }, + }); +} + +/** + * Unpauses the queue and flushes all the requests that were in it or were added to it while paused + */ +function unpause() { + if (!isQueuePaused) { + return; + } + + const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; + console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); + isQueuePaused = false; + flushOnyxUpdatesQueue(); + flush(); +} + /** * @returns {Boolean} */ From 7ab2d523417623d50a4ca91c76c9f19f567423a0 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 6 Sep 2023 19:18:56 +0200 Subject: [PATCH 68/79] fixing resolve response --- src/libs/PusherUtils.js | 2 +- src/libs/actions/OnyxUpdates.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/PusherUtils.js b/src/libs/PusherUtils.js index 7fdd12cf6296..b4615d3c7d8b 100644 --- a/src/libs/PusherUtils.js +++ b/src/libs/PusherUtils.js @@ -22,7 +22,7 @@ function subscribeToMultiEvent(eventType, callback) { */ function triggerMultiEventHandler(eventType, data) { if (!multiEventCallbackMapping[eventType]) { - return new Promise().resolve(); + return Promise.resolve(); } return multiEventCallbackMapping[eventType](data); } diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 29c3f39d16dd..bd313c221575 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -82,7 +82,7 @@ function apply({lastUpdateID, type, data}) { if (lastUpdateID < lastUpdateIDAppliedToClient) { console.debug('[OnyxUpdateManager] Update received was older than current state, returning without applying the updates'); - return new Promise().resolve(); + return Promise.resolve(); } if (lastUpdateID && lastUpdateID > lastUpdateIDAppliedToClient) { Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateID); From 30d8eafe4df5d310bade33b2b4301a583c053cae Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Thu, 7 Sep 2023 00:49:19 +0200 Subject: [PATCH 69/79] fixing check to apply updates in case lastUpdateID is 0 --- src/libs/actions/OnyxUpdates.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index bd313c221575..27b18cdec9c7 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -80,7 +80,7 @@ function applyPusherOnyxUpdates({updates}) { function apply({lastUpdateID, type, data}) { console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, data); - if (lastUpdateID < lastUpdateIDAppliedToClient) { + if (lastUpdateID && lastUpdateID < lastUpdateIDAppliedToClient) { console.debug('[OnyxUpdateManager] Update received was older than current state, returning without applying the updates'); return Promise.resolve(); } From 05938e2942a589a4217e553981e0b82762ec760e Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 8 Sep 2023 15:14:45 +0200 Subject: [PATCH 70/79] removing comment about the circular reference --- src/libs/actions/User.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index a54568463c70..6fccf5220bc8 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -572,8 +572,7 @@ function subscribeToUserEvents() { return; } - // If we reached this point, we need to pause the queue while we prepare to fetch older OnyxUpdates. This needs to happen here since adding it on OnyxUpdates - // would cause a circular reference issue. + // If we reached this point, we need to pause the queue while we prepare to fetch older OnyxUpdates. SequentialQueue.pause(); OnyxUpdates.saveUpdateInformation(updates, Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0)); }); From ead5186fd0ecc3ee93d6ba0bb148e22dee7ddae8 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 8 Sep 2023 15:18:43 +0200 Subject: [PATCH 71/79] updating signature for ts mapping --- src/types/onyx/OnyxUpdatesFromServer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index 781a6b8564aa..8ced8dd12757 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -10,8 +10,8 @@ type OnyxUpdatesFromServerData = { }; type OnyxUpdatesFromServer = { - lastUpdateID: number | string; - previousUpdateID: number | string; + lastUpdateIDFromServer: number | string; + previousUpdateIDFromServer: number | string; type: 'https' | 'pusher'; data: OnyxUpdatesFromServerData; }; From 0b8a8ccc83b2f17e19c3788263550dc7a15c58ed Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 8 Sep 2023 15:20:41 +0200 Subject: [PATCH 72/79] renaming responseData to response --- src/libs/Middleware/SaveResponseInOnyx.js | 20 ++++++++++---------- src/libs/actions/OnyxUpdates.js | 14 +++++++------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index ce178a2b2e16..4241df5abf23 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -14,17 +14,17 @@ const requestsToIgnoreLastUpdateID = ['OpenApp', 'ReconnectApp', 'GetMissingOnyx * @returns {Promise} */ function SaveResponseInOnyx(response, request) { - return response.then((responseData) => { - // Make sure we have response data (i.e. response isn't a promise being passed down to us by a failed retry request and responseData undefined) - if (!responseData) { + return response.then((response) => { + // Make sure we have response data (i.e. response isn't a promise being passed down to us by a failed retry request and response undefined) + if (!response) { return; } - const onyxUpdates = responseData.onyxData; + const onyxUpdates = response.onyxData; // Sometimes we call requests that are successfull but they don't have any response or any success/failure data to set. Let's return early since // we don't need to store anything here. if (!onyxUpdates && !request.successData && !request.failureData) { - return Promise.resolve(responseData); + return Promise.resolve(response); } // If there is an OnyxUpdate for using memory only keys, enable them @@ -39,23 +39,23 @@ function SaveResponseInOnyx(response, request) { const responseToApply = { type: CONST.ONYX_UPDATE_TYPES.HTTPS, - lastUpdateID: Number(responseData.lastUpdateID || 0), + lastUpdateID: Number(response.lastUpdateID || 0), data: { request, - responseData, + response, }, }; - if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.doesClientNeedToBeUpdated(Number(responseData.previousUpdateID || 0))) { + if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.doesClientNeedToBeUpdated(Number(response.previousUpdateID || 0))) { return OnyxUpdates.apply(responseToApply); } // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server - OnyxUpdates.saveUpdateInformation(responseToApply, Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0)); + OnyxUpdates.saveUpdateInformation(responseToApply, Number(response.lastUpdateID || 0), Number(response.previousUpdateID || 0)); // Ensure the queue is paused while the client resolves the gap in onyx updates so that updates are guaranteed to happen in a specific order. return Promise.resolve({ - ...responseData, + ...response, shouldPauseQueue: true, }); }); diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 27b18cdec9c7..58220109b568 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -16,10 +16,10 @@ Onyx.connect({ /** * @param {Object} data * @param {Object} data.request - * @param {Object} data.responseData + * @param {Object} data.response * @returns {Promise} */ -function applyHTTPSOnyxUpdates({request, responseData}) { +function applyHTTPSOnyxUpdates({request, response}) { console.debug('[OnyxUpdateManager] Applying https update'); // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. @@ -28,22 +28,22 @@ function applyHTTPSOnyxUpdates({request, responseData}) { // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained // in successData/failureData until after the component has received and API data. - const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); + const onyxDataUpdatePromise = response.onyxData ? updateHandler(response.onyxData) : Promise.resolve(); return onyxDataUpdatePromise .then(() => { // Handle the request's success/failure data (client-side data) - if (responseData.jsonCode === 200 && request.successData) { + if (response.jsonCode === 200 && request.successData) { return updateHandler(request.successData); } - if (responseData.jsonCode !== 200 && request.failureData) { + if (response.jsonCode !== 200 && request.failureData) { return updateHandler(request.failureData); } return Promise.resolve(); }) .then(() => { console.debug('[OnyxUpdateManager] Done applying HTTPS update'); - return responseData; + return response; }); } @@ -100,7 +100,7 @@ function apply({lastUpdateID, type, data}) { * @param {String} updateParams.type * @param {Object} updateParams.data * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.responseData] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' * @param {Number} [lastUpdateID] * @param {Number} [previousUpdateID] From ec61746f80a944e8152f00173d48dc8a95477129 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 8 Sep 2023 15:23:59 +0200 Subject: [PATCH 73/79] changing from reduce to map --- src/libs/actions/OnyxUpdates.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 58220109b568..1ddbf066465d 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -54,14 +54,7 @@ function applyHTTPSOnyxUpdates({request, response}) { */ function applyPusherOnyxUpdates({updates}) { console.debug('[OnyxUpdateManager] Applying pusher update'); - const pusherEventPromises = _.reduce( - updates, - (result, update) => { - result.push(PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); - return result; - }, - [], - ); + const pusherEventPromises = _.map(updates, update => PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); return Promise.all(pusherEventPromises).then(() => { console.debug('[OnyxUpdateManager] Done applying Pusher update'); }); From c5a8eb542758b578a112fb2fa94d73243d5b1111 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 8 Sep 2023 15:39:41 +0200 Subject: [PATCH 74/79] linter and fixing signature again --- src/libs/Middleware/SaveResponseInOnyx.js | 4 ++-- src/libs/Network/SequentialQueue.js | 4 ++-- src/libs/actions/OnyxUpdates.js | 2 +- src/types/onyx/OnyxUpdatesFromServer.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 4241df5abf23..8ee5d167589c 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -13,8 +13,8 @@ const requestsToIgnoreLastUpdateID = ['OpenApp', 'ReconnectApp', 'GetMissingOnyx * @param {Object} request * @returns {Promise} */ -function SaveResponseInOnyx(response, request) { - return response.then((response) => { +function SaveResponseInOnyx(requestResponse, request) { + return requestResponse.then((response) => { // Make sure we have response data (i.e. response isn't a promise being passed down to us by a failed retry request and response undefined) if (!response) { return; diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 56a11b492442..e53515fb5e87 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -68,10 +68,10 @@ function process() { // Set the current request to a promise awaiting its processing so that getCurrentRequest can be used to take some action after the current request has processed. currentRequest = Request.processWithMiddleware(requestToProcess, true) - .then((responseData) => { + .then((response) => { // A response might indicate that the queue should be paused. This happens when a gap in onyx updates is detected between the client and the server and // that gap needs resolved before the queue can continue. - if (responseData.shouldPauseQueue) { + if (response.shouldPauseQueue) { pause(); } PersistedRequests.remove(requestToProcess); diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 1ddbf066465d..906dda2eb6dc 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -54,7 +54,7 @@ function applyHTTPSOnyxUpdates({request, response}) { */ function applyPusherOnyxUpdates({updates}) { console.debug('[OnyxUpdateManager] Applying pusher update'); - const pusherEventPromises = _.map(updates, update => PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); + const pusherEventPromises = _.map(updates, (update) => PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); return Promise.all(pusherEventPromises).then(() => { console.debug('[OnyxUpdateManager] Done applying Pusher update'); }); diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index 8ced8dd12757..c653a5054673 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -13,7 +13,7 @@ type OnyxUpdatesFromServer = { lastUpdateIDFromServer: number | string; previousUpdateIDFromServer: number | string; type: 'https' | 'pusher'; - data: OnyxUpdatesFromServerData; + updateParams: OnyxUpdatesFromServerData; }; export default OnyxUpdatesFromServer; From 20fead9f220d38864b1618009ee7ca74a564a489 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Fri, 8 Sep 2023 15:44:28 +0200 Subject: [PATCH 75/79] fixing lint --- src/libs/Middleware/SaveResponseInOnyx.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index 8ee5d167589c..af407294a01d 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -9,7 +9,7 @@ import * as OnyxUpdates from '../actions/OnyxUpdates'; const requestsToIgnoreLastUpdateID = ['OpenApp', 'ReconnectApp', 'GetMissingOnyxMessages']; /** - * @param {Promise} response + * @param {Promise} requestResponse * @param {Object} request * @returns {Promise} */ From 250aa1797a7f1b5d17224c9aa71aa0510dd93b54 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Sat, 9 Sep 2023 00:47:43 +0300 Subject: [PATCH 76/79] changing object signature to simplify it --- src/libs/Middleware/SaveResponseInOnyx.js | 9 +++-- src/libs/actions/OnyxUpdateManager.js | 6 ++-- src/libs/actions/OnyxUpdates.js | 44 ++++++++++------------- src/libs/actions/User.js | 7 ++-- src/types/onyx/OnyxUpdatesFromServer.ts | 10 ++---- 5 files changed, 31 insertions(+), 45 deletions(-) diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index af407294a01d..8cb66c0c10d0 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -40,10 +40,9 @@ function SaveResponseInOnyx(requestResponse, request) { const responseToApply = { type: CONST.ONYX_UPDATE_TYPES.HTTPS, lastUpdateID: Number(response.lastUpdateID || 0), - data: { - request, - response, - }, + previousUpdateID: Number(response.previousUpdateID || 0), + request, + response, }; if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.doesClientNeedToBeUpdated(Number(response.previousUpdateID || 0))) { @@ -51,7 +50,7 @@ function SaveResponseInOnyx(requestResponse, request) { } // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server - OnyxUpdates.saveUpdateInformation(responseToApply, Number(response.lastUpdateID || 0), Number(response.previousUpdateID || 0)); + OnyxUpdates.saveUpdateInformation(responseToApply); // Ensure the queue is paused while the client resolves the gap in onyx updates so that updates are guaranteed to happen in a specific order. return Promise.resolve({ diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js index 8d0335ed03b7..f0051b85f302 100644 --- a/src/libs/actions/OnyxUpdateManager.js +++ b/src/libs/actions/OnyxUpdateManager.js @@ -35,7 +35,9 @@ export default () => { return; } - const {lastUpdateIDFromServer, previousUpdateIDFromServer, updateParams} = val; + const updateParams = val; + const lastUpdateIDFromServer = val.lastUpdateID; + const previousUpdateIDFromServer = val.previousUpdateID; // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient // we need to perform one of the 2 possible cases: @@ -69,7 +71,7 @@ export default () => { } canUnpauseQueuePromise.finally(() => { - OnyxUpdates.apply({...updateParams, lastUpdateID: lastUpdateIDFromServer}).finally(() => { + OnyxUpdates.apply(updateParams).finally(() => { console.debug('[OnyxUpdateManager] Done applying all updates'); SequentialQueue.unpause(); }); diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 906dda2eb6dc..6c210f1abadc 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -14,12 +14,11 @@ Onyx.connect({ }); /** - * @param {Object} data - * @param {Object} data.request - * @param {Object} data.response + * @param {Object} request + * @param {Object} response * @returns {Promise} */ -function applyHTTPSOnyxUpdates({request, response}) { +function applyHTTPSOnyxUpdates(request, response) { console.debug('[OnyxUpdateManager] Applying https update'); // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. @@ -48,11 +47,10 @@ function applyHTTPSOnyxUpdates({request, response}) { } /** - * @param {Object} data - * @param {Object} data.updates + * @param {Array} updates * @returns {Promise} */ -function applyPusherOnyxUpdates({updates}) { +function applyPusherOnyxUpdates(updates) { console.debug('[OnyxUpdateManager] Applying pusher update'); const pusherEventPromises = _.map(updates, (update) => PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); return Promise.all(pusherEventPromises).then(() => { @@ -64,13 +62,12 @@ function applyPusherOnyxUpdates({updates}) { * @param {Object[]} updateParams * @param {String} updateParams.type * @param {Number} updateParams.lastUpdateID - * @param {Object} updateParams.data - * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' + * @param {Object} [updateParams.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.updates] Exists if updateParams.type === 'pusher' * @returns {Promise} */ -function apply({lastUpdateID, type, data}) { +function apply({lastUpdateID, type, request, response, updates}) { console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, data); if (lastUpdateID && lastUpdateID < lastUpdateIDAppliedToClient) { @@ -81,30 +78,25 @@ function apply({lastUpdateID, type, data}) { Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateID); } if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { - return applyHTTPSOnyxUpdates(data); + return applyHTTPSOnyxUpdates(request, response); } if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { - return applyPusherOnyxUpdates(data); + return applyPusherOnyxUpdates(updates); } } /** * @param {Object[]} updateParams * @param {String} updateParams.type - * @param {Object} updateParams.data - * @param {Object} [updateParams.data.request] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.response] Exists if updateParams.type === 'https' - * @param {Object} [updateParams.data.updates] Exists if updateParams.type === 'pusher' - * @param {Number} [lastUpdateID] - * @param {Number} [previousUpdateID] + * @param {Object} [updateParams.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.updates] Exists if updateParams.type === 'pusher' + * @param {Number} [updateParams.lastUpdateID] + * @param {Number} [updateParams.previousUpdateID] */ -function saveUpdateInformation(updateParams, lastUpdateID = 0, previousUpdateID = 0) { +function saveUpdateInformation(updateParams) { // Always use set() here so that the updateParams are never merged and always unique to the request that came in - Onyx.set(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, { - lastUpdateIDFromServer: lastUpdateID, - previousUpdateIDFromServer: previousUpdateID, - updateParams, - }); + Onyx.set(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, updateParams); } /** diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index 6fccf5220bc8..ee93c6acb1e5 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -563,9 +563,8 @@ function subscribeToUserEvents() { const updates = { type: CONST.ONYX_UPDATE_TYPES.PUSHER, lastUpdateID: Number(pushJSON.lastUpdateID || 0), - data: { - updates: pushJSON.updates, - }, + updates: pushJSON.updates, + previousUpdateID: Number(pushJSON.previousUpdateID || 0), }; if (!OnyxUpdates.doesClientNeedToBeUpdated(Number(pushJSON.previousUpdateID || 0))) { OnyxUpdates.apply(updates); @@ -574,7 +573,7 @@ function subscribeToUserEvents() { // If we reached this point, we need to pause the queue while we prepare to fetch older OnyxUpdates. SequentialQueue.pause(); - OnyxUpdates.saveUpdateInformation(updates, Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0)); + OnyxUpdates.saveUpdateInformation(updates); }); // Handles Onyx updates coming from Pusher through the mega multipleEvents. diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index c653a5054673..c50362f3e6c7 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -2,18 +2,12 @@ import {OnyxUpdate} from 'react-native-onyx'; import Request from './Request'; import Response from './Response'; -type OnyxUpdatesFromServerData = { +type OnyxUpdatesFromServer = { + type: 'https' | 'pusher'; lastUpdateID: number | string; request?: Request; response?: Response; updates?: OnyxUpdate[]; }; -type OnyxUpdatesFromServer = { - lastUpdateIDFromServer: number | string; - previousUpdateIDFromServer: number | string; - type: 'https' | 'pusher'; - updateParams: OnyxUpdatesFromServerData; -}; - export default OnyxUpdatesFromServer; From e07c47c1b087b7201607d38079408fbbfd2ef665 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Sat, 9 Sep 2023 01:19:45 +0300 Subject: [PATCH 77/79] removing data reference --- src/libs/actions/OnyxUpdates.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 6c210f1abadc..5c4d3bf93a65 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -42,7 +42,7 @@ function applyHTTPSOnyxUpdates(request, response) { }) .then(() => { console.debug('[OnyxUpdateManager] Done applying HTTPS update'); - return response; + return Promise.resolve(response);; }); } @@ -68,7 +68,7 @@ function applyPusherOnyxUpdates(updates) { * @returns {Promise} */ function apply({lastUpdateID, type, request, response, updates}) { - console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, data); + console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, { request, response, updates } ); if (lastUpdateID && lastUpdateID < lastUpdateIDAppliedToClient) { console.debug('[OnyxUpdateManager] Update received was older than current state, returning without applying the updates'); From 9b700142137b36d990bd87326e6bd25eacab06e8 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Sat, 9 Sep 2023 03:40:41 +0300 Subject: [PATCH 78/79] prettier --- src/libs/actions/OnyxUpdates.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index 5c4d3bf93a65..8e45e7dd2e66 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -42,7 +42,7 @@ function applyHTTPSOnyxUpdates(request, response) { }) .then(() => { console.debug('[OnyxUpdateManager] Done applying HTTPS update'); - return Promise.resolve(response);; + return Promise.resolve(response); }); } @@ -68,7 +68,7 @@ function applyPusherOnyxUpdates(updates) { * @returns {Promise} */ function apply({lastUpdateID, type, request, response, updates}) { - console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, { request, response, updates } ); + console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, {request, response, updates}); if (lastUpdateID && lastUpdateID < lastUpdateIDAppliedToClient) { console.debug('[OnyxUpdateManager] Update received was older than current state, returning without applying the updates'); From d012c5499e2500dd4cf16bd8b3a1fd19deb95a73 Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Sun, 10 Sep 2023 10:55:11 +0800 Subject: [PATCH 79/79] adding previousUpdateID --- src/types/onyx/OnyxUpdatesFromServer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts index c50362f3e6c7..02a96d4ce230 100644 --- a/src/types/onyx/OnyxUpdatesFromServer.ts +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -5,6 +5,7 @@ import Response from './Response'; type OnyxUpdatesFromServer = { type: 'https' | 'pusher'; lastUpdateID: number | string; + previousUpdateID: number | string; request?: Request; response?: Response; updates?: OnyxUpdate[];