diff --git a/src/App.js b/src/App.js index c432a0b666c8..7ec82b9a4f8a 100644 --- a/src/App.js +++ b/src/App.js @@ -24,6 +24,7 @@ import {CurrentReportIDContextProvider} from './components/withCurrentReportID'; import {EnvironmentProvider} from './components/withEnvironment'; import * as Session from './libs/actions/Session'; import useDefaultDragAndDrop from './hooks/useDefaultDragAndDrop'; +import OnyxUpdateManager from './libs/actions/OnyxUpdateManager'; // For easier debugging and development, when we are in web we expose Onyx to the window, so you can more easily set data into Onyx if (window && Environment.isDevelopment()) { @@ -42,6 +43,7 @@ const fill = {flex: 1}; function App() { useDefaultDragAndDrop(); + OnyxUpdateManager(); return ( { - // Make sure we have response data (i.e. response isn't a promise being passed down to us by a failed retry request and responseData undefined) - if (!responseData) { +function SaveResponseInOnyx(requestResponse, request) { + return requestResponse.then((response) => { + // Make sure we have response data (i.e. response isn't a promise being passed down to us by a failed retry request and response undefined) + if (!response) { return; } + const onyxUpdates = response.onyxData; - // The data for this response comes in two different formats: - // 1. Original format - this is what was sent before the RELIABLE_UPDATES project and will go away once RELIABLE_UPDATES is fully complete - // - The data is an array of objects, where each object is an onyx update - // Example: [{onyxMethod: 'whatever', key: 'foo', value: 'bar'}] - // 1. Reliable updates format - this is what was sent with the RELIABLE_UPDATES project and will be the format from now on - // - The data is an object, containing updateIDs from the server and an array of onyx updates (this array is the same format as the original format above) - // Example: {lastUpdateID: 1, previousUpdateID: 0, onyxData: [{onyxMethod: 'whatever', key: 'foo', value: 'bar'}]} - // NOTE: This is slightly different than the format of the pusher event data, where pusher has "updates" and HTTPS responses have "onyxData" (long story) + // Sometimes we call requests that are successfull but they don't have any response or any success/failure data to set. Let's return early since + // we don't need to store anything here. + if (!onyxUpdates && !request.successData && !request.failureData) { + return Promise.resolve(response); + } - // Supports both the old format and the new format - const onyxUpdates = _.isArray(responseData) ? responseData : responseData.onyxData; // If there is an OnyxUpdate for using memory only keys, enable them _.find(onyxUpdates, ({key, value}) => { if (key !== ONYXKEYS.IS_USING_MEMORY_ONLY_KEYS || !value) { @@ -39,30 +37,26 @@ function SaveResponseInOnyx(response, request) { return true; }); - // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server - OnyxUpdates.saveUpdateIDs(Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0)); + const responseToApply = { + type: CONST.ONYX_UPDATE_TYPES.HTTPS, + lastUpdateID: Number(response.lastUpdateID || 0), + previousUpdateID: Number(response.previousUpdateID || 0), + request, + response, + }; - // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in - // the UI. See https://github.com/Expensify/App/issues/12775 for more info. - const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; + if (_.includes(requestsToIgnoreLastUpdateID, request.command) || !OnyxUpdates.doesClientNeedToBeUpdated(Number(response.previousUpdateID || 0))) { + return OnyxUpdates.apply(responseToApply); + } - // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then - // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained - // in successData/failureData until after the component has received and API data. - const onyxDataUpdatePromise = responseData.onyxData ? updateHandler(responseData.onyxData) : Promise.resolve(); + // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server + OnyxUpdates.saveUpdateInformation(responseToApply); - return onyxDataUpdatePromise - .then(() => { - // Handle the request's success/failure data (client-side data) - if (responseData.jsonCode === 200 && request.successData) { - return updateHandler(request.successData); - } - if (responseData.jsonCode !== 200 && request.failureData) { - return updateHandler(request.failureData); - } - return Promise.resolve(); - }) - .then(() => responseData); + // Ensure the queue is paused while the client resolves the gap in onyx updates so that updates are guaranteed to happen in a specific order. + return Promise.resolve({ + ...response, + shouldPauseQueue: true, + }); }); } diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index f8ea396663a5..e53515fb5e87 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -21,6 +21,30 @@ let isSequentialQueueRunning = false; let currentRequest = null; let isQueuePaused = false; +/** + * Puts the queue into a paused state so that no requests will be processed + */ +function pause() { + if (isQueuePaused) { + return; + } + + console.debug('[SequentialQueue] Pausing the queue'); + isQueuePaused = true; +} + +/** + * Gets the current Onyx queued updates, apply them and clear the queue if the queue is not paused. + */ +function flushOnyxUpdatesQueue() { + // The only situation where the queue is paused is if we found a gap between the app current data state and our server's. If that happens, + // we'll trigger async calls to make the client updated again. While we do that, we don't want to insert anything in Onyx. + if (isQueuePaused) { + return; + } + QueuedOnyxUpdates.flushQueue(); +} + /** * Process any persisted requests, when online, one at a time until the queue is empty. * @@ -44,7 +68,12 @@ function process() { // Set the current request to a promise awaiting its processing so that getCurrentRequest can be used to take some action after the current request has processed. currentRequest = Request.processWithMiddleware(requestToProcess, true) - .then(() => { + .then((response) => { + // A response might indicate that the queue should be paused. This happens when a gap in onyx updates is detected between the client and the server and + // that gap needs resolved before the queue can continue. + if (response.shouldPauseQueue) { + pause(); + } PersistedRequests.remove(requestToProcess); RequestThrottle.clear(); return process(); @@ -94,12 +123,27 @@ function flush() { isSequentialQueueRunning = false; resolveIsReadyPromise(); currentRequest = null; - Onyx.update(QueuedOnyxUpdates.getQueuedUpdates()).then(QueuedOnyxUpdates.clear); + flushOnyxUpdatesQueue(); }); }, }); } +/** + * Unpauses the queue and flushes all the requests that were in it or were added to it while paused + */ +function unpause() { + if (!isQueuePaused) { + return; + } + + const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; + console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); + isQueuePaused = false; + flushOnyxUpdatesQueue(); + flush(); +} + /** * @returns {Boolean} */ @@ -149,30 +193,4 @@ function waitForIdle() { return isReadyPromise; } -/** - * Puts the queue into a paused state so that no requests will be processed - */ -function pause() { - if (isQueuePaused) { - return; - } - - console.debug('[SequentialQueue] Pausing the queue'); - isQueuePaused = true; -} - -/** - * Unpauses the queue and flushes all the requests that were in it or were added to it while paused - */ -function unpause() { - if (!isQueuePaused) { - return; - } - - const numberOfPersistedRequests = PersistedRequests.getAll().length || 0; - console.debug(`[SequentialQueue] Unpausing the queue and flushing ${numberOfPersistedRequests} requests`); - isQueuePaused = false; - flush(); -} - export {flush, getCurrentRequest, isRunning, push, waitForIdle, pause, unpause}; diff --git a/src/libs/PusherUtils.js b/src/libs/PusherUtils.js index 9d84bd4012fe..b4615d3c7d8b 100644 --- a/src/libs/PusherUtils.js +++ b/src/libs/PusherUtils.js @@ -18,12 +18,13 @@ function subscribeToMultiEvent(eventType, callback) { /** * @param {String} eventType * @param {Mixed} data + * @returns {Promise} */ function triggerMultiEventHandler(eventType, data) { if (!multiEventCallbackMapping[eventType]) { - return; + return Promise.resolve(); } - multiEventCallbackMapping[eventType](data); + return multiEventCallbackMapping[eventType](data); } /** diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index 6028e0468696..90c2a9ec4f16 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -18,7 +18,6 @@ import * as Session from './Session'; import * as ReportActionsUtils from '../ReportActionsUtils'; import Timing from './Timing'; import * as Browser from '../Browser'; -import * as SequentialQueue from '../Network/SequentialQueue'; let currentUserAccountID; let currentUserEmail; @@ -208,6 +207,35 @@ function reconnectApp(updateIDFrom = 0) { }); } +/** + * Fetches data when the app will call reconnectApp without params for the last time. This is a separate function + * because it will follow patterns that are not recommended so we can be sure we're not putting the app in a unusable + * state because of race conditions between reconnectApp and other pusher updates being applied at the same time. + * @return {Promise} + */ +function finalReconnectAppAfterActivatingReliableUpdates() { + console.debug(`[OnyxUpdates] Executing last reconnect app with promise`); + return getPolicyParamsForOpenOrReconnect().then((policyParams) => { + const params = {...policyParams}; + + // When the app reconnects we do a fast "sync" of the LHN and only return chats that have new messages. We achieve this by sending the most recent reportActionID. + // we have locally. And then only update the user about chats with messages that have occurred after that reportActionID. + // + // - Look through the local report actions and reports to find the most recently modified report action or report. + // - We send this to the server so that it can compute which new chats the user needs to see and return only those as an optimization. + Timing.start(CONST.TIMING.CALCULATE_MOST_RECENT_LAST_MODIFIED_ACTION); + params.mostRecentReportActionLastModified = ReportActionsUtils.getMostRecentReportActionLastModified(); + Timing.end(CONST.TIMING.CALCULATE_MOST_RECENT_LAST_MODIFIED_ACTION, '', 500); + + // It is SUPER BAD FORM to return promises from action methods. + // DO NOT FOLLOW THIS PATTERN!!!!! + // It was absolutely necessary in order to not break the app while migrating to the new reliable updates pattern. This method will be removed + // as soon as we have everyone migrated to the reliableUpdate beta. + // eslint-disable-next-line rulesdir/no-api-side-effects-method + return API.makeRequestWithSideEffects('ReconnectApp', params, getOnyxDataForOpenOrReconnect()); + }); +} + /** * Fetches data when the client has discovered it missed some Onyx updates from the server * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from @@ -231,48 +259,6 @@ function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) { ); } -// The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has. -// When a gap is detected, the missing updates are fetched from the API. - -// These key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated -let lastUpdateIDAppliedToClient = 0; -Onyx.connect({ - key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, - callback: (val) => (lastUpdateIDAppliedToClient = val), -}); - -Onyx.connect({ - key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, - callback: (val) => { - if (!val) { - return; - } - - const {lastUpdateIDFromServer, previousUpdateIDFromServer} = val; - console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); - console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); - console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); - - // If the previous update from the server does not match the last update the client got, then the client is missing some updates. - // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. - if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { - console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates'); - Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { - lastUpdateIDFromServer, - previousUpdateIDFromServer, - lastUpdateIDAppliedToClient, - }); - SequentialQueue.pause(); - getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer).finally(SequentialQueue.unpause); - } - - if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { - // Update this value so that it matches what was just received from the server - Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); - } - }, -}); - /** * This promise is used so that deeplink component know when a transition is end. * This is necessary because we want to begin deeplink redirection after the transition is end. @@ -484,4 +470,6 @@ export { beginDeepLinkRedirect, beginDeepLinkRedirectAfterTransition, createWorkspaceAndNavigateToIt, + getMissingOnyxUpdates, + finalReconnectAppAfterActivatingReliableUpdates, }; diff --git a/src/libs/actions/OnyxUpdateManager.js b/src/libs/actions/OnyxUpdateManager.js new file mode 100644 index 000000000000..f0051b85f302 --- /dev/null +++ b/src/libs/actions/OnyxUpdateManager.js @@ -0,0 +1,81 @@ +import Onyx from 'react-native-onyx'; +import ONYXKEYS from '../../ONYXKEYS'; +import Log from '../Log'; +import * as SequentialQueue from '../Network/SequentialQueue'; +import * as App from './App'; +import * as OnyxUpdates from './OnyxUpdates'; + +// This file is in charge of looking at the updateIDs coming from the server and comparing them to the last updateID that the client has. +// If the client is behind the server, then we need to +// 1. Pause all sequential queue requests +// 2. Pause all Onyx updates from Pusher +// 3. Get the missing updates from the server +// 4. Apply those updates +// 5. Apply the original update that triggered this request (it could have come from either HTTPS or Pusher) +// 6. Restart the sequential queue +// 7. Restart the Onyx updates from Pusher +// This will ensure that the client is up-to-date with the server and all the updates have been applied in the correct order. +// It's important that this file is separate and not imported by OnyxUpdates.js, so that there are no circular dependencies. Onyx +// is used as a pub/sub mechanism to break out of the circular dependency. +// The circular dependency happens because this file calls API.GetMissingOnyxUpdates() which uses the SaveResponseInOnyx.js file +// (as a middleware). Therefore, SaveResponseInOnyx.js can't import and use this file directly. + +let lastUpdateIDAppliedToClient = 0; +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, + callback: (val) => (lastUpdateIDAppliedToClient = val), +}); + +export default () => { + console.debug('[OnyxUpdateManager] Listening for updates from the server'); + Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, + callback: (val) => { + if (!val) { + return; + } + + const updateParams = val; + const lastUpdateIDFromServer = val.lastUpdateID; + const previousUpdateIDFromServer = val.previousUpdateID; + + // In cases where we received a previousUpdateID and it doesn't match our lastUpdateIDAppliedToClient + // we need to perform one of the 2 possible cases: + // + // 1. This is the first time we're receiving an lastUpdateID, so we need to do a final reconnectApp before + // fully migrating to the reliable updates mode. + // 2. This client already has the reliable updates mode enabled, but it's missing some updates and it + // needs to fetch those. + // + // For both of those, we need to pause the sequential queue. This is important so that the updates are + // applied in their correct and specific order. If this queue was not paused, then there would be a lot of + // onyx data being applied while we are fetching the missing updates and that would put them all out of order. + SequentialQueue.pause(); + let canUnpauseQueuePromise; + + // The flow below is setting the promise to a reconnect app to address flow (1) explained above. + if (!lastUpdateIDAppliedToClient) { + Log.info('Client has not gotten reliable updates before so reconnecting the app to start the process'); + + // Since this is a full reconnectApp, we'll not apply the updates we received - those will come in the reconnect app request. + canUnpauseQueuePromise = App.finalReconnectAppAfterActivatingReliableUpdates(); + } else { + // The flow below is setting the promise to a getMissingOnyxUpdates to address flow (2) explained above. + console.debug(`[OnyxUpdateManager] Client is behind the server by ${previousUpdateIDFromServer - lastUpdateIDAppliedToClient} so fetching incremental updates`); + Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { + lastUpdateIDFromServer, + previousUpdateIDFromServer, + lastUpdateIDAppliedToClient, + }); + canUnpauseQueuePromise = App.getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); + } + + canUnpauseQueuePromise.finally(() => { + OnyxUpdates.apply(updateParams).finally(() => { + console.debug('[OnyxUpdateManager] Done applying all updates'); + SequentialQueue.unpause(); + }); + }); + }, + }); +}; diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js index e582016f0109..8e45e7dd2e66 100644 --- a/src/libs/actions/OnyxUpdates.js +++ b/src/libs/actions/OnyxUpdates.js @@ -1,22 +1,123 @@ import Onyx from 'react-native-onyx'; +import _ from 'underscore'; +import PusherUtils from '../PusherUtils'; import ONYXKEYS from '../../ONYXKEYS'; +import * as QueuedOnyxUpdates from './QueuedOnyxUpdates'; +import CONST from '../../CONST'; + +// This key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated. If that +// callback were triggered it would lead to duplicate processing of server updates. +let lastUpdateIDAppliedToClient = 0; +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, + callback: (val) => (lastUpdateIDAppliedToClient = val), +}); /** - * - * @param {Number} [lastUpdateID] - * @param {Number} [previousUpdateID] + * @param {Object} request + * @param {Object} response + * @returns {Promise} */ -function saveUpdateIDs(lastUpdateID = 0, previousUpdateID = 0) { - // Return early if there were no updateIDs - if (!lastUpdateID) { - return; - } +function applyHTTPSOnyxUpdates(request, response) { + console.debug('[OnyxUpdateManager] Applying https update'); + // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in + // the UI. See https://github.com/Expensify/App/issues/12775 for more info. + const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; - Onyx.merge(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, { - lastUpdateIDFromServer: lastUpdateID, - previousUpdateIDFromServer: previousUpdateID, + // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then + // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained + // in successData/failureData until after the component has received and API data. + const onyxDataUpdatePromise = response.onyxData ? updateHandler(response.onyxData) : Promise.resolve(); + + return onyxDataUpdatePromise + .then(() => { + // Handle the request's success/failure data (client-side data) + if (response.jsonCode === 200 && request.successData) { + return updateHandler(request.successData); + } + if (response.jsonCode !== 200 && request.failureData) { + return updateHandler(request.failureData); + } + return Promise.resolve(); + }) + .then(() => { + console.debug('[OnyxUpdateManager] Done applying HTTPS update'); + return Promise.resolve(response); + }); +} + +/** + * @param {Array} updates + * @returns {Promise} + */ +function applyPusherOnyxUpdates(updates) { + console.debug('[OnyxUpdateManager] Applying pusher update'); + const pusherEventPromises = _.map(updates, (update) => PusherUtils.triggerMultiEventHandler(update.eventType, update.data)); + return Promise.all(pusherEventPromises).then(() => { + console.debug('[OnyxUpdateManager] Done applying Pusher update'); }); } +/** + * @param {Object[]} updateParams + * @param {String} updateParams.type + * @param {Number} updateParams.lastUpdateID + * @param {Object} [updateParams.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.updates] Exists if updateParams.type === 'pusher' + * @returns {Promise} + */ +function apply({lastUpdateID, type, request, response, updates}) { + console.debug(`[OnyxUpdateManager] Applying update type: ${type} with lastUpdateID: ${lastUpdateID}`, {request, response, updates}); + + if (lastUpdateID && lastUpdateID < lastUpdateIDAppliedToClient) { + console.debug('[OnyxUpdateManager] Update received was older than current state, returning without applying the updates'); + return Promise.resolve(); + } + if (lastUpdateID && lastUpdateID > lastUpdateIDAppliedToClient) { + Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateID); + } + if (type === CONST.ONYX_UPDATE_TYPES.HTTPS) { + return applyHTTPSOnyxUpdates(request, response); + } + if (type === CONST.ONYX_UPDATE_TYPES.PUSHER) { + return applyPusherOnyxUpdates(updates); + } +} + +/** + * @param {Object[]} updateParams + * @param {String} updateParams.type + * @param {Object} [updateParams.request] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.response] Exists if updateParams.type === 'https' + * @param {Object} [updateParams.updates] Exists if updateParams.type === 'pusher' + * @param {Number} [updateParams.lastUpdateID] + * @param {Number} [updateParams.previousUpdateID] + */ +function saveUpdateInformation(updateParams) { + // Always use set() here so that the updateParams are never merged and always unique to the request that came in + Onyx.set(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, updateParams); +} + +/** + * This function will receive the previousUpdateID from any request/pusher update that has it, compare to our current app state + * and return if an update is needed + * @param {Number} previousUpdateID The previousUpdateID contained in the response object + * @returns {Boolean} + */ +function doesClientNeedToBeUpdated(previousUpdateID = 0) { + // If no previousUpdateID is sent, this is not a WRITE request so we don't need to update our current state + if (!previousUpdateID) { + return false; + } + + // If we don't have any value in lastUpdateIDAppliedToClient, this is the first time we're receiving anything, so we need to do a last reconnectApp + if (!lastUpdateIDAppliedToClient) { + return true; + } + + return lastUpdateIDAppliedToClient < previousUpdateID; +} + // eslint-disable-next-line import/prefer-default-export -export {saveUpdateIDs}; +export {saveUpdateInformation, doesClientNeedToBeUpdated, apply}; diff --git a/src/libs/actions/QueuedOnyxUpdates.js b/src/libs/actions/QueuedOnyxUpdates.js index 486108dd56cf..06f15be1340f 100644 --- a/src/libs/actions/QueuedOnyxUpdates.js +++ b/src/libs/actions/QueuedOnyxUpdates.js @@ -22,10 +22,10 @@ function clear() { } /** - * @returns {Array} + * @returns {Promise} */ -function getQueuedUpdates() { - return queuedOnyxUpdates; +function flushQueue() { + return Onyx.update(queuedOnyxUpdates).then(clear); } -export {queueOnyxUpdates, clear, getQueuedUpdates}; +export {queueOnyxUpdates, flushQueue}; diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index b77c5b278bc9..ee93c6acb1e5 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -546,8 +546,6 @@ function subscribeToUserEvents() { // Handles the mega multipleEvents from Pusher which contains an array of single events. // Each single event is passed to PusherUtils in order to trigger the callbacks for that event PusherUtils.subscribeToPrivateUserChannelEvent(Pusher.TYPE.MULTIPLE_EVENTS, currentUserAccountID, (pushJSON) => { - let updates; - // The data for this push event comes in two different formats: // 1. Original format - this is what was sent before the RELIABLE_UPDATES project and will go away once RELIABLE_UPDATES is fully complete // - The data is an array of objects, where each object is an onyx update @@ -556,28 +554,44 @@ function subscribeToUserEvents() { // - The data is an object, containing updateIDs from the server and an array of onyx updates (this array is the same format as the original format above) // Example: {lastUpdateID: 1, previousUpdateID: 0, updates: [{onyxMethod: 'whatever', key: 'foo', value: 'bar'}]} if (_.isArray(pushJSON)) { - updates = pushJSON; - } else { - updates = pushJSON.updates; - OnyxUpdates.saveUpdateIDs(Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0)); + _.each(pushJSON, (multipleEvent) => { + PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); + }); + return; + } + + const updates = { + type: CONST.ONYX_UPDATE_TYPES.PUSHER, + lastUpdateID: Number(pushJSON.lastUpdateID || 0), + updates: pushJSON.updates, + previousUpdateID: Number(pushJSON.previousUpdateID || 0), + }; + if (!OnyxUpdates.doesClientNeedToBeUpdated(Number(pushJSON.previousUpdateID || 0))) { + OnyxUpdates.apply(updates); + return; } - _.each(updates, (multipleEvent) => { - PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data); - }); + + // If we reached this point, we need to pause the queue while we prepare to fetch older OnyxUpdates. + SequentialQueue.pause(); + OnyxUpdates.saveUpdateInformation(updates); }); // Handles Onyx updates coming from Pusher through the mega multipleEvents. - PusherUtils.subscribeToMultiEvent(Pusher.TYPE.MULTIPLE_EVENT_TYPE.ONYX_API_UPDATE, (pushJSON) => { + PusherUtils.subscribeToMultiEvent(Pusher.TYPE.MULTIPLE_EVENT_TYPE.ONYX_API_UPDATE, (pushJSON) => SequentialQueue.getCurrentRequest().then(() => { // If we don't have the currentUserAccountID (user is logged out) we don't want to update Onyx with data from Pusher if (!currentUserAccountID) { return; } - Onyx.update(pushJSON); + const onyxUpdatePromise = Onyx.update(pushJSON); triggerNotifications(pushJSON); - }); - }); + + // Return a promise when Onyx is done updating so that the OnyxUpdatesManager can properly apply all + // the onyx updates in order + return onyxUpdatePromise; + }), + ); } /** diff --git a/src/types/onyx/OnyxUpdatesFromServer.ts b/src/types/onyx/OnyxUpdatesFromServer.ts new file mode 100644 index 000000000000..02a96d4ce230 --- /dev/null +++ b/src/types/onyx/OnyxUpdatesFromServer.ts @@ -0,0 +1,14 @@ +import {OnyxUpdate} from 'react-native-onyx'; +import Request from './Request'; +import Response from './Response'; + +type OnyxUpdatesFromServer = { + type: 'https' | 'pusher'; + lastUpdateID: number | string; + previousUpdateID: number | string; + request?: Request; + response?: Response; + updates?: OnyxUpdate[]; +}; + +export default OnyxUpdatesFromServer; diff --git a/src/types/onyx/Request.ts b/src/types/onyx/Request.ts index e730dfd807fb..1df20cfb28fe 100644 --- a/src/types/onyx/Request.ts +++ b/src/types/onyx/Request.ts @@ -1,8 +1,12 @@ +import {OnyxUpdate} from 'react-native-onyx'; + type Request = { command?: string; data?: Record; type?: string; shouldUseSecure?: boolean; + successData?: OnyxUpdate[]; + failureData?: OnyxUpdate[]; }; export default Request; diff --git a/src/types/onyx/Response.ts b/src/types/onyx/Response.ts new file mode 100644 index 000000000000..d9395b6e8dab --- /dev/null +++ b/src/types/onyx/Response.ts @@ -0,0 +1,11 @@ +import {OnyxUpdate} from 'react-native-onyx'; + +type Response = { + previousUpdateID?: number; + lastUpdateID?: number; + jsonCode?: number; + onyxData?: OnyxUpdate[]; + requestID?: string; +}; + +export default Response; diff --git a/src/types/onyx/index.ts b/src/types/onyx/index.ts index 039448fac531..61b0de27000c 100644 --- a/src/types/onyx/index.ts +++ b/src/types/onyx/index.ts @@ -33,6 +33,7 @@ import ReimbursementAccountDraft from './ReimbursementAccountDraft'; import WalletTransfer from './WalletTransfer'; import ReceiptModal from './ReceiptModal'; import MapboxAccessToken from './MapboxAccessToken'; +import OnyxUpdatesFromServer from './OnyxUpdatesFromServer'; import Download from './Download'; import PolicyMember from './PolicyMember'; import Policy from './Policy'; @@ -90,5 +91,6 @@ export type { Transaction, Form, AddDebitCardForm, + OnyxUpdatesFromServer, RecentWaypoints, }; diff --git a/tests/actions/IOUTest.js b/tests/actions/IOUTest.js index 6fbbe19cec8e..afb06cdb6fb3 100644 --- a/tests/actions/IOUTest.js +++ b/tests/actions/IOUTest.js @@ -9,6 +9,7 @@ import DateUtils from '../../src/libs/DateUtils'; import * as NumberUtils from '../../src/libs/NumberUtils'; import * as ReportActions from '../../src/libs/actions/ReportActions'; import * as Report from '../../src/libs/actions/Report'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; const CARLOS_EMAIL = 'cmartins@expensifail.com'; const CARLOS_ACCOUNT_ID = 1; @@ -19,6 +20,7 @@ const RORY_ACCOUNT_ID = 3; const VIT_EMAIL = 'vit@expensifail.com'; const VIT_ACCOUNT_ID = 4; +OnyxUpdateManager(); describe('actions/IOU', () => { beforeAll(() => { Onyx.init({ diff --git a/tests/actions/ReportTest.js b/tests/actions/ReportTest.js index c06d3bc83766..978186fcf9c4 100644 --- a/tests/actions/ReportTest.js +++ b/tests/actions/ReportTest.js @@ -14,6 +14,7 @@ import * as PersistedRequests from '../../src/libs/actions/PersistedRequests'; import * as User from '../../src/libs/actions/User'; import * as ReportUtils from '../../src/libs/ReportUtils'; import DateUtils from '../../src/libs/DateUtils'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; jest.mock('../../src/libs/actions/Report', () => { const originalModule = jest.requireActual('../../src/libs/actions/Report'); @@ -24,6 +25,7 @@ jest.mock('../../src/libs/actions/Report', () => { }; }); +OnyxUpdateManager(); describe('actions/Report', () => { beforeAll(() => { PusherHelper.setup(); diff --git a/tests/actions/SessionTest.js b/tests/actions/SessionTest.js index d8bfa144e358..59a7441679ea 100644 --- a/tests/actions/SessionTest.js +++ b/tests/actions/SessionTest.js @@ -7,6 +7,7 @@ import * as TestHelper from '../utils/TestHelper'; import CONST from '../../src/CONST'; import PushNotification from '../../src/libs/Notification/PushNotification'; import * as App from '../../src/libs/actions/App'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; // This lib needs to be imported, but it has nothing to export since all it contains is an Onyx connection // eslint-disable-next-line no-unused-vars @@ -24,6 +25,7 @@ Onyx.init({ registerStorageEventListener: () => {}, }); +OnyxUpdateManager(); beforeEach(() => Onyx.clear().then(waitForPromisesToResolve)); describe('Session', () => { diff --git a/tests/unit/NetworkTest.js b/tests/unit/NetworkTest.js index c8dcda0e2af5..7d8c4f23197c 100644 --- a/tests/unit/NetworkTest.js +++ b/tests/unit/NetworkTest.js @@ -14,6 +14,7 @@ import Log from '../../src/libs/Log'; import * as MainQueue from '../../src/libs/Network/MainQueue'; import * as App from '../../src/libs/actions/App'; import NetworkConnection from '../../src/libs/NetworkConnection'; +import OnyxUpdateManager from '../../src/libs/actions/OnyxUpdateManager'; jest.mock('../../src/libs/Log'); jest.useFakeTimers(); @@ -22,6 +23,7 @@ Onyx.init({ keys: ONYXKEYS, }); +OnyxUpdateManager(); const originalXHR = HttpUtils.xhr; beforeEach(() => {