Allow priority requests to go to the front of the sequential queue

src/ONYXKEYS.js

@@ -243,16 +243,14 @@ export default {
     // Experimental memory only Onyx mode flag
     IS_USING_MEMORY_ONLY_KEYS: 'isUsingMemoryOnlyKeys',
 
-    // The access token to be used with the Mapbox library
-    MAPBOX_ACCESS_TOKEN: 'mapboxAccessToken',
+    // Information about the onyx updates IDs that were received from the server
+    ONYX_UPDATES_FROM_SERVER: 'onyxUpdatesFromServer',
 
-    ONYX_UPDATES: {
-        // The ID of the last Onyx update that was applied to this client
-        LAST_UPDATE_ID: 'onyxUpdatesLastUpdateID',
+    // The last update ID that was applied to the client
+    ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT: 'OnyxUpdatesLastUpdateIDAppliedToClient',
 
-        // The ID of the previous Onyx update that was applied to this client
-        PREVIOUS_UPDATE_ID: 'onyxUpdatesPreviousUpdateID',
-    },
+    // The access token to be used with the Mapbox library
+    MAPBOX_ACCESS_TOKEN: 'mapboxAccessToken',
 
     // Manual request tab selector
     SELECTED_TAB: 'selectedTab',

src/libs/API.js

@@ -36,8 +36,9 @@ Request.use(Middleware.SaveResponseInOnyx);
  * @param {Object} [onyxData.optimisticData] - Onyx instructions that will be passed to Onyx.update() before the request is made.
  * @param {Object} [onyxData.successData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200.
  * @param {Object} [onyxData.failureData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode !== 200.
+ * @param {Boolean} [prioritizeRequest] Whether or not the request should be prioritized at the front of the queue or placed onto the back of the queue
  */
-function write(command, apiCommandParameters = {}, onyxData = {}) {
+function write(command, apiCommandParameters = {}, onyxData = {}, prioritizeRequest = false) {
     Log.info('Called API write', false, {command, ...apiCommandParameters});
 
     // Optimistically update Onyx
@@ -70,7 +71,7 @@ function write(command, apiCommandParameters = {}, onyxData = {}) {
     };
 
     // Write commands can be saved and retried, so push it to the SequentialQueue
-    SequentialQueue.push(request);
+    SequentialQueue.push(request, prioritizeRequest);
 }
 
 /**

src/libs/Middleware/SaveResponseInOnyx.js

@@ -1,6 +1,7 @@
 import Onyx from 'react-native-onyx';
 import CONST from '../../CONST';
 import * as QueuedOnyxUpdates from '../actions/QueuedOnyxUpdates';
+import * as OnyxUpdates from '../actions/OnyxUpdates';
 
 /**
  * @param {Promise} response
@@ -14,6 +15,9 @@ function SaveResponseInOnyx(response, request) {
             return;
         }
 
+        // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server
+        OnyxUpdates.saveUpdateIDs(Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0));
+
         // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in
         // the UI. See https://github.com/Expensify/App/issues/12775 for more info.
         const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update;

src/libs/Navigation/AppNavigator/AuthScreens.js

@@ -96,8 +96,8 @@ const propTypes = {
     /** Opt-in experimental mode that prevents certain Onyx keys from persisting to disk */
     isUsingMemoryOnlyKeys: PropTypes.bool,
 
-    /** The last Onyx update ID that is stored in Onyx (used for getting incremental updates when reconnecting) */
-    onyxUpdatesLastUpdateID: PropTypes.number,
+    /** The last Onyx update ID was applied to the client */
+    lastUpdateIDAppliedToClient: PropTypes.number,
 
     ...windowDimensionsPropTypes,
 };
@@ -108,7 +108,7 @@ const defaultProps = {
         email: null,
     },
     lastOpenedPublicRoomID: null,
-    onyxUpdatesLastUpdateID: 0,
+    lastUpdateIDAppliedToClient: null,
 };
 
 class AuthScreens extends React.Component {
@@ -120,7 +120,7 @@ class AuthScreens extends React.Component {
 
     componentDidMount() {
         NetworkConnection.listenForReconnect();
-        NetworkConnection.onReconnect(() => App.reconnectApp(this.props.onyxUpdatesLastUpdateID));
+        NetworkConnection.onReconnect(() => App.reconnectApp(this.props.lastUpdateIDAppliedToClient));
         PusherConnectionManager.init();
         Pusher.init({
             appKey: CONFIG.PUSHER.APP_KEY,
@@ -131,16 +131,15 @@ class AuthScreens extends React.Component {
         });
 
         // If we are on this screen then we are "logged in", but the user might not have "just logged in". They could be reopening the app
-        // or returning from background. If so, we'll assume they have some app data already and we can call
-        // reconnectApp(onyxUpdatesLastUpdateID) instead of openApp().
+        // or returning from background. If so, we'll assume they have some app data already and we can call reconnectApp() instead of openApp().
         // Note: If a Guide has enabled the memory only key mode then we do want to run OpenApp as their app will not be rehydrated with
         // the correct state on refresh. They are explicitly opting out of storing data they would need (i.e. reports_) to take advantage of
         // the optimizations performed during ReconnectApp.
         const shouldGetAllData = this.props.isUsingMemoryOnlyKeys || SessionUtils.didUserLogInDuringSession();
         if (shouldGetAllData) {
             App.openApp();
         } else {
-            App.reconnectApp(this.props.onyxUpdatesLastUpdateID);
+            App.reconnectApp(this.props.lastUpdateIDAppliedToClient);
         }
 
         App.setUpPoliciesAndNavigate(this.props.session, !this.props.isSmallScreenWidth);
@@ -329,8 +328,8 @@ export default compose(
         isUsingMemoryOnlyKeys: {
             key: ONYXKEYS.IS_USING_MEMORY_ONLY_KEYS,
         },
-        onyxUpdatesLastUpdateID: {
-            key: ONYXKEYS.ONYX_UPDATES.LAST_UPDATE_ID,
+        lastUpdateIDAppliedToClient: {
+            key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT,
         },
     }),
 )(AuthScreens);

src/libs/Network/SequentialQueue.js

@@ -101,10 +101,11 @@ NetworkStore.onReconnection(flush);
 
 /**
  * @param {Object} request
+ * @param {Boolean} [front] whether or not the request should be placed in the front of the queue
  */
-function push(request) {
+function push(request, front = false) {
     // Add request to Persisted Requests so that it can be retried if it fails
-    PersistedRequests.save([request]);
+    PersistedRequests.save([request], front);
 
     // If we are offline we don't need to trigger the queue to empty as it will happen when we come back online
     if (NetworkStore.isOffline()) {

src/libs/Pusher/pusher.js

@@ -136,7 +136,7 @@ function bindEventToChannel(channel, eventName, eventCallback = () => {}) {
         try {
             data = _.isObject(eventData) ? eventData : JSON.parse(eventData);
         } catch (err) {
-            Log.alert('[Pusher] Unable to parse JSON response from Pusher', {error: err, eventData});
+            Log.alert('[Pusher] Unable to parse single JSON event data from Pusher', {error: err, eventData});
             return;
         }
         if (data.id === undefined || data.chunk === undefined || data.final === undefined) {
@@ -172,6 +172,9 @@ function bindEventToChannel(channel, eventName, eventCallback = () => {}) {
                     error: err,
                     eventData: chunkedEvent.chunks.join(''),
                 });
+
+                // Using console.error is helpful here because it will print a usable stack trace to the console to debug where the error comes from
+                console.error(err);
             }
 
             delete chunkedDataEvents[data.id];

src/libs/actions/App.js

@@ -182,10 +182,9 @@ function openApp() {
 /**
  * Fetches data when the app reconnects to the network
  * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from
- * @param {Number} [updateIDTo] the ID of the Onyx update that we want to fetch up to
  */
-function reconnectApp(updateIDFrom = 0, updateIDTo = 0) {
-    console.debug(`[OnyxUpdates] App reconnecting with updateIDFrom: ${updateIDFrom} and updateIDTo: ${updateIDTo}`);
+function reconnectApp(updateIDFrom = 0) {
+    console.debug(`[OnyxUpdates] App reconnecting with updateIDFrom: ${updateIDFrom}`);
     getPolicyParamsForOpenOrReconnect().then((policyParams) => {
         const params = {...policyParams};
 
@@ -204,14 +203,72 @@ function reconnectApp(updateIDFrom = 0, updateIDTo = 0) {
             params.updateIDFrom = updateIDFrom;
         }
 
-        if (updateIDTo) {
-            params.updateIDTo = updateIDTo;
-        }
-
         API.write('ReconnectApp', params, getOnyxDataForOpenOrReconnect());
     });
 }
 
+/**
+ * Fetches data when the client has discovered it missed some Onyx updates from the server
+ * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from
+ * @param {Number} [updateIDTo] the ID of the Onyx update that we want to fetch up to
+ */
+function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) {
+    console.debug(`[OnyxUpdates] Fetching missing updates updateIDFrom: ${updateIDFrom} and updateIDTo: ${updateIDTo}`);
+
+    API.write(
+        'GetMissingOnyxMessages',
+        {
+            updateIDFrom,
+            updateIDTo,
+        },
+        getOnyxDataForOpenOrReconnect(),
+
+        // Set this to true so that the request will be prioritized at the front of the sequential queue
+        true,
+    );
+}
+
+// The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has.
+// When a gap is detected, the missing updates are fetched from the API.
+
+// These key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated
+let lastUpdateIDAppliedToClient = 0;
+Onyx.connect({
+    key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT,
+    callback: (val) => (lastUpdateIDAppliedToClient = val),
+});
+
+Onyx.connect({
+    key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER,
+    callback: (val) => {
+        if (!val) {
+            return;
+        }
+
+        const {lastUpdateIDFromServer, previousUpdateIDFromServer} = val;
+        console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer);
+        console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer);
+        console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient);
+
+        // If the previous update from the server does not match the last update the client got, then the client is missing some updates.
+        // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent.
+        if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) {
+            console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates');
+            Log.info('Gap detected in update IDs from server so fetching incremental updates', true, {
+                lastUpdateIDFromServer,
+                previousUpdateIDFromServer,
+                lastUpdateIDAppliedToClient,
+            });
+            getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer);
+        }
+
+        if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) {
+            // Update this value so that it matches what was just received from the server
+            Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0);
+        }
+    },
+});
+
 /**
  * This promise is used so that deeplink component know when a transition is end.
  * This is necessary because we want to begin deeplink redirection after the transition is end.

src/libs/actions/OnyxUpdates.js

@@ -0,0 +1,22 @@
+import Onyx from 'react-native-onyx';
+import ONYXKEYS from '../../ONYXKEYS';
+
+/**
+ *
+ * @param {Number} [lastUpdateID]
+ * @param {Number} [previousUpdateID]
+ */
+function saveUpdateIDs(lastUpdateID = 0, previousUpdateID = 0) {
+    // Return early if there were no updateIDs
+    if (!lastUpdateID) {
+        return;
+    }
+
+    Onyx.merge(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, {
+        lastUpdateIDFromServer: lastUpdateID,
+        previousUpdateIDFromServer: previousUpdateID,
+    });
+}
+
+// eslint-disable-next-line import/prefer-default-export
+export {saveUpdateIDs};

src/libs/actions/PersistedRequests.js

@@ -15,9 +15,14 @@ function clear() {
 
 /**
  * @param {Array} requestsToPersist
+ * @param {Boolean} [front] whether or not the request should go in the front of the queue
  */
-function save(requestsToPersist) {
-    persistedRequests = persistedRequests.concat(requestsToPersist);
+function save(requestsToPersist, front = false) {
+    if (persistedRequests.length) {
+        persistedRequests = front ? persistedRequests.unshift(...requestsToPersist) : persistedRequests.concat(requestsToPersist);
+    } else {
+        persistedRequests = requestsToPersist;
+    }
     Onyx.set(ONYXKEYS.PERSISTED_REQUESTS, persistedRequests);
 }
 

src/libs/actions/User.js

@@ -18,6 +18,7 @@ import * as ReportActionsUtils from '../ReportActionsUtils';
 import * as ErrorUtils from '../ErrorUtils';
 import * as Session from './Session';
 import * as PersonalDetails from './PersonalDetails';
+import * as OnyxUpdates from './OnyxUpdates';
 
 let currentUserAccountID = '';
 let currentEmail = '';
@@ -553,21 +554,7 @@ function subscribeToUserEvents() {
             updates = pushJSON;
         } else {
             updates = pushJSON.updates;
-
-            // Not always we'll have the lastUpdateID and previousUpdateID properties in the pusher update
-            // until we finish the migration to reliable updates. So let's check it before actually updating
-            // the properties in Onyx
-            if (pushJSON.lastUpdateID && pushJSON.previousUpdateID) {
-                console.debug('[OnyxUpdates] Received lastUpdateID from pusher', pushJSON.lastUpdateID);
-                console.debug('[OnyxUpdates] Received previousUpdateID from pusher', pushJSON.previousUpdateID);
-                // Store these values in Onyx to allow App.reconnectApp() to fetch incremental updates from the server when a previous session is being reconnected to.
-                Onyx.multiSet({
-                    [ONYXKEYS.ONYX_UPDATES.LAST_UPDATE_ID]: Number(pushJSON.lastUpdateID || 0),
-                    [ONYXKEYS.ONYX_UPDATES.PREVIOUS_UPDATE_ID]: Number(pushJSON.previousUpdateID || 0),
-                });
-            } else {
-                console.debug('[OnyxUpdates] No lastUpdateID and previousUpdateID provided');
-            }
+            OnyxUpdates.saveUpdateIDs(Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0));
         }
         _.each(updates, (multipleEvent) => {
             PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data);