Allow priority requests to go to the front of the sequential queue

src/ONYXKEYS.js

@@ -243,16 +243,14 @@ export default {
     // Experimental memory only Onyx mode flag
     IS_USING_MEMORY_ONLY_KEYS: 'isUsingMemoryOnlyKeys',
 
-    // The access token to be used with the Mapbox library
-    MAPBOX_ACCESS_TOKEN: 'mapboxAccessToken',
+    // Information about the onyx updates IDs that were received from the server
+    ONYX_UPDATES: 'onyxUpdates',
 
-    ONYX_UPDATES: {
-        // The ID of the last Onyx update that was applied to this client
-        LAST_UPDATE_ID: 'onyxUpdatesLastUpdateID',
+    // The last update ID that was applied to the client
+    ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT: 'OnyxUpdatesLastUpdateIDAppliedToClient',
 
-        // The ID of the previous Onyx update that was applied to this client
-        PREVIOUS_UPDATE_ID: 'onyxUpdatesPreviousUpdateID',
-    },
+    // The access token to be used with the Mapbox library
+    MAPBOX_ACCESS_TOKEN: 'mapboxAccessToken',
 
     // Manual request tab selector
     SELECTED_TAB: 'selectedTab',

src/libs/API.js

@@ -36,8 +36,9 @@ Request.use(Middleware.SaveResponseInOnyx);
  * @param {Object} [onyxData.optimisticData] - Onyx instructions that will be passed to Onyx.update() before the request is made.
  * @param {Object} [onyxData.successData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200.
  * @param {Object} [onyxData.failureData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode !== 200.
+ * @param {Boolean} [prioritizeRequest] Whether or not the request should be prioritized at the front of the queue or placed onto the back of the queue
  */
-function write(command, apiCommandParameters = {}, onyxData = {}) {
+function write(command, apiCommandParameters = {}, onyxData = {}, prioritizeRequest = false) {
     Log.info('Called API write', false, {command, ...apiCommandParameters});
 
     // Optimistically update Onyx
@@ -70,7 +71,7 @@ function write(command, apiCommandParameters = {}, onyxData = {}) {
     };
 
     // Write commands can be saved and retried, so push it to the SequentialQueue
-    SequentialQueue.push(request);
+    SequentialQueue.push(request, prioritizeRequest);
 }
 
 /**

src/libs/Middleware/SaveResponseInOnyx.js

@@ -1,6 +1,25 @@
 import Onyx from 'react-native-onyx';
 import CONST from '../../CONST';
 import * as QueuedOnyxUpdates from '../actions/QueuedOnyxUpdates';
+import * as OnyxUpdates from '../actions/OnyxUpdates';
+
+/**
+ * @param {Object} updates
+ * @param {Object[]} [updates.onyxData] an array of Onyx update instruction objects
+ * @param {Number} [updates.previousUpdateID]
+ * @param {Number} [updates.lastUpdateID]
+ * @returns {Promise}
+ */
+function updateOnyx(updates) {
+    // `updates.onyxData` only exists in the latest format of the OnyxUpdates provided by the RELIABLE_UPDATES beta.
+    // If the value doesn't exist, then onyx is updated with the old format of updates.
+    if (!updates.onyxData) {
+        return Onyx.update(updates);
+    }
+
+    OnyxUpdates.saveUpdateIDs(Number(updates.lastUpdateID || 0), Number(updates.previousUpdateID || 0));
+    return Onyx.update(updates.onyxData);
+}
 
 /**
  * @param {Promise} response
@@ -16,7 +35,7 @@ function SaveResponseInOnyx(response, request) {
 
         // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in
         // the UI. See https://github.com/Expensify/App/issues/12775 for more info.
-        const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update;
+        const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : updateOnyx;
 
         // First apply any onyx data updates that are being sent back from the API. We wait for this to complete and then
         // apply successData or failureData. This ensures that we do not update any pending, loading, or other UI states contained

src/libs/Navigation/AppNavigator/AuthScreens.js

@@ -96,8 +96,8 @@ const propTypes = {
     /** Opt-in experimental mode that prevents certain Onyx keys from persisting to disk */
     isUsingMemoryOnlyKeys: PropTypes.bool,
 
-    /** The last Onyx update ID that is stored in Onyx (used for getting incremental updates when reconnecting) */
-    onyxUpdatesLastUpdateID: PropTypes.number,
+    /** The last Onyx update ID was applied to the client */
+    lastUpdateIDAppliedToClient: PropTypes.number,
 
     ...windowDimensionsPropTypes,
 };
@@ -108,7 +108,7 @@ const defaultProps = {
         email: null,
     },
     lastOpenedPublicRoomID: null,
-    onyxUpdatesLastUpdateID: 0,
+    lastUpdateIDAppliedToClient: null,
 };
 
 class AuthScreens extends React.Component {
@@ -120,7 +120,7 @@ class AuthScreens extends React.Component {
 
     componentDidMount() {
         NetworkConnection.listenForReconnect();
-        NetworkConnection.onReconnect(() => App.reconnectApp(this.props.onyxUpdatesLastUpdateID));
+        NetworkConnection.onReconnect(() => App.reconnectApp(this.props.lastUpdateIDAppliedToClient));
         PusherConnectionManager.init();
         Pusher.init({
             appKey: CONFIG.PUSHER.APP_KEY,
@@ -131,16 +131,15 @@ class AuthScreens extends React.Component {
         });
 
         // If we are on this screen then we are "logged in", but the user might not have "just logged in". They could be reopening the app
-        // or returning from background. If so, we'll assume they have some app data already and we can call
-        // reconnectApp(onyxUpdatesLastUpdateID) instead of openApp().
+        // or returning from background. If so, we'll assume they have some app data already and we can call reconnectApp() instead of openApp().
         // Note: If a Guide has enabled the memory only key mode then we do want to run OpenApp as their app will not be rehydrated with
         // the correct state on refresh. They are explicitly opting out of storing data they would need (i.e. reports_) to take advantage of
         // the optimizations performed during ReconnectApp.
         const shouldGetAllData = this.props.isUsingMemoryOnlyKeys || SessionUtils.didUserLogInDuringSession();
         if (shouldGetAllData) {
             App.openApp();
         } else {
-            App.reconnectApp(this.props.onyxUpdatesLastUpdateID);
+            App.reconnectApp(this.props.lastUpdateIDAppliedToClient);
         }
 
         App.setUpPoliciesAndNavigate(this.props.session, !this.props.isSmallScreenWidth);
@@ -329,8 +328,8 @@ export default compose(
         isUsingMemoryOnlyKeys: {
             key: ONYXKEYS.IS_USING_MEMORY_ONLY_KEYS,
         },
-        onyxUpdatesLastUpdateID: {
-            key: ONYXKEYS.ONYX_UPDATES.LAST_UPDATE_ID,
+        lastUpdateIDAppliedToClient: {
+            key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT,
         },
     }),
 )(AuthScreens);

src/libs/Network/SequentialQueue.js

@@ -101,10 +101,11 @@ NetworkStore.onReconnection(flush);
 
 /**
  * @param {Object} request
+ * @param {Boolean} [front] whether or not the request should be placed in the front of the queue
  */
-function push(request) {
+function push(request, front = false) {
     // Add request to Persisted Requests so that it can be retried if it fails
-    PersistedRequests.save([request]);
+    PersistedRequests.save([request], front);
 
     // If we are offline we don't need to trigger the queue to empty as it will happen when we come back online
     if (NetworkStore.isOffline()) {

src/libs/Pusher/pusher.js

@@ -136,7 +136,7 @@ function bindEventToChannel(channel, eventName, eventCallback = () => {}) {
         try {
             data = _.isObject(eventData) ? eventData : JSON.parse(eventData);
         } catch (err) {
-            Log.alert('[Pusher] Unable to parse JSON response from Pusher', {error: err, eventData});
+            Log.alert('[Pusher] Unable to parse JSON response from Pusher 1', {error: err, eventData});
             return;
         }
         if (data.id === undefined || data.chunk === undefined || data.final === undefined) {
@@ -168,10 +168,11 @@ function bindEventToChannel(channel, eventName, eventCallback = () => {}) {
             try {
                 eventCallback(JSON.parse(chunkedEvent.chunks.join('')));
             } catch (err) {
-                Log.alert('[Pusher] Unable to parse chunked JSON response from Pusher', {
+                Log.alert('[Pusher] Unable to parse chunked JSON response from Pusher 2', {
                     error: err,
                     eventData: chunkedEvent.chunks.join(''),
                 });
+                console.error(err);
             }
 
             delete chunkedDataEvents[data.id];

src/libs/actions/App.js

@@ -182,10 +182,9 @@ function openApp() {
 /**
  * Fetches data when the app reconnects to the network
  * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from
- * @param {Number} [updateIDTo] the ID of the Onyx update that we want to fetch up to
  */
-function reconnectApp(updateIDFrom = 0, updateIDTo = 0) {
-    console.debug(`[OnyxUpdates] App reconnecting with updateIDFrom: ${updateIDFrom} and updateIDTo: ${updateIDTo}`);
+function reconnectApp(updateIDFrom = 0) {
+    console.debug(`[OnyxUpdates] App reconnecting with updateIDFrom: ${updateIDFrom}`);
     getPolicyParamsForOpenOrReconnect().then((policyParams) => {
         const params = {...policyParams};
 
@@ -204,14 +203,79 @@ function reconnectApp(updateIDFrom = 0, updateIDTo = 0) {
             params.updateIDFrom = updateIDFrom;
         }
 
-        if (updateIDTo) {
-            params.updateIDTo = updateIDTo;
-        }
-
         API.write('ReconnectApp', params, getOnyxDataForOpenOrReconnect());
     });
 }
 
+/**
+ * Fetches data when the client has discovered it missed some Onyx updates from the server
+ * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from
+ * @param {Number} [updateIDTo] the ID of the Onyx update that we want to fetch up to
+ */
+function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) {
+    console.debug(`[OnyxUpdates] Fetching missing updates updateIDFrom: ${updateIDFrom} and updateIDTo: ${updateIDTo}`);
+
+    API.write(
+        'GetMissingOnyxMessages',
+        {
+            updateIDFrom,
+            updateIDTo,
+        },
+        getOnyxDataForOpenOrReconnect(),
+
+        // Set this to true so that the request will be prioritized at the front of the sequential queue
+        true,
+    );
+}
+
+// The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has.
+// When a gap is detected, the missing updates are fetched from the API.
+
+// These key needs to be separate from ONYXKEYS.ONYX_UPDATES so that it can be updated without triggering the callback when the server IDs are updated
+let lastUpdateIDAppliedToClient = 0;
+Onyx.connect({
+    key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT,
+    callback: (val) => (lastUpdateIDAppliedToClient = val),
+});
+
+// This object is stored in Onyx and the callback is triggered anytime new update IDs are received either from Pusher or from HTTPs (it was the only way to keep the code DRY and to prevent circular dependencies)
+const onyxUpdates = {
+    lastUpdateIDFromServer: 0,
+    previousUpdateIDFromServer: 0,
+};
+Onyx.connect({
+    key: ONYXKEYS.ONYX_UPDATES,
+    callback: (val) => {
+        if (!val) {
+            return;
+        }
+
+        const {lastUpdateIDFromServer, previousUpdateIDFromServer} = val;
+        console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer);
+        console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer);
+        console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient);
+
+        // If the previous update from the server does not match the last update the client got, then the client is missing some updates.
+        // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent.
+        if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) {
+            console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates');
+            Log.info('Gap detected in update IDs from Pusher so fetching incremental updates', true, {
+                lastUpdateIDFromServer,
+                previousUpdateIDFromServer,
+                lastUpdateIDAppliedToClient,
+            });
+            getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer);
+        }
+
+        // Update the local values to be the same as the values stored in Onyx
+        onyxUpdates.lastUpdateIDFromServer = lastUpdateIDFromServer || 0;
+        onyxUpdates.previousUpdateIDFromServer = previousUpdateIDFromServer || 0;
+
+        // Update this value so that it matches what was just received from the server
+        Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0);
+    },
+});
+
 /**
  * This promise is used so that deeplink component know when a transition is end.
  * This is necessary because we want to begin deeplink redirection after the transition is end.

src/libs/actions/OnyxUpdates.js

@@ -0,0 +1,22 @@
+import Onyx from 'react-native-onyx';
+import ONYXKEYS from '../../ONYXKEYS';
+
+/**
+ *
+ * @param {Number} [lastUpdateID]
+ * @param {Number} [previousUpdateID]
+ */
+function saveUpdateIDs(lastUpdateID = 0, previousUpdateID = 0) {
+    // Return early if there were no updateIDs
+    if (!lastUpdateID) {
+        return;
+    }
+
+    Onyx.merge(ONYXKEYS.ONYX_UPDATES, {
+        lastUpdateIDFromServer: lastUpdateID,
+        previousUpdateIDFromServer: previousUpdateID,
+    });
+}
+
+// eslint-disable-next-line import/prefer-default-export
+export {saveUpdateIDs};

src/libs/actions/PersistedRequests.js

@@ -15,9 +15,14 @@ function clear() {
 
 /**
  * @param {Array} requestsToPersist
+ * @param {Boolean} [front] whether or not the request should go in the front of the queue
  */
-function save(requestsToPersist) {
-    persistedRequests = persistedRequests.concat(requestsToPersist);
+function save(requestsToPersist, front = false) {
+    if (persistedRequests.length) {
+        persistedRequests = front ? persistedRequests.unshift(...requestsToPersist) : persistedRequests.concat(requestsToPersist);
+    } else {
+        persistedRequests = requestsToPersist;
+    }
     Onyx.set(ONYXKEYS.PERSISTED_REQUESTS, persistedRequests);
 }
 

src/libs/actions/User.js

@@ -18,6 +18,7 @@ import * as ReportActionsUtils from '../ReportActionsUtils';
 import * as ErrorUtils from '../ErrorUtils';
 import * as Session from './Session';
 import * as PersonalDetails from './PersonalDetails';
+import * as OnyxUpdates from './OnyxUpdates';
 
 let currentUserAccountID = '';
 let currentEmail = '';
@@ -553,21 +554,7 @@ function subscribeToUserEvents() {
             updates = pushJSON;
         } else {
             updates = pushJSON.updates;
-
-            // Not always we'll have the lastUpdateID and previousUpdateID properties in the pusher update
-            // until we finish the migration to reliable updates. So let's check it before actually updating
-            // the properties in Onyx
-            if (pushJSON.lastUpdateID && pushJSON.previousUpdateID) {
-                console.debug('[OnyxUpdates] Received lastUpdateID from pusher', pushJSON.lastUpdateID);
-                console.debug('[OnyxUpdates] Received previousUpdateID from pusher', pushJSON.previousUpdateID);
-                // Store these values in Onyx to allow App.reconnectApp() to fetch incremental updates from the server when a previous session is being reconnected to.
-                Onyx.multiSet({
-                    [ONYXKEYS.ONYX_UPDATES.LAST_UPDATE_ID]: Number(pushJSON.lastUpdateID || 0),
-                    [ONYXKEYS.ONYX_UPDATES.PREVIOUS_UPDATE_ID]: Number(pushJSON.previousUpdateID || 0),
-                });
-            } else {
-                console.debug('[OnyxUpdates] No lastUpdateID and previousUpdateID provided');
-            }
+            OnyxUpdates.saveUpdateIDs(Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0));
         }
         _.each(updates, (multipleEvent) => {
             PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data);