GitLab backend built with cursor API

example/index.html

@@ -63,7 +63,7 @@
 
   var ONE_DAY = 60 * 60 * 24 * 1000;
 
-  for (var i=1; i<=10; i++) {
+  for (var i=1; i<=20; i++) {
     var date = new Date();
 
     date.setTime(date.getTime() + ONE_DAY);

src/actions/entries.js

@@ -1,12 +1,15 @@
-import { List } from 'immutable';
+import { fromJS, List, Set } from 'immutable';
 import { actions as notifActions } from 'redux-notifications';
 import { serializeValues } from 'Lib/serializeEntryValues';
 import { currentBackend } from 'Backends/backend';
 import { getIntegrationProvider } from 'Integrations';
 import { getAsset, selectIntegration } from 'Reducers';
 import { selectFields } from 'Reducers/collections';
+import { selectCollectionEntriesCursor } from 'Reducers/cursors';
+import Cursor from 'ValueObjects/Cursor';
 import { createEntry } from 'ValueObjects/Entry';
 import ValidationErrorTypes from 'Constants/validationErrorTypes';
+import isArray from 'lodash/isArray';
 
 const { notifSend } = notifActions;
 
@@ -80,13 +83,15 @@ export function entriesLoading(collection) {
   };
 }
 
-export function entriesLoaded(collection, entries, pagination) {
+export function entriesLoaded(collection, entries, pagination, cursor, append = true) {
   return {
     type: ENTRIES_SUCCESS,
     payload: {
       collection: collection.get('name'),
       entries,
       page: pagination,
+      cursor: Cursor.create(cursor),
+      append,
     },
   };
 }
@@ -238,6 +243,16 @@ export function loadEntry(collection, slug) {
   };
 }
 
+const appendActions = fromJS({
+  ["append_next"]: { action: "next", append: true },
+});
+
+const addAppendActionsToCursor = cursor => Cursor
+  .create(cursor)
+  .updateStore("actions", actions => actions.union(
+    appendActions.filter(v => actions.has(v.get("action"))).keySeq()
+  ));
+
 export function loadEntries(collection, page = 0) {
   return (dispatch, getState) => {
     if (collection.get('isFetching')) {
@@ -247,14 +262,87 @@ export function loadEntries(collection, page = 0) {
     const backend = currentBackend(state.config);
     const integration = selectIntegration(state, collection.get('name'), 'listEntries');
     const provider = integration ? getIntegrationProvider(state.integrations, backend.getToken, integration) : backend;
+    const append = !!(page && !isNaN(page) && page > 0);
     dispatch(entriesLoading(collection));
-    provider.listEntries(collection, page).then(
-      response => dispatch(entriesLoaded(collection, response.entries.reverse(), response.pagination)),
-      error => dispatch(entriesFailed(collection, error))
-    );
+    provider.listEntries(collection, page)
+    .then(response => ({
+      ...response,
+
+      // The only existing backend using the pagination system is the
+      // Algolia integration, which is also the only integration used
+      // to list entries. Thus, this checking for an integration can
+      // determine whether or not this is using the old integer-based
+      // pagination API. Other backends will simply store an empty
+      // cursor, which behaves identically to no cursor at all.
+      cursor: integration
+        ? Cursor.create({ meta: { usingOldPaginationAPI: true }, data: { nextPage: page + 1 } })
+        : Cursor.create(response.cursor),
+    }))
+    .then(response => dispatch(entriesLoaded(
+      collection,
+      response.cursor.meta.get('usingOldPaginationAPI')
+        ? response.entries.reverse()
+        : response.entries,
+      response.pagination,
+      addAppendActionsToCursor(response.cursor),
+      append,
+    )))
+    .catch(err => {
+      dispatch(notifSend({
+        message: `Failed to load entries: ${ err }`,
+        kind: 'danger',
+        dismissAfter: 8000,
+      }));
+      return Promise.reject(dispatch(entriesFailed(collection, err)));
+    });
   };
 }
 
+function traverseCursor(backend, cursor, action) {
+  if (!cursor.actions.has(action)) {
+    throw new Error(`The current cursor does not support the pagination action "${ action }".`);
+  }
+  return backend.traverseCursor(cursor, action);
+}
+
+export function traverseCollectionCursor(collection, action) {
+  return async (dispatch, getState) => {
+    const state = getState();
+    if (state.entries.getIn(['pages', `${ collection.get('name') }`, 'isFetching',])) {
+      return;
+    }
+    const backend = currentBackend(state.config);
+
+    const { action: realAction, append } = appendActions.has(action)
+      ? appendActions.get(action).toJS()
+      : { action, append: false };
+    const cursor = selectCollectionEntriesCursor(state.cursors, collection.get('name'));
+
+    // Handle cursors representing pages in the old, integer-based
+    // pagination API
+    if (cursor.meta.get("usingOldPaginationAPI", false)) {
+      const [data] = cursor.unwrapData();
+      return loadEntries(collection, data.get("nextPage"));
+    }
+
+    try {
+      dispatch(entriesLoading(collection));
+      const { entries, cursor: newCursor } = await traverseCursor(backend, cursor, realAction);
+      // Pass null for the old pagination argument - this will
+      // eventually be removed.
+      return dispatch(entriesLoaded(collection, entries, null, addAppendActionsToCursor(newCursor), append));
+    } catch (err) {
+      console.error(err);
+      dispatch(notifSend({
+        message: `Failed to persist entry: ${ err }`,
+        kind: 'danger',
+        dismissAfter: 8000,
+      }));
+      return Promise.reject(dispatch(entriesFailed(collection, err)));
+    }
+  }
+}
+
 export function createEmptyDraft(collection) {
   return (dispatch) => {
     const dataFields = {};

src/actions/search.js

@@ -105,121 +105,44 @@ export function clearSearch() {
 // SearchEntries will search for complete entries in all collections.
 export function searchEntries(searchTerm, page = 0) {
   return (dispatch, getState) => {
+    dispatch(searchingEntries(searchTerm));
+
     const state = getState();
+    const backend = currentBackend(state.config);
     const allCollections = state.collections.keySeq().toArray();
     const collections = allCollections.filter(collection => selectIntegration(state, collection, 'search'));
     const integration = selectIntegration(state, collections[0], 'search');
-    if (!integration) {
-      localSearch(searchTerm, getState, dispatch);
-    } else {
-      const provider = getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration);
-      dispatch(searchingEntries(searchTerm));
-      provider.search(collections, searchTerm, page).then(
-        response => dispatch(searchSuccess(searchTerm, response.entries, response.pagination)),
-        error => dispatch(searchFailure(searchTerm, error))
-      );
-    }
+
+    const searchPromise = integration
+      ? getIntegrationProvider(state.integrations, backend.getToken, integration).search(collections, searchTerm, page)
+      : backend.search(state.collections.valueSeq().toArray(), searchTerm);
+
+    return searchPromise.then(
+      response => dispatch(searchSuccess(searchTerm, response.entries, response.pagination)),
+      error => dispatch(searchFailure(searchTerm, error))
+    );
   };
 }
 
 // Instead of searching for complete entries, query will search for specific fields
 // in specific collections and return raw data (no entries).
-export function query(namespace, collection, searchFields, searchTerm) {
+export function query(namespace, collectionName, searchFields, searchTerm) {
   return (dispatch, getState) => {
-    const state = getState();
-    const integration = selectIntegration(state, collection, 'search');
     dispatch(querying(namespace, collection, searchFields, searchTerm));
-    if (!integration) {
-      localQuery(namespace, collection, searchFields, searchTerm, state, dispatch);
-    } else {
-      const provider = getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration);
-      provider.searchBy(searchFields.map(f => `data.${ f }`), collection, searchTerm).then(
-        response => dispatch(querySuccess(namespace, collection, searchFields, searchTerm, response)),
-        error => dispatch(queryFailure(namespace, collection, searchFields, searchTerm, error))
-      );
-    }
-  };
-}
 
-// Local Query & Search functions
-
-function localSearch(searchTerm, getState, dispatch) {
-  return (function acc(localResults = { entries: [] }) {
-    function processCollection(collection, collectionKey) {
-      const state = getState();
-      if (state.entries.hasIn(['pages', collectionKey, 'ids'])) {
-        const searchFields = [
-          selectInferedField(collection, 'title'),
-          selectInferedField(collection, 'shortTitle'),
-          selectInferedField(collection, 'author'),
-        ];
-        const collectionEntries = selectEntries(state, collectionKey).toJS();
-        const filteredEntries = fuzzy.filter(searchTerm, collectionEntries, {
-          extract: entry => searchFields.reduce((acc, field) => {
-            const f = entry.data[field];
-            return f ? `${ acc } ${ f }` : acc;
-          }, ""),
-        }).filter(entry => entry.score > 5);
-        localResults[collectionKey] = true;
-        localResults.entries = localResults.entries.concat(filteredEntries);
-
-        const returnedKeys = Object.keys(localResults);
-        const allCollections = state.collections.keySeq().toArray();
-        if (allCollections.every(v => returnedKeys.indexOf(v) !== -1)) {
-          const sortedResults = localResults.entries.sort((a, b) => {
-            if (a.score > b.score) return -1;
-            if (a.score < b.score) return 1;
-            return 0;
-          }).map(f => f.original);
-          if (allCollections.size > 3 || localResults.entries.length > 30) {
-            console.warn('The Netlify CMS is currently using a Built-in search.' +
-            '\nWhile this works great for small sites, bigger projects might benefit from a separate search integration.' + 
-            '\nPlease refer to the documentation for more information');
-          }
-          dispatch(searchSuccess(searchTerm, sortedResults, 0));
-        }
-      } else {
-        // Collection entries aren't loaded yet.
-        // Dispatch loadEntries and wait before redispatching this action again.
-        dispatch({
-          type: WAIT_UNTIL_ACTION,
-          predicate: action => (action.type === ENTRIES_SUCCESS && action.payload.collection === collectionKey),
-          run: () => processCollection(collection, collectionKey),
-        });
-        dispatch(loadEntries(collection));
-      }
-    }
-    getState().collections.forEach(processCollection);
-  }());
-}
+    const state = getState();
+    const backend = currentBackend(state.config);
+    const collection = state.collections.find(collection => collection.get('name') === collectionName);
+    const integration = selectIntegration(state, collection, 'search');
 
+    const queryPromise = integration
+      ? getIntegrationProvider(state.integrations, backend.getToken, integration)
+          .searchBy(searchFields.map(f => `data.${ f }`), collectionName, searchTerm)
+      : backend.query(collection, searchFields, searchTerm);
 
-function localQuery(namespace, collection, searchFields, searchTerm, state, dispatch) {
-  // Check if entries in this collection were already loaded
-  if (state.entries.hasIn(['pages', collection, 'ids'])) {
-    const entries = selectEntries(state, collection).toJS();
-    const filteredEntries = fuzzy.filter(searchTerm, entries, {
-      extract: entry => searchFields.reduce((acc, field) => {
-        const f = entry.data[field];
-        return f ? `${ acc } ${ f }` : acc;
-      }, ""),
-    }).filter(entry => entry.score > 5);
-
-    const resultObj = {
-      query: searchTerm,
-      hits: [],
-    };
-
-    resultObj.hits = filteredEntries.map(f => f.original);
-    dispatch(querySuccess(namespace, collection, searchFields, searchTerm, resultObj));
-  } else {
-    // Collection entries aren't loaded yet.
-    // Dispatch loadEntries and wait before redispatching this action again.
-    dispatch({
-      type: WAIT_UNTIL_ACTION,
-      predicate: action => (action.type === ENTRIES_SUCCESS && action.payload.collection === collection),
-      run: dispatch => dispatch(query(namespace, collection, searchFields, searchTerm)),
-    });
-    dispatch(loadEntries(state.collections.get(collection)));
-  }
+    return queryPromise.then(
+      response => dispatch(querySuccess(namespace, collectionName, searchFields, searchTerm, response)),
+      error => dispatch(queryFailure(namespace, collectionName, searchFields, searchTerm, error))
+    );
+  };
 }