Skip to content

Commit

Permalink
GitLab backend built with cursor API (decaporg#1343)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Benaiah @erquhart
Benaiah authored and erquhart committed Jun 12, 2018
1 parent 50238a0 commit c0dc8f3
Show file tree
Hide file tree
Showing 29 changed files with 1,362 additions and 278 deletions.
2 changes: 1 addition & 1 deletion example/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@

var ONE_DAY = 60 * 60 * 24 * 1000;

for (var i=1; i<=10; i++) {
for (var i=1; i<=20; i++) {
var date = new Date();

date.setTime(date.getTime() + ONE_DAY);
Expand Down
99 changes: 93 additions & 6 deletions src/actions/entries.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import { List } from 'immutable';
import { fromJS, List, Set } from 'immutable';
import { actions as notifActions } from 'redux-notifications';
import { serializeValues } from 'Lib/serializeEntryValues';
import { currentBackend } from 'Backends/backend';
import { getIntegrationProvider } from 'Integrations';
import { getAsset, selectIntegration } from 'Reducers';
import { selectFields } from 'Reducers/collections';
import { selectCollectionEntriesCursor } from 'Reducers/cursors';
import Cursor from 'ValueObjects/Cursor';
import { createEntry } from 'ValueObjects/Entry';
import ValidationErrorTypes from 'Constants/validationErrorTypes';
import isArray from 'lodash/isArray';

const { notifSend } = notifActions;

Expand Down Expand Up @@ -80,13 +83,15 @@ export function entriesLoading(collection) {
};
}

export function entriesLoaded(collection, entries, pagination) {
export function entriesLoaded(collection, entries, pagination, cursor, append = true) {
return {
type: ENTRIES_SUCCESS,
payload: {
collection: collection.get('name'),
entries,
page: pagination,
cursor: Cursor.create(cursor),
append,
},
};
}
Expand Down Expand Up @@ -238,6 +243,16 @@ export function loadEntry(collection, slug) {
};
}

const appendActions = fromJS({
["append_next"]: { action: "next", append: true },
});

const addAppendActionsToCursor = cursor => Cursor
.create(cursor)
.updateStore("actions", actions => actions.union(
appendActions.filter(v => actions.has(v.get("action"))).keySeq()
));

export function loadEntries(collection, page = 0) {
return (dispatch, getState) => {
if (collection.get('isFetching')) {
Expand All @@ -247,14 +262,86 @@ export function loadEntries(collection, page = 0) {
const backend = currentBackend(state.config);
const integration = selectIntegration(state, collection.get('name'), 'listEntries');
const provider = integration ? getIntegrationProvider(state.integrations, backend.getToken, integration) : backend;
const append = !!(page && !isNaN(page) && page > 0);
dispatch(entriesLoading(collection));
provider.listEntries(collection, page).then(
response => dispatch(entriesLoaded(collection, response.entries.reverse(), response.pagination)),
error => dispatch(entriesFailed(collection, error))
);
provider.listEntries(collection, page)
.then(response => ({
...response,

// The only existing backend using the pagination system is the
// Algolia integration, which is also the only integration used
// to list entries. Thus, this checking for an integration can
// determine whether or not this is using the old integer-based
// pagination API. Other backends will simply store an empty
// cursor, which behaves identically to no cursor at all.
cursor: integration
? Cursor.create({ actions: ["next"], meta: { usingOldPaginationAPI: true }, data: { nextPage: page + 1 } })
: Cursor.create(response.cursor),
}))
.then(response => dispatch(entriesLoaded(
collection,
response.cursor.meta.get('usingOldPaginationAPI')
? response.entries.reverse()
: response.entries,
response.pagination,
addAppendActionsToCursor(response.cursor),
append,
)))
.catch(err => {
dispatch(notifSend({
message: `Failed to load entries: ${ err }`,
kind: 'danger',
dismissAfter: 8000,
}));
return Promise.reject(dispatch(entriesFailed(collection, err)));
});
};
}

function traverseCursor(backend, cursor, action) {
if (!cursor.actions.has(action)) {
throw new Error(`The current cursor does not support the pagination action "${ action }".`);
}
return backend.traverseCursor(cursor, action);
}

export function traverseCollectionCursor(collection, action) {
return async (dispatch, getState) => {
const state = getState();
if (state.entries.getIn(['pages', `${ collection.get('name') }`, 'isFetching',])) {
return;
}
const backend = currentBackend(state.config);

const { action: realAction, append } = appendActions.has(action)
? appendActions.get(action).toJS()
: { action, append: false };
const cursor = selectCollectionEntriesCursor(state.cursors, collection.get('name'));

// Handle cursors representing pages in the old, integer-based
// pagination API
if (cursor.meta.get("usingOldPaginationAPI", false)) {
return dispatch(loadEntries(collection, cursor.data.get("nextPage")));
}

try {
dispatch(entriesLoading(collection));
const { entries, cursor: newCursor } = await traverseCursor(backend, cursor, realAction);
// Pass null for the old pagination argument - this will
// eventually be removed.
return dispatch(entriesLoaded(collection, entries, null, addAppendActionsToCursor(newCursor), append));
} catch (err) {
console.error(err);
dispatch(notifSend({
message: `Failed to persist entry: ${ err }`,
kind: 'danger',
dismissAfter: 8000,
}));
return Promise.reject(dispatch(entriesFailed(collection, err)));
}
}
}

export function createEmptyDraft(collection) {
return (dispatch) => {
const dataFields = {};
Expand Down
133 changes: 28 additions & 105 deletions src/actions/search.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,121 +105,44 @@ export function clearSearch() {
// SearchEntries will search for complete entries in all collections.
export function searchEntries(searchTerm, page = 0) {
return (dispatch, getState) => {
dispatch(searchingEntries(searchTerm));

const state = getState();
const backend = currentBackend(state.config);
const allCollections = state.collections.keySeq().toArray();
const collections = allCollections.filter(collection => selectIntegration(state, collection, 'search'));
const integration = selectIntegration(state, collections[0], 'search');
if (!integration) {
localSearch(searchTerm, getState, dispatch);
} else {
const provider = getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration);
dispatch(searchingEntries(searchTerm));
provider.search(collections, searchTerm, page).then(
response => dispatch(searchSuccess(searchTerm, response.entries, response.pagination)),
error => dispatch(searchFailure(searchTerm, error))
);
}

const searchPromise = integration
? getIntegrationProvider(state.integrations, backend.getToken, integration).search(collections, searchTerm, page)
: backend.search(state.collections.valueSeq().toArray(), searchTerm);

return searchPromise.then(
response => dispatch(searchSuccess(searchTerm, response.entries, response.pagination)),
error => dispatch(searchFailure(searchTerm, error))
);
};
}

// Instead of searching for complete entries, query will search for specific fields
// in specific collections and return raw data (no entries).
export function query(namespace, collection, searchFields, searchTerm) {
export function query(namespace, collectionName, searchFields, searchTerm) {
return (dispatch, getState) => {
dispatch(querying(namespace, collectionName, searchFields, searchTerm));

const state = getState();
const integration = selectIntegration(state, collection, 'search');
dispatch(querying(namespace, collection, searchFields, searchTerm));
if (!integration) {
localQuery(namespace, collection, searchFields, searchTerm, state, dispatch);
} else {
const provider = getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration);
provider.searchBy(searchFields.map(f => `data.${ f }`), collection, searchTerm).then(
response => dispatch(querySuccess(namespace, collection, searchFields, searchTerm, response)),
error => dispatch(queryFailure(namespace, collection, searchFields, searchTerm, error))
);
}
const backend = currentBackend(state.config);
const integration = selectIntegration(state, collectionName, 'search');
const collection = state.collections.find(collection => collection.get('name') === collectionName);

const queryPromise = integration
? getIntegrationProvider(state.integrations, backend.getToken, integration)
.searchBy(searchFields.map(f => `data.${ f }`), collectionName, searchTerm)
: backend.query(collection, searchFields, searchTerm);

return queryPromise.then(
response => dispatch(querySuccess(namespace, collectionName, searchFields, searchTerm, response)),
error => dispatch(queryFailure(namespace, collectionName, searchFields, searchTerm, error))
);
};
}

// Local Query & Search functions

function localSearch(searchTerm, getState, dispatch) {
return (function acc(localResults = { entries: [] }) {
function processCollection(collection, collectionKey) {
const state = getState();
if (state.entries.hasIn(['pages', collectionKey, 'ids'])) {
const searchFields = [
selectInferedField(collection, 'title'),
selectInferedField(collection, 'shortTitle'),
selectInferedField(collection, 'author'),
];
const collectionEntries = selectEntries(state, collectionKey).toJS();
const filteredEntries = fuzzy.filter(searchTerm, collectionEntries, {
extract: entry => searchFields.reduce((acc, field) => {
const f = entry.data[field];
return f ? `${ acc } ${ f }` : acc;
}, ""),
}).filter(entry => entry.score > 5);
localResults[collectionKey] = true;
localResults.entries = localResults.entries.concat(filteredEntries);

const returnedKeys = Object.keys(localResults);
const allCollections = state.collections.keySeq().toArray();
if (allCollections.every(v => returnedKeys.indexOf(v) !== -1)) {
const sortedResults = localResults.entries.sort((a, b) => {
if (a.score > b.score) return -1;
if (a.score < b.score) return 1;
return 0;
}).map(f => f.original);
if (allCollections.size > 3 || localResults.entries.length > 30) {
console.warn('The Netlify CMS is currently using a Built-in search.' +
'\nWhile this works great for small sites, bigger projects might benefit from a separate search integration.' +
'\nPlease refer to the documentation for more information');
}
dispatch(searchSuccess(searchTerm, sortedResults, 0));
}
} else {
// Collection entries aren't loaded yet.
// Dispatch loadEntries and wait before redispatching this action again.
dispatch({
type: WAIT_UNTIL_ACTION,
predicate: action => (action.type === ENTRIES_SUCCESS && action.payload.collection === collectionKey),
run: () => processCollection(collection, collectionKey),
});
dispatch(loadEntries(collection));
}
}
getState().collections.forEach(processCollection);
}());
}


function localQuery(namespace, collection, searchFields, searchTerm, state, dispatch) {
// Check if entries in this collection were already loaded
if (state.entries.hasIn(['pages', collection, 'ids'])) {
const entries = selectEntries(state, collection).toJS();
const filteredEntries = fuzzy.filter(searchTerm, entries, {
extract: entry => searchFields.reduce((acc, field) => {
const f = entry.data[field];
return f ? `${ acc } ${ f }` : acc;
}, ""),
}).filter(entry => entry.score > 5);

const resultObj = {
query: searchTerm,
hits: [],
};

resultObj.hits = filteredEntries.map(f => f.original);
dispatch(querySuccess(namespace, collection, searchFields, searchTerm, resultObj));
} else {
// Collection entries aren't loaded yet.
// Dispatch loadEntries and wait before redispatching this action again.
dispatch({
type: WAIT_UNTIL_ACTION,
predicate: action => (action.type === ENTRIES_SUCCESS && action.payload.collection === collection),
run: dispatch => dispatch(query(namespace, collection, searchFields, searchTerm)),
});
dispatch(loadEntries(state.collections.get(collection)));
}
}
Loading

0 comments on commit c0dc8f3

Please sign in to comment.