-
Notifications
You must be signed in to change notification settings - Fork 2.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(plugin-search): added support for reindexing collections on dema…
…nd (#9391) ### What? This PR aims to add reindexing capabilities to `plugin-search` to allow users to reindex entire searchable collections on demand. ### Why? As it stands, end users must either perform document reindexing manually one-by-one or via bulk operations. Both of these approaches are undesirable because they result in new versions being published on existing documents. Consider the case when `plugin-search` is only added _after_ the project has started and documents have been added to existing collections. It would be nice if users could simply click a button, choose the searchable collections to reindex, and have the custom endpoint handle the rest. ### How? This PR adds on to the existing plugin configuration, creating a custom endpoint and a custom `beforeListTable` component in the form of a popup button. Upon clicking the button, a dropdown/popup is opened with options to select which collection to reindex, as well as a useful `All Collections` option to run reindexing on all configured search collections. It also adds a `reindexBatchSize` option in the config to allow users to specify in what quantity to batch documents to sync with search. Big shoutout to @paulpopus & @r1tsuu for the triple-A level support on this one! Fixes #8902 See it in action: https://github.com/user-attachments/assets/ee8dd68c-ea89-49cd-adc3-151973eea28b Notes: - Traditionally these kinds of long-running tasks would be better suited for a job. However, given how many users enjoy deploying to serverless environments, it would be problematic to offer this feature exclusive to jobs queues. I thought a significant amount about this and decided it would be best to ship the feature as-is with the intention of creating an opt-in method to use job queues in the future if/when this gets merged. - In my testing, the collection description somehow started to appear in the document views after the on-demand RSC merge. I haven't reproduced this, but this PR has an example of that problem. Super strange. --------- Co-authored-by: Sasha <64744993+r1tsuu@users.noreply.github.com> Co-authored-by: Paul Popus <paul@nouance.io>
- Loading branch information
1 parent
bffd98f
commit defa13e
Showing
55 changed files
with
1,424 additions
and
196 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
183 changes: 4 additions & 179 deletions
183
packages/plugin-search/src/Search/hooks/syncWithSearch.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,182 +1,7 @@ | ||
import type { DocToSync, SyncWithSearch } from '../../types.js' | ||
import type { SyncWithSearch } from '../../types.js' | ||
|
||
export const syncWithSearch: SyncWithSearch = async (args) => { | ||
const { | ||
collection, | ||
doc, | ||
operation, | ||
pluginConfig, | ||
req: { payload }, | ||
req, | ||
} = args | ||
import { syncDocAsSearchIndex } from '../../utilities/syncDocAsSearchIndex.js' | ||
|
||
const { id, _status: status, title } = doc || {} | ||
|
||
const { beforeSync, defaultPriorities, deleteDrafts, searchOverrides, syncDrafts } = pluginConfig | ||
|
||
const searchSlug = searchOverrides?.slug || 'search' | ||
|
||
let dataToSave: DocToSync = { | ||
doc: { | ||
relationTo: collection, | ||
value: id, | ||
}, | ||
title, | ||
} | ||
|
||
if (typeof beforeSync === 'function') { | ||
let docToSyncWith = doc | ||
if (payload.config?.localization) { | ||
docToSyncWith = await payload.findByID({ | ||
id, | ||
collection, | ||
locale: req.locale, | ||
req, | ||
}) | ||
} | ||
dataToSave = await beforeSync({ | ||
originalDoc: docToSyncWith, | ||
payload, | ||
req, | ||
searchDoc: dataToSave, | ||
}) | ||
} | ||
|
||
let defaultPriority = 0 | ||
if (defaultPriorities) { | ||
const { [collection]: priority } = defaultPriorities | ||
|
||
if (typeof priority === 'function') { | ||
try { | ||
defaultPriority = await priority(doc) | ||
} catch (err: unknown) { | ||
payload.logger.error(err) | ||
payload.logger.error( | ||
`Error gathering default priority for ${searchSlug} documents related to ${collection}`, | ||
) | ||
} | ||
} else { | ||
defaultPriority = priority | ||
} | ||
} | ||
|
||
const doSync = syncDrafts || (!syncDrafts && status !== 'draft') | ||
|
||
try { | ||
if (operation === 'create') { | ||
if (doSync) { | ||
await payload.create({ | ||
collection: searchSlug, | ||
data: { | ||
...dataToSave, | ||
priority: defaultPriority, | ||
}, | ||
locale: req.locale, | ||
req, | ||
}) | ||
} | ||
} | ||
|
||
if (operation === 'update') { | ||
try { | ||
// find the correct doc to sync with | ||
const searchDocQuery = await payload.find({ | ||
collection: searchSlug, | ||
depth: 0, | ||
locale: req.locale, | ||
req, | ||
where: { | ||
'doc.relationTo': { | ||
equals: collection, | ||
}, | ||
'doc.value': { | ||
equals: id, | ||
}, | ||
}, | ||
}) | ||
|
||
const docs: Array<{ | ||
id: number | string | ||
priority?: number | ||
}> = searchDocQuery?.docs || [] | ||
|
||
const [foundDoc, ...duplicativeDocs] = docs | ||
|
||
// delete all duplicative search docs (docs that reference the same page) | ||
// to ensure the same, out-of-date result does not appear twice (where only syncing the first found doc) | ||
if (duplicativeDocs.length > 0) { | ||
try { | ||
const duplicativeDocIDs = duplicativeDocs.map(({ id }) => id) | ||
await payload.delete({ | ||
collection: searchSlug, | ||
req, | ||
where: { id: { in: duplicativeDocIDs } }, | ||
}) | ||
} catch (err: unknown) { | ||
payload.logger.error({ | ||
err, | ||
msg: `Error deleting duplicative ${searchSlug} documents.`, | ||
}) | ||
} | ||
} | ||
|
||
if (foundDoc) { | ||
const { id: searchDocID } = foundDoc | ||
|
||
if (doSync) { | ||
// update the doc normally | ||
try { | ||
await payload.update({ | ||
id: searchDocID, | ||
collection: searchSlug, | ||
data: { | ||
...dataToSave, | ||
priority: foundDoc.priority || defaultPriority, | ||
}, | ||
locale: req.locale, | ||
req, | ||
}) | ||
} catch (err: unknown) { | ||
payload.logger.error({ err, msg: `Error updating ${searchSlug} document.` }) | ||
} | ||
} | ||
if (deleteDrafts && status === 'draft') { | ||
// do not include draft docs in search results, so delete the record | ||
try { | ||
await payload.delete({ | ||
id: searchDocID, | ||
collection: searchSlug, | ||
req, | ||
}) | ||
} catch (err: unknown) { | ||
payload.logger.error({ err, msg: `Error deleting ${searchSlug} document.` }) | ||
} | ||
} | ||
} else if (doSync) { | ||
try { | ||
await payload.create({ | ||
collection: searchSlug, | ||
data: { | ||
...dataToSave, | ||
priority: defaultPriority, | ||
}, | ||
locale: req.locale, | ||
req, | ||
}) | ||
} catch (err: unknown) { | ||
payload.logger.error({ err, msg: `Error creating ${searchSlug} document.` }) | ||
} | ||
} | ||
} catch (err: unknown) { | ||
payload.logger.error({ err, msg: `Error finding ${searchSlug} document.` }) | ||
} | ||
} | ||
} catch (err: unknown) { | ||
payload.logger.error({ | ||
err, | ||
msg: `Error syncing ${searchSlug} document related to ${collection} with id: '${id}'.`, | ||
}) | ||
} | ||
|
||
return doc | ||
export const syncWithSearch: SyncWithSearch = (args) => { | ||
return syncDocAsSearchIndex(args) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
12 changes: 12 additions & 0 deletions
12
packages/plugin-search/src/Search/ui/ReindexButton/ReindexButtonLabel/index.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
import { ChevronIcon, Pill, useTranslation } from '@payloadcms/ui' | ||
|
||
export const ReindexButtonLabel = () => { | ||
const { | ||
i18n: { t }, | ||
} = useTranslation() | ||
return ( | ||
<Pill className="pill--has-action" icon={<ChevronIcon />} pillStyle="light"> | ||
{t('general:reindex')} | ||
</Pill> | ||
) | ||
} |
39 changes: 39 additions & 0 deletions
39
packages/plugin-search/src/Search/ui/ReindexButton/ReindexConfirmModal/index.scss
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
@import '../../../../../../ui/src/scss/styles.scss'; | ||
|
||
@layer payload-default { | ||
.reindex-confirm-modal { | ||
@include blur-bg; | ||
display: flex; | ||
align-items: center; | ||
justify-content: center; | ||
height: 100%; | ||
|
||
&__wrapper { | ||
z-index: 1; | ||
position: relative; | ||
display: flex; | ||
flex-direction: column; | ||
gap: base(2); | ||
padding: base(2); | ||
} | ||
|
||
&__content { | ||
display: flex; | ||
flex-direction: column; | ||
gap: base(1); | ||
|
||
> * { | ||
margin: 0; | ||
} | ||
} | ||
|
||
&__controls { | ||
display: flex; | ||
gap: base(0.4); | ||
|
||
.btn { | ||
margin: 0; | ||
} | ||
} | ||
} | ||
} |
37 changes: 37 additions & 0 deletions
37
packages/plugin-search/src/Search/ui/ReindexButton/ReindexConfirmModal/index.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
import { Button, Modal, useTranslation } from '@payloadcms/ui' | ||
|
||
import './index.scss' | ||
|
||
type Props = { | ||
description: string | ||
onCancel: () => void | ||
onConfirm: () => void | ||
slug: string | ||
title: string | ||
} | ||
|
||
const baseClass = 'reindex-confirm-modal' | ||
|
||
export const ReindexConfirmModal = ({ slug, description, onCancel, onConfirm, title }: Props) => { | ||
const { | ||
i18n: { t }, | ||
} = useTranslation() | ||
return ( | ||
<Modal className={baseClass} slug={slug}> | ||
<div className={`${baseClass}__wrapper`}> | ||
<div className={`${baseClass}__content`}> | ||
<h1>{title}</h1> | ||
<p>{description}</p> | ||
</div> | ||
<div className={`${baseClass}__controls`}> | ||
<Button buttonStyle="secondary" onClick={onCancel} size="large"> | ||
{t('general:cancel')} | ||
</Button> | ||
<Button onClick={onConfirm} size="large"> | ||
{t('general:confirm')} | ||
</Button> | ||
</div> | ||
</div> | ||
</Modal> | ||
) | ||
} |
Oops, something went wrong.