feat(default-layout): sortable search results, search phrases, experi…

…mental config (#3652)

This commit updates search with the following features:
- Order search results by relevance, created and last updated date.
- Search for documents containing an exact phrase by wrapping text inside double quotes.
- Use the experimental config `__experimental_omnisearch_visibility` to hide documents from global search.
- Display relevance scores in search results via a debug flag (`#_debug_search_score`)

It comes with the following fixes:
- Fixes an issue where the scroll position of search results wasn’t being correctly retained in some instances.
- Fixes an issue where search would cause the studio to crash if Local Storage is unavailable.

dev/test-studio/parts/deskStructure.js

@@ -97,6 +97,7 @@ const DEBUG_INPUT_TYPES = [
   'withParentTest',
   'arrayExperimentalSearch',
   'arrayPreviewSelect',
+  'experimentalOmnisearchVisibilityTest',
 ]
 
 const CI_INPUT_TYPES = ['conditionalFieldset', 'validationCI']

dev/test-studio/schema/debug/experimentalOmnisearchVisibilityTest.js

@@ -0,0 +1,18 @@
+/**
+ * Documents of this type should never be visible in omnisearch results,
+ * nor should they appear in the omnisearch document filter list.
+ */
+export default {
+  // eslint-disable-next-line camelcase
+  __experimental_omnisearch_visibility: false,
+  type: 'document',
+  name: 'experimentalOmnisearchVisibilityTest',
+  title: 'Experimental omnisearch visibility test',
+  fields: [
+    {
+      type: 'string',
+      name: 'title',
+      title: 'Title',
+    },
+  ],
+}

dev/test-studio/schema/index.js

@@ -41,6 +41,7 @@ import customNumber from './debug/customNumber'
 import documentActions from './debug/documentActions'
 import empty from './debug/empty'
 import experiment from './debug/experiment'
+import experimentalOmnisearchVisibilityTest from './debug/experimentalOmnisearchVisibilityTest'
 import fieldsets from './debug/fieldsets'
 import {
   fieldValidationInferReproSharedObject,
@@ -124,6 +125,7 @@ export default createSchema({
     emails,
     empty,
     experiment,
+    experimentalOmnisearchVisibilityTest,
     fieldValidationInferReproDoc,
     fieldValidationInferReproSharedObject,
     fieldsets,

jest.config.js

@@ -2,6 +2,7 @@ const JEST_PROJECTS = [
   '@sanity/core',
   '@sanity/base',
   '@sanity/block-tools',
+  '@sanity/default-layout',
   '@sanity/desk-tool',
   '@sanity/export',
   '@sanity/form-builder',

packages/@sanity/base/src/_exports/index.ts

@@ -17,4 +17,5 @@ export type {
   DocumentActionProps,
 } from '../actions/utils/types'
 
-export type {SearchTerms, SearchableType} from '../search'
+// Export search typings
+export type {SearchOptions, SearchSort, SearchTerms, SearchableType, WeightedHit} from '../search'

packages/@sanity/base/test/search-tokenize.test.js → packages/@sanity/base/src/search/common/tokenize.test.ts

@@ -1,6 +1,6 @@
-import {tokenize} from '../src/search/common/tokenize'
+import {tokenize} from './tokenize'
 
-const tests = [
+const tests: [string, string[]][] = [
   ['', []],
   ['foo', ['foo']],
   ['0foo', ['0foo']],

packages/@sanity/base/src/search/index.ts

@@ -6,13 +6,19 @@ import {versionedClient} from '../client/versionedClient'
 import {getSearchableTypes} from './common/utils'
 import {createWeightedSearch} from './weighted/createWeightedSearch'
 
+export type {
+  SearchOptions,
+  SearchSort,
+  SearchTerms,
+  SearchableType,
+  WeightedHit,
+} from './weighted/types'
+
 // Use >= 2021-03-25 for pt::text() support
 const searchClient = versionedClient.withConfig({
   apiVersion: '2021-03-25',
 })
 
-export type {SearchTerms, SearchableType} from './weighted/types'
-
 export default createWeightedSearch(getSearchableTypes(schema), searchClient, {
   unique: true,
   tag: 'search.global',

packages/@sanity/base/src/search/weighted/applyWeights.test.ts

@@ -0,0 +1,49 @@
+import {
+  calculatePhraseScore,
+  calculateWordScore,
+  partitionAndSanitizeSearchTerms,
+} from './applyWeights'
+
+describe('calculatePhraseScore', () => {
+  it('should handle exact matches', () => {
+    expect(calculatePhraseScore(['the fox'], 'the fox')).toEqual([1, '[Phrase] Exact match'])
+  })
+  it('should handle partial matches', () => {
+    expect(calculatePhraseScore(['the fox'], 'the fox of foo')).toEqual([
+      0.25,
+      '[Phrase] Matched 7 of 14 characters',
+    ])
+  })
+})
+
+describe('calculateWordScore', () => {
+  it('should handle exact matches', () => {
+    expect(calculateWordScore(['foo'], 'foo')).toEqual([1, '[Word] Exact match'])
+    expect(calculateWordScore(['foo', 'foo'], 'foo foo')).toEqual([1, '[Word] Exact match'])
+    expect(calculateWordScore(['bar', 'foo'], 'foo bar')).toEqual([1, '[Word] Exact match'])
+    expect(calculateWordScore(['foo', 'bar'], 'bar, foo')).toEqual([1, '[Word] Exact match'])
+    expect(calculateWordScore(['foo', 'bar'], 'bar & foo')).toEqual([1, '[Word] Exact match'])
+  })
+  it('should handle partial matches', () => {
+    expect(calculateWordScore(['foo'], 'bar foo')).toEqual([
+      0.25,
+      '[Word] Matched 1 of 2 terms: [foo]',
+    ])
+    expect(calculateWordScore(['foo', 'bar'], 'foo')).toEqual([
+      0.25,
+      `[Word] Matched 1 of 2 terms: [foo]`,
+    ])
+    expect(calculateWordScore(['foo', 'bar', 'baz'], 'foo foo bar')).toEqual([
+      1 / 3,
+      `[Word] Matched 2 of 3 terms: [foo, bar]`,
+    ])
+  })
+})
+
+describe('partitionAndSanitizeSearchTerms', () => {
+  it('should separate words and phrases', () => {
+    const {phrases, words} = partitionAndSanitizeSearchTerms(['foo', 'bar', `"foo bar"`])
+    expect(phrases).toEqual(['foo bar'])
+    expect(words).toEqual(['foo', 'bar'])
+  })
+})

packages/@sanity/base/src/search/weighted/applyWeights.ts

@@ -1,16 +1,19 @@
-import {words, uniq, compact, union, intersection, keyBy, toLower} from 'lodash'
+import {compact, intersection, keyBy, partition, toLower, union, uniq, words} from 'lodash'
 import {SearchHit, WeightedHit, SearchSpec} from './types'
 
+type SearchScore = [number, string]
+
 // takes a set of terms and a value and returns a [score, story] pair where score is a value between 0, 1 and story is the explanation
-export const calculateScore = (searchTerms: string[], value: string): [number, string] => {
-  const uniqueValueTerms = uniq(compact(words(toLower(value))))
-  const uniqueSearchTerms = uniq(searchTerms.map(toLower))
-  const matches = intersection(uniqueSearchTerms, uniqueValueTerms)
-  const all = union(uniqueValueTerms, uniqueSearchTerms)
-  const fieldScore = matches.length / all.length
-  return fieldScore === 1
-    ? [1, 'Exact match']
-    : [fieldScore / 2, `Matched ${matches.length} of ${all.length} terms: [${matches.join(', ')}]`]
+export const calculateScore = (searchTerms: string[], value: string): SearchScore => {
+  // Separate search terms by phrases (wrapped with quotes) and words.
+  const {phrases: uniqueSearchPhrases, words: uniqueSearchWords} = partitionAndSanitizeSearchTerms(
+    searchTerms
+  )
+
+  // Calculate an aggregated score of both phrase and word matches.
+  const [phraseScore, phraseWhy] = calculatePhraseScore(uniqueSearchPhrases, value)
+  const [wordScore, wordWhy] = calculateWordScore(uniqueSearchWords, value)
+  return [phraseScore + wordScore, [wordWhy, phraseWhy].join(', ')]
 }
 
 const stringify = (value: unknown): string =>
@@ -42,3 +45,67 @@ export function applyWeights(
     return {hit, resultIndex: hits.length - index, score: totalScore, stories: stories}
   })
 }
+/**
+ * For phrases: score on the total number of matching characters.
+ * E.g. given the phrases ["the fox", "of london"] for the target value "the wily fox of london"
+ *
+ * - "the fox" isn't included in the target value (score: 0)
+ * - "of london" is included in the target value, and 9 out of 22 characters match (score: 9/22 = ~0.408)
+ * - non-exact matches have their score divided in half (final score: ~0.204)
+ */
+export function calculatePhraseScore(uniqueSearchPhrases: string[], value: string): SearchScore {
+  const sanitizedValue = value.toLowerCase().trim()
+
+  let fieldScore = 0
+  let matchCount = 0
+  uniqueSearchPhrases.forEach((term) => {
+    if (sanitizedValue.includes(term)) {
+      fieldScore += term.length / sanitizedValue.length
+      matchCount += term.length
+    }
+  })
+
+  return fieldScore === 1
+    ? [1, '[Phrase] Exact match']
+    : [fieldScore / 2, `[Phrase] Matched ${matchCount} of ${sanitizedValue.length} characters`]
+}
+
+/**
+ * For words: score on the total number of matching words.
+ * E.g. given the words ["the", "fox", "of", "london"] for the target value "the wily fox of london"
+ *
+ * - 4 out of 5 words match (score: 4/5 = 0.8)
+ * - non-exact matches have their score divided in half (final score: 0.4)
+ */
+export function calculateWordScore(uniqueSearchTerms: string[], value: string): SearchScore {
+  const uniqueValueTerms = uniq(compact(words(toLower(value))))
+
+  const matches = intersection(uniqueSearchTerms, uniqueValueTerms)
+  const all = union(uniqueValueTerms, uniqueSearchTerms)
+  const fieldScore = matches.length / all.length
+  return fieldScore === 1
+    ? [1, '[Word] Exact match']
+    : [
+        fieldScore / 2,
+        `[Word] Matched ${matches.length} of ${all.length} terms: [${matches.join(', ')}]`,
+      ]
+}
+
+export function partitionAndSanitizeSearchTerms(
+  searchTerms: string[]
+): {
+  phrases: string[]
+  words: string[]
+} {
+  const uniqueSearchTerms = uniq(searchTerms.map(toLower))
+
+  const [searchPhrases, searchWords] = partition(uniqueSearchTerms, (term) => /^".*"$/.test(term))
+  return {
+    phrases: uniq(searchPhrases).map(toLower).map(stripWrappingQuotes), //
+    words: uniq(searchWords.map(toLower)),
+  }
+}
+
+function stripWrappingQuotes(str: string) {
+  return str.replace(/^"(.*)"$/, '$1')
+}