From dd8272c0e5a76e9eff60902439111c28d7e72438 Mon Sep 17 00:00:00 2001 From: meili-bot <74670311+meili-bot@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:09:09 +0200 Subject: [PATCH 1/8] Update README.md From 1d6507e13dfa63b97b52ef62e0f4fe70afbcf447 Mon Sep 17 00:00:00 2001 From: Shalabh Agarwal <34604329+the-sinner@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:33:04 +0530 Subject: [PATCH 2/8] Add frequency matching strategy (#1670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add frequency matching strategy * Update tests/search.test.ts --------- Co-authored-by: Clémentine --- src/types/types.ts | 1 + tests/search.test.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/types/types.ts b/src/types/types.ts index 5b012f3b9..0f7d7efe2 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -63,6 +63,7 @@ export type IndexesResults = ResourceResults & {}; export const MatchingStrategies = { ALL: 'all', LAST: 'last', + FREQUENCY: 'frequency', } as const; export type MatchingStrategies = diff --git a/tests/search.test.ts b/tests/search.test.ts index 0f9763b5f..ffb81027a 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -191,6 +191,18 @@ describe.each([ expect(response.hits.length).toEqual(2); }); + test(`${permission} key: Basic phrase search with matchingStrategy at FREQUENCY`, async () => { + const client = await getClient(permission); + const response = await client.index(index.uid).search('french book', { + matchingStrategy: MatchingStrategies.FREQUENCY, + }); + + expect(response).toHaveProperty('hits', expect.any(Array)); + expect(response).toHaveProperty('offset', 0); + expect(response).toHaveProperty('limit', 20); + expect(response.hits.length).toEqual(2); + }); + test(`${permission} key: Search with query in searchParams overwriting query`, async () => { const client = await getClient(permission); const response = await client From 5f4d297b94f42b2c4172946fbc0e30d735ed714c Mon Sep 17 00:00:00 2001 From: Shalabh Agarwal <34604329+the-sinner@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:13:38 +0530 Subject: [PATCH 3/8] Add rankingScoreThreshold in search (#1669) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add rankingScoreThreshold in search * Update src/types/types.ts * Update src/types/types.ts * Apply suggestions from code review * Update src/types/types.ts --------- Co-authored-by: Clémentine --- src/types/types.ts | 9 +++++++++ tests/search.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/types/types.ts b/src/types/types.ts index 0f7d7efe2..e19579e0f 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -127,6 +127,7 @@ export type SearchParams = Query & showRankingScoreDetails?: boolean; attributesToSearchOn?: string[] | null; hybrid?: HybridSearch; + rankingScoreThreshold?: number; }; // Search parameters for searches made with the GET method @@ -1011,6 +1012,14 @@ export const ErrorStatusCode = { /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_facet_search_facet_query */ INVALID_FACET_SEARCH_FACET_QUERY: 'invalid_facet_search_facet_query', + + /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_search_ranking_score_threshold */ + INVALID_SEARCH_RANKING_SCORE_THRESHOLD: + 'invalid_search_ranking_score_threshold', + + /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_similar_ranking_score_threshold */ + INVALID_SIMILAR_RANKING_SCORE_THRESHOLD: + 'invalid_similar_ranking_score_threshold', }; export type ErrorStatusCode = diff --git a/tests/search.test.ts b/tests/search.test.ts index ffb81027a..3f8072e49 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -306,6 +306,29 @@ describe.each([ ]); }); + test(`${permission} key: search with rankingScoreThreshold filter`, async () => { + const client = await getClient(permission); + + const response = await client.index(index.uid).search('prince', { + showRankingScore: true, + rankingScoreThreshold: 0.8, + }); + + const hit = response.hits[0]; + + expect(response).toHaveProperty('hits', expect.any(Array)); + expect(response).toHaveProperty('query', 'prince'); + expect(hit).toHaveProperty('_rankingScore'); + expect(hit['_rankingScore']).toBeGreaterThanOrEqual(0.8); + + const response2 = await client.index(index.uid).search('prince', { + showRankingScore: true, + rankingScoreThreshold: 0.9, + }); + + expect(response2.hits.length).toBeLessThanOrEqual(0); + }); + test(`${permission} key: search with array options`, async () => { const client = await getClient(permission); From f3bb65b90a1cb3cd8f821f7b6a6169f3b8580a6f Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:32:06 +0200 Subject: [PATCH 4/8] feat: add rankingScoreThreshold to searchGet (#1673) --- src/types/types.ts | 3 ++- tests/get_search.test.ts | 25 ++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/types/types.ts b/src/types/types.ts index e19579e0f..c031a1cca 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -125,9 +125,9 @@ export type SearchParams = Query & vector?: number[] | null; showRankingScore?: boolean; showRankingScoreDetails?: boolean; + rankingScoreThreshold?: number; attributesToSearchOn?: string[] | null; hybrid?: HybridSearch; - rankingScoreThreshold?: number; }; // Search parameters for searches made with the GET method @@ -147,6 +147,7 @@ export type SearchRequestGET = Pagination & attributesToSearchOn?: string | null; hybridEmbedder?: string; hybridSemanticRatio?: number; + rankingScoreThreshold?: number; }; export type MultiSearchQuery = SearchParams & { indexUid: string }; diff --git a/tests/get_search.test.ts b/tests/get_search.test.ts index 4e5edd61f..8b78e5714 100644 --- a/tests/get_search.test.ts +++ b/tests/get_search.test.ts @@ -483,11 +483,34 @@ describe.each([ test(`${permission} key: search without vectors`, async () => { const client = await getClient(permission); - const response = await client.index(index.uid).search('prince', {}); + const response = await client.index(index.uid).searchGet('prince', {}); expect(response).not.toHaveProperty('semanticHitCount'); }); + test(`${permission} key: search with rankingScoreThreshold filter`, async () => { + const client = await getClient(permission); + + const response = await client.index(index.uid).searchGet('prince', { + showRankingScore: true, + rankingScoreThreshold: 0.8, + }); + + const hit = response.hits[0]; + + expect(response).toHaveProperty('hits', expect.any(Array)); + expect(response).toHaveProperty('query', 'prince'); + expect(hit).toHaveProperty('_rankingScore'); + expect(hit['_rankingScore']).toBeGreaterThanOrEqual(0.8); + + const response2 = await client.index(index.uid).search('prince', { + showRankingScore: true, + rankingScoreThreshold: 0.9, + }); + + expect(response2.hits.length).toBeLessThanOrEqual(0); + }); + test(`${permission} key: Try to search on deleted index and fail`, async () => { const client = await getClient(permission); const masterClient = await getClient('Master'); From 1716ac8889eb5a29e5a1ee4f53d55ec2a59ca13e Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:00:33 +0200 Subject: [PATCH 5/8] feat: add attribute at search (#1676) --- src/types/types.ts | 2 ++ tests/get_search.test.ts | 9 +++++++++ tests/search.test.ts | 9 +++++++++ 3 files changed, 20 insertions(+) diff --git a/src/types/types.ts b/src/types/types.ts index c031a1cca..81b1f83f9 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -128,6 +128,7 @@ export type SearchParams = Query & rankingScoreThreshold?: number; attributesToSearchOn?: string[] | null; hybrid?: HybridSearch; + distinct?: string; }; // Search parameters for searches made with the GET method @@ -148,6 +149,7 @@ export type SearchRequestGET = Pagination & hybridEmbedder?: string; hybridSemanticRatio?: number; rankingScoreThreshold?: number; + distinct?: string; }; export type MultiSearchQuery = SearchParams & { indexUid: string }; diff --git a/tests/get_search.test.ts b/tests/get_search.test.ts index 8b78e5714..bb8af3250 100644 --- a/tests/get_search.test.ts +++ b/tests/get_search.test.ts @@ -511,6 +511,15 @@ describe.each([ expect(response2.hits.length).toBeLessThanOrEqual(0); }); + test(`${permission} key: search with distinct`, async () => { + const client = await getClient(permission); + const response = await client + .index(index.uid) + .search('', { distinct: 'genre' }); + + expect(response.hits.length).toEqual(4); + }); + test(`${permission} key: Try to search on deleted index and fail`, async () => { const client = await getClient(permission); const masterClient = await getClient('Master'); diff --git a/tests/search.test.ts b/tests/search.test.ts index 3f8072e49..41e5bcb80 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -911,6 +911,15 @@ describe.each([ expect(response).not.toHaveProperty('semanticHitCount'); }); + test(`${permission} key: search with distinct`, async () => { + const client = await getClient(permission); + const response = await client + .index(index.uid) + .search('', { distinct: 'genre' }); + + expect(response.hits.length).toEqual(4); + }); + test(`${permission} key: Try to search on deleted index and fail`, async () => { const client = await getClient(permission); const masterClient = await getClient('Master'); From 65bc592f9884f3be14510fac611794a85b0b3125 Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:06:32 +0200 Subject: [PATCH 6/8] feat: get similar documents (#1677) --- src/indexes.ts | 20 +++++++++++++ src/types/types.ts | 12 ++++++++ tests/embedders.test.ts | 65 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/src/indexes.ts b/src/indexes.ts index 3fb051111..d1fe24c0f 100644 --- a/src/indexes.ts +++ b/src/indexes.ts @@ -53,6 +53,7 @@ import { ProximityPrecision, Embedders, SearchCutoffMs, + SearchSimilarDocumentsParams, } from './types'; import { removeUndefinedFromObject } from './utils'; import { HttpRequests } from './http-requests'; @@ -177,6 +178,25 @@ class Index = Record> { ); } + /** + * Search for similar documents + * + * @param params - Parameters used to search for similar documents + * @returns Promise containing the search response + */ + async searchSimilarDocuments< + D extends Record = T, + S extends SearchParams = SearchParams, + >(params: SearchSimilarDocumentsParams): Promise> { + const url = `indexes/${this.uid}/similar`; + + return await this.httpRequest.post( + url, + removeUndefinedFromObject(params), + undefined, + ); + } + /// /// INDEX /// diff --git a/src/types/types.ts b/src/types/types.ts index 81b1f83f9..11f36c808 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -267,6 +267,18 @@ export type FieldDistribution = { [field: string]: number; }; +export type SearchSimilarDocumentsParams = { + id: string | number; + offset?: number; + limit?: number; + filter?: Filter; + embedder?: string; + attributesToRetrieve?: string[]; + showRankingScore?: boolean; + showRankingScoreDetails?: boolean; + rankingScoreThreshold?: number; +}; + /* ** Documents */ diff --git a/tests/embedders.test.ts b/tests/embedders.test.ts index d6691581f..60748756f 100644 --- a/tests/embedders.test.ts +++ b/tests/embedders.test.ts @@ -14,6 +14,39 @@ const index = { uid: 'movies_test', }; +const datasetSimilarSearch = [ + { + title: 'Shazam!', + release_year: 2019, + id: '287947', + _vectors: { manual: [0.8, 0.4, -0.5] }, + }, + { + title: 'Captain Marvel', + release_year: 2019, + id: '299537', + _vectors: { manual: [0.6, 0.8, -0.2] }, + }, + { + title: 'Escape Room', + release_year: 2019, + id: '522681', + _vectors: { manual: [0.1, 0.6, 0.8] }, + }, + { + title: 'How to Train Your Dragon: The Hidden World', + release_year: 2019, + id: '166428', + _vectors: { manual: [0.7, 0.7, -0.4] }, + }, + { + title: 'All Quiet on the Western Front', + release_year: 1930, + id: '143', + _vectors: { manual: [-0.5, 0.3, 0.85] }, + }, +]; + jest.setTimeout(100 * 1000); afterAll(() => { @@ -223,6 +256,38 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( expect(response).toEqual(null); }); + + test(`${permission} key: search for similar documents`, async () => { + const client = await getClient(permission); + + const newEmbedder: Embedders = { + manual: { + source: 'userProvided', + dimensions: 3, + }, + }; + const { taskUid: updateEmbeddersTask }: EnqueuedTask = await client + .index(index.uid) + .updateEmbedders(newEmbedder); + + await client.waitForTask(updateEmbeddersTask); + + const { taskUid: documentAdditionTask } = await client + .index(index.uid) + .addDocuments(datasetSimilarSearch); + + await client.waitForTask(documentAdditionTask); + + const response = await client.index(index.uid).searchSimilarDocuments({ + id: '143', + }); + + expect(response).toHaveProperty('hits'); + expect(response.hits.length).toEqual(4); + expect(response).toHaveProperty('offset', 0); + expect(response).toHaveProperty('limit', 20); + expect(response).toHaveProperty('estimatedTotalHits', 4); + }); }, ); From 8e56525136748c0c309f315daea20a2f984a2509 Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Tue, 25 Jun 2024 15:20:56 +0200 Subject: [PATCH 7/8] feat: hybrid search changes (#1679) --- src/types/types.ts | 3 ++ tests/documents.test.ts | 86 ++++++++++++++++++++++++++++++++++++++++ tests/get_search.test.ts | 42 ++++++++++++++++++++ tests/search.test.ts | 42 ++++++++++++++++++++ 4 files changed, 173 insertions(+) diff --git a/src/types/types.ts b/src/types/types.ts index 11f36c808..71c635ef0 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -129,6 +129,7 @@ export type SearchParams = Query & attributesToSearchOn?: string[] | null; hybrid?: HybridSearch; distinct?: string; + retrieveVectors?: boolean; }; // Search parameters for searches made with the GET method @@ -150,6 +151,7 @@ export type SearchRequestGET = Pagination & hybridSemanticRatio?: number; rankingScoreThreshold?: number; distinct?: string; + retrieveVectors?: boolean; }; export type MultiSearchQuery = SearchParams & { indexUid: string }; @@ -311,6 +313,7 @@ export type DocumentsQuery> = ResourceQuery & { filter?: Filter; limit?: number; offset?: number; + retrieveVectors?: boolean; }; export type DocumentQuery> = { diff --git a/tests/documents.test.ts b/tests/documents.test.ts index ef3a54930..97967c044 100644 --- a/tests/documents.test.ts +++ b/tests/documents.test.ts @@ -218,6 +218,92 @@ Hint: It might not be working because maybe you're not up to date with the Meili expect(documents.results.length).toEqual(dataset.length); }); + test(`${permission} key: Get documents with retrieveVectors to true`, async () => { + const client = await getClient(permission); + const adminKey = await getKey('Admin'); + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }); + + const { taskUid } = await client + .index(indexPk.uid) + .addDocuments(dataset); + await client.index(indexPk.uid).waitForTask(taskUid); + + // Get documents with POST + const documentsPost = await client + .index(indexPk.uid) + .getDocuments({ retrieveVectors: true }); + + expect(documentsPost.results.length).toEqual(dataset.length); + expect(documentsPost.results[0]).toHaveProperty('_vectors'); + + // Get documents with GET + const res = await fetch( + `${HOST}/indexes/${indexPk.uid}/documents?retrieveVectors=true`, + { + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'GET', + }, + ); + const documentsGet = await res.json(); + + expect(documentsGet.results.length).toEqual(dataset.length); + expect(documentsGet.results[0]).toHaveProperty('_vectors'); + }); + + test(`${permission} key: Get documents without retrieveVectors`, async () => { + const client = await getClient(permission); + const adminKey = await getKey('Admin'); + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }); + + const { taskUid } = await client + .index(indexPk.uid) + .addDocuments(dataset); + await client.index(indexPk.uid).waitForTask(taskUid); + + // Get documents with POST + const documentsPost = await client + .index(indexPk.uid) + .getDocuments(); + + expect(documentsPost.results.length).toEqual(dataset.length); + expect(documentsPost.results[0]).not.toHaveProperty('_vectors'); + + // Get documents with GET + const res = await fetch( + `${HOST}/indexes/${indexPk.uid}/documents?retrieveVectors=false`, + { + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'GET', + }, + ); + const documentsGet = await res.json(); + + expect(documentsGet.results.length).toEqual(dataset.length); + expect(documentsGet.results[0]).not.toHaveProperty('_vectors'); + }); + test(`${permission} key: Replace documents from index that has NO primary key`, async () => { const client = await getClient(permission); const { taskUid: addDocTask } = await client diff --git a/tests/get_search.test.ts b/tests/get_search.test.ts index bb8af3250..209e557c8 100644 --- a/tests/get_search.test.ts +++ b/tests/get_search.test.ts @@ -520,6 +520,48 @@ describe.each([ expect(response.hits.length).toEqual(4); }); + test(`${permission} key: search with retrieveVectors to true`, async () => { + const client = await getClient(permission); + const adminKey = await getKey('Admin'); + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }); + + const response = await client.index(index.uid).searchGet('prince', { + retrieveVectors: true, + }); + + expect(response).toHaveProperty('hits', expect.any(Array)); + expect(response).toHaveProperty('query', 'prince'); + expect(response.hits[0]).toHaveProperty('_vectors'); + }); + + test(`${permission} key: search without retrieveVectors`, async () => { + const client = await getClient(permission); + const adminKey = await getKey('Admin'); + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }); + + const response = await client.index(index.uid).searchGet('prince'); + + expect(response).toHaveProperty('hits', expect.any(Array)); + expect(response).toHaveProperty('query', 'prince'); + expect(response.hits[0]).not.toHaveProperty('_vectors'); + }); + test(`${permission} key: Try to search on deleted index and fail`, async () => { const client = await getClient(permission); const masterClient = await getClient('Master'); diff --git a/tests/search.test.ts b/tests/search.test.ts index 41e5bcb80..f4d9f1feb 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -920,6 +920,48 @@ describe.each([ expect(response.hits.length).toEqual(4); }); + test(`${permission} key: search with retrieveVectors to true`, async () => { + const client = await getClient(permission); + const adminKey = await getKey('Admin'); + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }); + + const response = await client.index(index.uid).search('prince', { + retrieveVectors: true, + }); + + expect(response).toHaveProperty('hits', expect.any(Array)); + expect(response).toHaveProperty('query', 'prince'); + expect(response.hits[0]).toHaveProperty('_vectors'); + }); + + test(`${permission} key: search without retrieveVectors`, async () => { + const client = await getClient(permission); + const adminKey = await getKey('Admin'); + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${adminKey}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }); + + const response = await client.index(index.uid).search('prince'); + + expect(response).toHaveProperty('hits', expect.any(Array)); + expect(response).toHaveProperty('query', 'prince'); + expect(response.hits[0]).not.toHaveProperty('_vectors'); + }); + test(`${permission} key: Try to search on deleted index and fail`, async () => { const client = await getClient(permission); const masterClient = await getClient('Master'); From 7edd89cd2b986133353a2f74b89dd16aece38a77 Mon Sep 17 00:00:00 2001 From: curquiza Date: Wed, 26 Jun 2024 16:06:46 +0200 Subject: [PATCH 8/8] Update code samples --- .code-samples.meilisearch.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.code-samples.meilisearch.yaml b/.code-samples.meilisearch.yaml index a0909e542..9271498bd 100644 --- a/.code-samples.meilisearch.yaml +++ b/.code-samples.meilisearch.yaml @@ -754,3 +754,22 @@ negative_search_1: |- client.index('movies').search('-escape') negative_search_2: |- client.index('movies').search('-"escape"') +search_parameter_reference_ranking_score_threshold_1: |- + client.index('INDEX_NAME').search('badman', { rankingScoreThreshold: 0.2 }) +search_parameter_reference_retrieve_vectors_1: |- + client.index('INDEX_NAME').search('kitchen utensils', { + retrieveVectors: true, + hybrid: { embedder: 'default'} + }) +get_similar_post_1: |- + client.index('INDEX_NAME').searchSimilarDocuments({ id: 'TARGET_DOCUMENT_ID'}) +search_parameter_guide_matching_strategy_3: |- + client.index('movies').search('white shirt', { + matchingStrategy: 'frequency' + }) +search_parameter_reference_distinct_1: |- + client.index('INDEX_NAME').search('QUERY TERMS', { distinct: 'ATTRIBUTE_A' }) +distinct_attribute_guide_filterable_1: |- + client.index('products').updateFilterableAttributes(['product_id', 'sku', 'url']) +distinct_attribute_guide_distinct_parameter_1: |- + client.index('products').search('white shirt', { distinct: 'sku' })