Merge #1668

1668: Changes related to the next Meilisearch release (v1.9.0) r=curquiza a=meili-bot

Related to this issue: meilisearch/integration-guides#301

This PR:
- gathers the changes related to the next Meilisearch release (v1.9.0) so that this package is ready when the official release is out.
- should pass the tests against the [latest pre-release of Meilisearch](https://github.com/meilisearch/meilisearch/releases).
- might eventually contain test failures until the Meilisearch v1.9.0 is out.

⚠️ This PR should NOT be merged until the next release of Meilisearch (v1.9.0) is out.

_This PR is auto-generated for the [pre-release week](https://github.com/meilisearch/integration-guides/blob/main/resources/pre-release-week.md) purpose._


Co-authored-by: meili-bot <74670311+meili-bot@users.noreply.github.com>
Co-authored-by: Shalabh Agarwal <34604329+the-sinner@users.noreply.github.com>
Co-authored-by: Morgane Dubus <30866152+mdubus@users.noreply.github.com>
Co-authored-by: curquiza <clementine@meilisearch.com>
Co-authored-by: Clémentine <clementine@meilisearch.com>

.code-samples.meilisearch.yaml

@@ -754,3 +754,22 @@ negative_search_1: |-
   client.index('movies').search('-escape')
 negative_search_2: |-
   client.index('movies').search('-"escape"')
+search_parameter_reference_ranking_score_threshold_1: |-
+  client.index('INDEX_NAME').search('badman', { rankingScoreThreshold: 0.2 })
+search_parameter_reference_retrieve_vectors_1: |-
+  client.index('INDEX_NAME').search('kitchen utensils', {
+    retrieveVectors: true,
+    hybrid: { embedder: 'default'}
+  })
+get_similar_post_1: |-
+  client.index('INDEX_NAME').searchSimilarDocuments({ id: 'TARGET_DOCUMENT_ID'})
+search_parameter_guide_matching_strategy_3: |-
+  client.index('movies').search('white shirt', {
+    matchingStrategy: 'frequency'
+  })
+search_parameter_reference_distinct_1: |-
+  client.index('INDEX_NAME').search('QUERY TERMS', { distinct: 'ATTRIBUTE_A' })
+distinct_attribute_guide_filterable_1: |-
+  client.index('products').updateFilterableAttributes(['product_id', 'sku', 'url'])
+distinct_attribute_guide_distinct_parameter_1: |-
+  client.index('products').search('white shirt', { distinct: 'sku' })

src/indexes.ts

@@ -53,6 +53,7 @@ import {
   ProximityPrecision,
   Embedders,
   SearchCutoffMs,
+  SearchSimilarDocumentsParams,
 } from './types';
 import { removeUndefinedFromObject } from './utils';
 import { HttpRequests } from './http-requests';
@@ -177,6 +178,25 @@ class Index<T extends Record<string, any> = Record<string, any>> {
     );
   }
 
+  /**
+   * Search for similar documents
+   *
+   * @param params - Parameters used to search for similar documents
+   * @returns Promise containing the search response
+   */
+  async searchSimilarDocuments<
+    D extends Record<string, any> = T,
+    S extends SearchParams = SearchParams,
+  >(params: SearchSimilarDocumentsParams): Promise<SearchResponse<D, S>> {
+    const url = `indexes/${this.uid}/similar`;
+
+    return await this.httpRequest.post(
+      url,
+      removeUndefinedFromObject(params),
+      undefined,
+    );
+  }
+
   ///
   /// INDEX
   ///

src/types/types.ts

@@ -63,6 +63,7 @@ export type IndexesResults<T> = ResourceResults<T> & {};
 export const MatchingStrategies = {
   ALL: 'all',
   LAST: 'last',
+  FREQUENCY: 'frequency',
 } as const;
 
 export type MatchingStrategies =
@@ -124,8 +125,11 @@ export type SearchParams = Query &
     vector?: number[] | null;
     showRankingScore?: boolean;
     showRankingScoreDetails?: boolean;
+    rankingScoreThreshold?: number;
     attributesToSearchOn?: string[] | null;
     hybrid?: HybridSearch;
+    distinct?: string;
+    retrieveVectors?: boolean;
   };
 
 // Search parameters for searches made with the GET method
@@ -145,6 +149,9 @@ export type SearchRequestGET = Pagination &
     attributesToSearchOn?: string | null;
     hybridEmbedder?: string;
     hybridSemanticRatio?: number;
+    rankingScoreThreshold?: number;
+    distinct?: string;
+    retrieveVectors?: boolean;
   };
 
 export type MultiSearchQuery = SearchParams & { indexUid: string };
@@ -262,6 +269,18 @@ export type FieldDistribution = {
   [field: string]: number;
 };
 
+export type SearchSimilarDocumentsParams = {
+  id: string | number;
+  offset?: number;
+  limit?: number;
+  filter?: Filter;
+  embedder?: string;
+  attributesToRetrieve?: string[];
+  showRankingScore?: boolean;
+  showRankingScoreDetails?: boolean;
+  rankingScoreThreshold?: number;
+};
+
 /*
  ** Documents
  */
@@ -294,6 +313,7 @@ export type DocumentsQuery<T = Record<string, any>> = ResourceQuery & {
   filter?: Filter;
   limit?: number;
   offset?: number;
+  retrieveVectors?: boolean;
 };
 
 export type DocumentQuery<T = Record<string, any>> = {
@@ -1010,6 +1030,14 @@ export const ErrorStatusCode = {
 
   /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_facet_search_facet_query */
   INVALID_FACET_SEARCH_FACET_QUERY: 'invalid_facet_search_facet_query',
+
+  /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_search_ranking_score_threshold */
+  INVALID_SEARCH_RANKING_SCORE_THRESHOLD:
+    'invalid_search_ranking_score_threshold',
+
+  /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_similar_ranking_score_threshold */
+  INVALID_SIMILAR_RANKING_SCORE_THRESHOLD:
+    'invalid_similar_ranking_score_threshold',
 };
 
 export type ErrorStatusCode =

tests/documents.test.ts

@@ -218,6 +218,92 @@ Hint: It might not be working because maybe you're not up to date with the Meili
         expect(documents.results.length).toEqual(dataset.length);
       });
 
+      test(`${permission} key: Get documents with retrieveVectors to true`, async () => {
+        const client = await getClient(permission);
+        const adminKey = await getKey('Admin');
+
+        await fetch(`${HOST}/experimental-features`, {
+          body: JSON.stringify({ vectorStore: true }),
+          headers: {
+            Authorization: `Bearer ${adminKey}`,
+            'Content-Type': 'application/json',
+          },
+          method: 'PATCH',
+        });
+
+        const { taskUid } = await client
+          .index(indexPk.uid)
+          .addDocuments(dataset);
+        await client.index(indexPk.uid).waitForTask(taskUid);
+
+        // Get documents with POST
+        const documentsPost = await client
+          .index(indexPk.uid)
+          .getDocuments<Book>({ retrieveVectors: true });
+
+        expect(documentsPost.results.length).toEqual(dataset.length);
+        expect(documentsPost.results[0]).toHaveProperty('_vectors');
+
+        // Get documents with GET
+        const res = await fetch(
+          `${HOST}/indexes/${indexPk.uid}/documents?retrieveVectors=true`,
+          {
+            headers: {
+              Authorization: `Bearer ${adminKey}`,
+              'Content-Type': 'application/json',
+            },
+            method: 'GET',
+          },
+        );
+        const documentsGet = await res.json();
+
+        expect(documentsGet.results.length).toEqual(dataset.length);
+        expect(documentsGet.results[0]).toHaveProperty('_vectors');
+      });
+
+      test(`${permission} key: Get documents without retrieveVectors`, async () => {
+        const client = await getClient(permission);
+        const adminKey = await getKey('Admin');
+
+        await fetch(`${HOST}/experimental-features`, {
+          body: JSON.stringify({ vectorStore: true }),
+          headers: {
+            Authorization: `Bearer ${adminKey}`,
+            'Content-Type': 'application/json',
+          },
+          method: 'PATCH',
+        });
+
+        const { taskUid } = await client
+          .index(indexPk.uid)
+          .addDocuments(dataset);
+        await client.index(indexPk.uid).waitForTask(taskUid);
+
+        // Get documents with POST
+        const documentsPost = await client
+          .index(indexPk.uid)
+          .getDocuments<Book>();
+
+        expect(documentsPost.results.length).toEqual(dataset.length);
+        expect(documentsPost.results[0]).not.toHaveProperty('_vectors');
+
+        // Get documents with GET
+        const res = await fetch(
+          `${HOST}/indexes/${indexPk.uid}/documents?retrieveVectors=false`,
+          {
+            headers: {
+              Authorization: `Bearer ${adminKey}`,
+              'Content-Type': 'application/json',
+            },
+            method: 'GET',
+          },
+        );
+        const documentsGet = await res.json();
+
+        expect(documentsGet.results.length).toEqual(dataset.length);
+        expect(documentsGet.results[0]).not.toHaveProperty('_vectors');
+      });
+
       test(`${permission} key: Replace documents from index that has NO primary key`, async () => {
         const client = await getClient(permission);
         const { taskUid: addDocTask } = await client

tests/embedders.test.ts

@@ -14,6 +14,39 @@ const index = {
   uid: 'movies_test',
 };
 
+const datasetSimilarSearch = [
+  {
+    title: 'Shazam!',
+    release_year: 2019,
+    id: '287947',
+    _vectors: { manual: [0.8, 0.4, -0.5] },
+  },
+  {
+    title: 'Captain Marvel',
+    release_year: 2019,
+    id: '299537',
+    _vectors: { manual: [0.6, 0.8, -0.2] },
+  },
+  {
+    title: 'Escape Room',
+    release_year: 2019,
+    id: '522681',
+    _vectors: { manual: [0.1, 0.6, 0.8] },
+  },
+  {
+    title: 'How to Train Your Dragon: The Hidden World',
+    release_year: 2019,
+    id: '166428',
+    _vectors: { manual: [0.7, 0.7, -0.4] },
+  },
+  {
+    title: 'All Quiet on the Western Front',
+    release_year: 1930,
+    id: '143',
+    _vectors: { manual: [-0.5, 0.3, 0.85] },
+  },
+];
+
 jest.setTimeout(100 * 1000);
 
 afterAll(() => {
@@ -223,6 +256,38 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])(
 
       expect(response).toEqual(null);
     });
+
+    test(`${permission} key: search for similar documents`, async () => {
+      const client = await getClient(permission);
+
+      const newEmbedder: Embedders = {
+        manual: {
+          source: 'userProvided',
+          dimensions: 3,
+        },
+      };
+      const { taskUid: updateEmbeddersTask }: EnqueuedTask = await client
+        .index(index.uid)
+        .updateEmbedders(newEmbedder);
+
+      await client.waitForTask(updateEmbeddersTask);
+
+      const { taskUid: documentAdditionTask } = await client
+        .index(index.uid)
+        .addDocuments(datasetSimilarSearch);
+
+      await client.waitForTask(documentAdditionTask);
+
+      const response = await client.index(index.uid).searchSimilarDocuments({
+        id: '143',
+      });
+
+      expect(response).toHaveProperty('hits');
+      expect(response.hits.length).toEqual(4);
+      expect(response).toHaveProperty('offset', 0);
+      expect(response).toHaveProperty('limit', 20);
+      expect(response).toHaveProperty('estimatedTotalHits', 4);
+    });
   },
 );
 

tests/get_search.test.ts

@@ -483,11 +483,85 @@ describe.each([
 
   test(`${permission} key: search without vectors`, async () => {
     const client = await getClient(permission);
-    const response = await client.index(index.uid).search('prince', {});
+    const response = await client.index(index.uid).searchGet('prince', {});
 
     expect(response).not.toHaveProperty('semanticHitCount');
   });
 
+  test(`${permission} key: search with rankingScoreThreshold filter`, async () => {
+    const client = await getClient(permission);
+
+    const response = await client.index(index.uid).searchGet('prince', {
+      showRankingScore: true,
+      rankingScoreThreshold: 0.8,
+    });
+
+    const hit = response.hits[0];
+
+    expect(response).toHaveProperty('hits', expect.any(Array));
+    expect(response).toHaveProperty('query', 'prince');
+    expect(hit).toHaveProperty('_rankingScore');
+    expect(hit['_rankingScore']).toBeGreaterThanOrEqual(0.8);
+
+    const response2 = await client.index(index.uid).search('prince', {
+      showRankingScore: true,
+      rankingScoreThreshold: 0.9,
+    });
+
+    expect(response2.hits.length).toBeLessThanOrEqual(0);
+  });
+
+  test(`${permission} key: search with distinct`, async () => {
+    const client = await getClient(permission);
+    const response = await client
+      .index(index.uid)
+      .search('', { distinct: 'genre' });
+
+    expect(response.hits.length).toEqual(4);
+  });
+
+  test(`${permission} key: search with retrieveVectors to true`, async () => {
+    const client = await getClient(permission);
+    const adminKey = await getKey('Admin');
+
+    await fetch(`${HOST}/experimental-features`, {
+      body: JSON.stringify({ vectorStore: true }),
+      headers: {
+        Authorization: `Bearer ${adminKey}`,
+        'Content-Type': 'application/json',
+      },
+      method: 'PATCH',
+    });
+
+    const response = await client.index(index.uid).searchGet('prince', {
+      retrieveVectors: true,
+    });
+
+    expect(response).toHaveProperty('hits', expect.any(Array));
+    expect(response).toHaveProperty('query', 'prince');
+    expect(response.hits[0]).toHaveProperty('_vectors');
+  });
+
+  test(`${permission} key: search without retrieveVectors`, async () => {
+    const client = await getClient(permission);
+    const adminKey = await getKey('Admin');
+
+    await fetch(`${HOST}/experimental-features`, {
+      body: JSON.stringify({ vectorStore: true }),
+      headers: {
+        Authorization: `Bearer ${adminKey}`,
+        'Content-Type': 'application/json',
+      },
+      method: 'PATCH',
+    });
+
+    const response = await client.index(index.uid).searchGet('prince');
+
+    expect(response).toHaveProperty('hits', expect.any(Array));
+    expect(response).toHaveProperty('query', 'prince');
+    expect(response.hits[0]).not.toHaveProperty('_vectors');
+  });
+
   test(`${permission} key: Try to search on deleted index and fail`, async () => {
     const client = await getClient(permission);
     const masterClient = await getClient('Master');