Skip to content

Commit

Permalink
Merge #1715
Browse files Browse the repository at this point in the history 
1715: Changes related to the next Meilisearch release (v1.11.0) r=brunoocasali a=meili-bot

Related to this issue: meilisearch/integration-guides#303

This PR:
- gathers the changes related to the next Meilisearch release (v1.11.0) so that this package is ready when the official release is out.
- should pass the tests against the [latest pre-release of Meilisearch](https://github.com/meilisearch/meilisearch/releases).
- might eventually contain test failures until the Meilisearch v1.11.0 is out.

⚠️ This PR should NOT be merged until the next release of Meilisearch (v1.11.0) is out.

_This PR is auto-generated for the [pre-release week](https://github.com/meilisearch/integration-guides/blob/main/resources/pre-release-week.md) purpose._


Co-authored-by: meili-bot <74670311+meili-bot@users.noreply.github.com>
Co-authored-by: Clémentine <clementine@meilisearch.com>
Co-authored-by: Morgane Dubus <30866152+mdubus@users.noreply.github.com>
Co-authored-by: curquiza <clementine@meilisearch.com>
  • Loading branch information
@meili-bors @meili-bot
@curquiza @mdubus
4 people authored Oct 28, 2024
2 parents c49c0b5 + e66c494 commit 4683cc6
Show file tree
Hide file tree
Showing 6 changed files with 278 additions and 84 deletions.
15 changes: 12 additions & 3 deletions .code-samples.meilisearch.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -759,10 +759,19 @@ search_parameter_reference_ranking_score_threshold_1: |-
search_parameter_reference_retrieve_vectors_1: |-
client.index('INDEX_NAME').search('kitchen utensils', {
retrieveVectors: true,
hybrid: { embedder: 'default'}
hybrid: {
embedder: 'EMBEDDER_NAME'
}
})
search_parameter_guide_hybrid_1: |-
client.index('INDEX_NAME').search('kitchen utensils', {
hybrid: {
semanticRatio: 0.9,
embedder: 'EMBEDDER_NAME'
}
})
get_similar_post_1: |-
client.index('INDEX_NAME').searchSimilarDocuments({ id: 'TARGET_DOCUMENT_ID'})
client.index('INDEX_NAME').searchSimilarDocuments({ id: 'TARGET_DOCUMENT_ID', embedder: 'default' })
search_parameter_guide_matching_strategy_3: |-
client.index('movies').search('white shirt', {
matchingStrategy: 'frequency'
Expand All @@ -788,7 +797,7 @@ multi_search_federated_1: |-
]
})
search_parameter_reference_locales_1: |-
client.index('INDEX_NAME').search('進撃の巨人', { locales: ['jpn'] })
client.index('INDEX_NAME').search('QUERY TEXT IN JAPANESE', { locales: ['jpn'] })
get_localized_attribute_settings_1: |-
client.index('INDEX_NAME').getLocalizedAttributes()
update_localized_attribute_settings_1: |-
Expand Down
31 changes: 29 additions & 2 deletions src/types/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export type SearchForFacetValuesResponse = {
};

export type HybridSearch = {
embedder?: string;
embedder: string;
semanticRatio?: number;
};

Expand Down Expand Up @@ -153,8 +153,17 @@ export type SearchRequestGET = Pagination &
locales?: Locale[];
};

export type MergeFacets = {
maxValuesPerFacet?: number | null;
};

export type FederationOptions = { weight: number };
export type MultiSearchFederation = { limit?: number; offset?: number };
export type MultiSearchFederation = {
limit?: number;
offset?: number;
facetsByIndex?: Record<string, string[]>;
mergeFacets?: MergeFacets | null;
};

export type MultiSearchQuery = SearchParams & { indexUid: string };
export type MultiSearchQueryWithFederation = MultiSearchQuery & {
Expand Down Expand Up @@ -229,6 +238,14 @@ export type Hits<T = Record<string, any>> = Array<Hit<T>>;
export type FacetStat = { min: number; max: number };
export type FacetStats = Record<string, FacetStat>;

export type FacetsByIndex = Record<
string,
{
distribution: FacetDistribution;
stats: FacetStats;
}
>;

export type SearchResponse<
T = Record<string, any>,
S extends SearchParams | undefined = undefined,
Expand All @@ -238,6 +255,7 @@ export type SearchResponse<
query: string;
facetDistribution?: FacetDistribution;
facetStats?: FacetStats;
facetsByIndex?: FacetsByIndex;
} & (undefined extends S
? Partial<FinitePagination & InfinitePagination>
: true extends IsFinitePagination<NonNullable<S>>
Expand Down Expand Up @@ -389,6 +407,8 @@ export type OpenAiEmbedder = {
dimensions?: number;
distribution?: Distribution;
url?: string;
documentTemplateMaxBytes?: number;
binaryQuantized?: boolean;
};

export type HuggingFaceEmbedder = {
Expand All @@ -397,12 +417,15 @@ export type HuggingFaceEmbedder = {
revision?: string;
documentTemplate?: string;
distribution?: Distribution;
documentTemplateMaxBytes?: number;
binaryQuantized?: boolean;
};

export type UserProvidedEmbedder = {
source: "userProvided";
dimensions: number;
distribution?: Distribution;
binaryQuantized?: boolean;
};

export type RestEmbedder = {
Expand All @@ -415,6 +438,8 @@ export type RestEmbedder = {
request: Record<string, any>;
response: Record<string, any>;
headers?: Record<string, string>;
documentTemplateMaxBytes?: number;
binaryQuantized?: boolean;
};

export type OllamaEmbedder = {
Expand All @@ -425,6 +450,8 @@ export type OllamaEmbedder = {
documentTemplate?: string;
distribution?: Distribution;
dimensions?: number;
documentTemplateMaxBytes?: number;
binaryQuantized?: boolean;
};

export type Embedder =
Expand Down
10 changes: 6 additions & 4 deletions tests/__snapshots__/settings.test.ts.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,9 @@ exports[`Test on settings > Admin key: Update embedders settings 1`] = `
"distinctAttribute": null,
"embedders": {
"default": {
"documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}
{% endfor %}",
"documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}
{% endif %}{% endfor %}",
"documentTemplateMaxBytes": 400,
"model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
"source": "huggingFace",
},
Expand Down Expand Up @@ -804,8 +805,9 @@ exports[`Test on settings > Master key: Update embedders settings 1`] = `
"distinctAttribute": null,
"embedders": {
"default": {
"documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}
{% endfor %}",
"documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}
{% endif %}{% endfor %}",
"documentTemplateMaxBytes": 400,
"model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
"source": "huggingFace",
},
Expand Down
63 changes: 63 additions & 0 deletions tests/embedders.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
mean: 0.7,
sigma: 0.3,
},
binaryQuantized: false,
},
};
const task: EnqueuedTask = await client
Expand All @@ -101,6 +102,7 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
const response: Embedders = await client.index(index.uid).getEmbedders();

expect(response).toEqual(newEmbedder);
expect(response).not.toHaveProperty("documentTemplateMaxBytes");
});

test(`${permission} key: Update embedders with 'openAi' source`, async () => {
Expand All @@ -118,6 +120,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
sigma: 0.3,
},
url: "https://api.openai.com/v1/embeddings",
documentTemplateMaxBytes: 500,
binaryQuantized: false,
},
};
const task: EnqueuedTask = await client
Expand Down Expand Up @@ -147,6 +151,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
mean: 0.7,
sigma: 0.3,
},
documentTemplateMaxBytes: 500,
binaryQuantized: false,
},
};
const task: EnqueuedTask = await client
Expand Down Expand Up @@ -188,6 +194,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
headers: {
"Custom-Header": "CustomValue",
},
documentTemplateMaxBytes: 500,
binaryQuantized: false,
},
};
const task: EnqueuedTask = await client
Expand Down Expand Up @@ -219,6 +227,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
sigma: 0.3,
},
dimensions: 512,
documentTemplateMaxBytes: 500,
binaryQuantized: false,
},
};
const task: EnqueuedTask = await client
Expand Down Expand Up @@ -266,6 +276,58 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
expect(response).toEqual(null);
});

test(`${permission} key: search (POST) with vectors`, async () => {
const client = await getClient(permission);

const { taskUid } = await client.index(index.uid).updateEmbedders({
default: {
source: "userProvided",
dimensions: 1,
},
});
await client.waitForTask(taskUid);

const response = await client.index(index.uid).search("", {
vector: [1],
hybrid: {
embedder: "default",
semanticRatio: 1.0,
},
});

expect(response).toHaveProperty("hits");
expect(response).toHaveProperty("semanticHitCount");
// Those fields are no longer returned by the search response
// We want to ensure that they don't appear in it anymore
expect(response).not.toHaveProperty("vector");
expect(response).not.toHaveProperty("_semanticScore");
});

test(`${permission} key: search (GET) with vectors`, async () => {
const client = await getClient(permission);

const { taskUid } = await client.index(index.uid).updateEmbedders({
default: {
source: "userProvided",
dimensions: 1,
},
});
await client.waitForTask(taskUid);

const response = await client.index(index.uid).searchGet("", {
vector: [1],
hybridEmbedder: "default",
hybridSemanticRatio: 1.0,
});

expect(response).toHaveProperty("hits");
expect(response).toHaveProperty("semanticHitCount");
// Those fields are no longer returned by the search response
// We want to ensure that they don't appear in it anymore
expect(response).not.toHaveProperty("vector");
expect(response).not.toHaveProperty("_semanticScore");
});

test(`${permission} key: search for similar documents`, async () => {
const client = await getClient(permission);

Expand All @@ -288,6 +350,7 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])(
await client.waitForTask(documentAdditionTask);

const response = await client.index(index.uid).searchSimilarDocuments({
embedder: "manual",
id: "143",
});

Expand Down
35 changes: 0 additions & 35 deletions tests/get_search.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,41 +457,6 @@ describe.each([
"The filter query parameter should be in string format when using searchGet",
);
});
test(`${permission} key: search with vectors`, async () => {
const client = await getClient(permission);
const adminClient = await getClient("Admin");
const adminKey = await getKey("Admin");

await fetch(`${HOST}/experimental-features`, {
body: JSON.stringify({ vectorStore: true }),
headers: {
Authorization: `Bearer ${adminKey}`,
"Content-Type": "application/json",
},
method: "PATCH",
});

const { taskUid } = await adminClient
.index(emptyIndex.uid)
.updateEmbedders({
default: {
source: "userProvided",
dimensions: 1,
},
});
await adminClient.waitForTask(taskUid);

const response = await client
.index(emptyIndex.uid)
.searchGet("", { vector: [1], hybridSemanticRatio: 1.0 });

expect(response).toHaveProperty("hits");
expect(response).toHaveProperty("semanticHitCount");
// Those fields are no longer returned by the search response
// We want to ensure that they don't appear in it anymore
expect(response).not.toHaveProperty("vector");
expect(response).not.toHaveProperty("_semanticScore");
});

test(`${permission} key: search without vectors`, async () => {
const client = await getClient(permission);
Expand Down
Loading

0 comments on commit 4683cc6

Please sign in to comment.