diff --git a/.netconfig b/.netconfig index 4d229c7..6719ea3 100644 --- a/.netconfig +++ b/.netconfig @@ -127,8 +127,8 @@ weak [file "src/GrokClient/chat.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/chat.proto - sha = 362749661fa2d340d234ad372fab920538897821 - etag = 0e35007f9056fd101983a35614525773ea215eec9a8fa6707c2ec4866da5b7f8 + sha = f6d4d709515fd666ebeb7dfcb89bd50e58c859d9 + etag = a3ef7183a220c08fd57b7a197090a6a8e6e89028e0477184bf73e61b4b7da6ff weak [file "src/GrokClient/deferred.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/deferred.proto @@ -137,8 +137,8 @@ weak [file "src/GrokClient/documents.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/documents.proto - sha = 736b835b0c0dd93698664732daad49f87a2fbc6f - etag = 3719cf7bc6280bc244ec25290be31fc925c95d0833f5fe282d9d0be805827ec6 + sha = f6d4d709515fd666ebeb7dfcb89bd50e58c859d9 + etag = 4a8a0f386f19acd6818a04c0729c940a4943054938800e92a8b3973ceedc06f4 weak [file "src/GrokClient/embed.proto"] url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/embed.proto diff --git a/src/GrokClient/chat.proto b/src/GrokClient/chat.proto index e07f0e7..e52ff00 100644 --- a/src/GrokClient/chat.proto +++ b/src/GrokClient/chat.proto @@ -5,6 +5,7 @@ package xai_api; import "google/protobuf/timestamp.proto"; import "deferred.proto"; +import "documents.proto"; import "image.proto"; import "sample.proto"; import "usage.proto"; @@ -666,6 +667,20 @@ message CollectionsSearch { // Optional number of chunks to be returned for each collections search. // Defaults to 10. optional int32 limit = 2; + + // User-defined instructions to be included in the search query. Defaults to generic search + // instructions used by the collections search backend if unset. + optional string instructions = 3; + + // How to perform the document search. Defaults to hybrid retrieval when unset. + oneof retrieval_mode { + // Perform hybrid retrieval combining keyword and semantic search. + HybridRetrieval hybrid_retrieval = 4; + // Perform pure semantic retrieval using dense embeddings. + SemanticRetrieval semantic_retrieval = 5; + // Perform keyword-based retrieval using sparse embeddings. + KeywordRetrieval keyword_retrieval = 6; + } } message AttachmentSearch { diff --git a/src/GrokClient/documents.proto b/src/GrokClient/documents.proto index 1a7f02e..2fb5742 100644 --- a/src/GrokClient/documents.proto +++ b/src/GrokClient/documents.proto @@ -7,13 +7,68 @@ service Documents { rpc Search(SearchRequest) returns (SearchResponse) {} } +// Document search using a combination of keyword and semantic search. +message HybridRetrieval { + // Overfetch multiplier applied to the requested search limit. + // When set, fetches (limit * search_multiplier) results from each retrieval method + // before reranking, then returns the top `limit` results after reranking. + // Valid range is [1, 100]. Defaults to 1 when unset. + optional int32 search_multiplier = 1; + + // Which reranker to use to limit results to the desired value. + oneof reranker { + // Use a reranker model to perform the reranking. + RerankerModel reranker_model = 2; + // Use RRF to perform the reranking. + ReciprocalRankFusion reciprocal_rank_fusion = 3; + } +} + +// Document search using keyword matching (sparse embeddings). +message KeywordRetrieval { + // Optional, but always used when doing search across multiple collections. + optional RerankerModel reranker = 1; +} + +// Document search using semantic similarity (dense embeddings). +message SemanticRetrieval { + // Optional, but always used when doing search across multiple collections. + optional RerankerModel reranker = 1; +} + +// Configuration for reciprocal rank fusion (RRF) reranking. +message ReciprocalRankFusion { + // The RRF constant k used in the reciprocal rank fusion formula. Defaults to 60. + optional int32 k = 1; + + // Weight for embedding (dense) search results. Should be between 0 and 1. Defaults to 0.5. + float embedding_weight = 3; + + // Weight for keyword (sparse) search results. Should be between 0 and 1. Defaults to 0.5. + float text_weight = 4; + + // Deprecated: Use embedding_weight and text_weight instead. + reserved 2; +} + +// Configuration for model-based reranking. +message RerankerModel { + // The model to use for reranking. Defaults to standard reranker model. + optional string model = 1; + + // Instructions for the reranking model. Defaults to generic reranking instructions. + optional string instructions = 2; +} + // RankingMetric is the metric to use for the search. +// Deprecated: Metric now comes from what is set in the collection creation. enum RankingMetric { RANKING_METRIC_UNKNOWN = 0; RANKING_METRIC_L2_DISTANCE = 1; RANKING_METRIC_COSINE_SIMILARITY = 2; } +// Message that contains settings needed to do a document search. message SearchRequest { // The query to search for which will be embedded using the // same embedding model as the one used for the source to query. @@ -22,14 +77,27 @@ message SearchRequest { DocumentsSource source = 2; // The number of chunks to return. // Will always return the top matching chunks. - // Optional, defaults to 10 + // Optional, defaults to 10. optional int32 limit = 3; - - // The ranking metric to use for the search. Defaults to RANK_METRIC_L2_DISTANCE. - optional RankingMetric ranking_metric = 4; - // User-defined instructions to be included in the search query. Defaults to generic search instructions. optional string instructions = 5; + + // How to perform the document search. Defaults to HybridRetrieval + oneof retrieval_mode { + HybridRetrieval hybrid_retrieval = 11; + SemanticRetrieval semantic_retrieval = 12; + KeywordRetrieval keyword_retrieval = 13; + } + + // Deprecated: Metric now comes from what is set during collection creation. + optional RankingMetric ranking_metric = 4 [deprecated = true]; + + // Deprecated: Use retrieval_mode instead + reserved 6; + reserved 7; + reserved 8; + reserved 9; + reserved 10; } // SearchResponse message contains the results of a document search operation.