⬆️ Add hybrid/semantic/keyword retrieval modes with reranker config

.netconfig

@@ -127,8 +127,8 @@
 	weak
 [file "src/GrokClient/chat.proto"]
 	url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/chat.proto
-	sha = 362749661fa2d340d234ad372fab920538897821
-	etag = 0e35007f9056fd101983a35614525773ea215eec9a8fa6707c2ec4866da5b7f8
+	sha = f6d4d709515fd666ebeb7dfcb89bd50e58c859d9
+	etag = a3ef7183a220c08fd57b7a197090a6a8e6e89028e0477184bf73e61b4b7da6ff
 	weak
 [file "src/GrokClient/deferred.proto"]
 	url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/deferred.proto
@@ -137,8 +137,8 @@
 	weak
 [file "src/GrokClient/documents.proto"]
 	url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/documents.proto
-	sha = 736b835b0c0dd93698664732daad49f87a2fbc6f
-	etag = 3719cf7bc6280bc244ec25290be31fc925c95d0833f5fe282d9d0be805827ec6
+	sha = f6d4d709515fd666ebeb7dfcb89bd50e58c859d9
+	etag = 4a8a0f386f19acd6818a04c0729c940a4943054938800e92a8b3973ceedc06f4
 	weak
 [file "src/GrokClient/embed.proto"]
 	url = https://github.com/xai-org/xai-proto/blob/main/proto/xai/api/v1/embed.proto

src/GrokClient/chat.proto

@@ -5,6 +5,7 @@ package xai_api;
 
 import "google/protobuf/timestamp.proto";
 import "deferred.proto";
+import "documents.proto";
 import "image.proto";
 import "sample.proto";
 import "usage.proto";
@@ -666,6 +667,20 @@ message CollectionsSearch {
   // Optional number of chunks to be returned for each collections search.
   // Defaults to 10.
   optional int32 limit = 2;
+
+  // User-defined instructions to be included in the search query. Defaults to generic search
+  // instructions used by the collections search backend if unset.
+  optional string instructions = 3;
+
+  // How to perform the document search. Defaults to hybrid retrieval when unset.
+  oneof retrieval_mode {
+    // Perform hybrid retrieval combining keyword and semantic search.
+    HybridRetrieval hybrid_retrieval = 4;
+    // Perform pure semantic retrieval using dense embeddings.
+    SemanticRetrieval semantic_retrieval = 5;
+    // Perform keyword-based retrieval using sparse embeddings.
+    KeywordRetrieval keyword_retrieval = 6;
+  }
 }
 
 message AttachmentSearch {

src/GrokClient/documents.proto

@@ -7,13 +7,68 @@ service Documents {
   rpc Search(SearchRequest) returns (SearchResponse) {}
 }
 
+// Document search using a combination of keyword and semantic search.
+message HybridRetrieval {
+  // Overfetch multiplier applied to the requested search limit.
+  // When set, fetches (limit * search_multiplier) results from each retrieval method
+  // before reranking, then returns the top `limit` results after reranking.
+  // Valid range is [1, 100]. Defaults to 1 when unset.
+  optional int32 search_multiplier = 1;
+
+  // Which reranker to use to limit results to the desired value.
+  oneof reranker {
+    // Use a reranker model to perform the reranking.
+    RerankerModel reranker_model = 2;
+    // Use RRF to perform the reranking.
+    ReciprocalRankFusion reciprocal_rank_fusion = 3;
+  }
+}
+
+// Document search using keyword matching (sparse embeddings).
+message KeywordRetrieval {
+  // Optional, but always used when doing search across multiple collections.
+  optional RerankerModel reranker = 1;
+}
+
+// Document search using semantic similarity (dense embeddings).
+message SemanticRetrieval {
+  // Optional, but always used when doing search across multiple collections.
+  optional RerankerModel reranker = 1;
+}
+
+// Configuration for reciprocal rank fusion (RRF) reranking.
+message ReciprocalRankFusion {
+  // The RRF constant k used in the reciprocal rank fusion formula. Defaults to 60.
+  optional int32 k = 1;
+
+  // Weight for embedding (dense) search results. Should be between 0 and 1. Defaults to 0.5.
+  float embedding_weight = 3;
+
+  // Weight for keyword (sparse) search results. Should be between 0 and 1. Defaults to 0.5.
+  float text_weight = 4;
+
+  // Deprecated: Use embedding_weight and text_weight instead.
+  reserved 2;
+}
+
+// Configuration for model-based reranking.
+message RerankerModel {
+  // The model to use for reranking. Defaults to standard reranker model.
+  optional string model = 1;
+
+  // Instructions for the reranking model. Defaults to generic reranking instructions.
+  optional string instructions = 2;
+}
+
 // RankingMetric is the metric to use for the search.
+// Deprecated: Metric now comes from what is set in the collection creation.
 enum RankingMetric {
   RANKING_METRIC_UNKNOWN = 0;
   RANKING_METRIC_L2_DISTANCE = 1;
   RANKING_METRIC_COSINE_SIMILARITY = 2;
 }
 
+// Message that contains settings needed to do a document search.
 message SearchRequest {
   // The query to search for which will be embedded using the
   // same embedding model as the one used for the source to query.
@@ -22,14 +77,27 @@ message SearchRequest {
   DocumentsSource source = 2;
   // The number of chunks to return.
   // Will always return the top matching chunks.
-  // Optional, defaults to 10
+  // Optional, defaults to 10.
   optional int32 limit = 3;
-
-  // The ranking metric to use for the search. Defaults to RANK_METRIC_L2_DISTANCE.
-  optional RankingMetric ranking_metric = 4;
-
   // User-defined instructions to be included in the search query. Defaults to generic search instructions.
   optional string instructions = 5;
+
+  // How to perform the document search. Defaults to HybridRetrieval
+  oneof retrieval_mode {
+    HybridRetrieval hybrid_retrieval = 11;
+    SemanticRetrieval semantic_retrieval = 12;
+    KeywordRetrieval keyword_retrieval = 13;
+  }
+
+  // Deprecated: Metric now comes from what is set during collection creation.
+  optional RankingMetric ranking_metric = 4 [deprecated = true];
+
+  // Deprecated: Use retrieval_mode instead
+  reserved 6;
+  reserved 7;
+  reserved 8;
+  reserved 9;
+  reserved 10;
 }
 
 // SearchResponse message contains the results of a document search operation.