diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index c5c00b20d18..e041cacddb7 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -4743,7 +4743,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "ownedbytes" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "stable_deref_trait", ] @@ -8175,7 +8175,7 @@ dependencies = [ [[package]] name = "tantivy" version = "0.23.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "aho-corasick", "arc-swap", @@ -8228,7 +8228,7 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.6.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "bitpacking", ] @@ -8236,7 +8236,7 @@ dependencies = [ [[package]] name = "tantivy-columnar" version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "downcast-rs", "fastdivide", @@ -8251,7 +8251,7 @@ dependencies = [ [[package]] name = "tantivy-common" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "async-trait", "byteorder", @@ -8274,7 +8274,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.22.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "nom", ] @@ -8282,7 +8282,7 @@ dependencies = [ [[package]] name = "tantivy-sstable" version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "tantivy-bitpacker", "tantivy-common", @@ -8293,7 +8293,7 @@ dependencies = [ [[package]] name = "tantivy-stacker" version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "murmurhash32", "rand_distr", @@ -8303,7 +8303,7 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" version = "0.3.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=9f81d59#9f81d59ecd283a4ce1b6527a447c705b5fba0b1a" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=55b0b52#55b0b52457c87df45250243c3310b4ffad85b088" dependencies = [ "serde", ] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index bcbf22ab2e3..c64877e524f 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -327,7 +327,7 @@ quickwit-serve = { path = "quickwit-serve" } quickwit-storage = { path = "quickwit-storage" } quickwit-telemetry = { path = "quickwit-telemetry" } -tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "9f81d59", default-features = false, features = [ +tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "55b0b52", default-features = false, features = [ "lz4-compression", "mmap", "quickwit", diff --git a/quickwit/quickwit-search/src/collector.rs b/quickwit/quickwit-search/src/collector.rs index 74c2cb8b331..afeb2e0a4b9 100644 --- a/quickwit/quickwit-search/src/collector.rs +++ b/quickwit/quickwit-search/src/collector.rs @@ -32,7 +32,7 @@ use quickwit_proto::types::SplitId; use serde::Deserialize; use tantivy::aggregation::agg_req::{get_fast_field_names, Aggregations}; use tantivy::aggregation::intermediate_agg_result::IntermediateAggregationResults; -use tantivy::aggregation::{AggregationLimits, AggregationSegmentCollector}; +use tantivy::aggregation::{AggregationLimitsGuard, AggregationSegmentCollector}; use tantivy::collector::{Collector, SegmentCollector}; use tantivy::columnar::{ColumnType, MonotonicallyMappableToU64}; use tantivy::fastfield::Column; @@ -717,7 +717,7 @@ pub(crate) struct QuickwitCollector { pub max_hits: usize, pub sort_by: SortByPair, pub aggregation: Option, - pub aggregation_limits: AggregationLimits, + pub aggregation_limits: AggregationLimitsGuard, search_after: Option, } @@ -1015,7 +1015,7 @@ pub(crate) fn sort_by_from_request(search_request: &SearchRequest) -> SortByPair pub(crate) fn make_collector_for_split( split_id: SplitId, search_request: &SearchRequest, - aggregation_limits: AggregationLimits, + aggregation_limits: AggregationLimitsGuard, ) -> crate::Result { let aggregation = match &search_request.aggregation_request { Some(aggregation) => Some(serde_json::from_str(aggregation)?), @@ -1036,7 +1036,7 @@ pub(crate) fn make_collector_for_split( /// Builds a QuickwitCollector that's only useful for merging fruits. pub(crate) fn make_merge_collector( search_request: &SearchRequest, - aggregation_limits: &AggregationLimits, + aggregation_limits: &AggregationLimitsGuard, ) -> crate::Result { let aggregation = match &search_request.aggregation_request { Some(aggregation) => Some(serde_json::from_str(aggregation)?), diff --git a/quickwit/quickwit-search/src/leaf.rs b/quickwit/quickwit-search/src/leaf.rs index cbad5d87728..d60259065e1 100644 --- a/quickwit/quickwit-search/src/leaf.rs +++ b/quickwit/quickwit-search/src/leaf.rs @@ -39,7 +39,7 @@ use quickwit_storage::{ StorageResolver, }; use tantivy::aggregation::agg_req::{AggregationVariants, Aggregations}; -use tantivy::aggregation::AggregationLimits; +use tantivy::aggregation::AggregationLimitsGuard; use tantivy::directory::FileSlice; use tantivy::fastfield::FastFieldReaders; use tantivy::schema::Field; @@ -350,7 +350,7 @@ async fn leaf_search_single_split( split: SplitIdAndFooterOffsets, doc_mapper: Arc, split_filter: Arc>, - aggregations_limits: AggregationLimits, + aggregations_limits: AggregationLimitsGuard, ) -> crate::Result { rewrite_request( &mut search_request, @@ -1157,7 +1157,7 @@ async fn resolve_storage_and_leaf_search( storage_resolver: StorageResolver, splits: Vec, doc_mapper: Arc, - aggregations_limits: AggregationLimits, + aggregations_limits: AggregationLimitsGuard, ) -> crate::Result { let storage = storage_resolver.resolve(&index_uri).await?; @@ -1208,7 +1208,7 @@ pub async fn leaf_search( index_storage: Arc, splits: Vec, doc_mapper: Arc, - aggregations_limits: AggregationLimits, + aggregations_limits: AggregationLimitsGuard, ) -> Result { info!(splits_num = splits.len(), split_offsets = ?PrettySample::new(&splits, 5)); @@ -1317,7 +1317,7 @@ async fn leaf_search_single_split_wrapper( split_filter: Arc>, incremental_merge_collector: Arc>, leaf_split_search_permit: tokio::sync::OwnedSemaphorePermit, - aggregations_limits: AggregationLimits, + aggregations_limits: AggregationLimitsGuard, ) { crate::SEARCH_METRICS.leaf_searches_splits_total.inc(); let timer = crate::SEARCH_METRICS diff --git a/quickwit/quickwit-search/src/root.rs b/quickwit/quickwit-search/src/root.rs index f7aa7395172..f2c636c1037 100644 --- a/quickwit/quickwit-search/src/root.rs +++ b/quickwit/quickwit-search/src/root.rs @@ -963,7 +963,7 @@ fn finalize_aggregation( Default::default() }; let final_aggregation_results: AggregationResults = intermediate_aggregation_results - .into_final_result(aggregations, &searcher_context.get_aggregation_limits())?; + .into_final_result(aggregations, searcher_context.get_aggregation_limits())?; serde_json::to_string(&final_aggregation_results)? } }; diff --git a/quickwit/quickwit-search/src/service.rs b/quickwit/quickwit-search/src/service.rs index 1fb529c3d42..2a57a7ab65b 100644 --- a/quickwit/quickwit-search/src/service.rs +++ b/quickwit/quickwit-search/src/service.rs @@ -39,7 +39,7 @@ use quickwit_proto::search::{ use quickwit_storage::{ MemorySizedCache, QuickwitCache, SplitCache, StorageCache, StorageResolver, }; -use tantivy::aggregation::AggregationLimits; +use tantivy::aggregation::AggregationLimitsGuard; use tokio::sync::Semaphore; use tokio_stream::wrappers::UnboundedReceiverStream; @@ -459,7 +459,7 @@ pub struct SearcherContext { /// List fields cache. Caches the list fields response for a given split. pub list_fields_cache: ListFieldsCache, /// The aggregation limits are passed to limit the memory usage. - pub aggregation_limit: AggregationLimits, + pub aggregation_limit: AggregationLimitsGuard, } impl std::fmt::Debug for SearcherContext { @@ -500,7 +500,7 @@ impl SearcherContext { LeafSearchCache::new(searcher_config.partial_request_cache_capacity.as_u64() as usize); let list_fields_cache = ListFieldsCache::new(searcher_config.partial_request_cache_capacity.as_u64() as usize); - let aggregation_limit = AggregationLimits::new( + let aggregation_limit = AggregationLimitsGuard::new( Some(searcher_config.aggregation_memory_limit.as_u64()), Some(searcher_config.aggregation_bucket_limit), ); @@ -519,7 +519,7 @@ impl SearcherContext { } /// Returns the shared instance to track the aggregation memory usage. - pub fn get_aggregation_limits(&self) -> AggregationLimits { + pub fn get_aggregation_limits(&self) -> AggregationLimitsGuard { self.aggregation_limit.clone() } }