remove tokenstream vec alloc

quickwit-oss · Oct 11, 2022 · 2efebdb · 2efebdb
1 parent 5f565e7
commit 2efebdb
Showing 1 changed file with 9 additions and 15 deletions.
diff --git a/src/indexer/segment_writer.rs b/src/indexer/segment_writer.rs
@@ -14,9 +14,7 @@ use crate::postings::{
 };
 use crate::schema::{FieldEntry, FieldType, FieldValue, Schema, Term, Value};
 use crate::store::{StoreReader, StoreWriter};
-use crate::tokenizer::{
-    BoxTokenStream, FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer,
-};
+use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
 use crate::{DatePrecision, DocId, Document, Opstamp, SegmentComponent};
 
 /// Computes the initial size of the hash table.
@@ -204,26 +202,22 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut token_streams: Vec<BoxTokenStream> = vec![];
-
+                    let mut indexing_position = IndexingPosition::default();
                     for value in values {
-                        match value {
+                        let mut token_stream = match value {
                             Value::PreTokStr(tok_str) => {
-                                token_streams
-                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
+                                PreTokenizedStream::from(tok_str.clone()).into()
                             }
                             Value::Str(ref text) => {
                                 let text_analyzer =
                                     &self.per_field_text_analyzers[field.field_id() as usize];
-                                token_streams.push(text_analyzer.token_stream(text));
+                                text_analyzer.token_stream(text)
                             }
-                            _ => (),
-                        }
-                    }
-
-                    let mut indexing_position = IndexingPosition::default();
+                            _ => {
+                                continue;
+                            }
+                        };
 
-                    for mut token_stream in token_streams {
                         assert_eq!(term_buffer.as_slice().len(), 5);
                         postings_writer.index_text(
                             doc_id,