fix: don't always include first doc

python/python/tests/test_scalar_index.py

@@ -245,6 +245,31 @@ def test_filter_with_fts_index(dataset):
  assert query == row.as_py()
 
 
+def test_indexed_filter_with_fts_index(tmp_path):
+ data = pa.table(
+ {
+ "text": [
+ "Frodo was a puppy",
+ "There were several kittens playing",
+ "Frodo was a happy puppy",
+ "Frodo was a very happy puppy",
+ ],
+ "sentiment": ["neutral", "neutral", "positive", "positive"],
+ }
+ )
+ ds = lance.write_dataset(data, tmp_path, mode="overwrite")
+ ds.create_scalar_index("text", "INVERTED")
+ ds.create_scalar_index("sentiment", "BITMAP")
+
+ results = ds.to_table(
+ full_text_query="puppy",
+ filter="sentiment='positive'",
+ prefilter=True,
+ with_row_id=True,
+ )
+ assert results["_rowid"].to_pylist() == [2, 3]
+
+
 def test_fts_with_postfilter(tmp_path):
  tab = pa.table({"text": ["Frodo the puppy"] * 100, "id": range(100)})
  dataset = lance.write_dataset(tab, tmp_path)

rust/lance-index/src/scalar/inverted/index.rs

@@ -151,7 +151,7 @@ impl InvertedIndex {
  position as i32,
  posting,
  self.docs.len(),
- mask.clone(),
+ mask,
  ))
  })
  // Use compute count since data hopefully cached

rust/lance-index/src/scalar/inverted/wand.rs

@@ -63,11 +63,19 @@ impl PostingIterator {
  Some(max_score) => max_score,
  None => idf(list.len(), num_doc) * (K1 + 1.0),
  };
+
+ // move the iterator to the first selected document. This is important
+ // because caller might directly call `doc()` without calling `next()`.
+ let mut index = 0;
+ while index < list.len() && !mask.selected(list.row_id(index)) {
+ index += 1;
+ }
+
  Self {
  token_id,
  position,
  list,
- index: 0,
+ index,
  mask,
  approximate_upper_bound,
  }