feat: filter() function return remain_columns and deleted_columns for statistics

src/query/datavalues/src/columns/boolean/mod.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use core::ops::Not;
 use std::sync::Arc;
 
 use common_arrow::arrow::array::*;
@@ -63,6 +64,10 @@ impl BooleanColumn {
     pub fn values(&self) -> &Bitmap {
         &self.values
     }
+
+    pub fn neg(&self) -> Self {
+        Self::from_arrow_data(Not::not(&self.values))
+    }
 }
 
 impl Column for BooleanColumn {

src/query/datavalues/src/columns/primitive/primitive.rs

@@ -232,11 +232,9 @@ impl<T: PrimitiveType> Column for PrimitiveColumn<T> {
             length -= n;
             values = &values[n..];
         }
-
         const CHUNK_SIZE: usize = 64;
         let mut chunks = values.chunks_exact(CHUNK_SIZE);
         let mut mask_chunks = BitChunksExact::<u64>::new(slice, length);
-
         chunks
             .by_ref()
             .zip(mask_chunks.by_ref())
@@ -257,7 +255,6 @@ impl<T: PrimitiveType> Column for PrimitiveColumn<T> {
                     }
                 }
             });
-
         chunks
             .remainder()
             .iter()

src/query/datavalues/tests/it/columns/boolean.rs

@@ -60,16 +60,19 @@ fn test_filter_column() {
     struct Test {
         filter: BooleanColumn,
         expect: Vec<bool>,
+        deleted: Option<Vec<bool>>,
     }
 
     let tests: Vec<Test> = vec![
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|_| true)),
             expect: (0..N).map(|e| e % 2 == 0).collect(),
+            deleted: None,
         },
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|_| false)),
             expect: vec![],
+            deleted: Some((0..N).map(|e| e % 2 == 0).collect()),
         },
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|i| i % 3 == 0)),
@@ -79,6 +82,14 @@ fn test_filter_column() {
                 .filter(|(i, _)| i % 3 == 0)
                 .map(|(_, e)| e)
                 .collect(),
+            deleted: Some(
+                (0..N)
+                    .map(|e| e % 2 == 0)
+                    .enumerate()
+                    .filter(|(i, _)| i % 3 != 0)
+                    .map(|(_, e)| e)
+                    .collect(),
+            ),
         },
     ];
 
@@ -93,5 +104,20 @@ fn test_filter_column() {
                 .values(),
             &bitmap
         );
+
+        if let Some(deleted) = test.deleted {
+            let iter = deleted.into_iter();
+            let bitmap: Bitmap = MutableBitmap::from_iter(iter).into();
+
+            let res = data_column.filter(&test.filter.neg());
+
+            assert_eq!(
+                res.as_any()
+                    .downcast_ref::<BooleanColumn>()
+                    .unwrap()
+                    .values(),
+                &bitmap
+            );
+        };
     }
 }

src/query/datavalues/tests/it/columns/primitive.rs

@@ -64,27 +64,37 @@ fn test_filter_column() {
     struct Test {
         filter: BooleanColumn,
         expect: Vec<i32>,
+        deleted: Option<Vec<i32>>,
     }
 
     let mut tests: Vec<Test> = vec![
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|_| true)),
             expect: (0..N).map(|i| i as i32).collect(),
+            deleted: None,
         },
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|_| false)),
             expect: vec![],
+            deleted: Some((0..N).map(|i| i as i32).collect()),
         },
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|i| i % 10 == 0)),
             expect: (0..N).map(|i| i as i32).filter(|i| i % 10 == 0).collect(),
+            deleted: Some((0..N).map(|i| i as i32).filter(|i| i % 10 != 0).collect()),
         },
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|i| !(100..=800).contains(&i))),
             expect: (0..N)
                 .map(|i| i as i32)
                 .filter(|&i| !(100..=800).contains(&i))
                 .collect(),
+            deleted: Some(
+                (0..N)
+                    .map(|i| i as i32)
+                    .filter(|&i| (100..=800).contains(&i))
+                    .collect(),
+            ),
         },
     ];
 
@@ -103,6 +113,12 @@ fn test_filter_column() {
             .map(|i| i as i32)
             .filter(|&i| !(100..=800).contains(&i))
             .collect(),
+        deleted: Some(
+            (0..N)
+                .map(|i| i as i32)
+                .filter(|&i| (100..=800).contains(&i))
+                .collect(),
+        ),
     });
 
     for test in tests {
@@ -114,5 +130,16 @@ fn test_filter_column() {
                 .values(),
             test.expect
         );
+
+        if let Some(deleted) = test.deleted {
+            let res = data_column.filter(&test.filter.neg());
+            assert_eq!(
+                res.as_any()
+                    .downcast_ref::<PrimitiveColumn<i32>>()
+                    .unwrap()
+                    .values(),
+                deleted
+            );
+        }
     }
 }

src/query/datavalues/tests/it/columns/string.rs

@@ -58,6 +58,7 @@ fn test_string_column() {
     struct Test {
         filter: BooleanColumn,
         expect: StringColumn,
+        deleted: Option<StringColumn>,
     }
 
     let empty_case: Vec<&str> = vec![];
@@ -67,6 +68,12 @@ fn test_string_column() {
         .filter(|(i, _)| i % 3 == 0)
         .map(|(_, e)| e)
         .collect();
+    let not_normal_case: Vec<&str> = (0..N)
+        .map(|i| if i % 2 == 0 { "你好" } else { "hello" })
+        .enumerate()
+        .filter(|(i, _)| i % 3 != 0)
+        .map(|(_, e)| e)
+        .collect();
 
     let tests: Vec<Test> = vec![
         Test {
@@ -78,14 +85,23 @@ fn test_string_column() {
                     "hello".as_bytes()
                 }
             })),
+            deleted: None,
         },
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|_| false)),
             expect: NewColumn::new_from_iter(empty_case.iter()),
+            deleted: Some(NewColumn::new_from_iter((0..N).map(|i| {
+                if i % 2 == 0 {
+                    "你好".as_bytes()
+                } else {
+                    "hello".as_bytes()
+                }
+            }))),
         },
         Test {
             filter: BooleanColumn::from_iterator((0..N).map(|i| i % 3 == 0)),
             expect: NewColumn::new_from_iter(normal_case.iter()),
+            deleted: Some(NewColumn::new_from_iter(not_normal_case.iter())),
         },
     ];
 
@@ -105,5 +121,22 @@ fn test_string_column() {
                 .unwrap()
                 .values()
         );
+
+        if let Some(deleted) = test.deleted {
+            let res = data_column.filter(&test.filter.neg());
+            let values = res
+                .as_any()
+                .downcast_ref::<StringColumn>()
+                .unwrap()
+                .values();
+            assert_eq!(
+                values,
+                deleted
+                    .as_any()
+                    .downcast_ref::<StringColumn>()
+                    .unwrap()
+                    .values()
+            );
+        }
     }
 }