feat: replace bloom with xor8 filter (#631)

* feat: add xor8

* fix test

* rename bloom

* remove unsed fns

* rename files

* rename sst filter to parquet filter

Cargo.toml

@@ -75,8 +75,10 @@ datafusion-optimizer = "17.0.0"
 datafusion-proto = "17.0.0"
 df_operator = { path = "df_operator" }
 env_logger = "0.6"
-ethbloom = "0.13.0"
 futures = "0.3"
+xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", features = [
+    "cbordata",
+], tag = "databend-alpha.4" }
 lazy_static = "1.4.0"
 log = "0.4"
 logger = { path = "components/logger" }

analytic_engine/Cargo.toml

@@ -25,7 +25,6 @@ common_types = { workspace = true }
 common_util = { workspace = true }
 datafusion = { workspace = true }
 datafusion-expr = { workspace = true }
-ethbloom = { workspace = true }
 futures = { workspace = true }
 lazy_static = { workspace = true }
 log = { workspace = true }
@@ -49,6 +48,7 @@ table_kv = { workspace = true }
 tempfile = { workspace = true, optional = true }
 tokio = { workspace = true }
 wal = { workspace = true }
+xorfilter-rs = { workspace = true }
 
 [dev-dependencies]
 common_types = { workspace = true, features = ["test"] }

analytic_engine/src/compaction/picker.rs

@@ -627,7 +627,7 @@ mod tests {
             time_range,
             max_sequence: 200,
             schema: build_schema(),
-            bloom_filter: Default::default(),
+            parquet_filter: Default::default(),
             collapsible_cols_idx: Vec::new(),
         };
 

analytic_engine/src/sst/meta_data/cache.rs

@@ -35,7 +35,7 @@ impl MetaData {
     /// contains no extended custom information.
     pub fn try_new(
         parquet_meta_data: &parquet_ext::ParquetMetaData,
-        ignore_bloom_filter: bool,
+        ignore_sst_filter: bool,
     ) -> Result<Self> {
         let file_meta_data = parquet_meta_data.file_metadata();
         let kv_metas = file_meta_data
@@ -46,8 +46,8 @@ impl MetaData {
         let custom = {
             let mut sst_meta =
                 encoding::decode_sst_meta_data(&kv_metas[0]).context(DecodeCustomMetaData)?;
-            if ignore_bloom_filter {
-                sst_meta.bloom_filter = None;
+            if ignore_sst_filter {
+                sst_meta.parquet_filter = None;
             }
 
             Arc::new(sst_meta)