1.2.307 hotfix 20240624

src/common/base/src/runtime/runtime.rs

@@ -159,6 +159,7 @@ impl Runtime {
 
             runtime_builder.thread_stack_size(20 * 1024 * 1024);
         }
+        runtime_builder.thread_stack_size(100 * 1024 * 1024);
 
         Self::create(None, mem_stat, &mut runtime_builder)
     }
@@ -185,6 +186,7 @@ impl Runtime {
 
             runtime_builder.thread_stack_size(20 * 1024 * 1024);
         }
+        runtime_builder.thread_stack_size(100 * 1024 * 1024);
 
         if let Some(thread_name) = &thread_name {
             runtime_builder.thread_name(thread_name);

src/common/hashtable/src/lib.rs

@@ -113,5 +113,6 @@ pub use partitioned_hashtable::hash2bucket;
 pub type HashJoinHashMap<K> = hashjoin_hashtable::HashJoinHashTable<K>;
 pub type BinaryHashJoinHashMap = hashjoin_string_hashtable::HashJoinStringHashTable;
 pub use traits::HashJoinHashtableLike;
+pub use utils::fast_memcmp;
 pub use utils::Interval;
 pub use utils::MergeIntoBlockInfoIndex;

src/common/hashtable/src/utils.rs

@@ -97,6 +97,18 @@ pub unsafe fn read_le(data: *const u8, len: usize) -> u64 {
     }
 }
 
+#[cfg(all(target_arch = "x86_64", target_feature = "sse4.2"))]
+#[inline]
+pub fn fast_memcmp(a: &[u8], b: &[u8]) -> bool {
+    unsafe { sse::memcmp_sse(a, b) }
+}
+
+#[cfg(not(all(any(target_arch = "x86_64"), target_feature = "sse4.2")))]
+#[inline]
+pub fn fast_memcmp(a: &[u8], b: &[u8]) -> bool {
+    a == b
+}
+
 #[cfg(all(target_arch = "x86_64", target_feature = "sse4.2"))]
 pub mod sse {
     use std::arch::x86_64::*;
@@ -137,6 +149,8 @@ pub mod sse {
             ))
     }
 
+    /// # Safety
+    /// This is safe that we compare bytes via addr
     #[inline(always)]
     pub unsafe fn memcmp_sse(a: &[u8], b: &[u8]) -> bool {
         let mut size = a.len();

src/common/metrics/src/metrics/transform.rs

@@ -22,10 +22,16 @@ use crate::Family;
 use crate::Histogram;
 use crate::VecLabels;
 
+pub static AGGREGATE_PARTIAL_CELL_COUNT: LazyLock<Counter> =
+    LazyLock::new(|| register_counter("transform_aggregate_partial_cell_count"));
+
 pub static AGGREGATE_PARTIAL_SPILL_CELL_COUNT: LazyLock<Counter> =
     LazyLock::new(|| register_counter("transform_aggregate_partial_spill_cell_count"));
 pub static AGGREGATE_PARTIAL_HASHTABLE_ALLOCATED_BYTES: LazyLock<Counter> =
     LazyLock::new(|| register_counter("transform_aggregate_partial_hashtable_allocated_bytes"));
+pub static AGGREGATE_PARTIAL_HASHTABLE_EXCHANGE_ROWS: LazyLock<Counter> =
+    LazyLock::new(|| register_counter("transform_aggregate_partial_hashtable_exchange_rows"));
+
 pub static SPILL_COUNT: LazyLock<Family<VecLabels, Counter>> =
     LazyLock::new(|| register_counter_family("transform_spill_count"));
 pub static SPILL_WRITE_COUNT: LazyLock<Family<VecLabels, Counter>> =
@@ -70,10 +76,27 @@ pub fn metrics_inc_aggregate_partial_spill_cell_count(c: u64) {
     AGGREGATE_PARTIAL_SPILL_CELL_COUNT.inc_by(c);
 }
 
+pub fn metrics_inc_aggregate_partial_hashtable_exchange_rows(c: u64) {
+    AGGREGATE_PARTIAL_HASHTABLE_EXCHANGE_ROWS.inc_by(c);
+}
+
 pub fn metrics_inc_aggregate_partial_hashtable_allocated_bytes(c: u64) {
     AGGREGATE_PARTIAL_HASHTABLE_ALLOCATED_BYTES.inc_by(c);
 }
 
+pub fn metrics_inc_group_by_partial_spill_count() {
+    let labels = &vec![("spill", "group_by_partial_spill".to_string())];
+    SPILL_COUNT.get_or_create(labels).inc();
+}
+
+pub fn metrics_inc_group_by_partial_spill_cell_count(c: u64) {
+    AGGREGATE_PARTIAL_SPILL_CELL_COUNT.inc_by(c);
+}
+
+pub fn metrics_inc_group_by_partial_hashtable_allocated_bytes(c: u64) {
+    AGGREGATE_PARTIAL_HASHTABLE_ALLOCATED_BYTES.inc_by(c);
+}
+
 pub fn metrics_inc_group_by_spill_write_count() {
     let labels = &vec![("spill", "group_by_spill".to_string())];
     SPILL_WRITE_COUNT.get_or_create(labels).inc();

src/meta/app/src/principal/user_privilege.rs

@@ -19,6 +19,7 @@ use enumflags2::bitflags;
 use enumflags2::make_bitflags;
 use enumflags2::BitFlags;
 
+// Note: If add new privilege type, need add forward test
 #[bitflags]
 #[repr(u64)]
 #[derive(

src/meta/proto-conv/src/share_from_to_protobuf_impl.rs

@@ -129,23 +129,19 @@ impl FromToProto for mt::ShareGrantEntry {
     where Self: Sized {
         reader_check_msg(p.ver, p.min_reader_ver)?;
 
-        let privileges = BitFlags::<mt::ShareGrantObjectPrivilege, u64>::from_bits(p.privileges);
-        match privileges {
-            Ok(privileges) => Ok(mt::ShareGrantEntry {
-                object: mt::ShareGrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
-                    reason: "ShareGrantEntry.object can not be None".to_string(),
-                })?)?,
-                privileges,
-                grant_on: DateTime::<Utc>::from_pb(p.grant_on)?,
-                update_on: match p.update_on {
-                    Some(t) => Some(DateTime::<Utc>::from_pb(t)?),
-                    None => None,
-                },
-            }),
-            Err(e) => Err(Incompatible {
-                reason: format!("UserPrivilegeType error: {}", e),
-            }),
-        }
+        let privileges =
+            BitFlags::<mt::ShareGrantObjectPrivilege, u64>::from_bits_truncate(p.privileges);
+        Ok(mt::ShareGrantEntry {
+            object: mt::ShareGrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
+                reason: "ShareGrantEntry.object can not be None".to_string(),
+            })?)?,
+            privileges,
+            grant_on: DateTime::<Utc>::from_pb(p.grant_on)?,
+            update_on: match p.update_on {
+                Some(t) => Some(DateTime::<Utc>::from_pb(t)?),
+                None => None,
+            },
+        })
     }
 
     fn to_pb(&self) -> Result<pb::ShareGrantEntry, Incompatible> {

src/meta/proto-conv/src/user_from_to_protobuf_impl.rs

@@ -241,18 +241,14 @@ impl FromToProto for mt::principal::GrantEntry {
     where Self: Sized {
         reader_check_msg(p.ver, p.min_reader_ver)?;
 
-        let privileges = BitFlags::<mt::principal::UserPrivilegeType, u64>::from_bits(p.privileges);
-        match privileges {
-            Ok(privileges) => Ok(mt::principal::GrantEntry::new(
-                mt::principal::GrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
-                    reason: "GrantEntry.object can not be None".to_string(),
-                })?)?,
-                privileges,
-            )),
-            Err(e) => Err(Incompatible {
-                reason: format!("UserPrivilegeType error: {}", e),
-            }),
-        }
+        let privileges =
+            BitFlags::<mt::principal::UserPrivilegeType, u64>::from_bits_truncate(p.privileges);
+        Ok(mt::principal::GrantEntry::new(
+            mt::principal::GrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
+                reason: "GrantEntry.object can not be None".to_string(),
+            })?)?,
+            privileges,
+        ))
     }
 
     fn to_pb(&self) -> Result<pb::GrantEntry, Incompatible> {

src/meta/proto-conv/src/util.rs

@@ -107,6 +107,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (75, "2024-01-15: ADD: user.proto/CsvFileFormatParams add field `binary_format` and `output_header`", ),
     (76, "2024-01-18: ADD: ownership.proto and role.proto", ),
     (77, "2024-01-22: Remove: allow_anonymous in S3 Config", ),
+    (78, "2024-01-29: Refactor: GrantEntry::UserPrivilegeType and ShareGrantEntry::ShareGrantObjectPrivilege use from_bits_truncate deserialize", ),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -81,3 +81,4 @@ mod v074_table_db_meta;
 mod v075_csv_format_params;
 mod v076_role_ownership_info;
 mod v077_s3_remove_allow_anonymous;
+mod v078_grantentry;

src/meta/proto-conv/tests/it/v078_grantentry.rs

@@ -0,0 +1,72 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use chrono::TimeZone;
+use chrono::Utc;
+use databend_common_meta_app as mt;
+use databend_common_meta_app::principal::UserPrivilegeType;
+use databend_common_meta_app::share::ShareGrantObjectPrivilege;
+use enumflags2::make_bitflags;
+use minitrace::func_name;
+
+use crate::common;
+
+// These bytes are built when a new version in introduced,
+// and are kept for backward compatibility test.
+//
+// *************************************************************
+// * These messages should never be updated,                   *
+// * only be added when a new version is added,                *
+// * or be removed when an old version is no longer supported. *
+// *************************************************************
+//
+
+#[test]
+fn test_decode_v78_grant_entry() -> anyhow::Result<()> {
+    let grant_entry_v78 = vec![
+        10, 8, 10, 0, 160, 6, 78, 168, 6, 24, 16, 254, 255, 55, 160, 6, 78, 168, 6, 24,
+    ];
+
+    let want = || {
+        mt::principal::GrantEntry::new(
+            mt::principal::GrantObject::Global,
+            make_bitflags!(UserPrivilegeType::{Create | Select | Insert | Update | Delete | Drop | Alter | Super | CreateUser | DropUser | CreateRole | DropRole | Grant | CreateStage | Set | CreateDataMask | Read | Write }),
+        )
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), grant_entry_v78.as_slice(), 78, want())?;
+
+    Ok(())
+}
+
+#[test]
+fn test_decode_v78_share_grant_entry() -> anyhow::Result<()> {
+    let share_grant_entry_v78 = vec![
+        10, 8, 16, 19, 160, 6, 78, 168, 6, 24, 16, 4, 26, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50,
+        56, 32, 49, 50, 58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 160, 6, 78, 168, 6, 24,
+    ];
+
+    let want = || mt::share::ShareGrantEntry {
+        object: mt::share::ShareGrantObject::Table(19),
+        privileges: make_bitflags!(ShareGrantObjectPrivilege::{Select}),
+        grant_on: Utc.with_ymd_and_hms(2014, 11, 28, 12, 0, 9).unwrap(),
+        update_on: None,
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), share_grant_entry_v78.as_slice(), 78, want())?;
+
+    Ok(())
+}

src/query/expression/Cargo.toml

@@ -51,6 +51,7 @@ rust_decimal = "1.26"
 serde = { workspace = true }
 serde_json = { workspace = true }
 simdutf8 = "0.1.4"
+strength_reduce = "0.2.4"
 terminal_size = "0.2.6"
 tonic = { workspace = true }
 typetag = { workspace = true }