feat: add non-cryptographic hash function

crates/sqlbuiltins/Cargo.toml

@@ -35,3 +35,5 @@ bson = "2.7.0"
 tokio-util = "0.7.10"
 bytes = "1.5.0"
 kdl = "5.0.0-alpha.1"
+siphasher = "1.0.0"
+fnv = "1.0.7"

crates/sqlbuiltins/src/functions/mod.rs

@@ -3,20 +3,23 @@ mod aggregates;
 mod scalars;
 mod table;
 
-use self::scalars::df_scalars::ArrowCastFunction;
-use self::scalars::kdl::{KDLMatches, KDLSelect};
-use self::scalars::{postgres::*, ConnectionId, Version};
-use self::table::{BuiltinTableFuncs, TableFunc};
+use std::collections::HashMap;
+use std::sync::Arc;
 
 use datafusion::logical_expr::{AggregateFunction, BuiltinScalarFunction, Expr, Signature};
 use once_cell::sync::Lazy;
+
+use scalars::df_scalars::ArrowCastFunction;
+use scalars::hashing::{FnvHash, SipHash};
+use scalars::kdl::{KDLMatches, KDLSelect};
+use scalars::postgres::*;
+use scalars::{ConnectionId, Version};
+use table::{BuiltinTableFuncs, TableFunc};
+
 use protogen::metastore::types::catalog::{
     EntryMeta, EntryType, FunctionEntry, FunctionType, RuntimePreference,
 };
 
-use std::collections::HashMap;
-use std::sync::Arc;
-
 /// Builtin table returning functions available for all sessions.
 static BUILTIN_TABLE_FUNCS: Lazy<BuiltinTableFuncs> = Lazy::new(BuiltinTableFuncs::new);
 pub static ARROW_CAST_FUNC: Lazy<ArrowCastFunction> = Lazy::new(|| ArrowCastFunction {});
@@ -187,12 +190,15 @@ impl FunctionRegistry {
             Arc::new(PgTableIsVisible),
             Arc::new(PgEncodingToChar),
             Arc::new(PgArrayToString),
+            // System functions
+            Arc::new(ConnectionId),
+            Arc::new(Version),
             // KDL functions
             Arc::new(KDLMatches),
             Arc::new(KDLSelect),
-            // Other functions
-            Arc::new(ConnectionId),
-            Arc::new(Version),
+            // Hashing/Sharding
+            Arc::new(SipHash),
+            Arc::new(FnvHash),
         ];
         let udfs = udfs
             .into_iter()

crates/sqlbuiltins/src/functions/scalars/hashing.rs

@@ -0,0 +1,96 @@
+use std::hash::{Hash, Hasher};
+
+use fnv::FnvHasher;
+use siphasher::sip::SipHasher24;
+
+use super::*;
+
+pub struct SipHash;
+
+impl ConstBuiltinFunction for SipHash {
+    const NAME: &'static str = "siphash";
+    const DESCRIPTION: &'static str =
+        "Calculates a 64bit non-cryptographic hash (SipHash24) of the value.";
+    const EXAMPLE: &'static str = "siphash(<value>)";
+    const FUNCTION_TYPE: FunctionType = FunctionType::Scalar;
+
+    fn signature(&self) -> Option<Signature> {
+        Some(Signature::new(
+            // args: <FIELD>
+            TypeSignature::Any(1),
+            Volatility::Immutable,
+        ))
+    }
+}
+impl BuiltinScalarUDF for SipHash {
+    fn as_expr(&self, args: Vec<Expr>) -> Expr {
+        let udf = ScalarUDF {
+            name: Self::NAME.to_string(),
+            signature: ConstBuiltinFunction::signature(self).unwrap(),
+            return_type: Arc::new(|_| Ok(Arc::new(DataType::Utf8))),
+            fun: Arc::new(move |input| match get_nth_scalar_value(input, 0) {
+                Some(value) => {
+                    let mut hasher = SipHasher24::new();
+
+                    value.hash(&mut hasher);
+
+                    Ok(ColumnarValue::Scalar(ScalarValue::UInt64(Some(
+                        hasher.finish(),
+                    ))))
+                }
+                None => Err(datafusion::error::DataFusionError::Execution(
+                    "must have exactly one value to hash".to_string(),
+                )),
+            }),
+        };
+        Expr::ScalarUDF(datafusion::logical_expr::expr::ScalarUDF::new(
+            Arc::new(udf),
+            args,
+        ))
+    }
+}
+
+pub struct FnvHash;
+
+impl ConstBuiltinFunction for FnvHash {
+    const NAME: &'static str = "fnv";
+    const DESCRIPTION: &'static str =
+        "Calculates a 64bit non-cryptographic hash (fnv1a) of the value.";
+    const EXAMPLE: &'static str = "fnv(<value>)";
+    const FUNCTION_TYPE: FunctionType = FunctionType::Scalar;
+
+    fn signature(&self) -> Option<Signature> {
+        Some(Signature::new(
+            // args: <FIELD>
+            TypeSignature::Any(1),
+            Volatility::Immutable,
+        ))
+    }
+}
+impl BuiltinScalarUDF for FnvHash {
+    fn as_expr(&self, args: Vec<Expr>) -> Expr {
+        let udf = ScalarUDF {
+            name: Self::NAME.to_string(),
+            signature: ConstBuiltinFunction::signature(self).unwrap(),
+            return_type: Arc::new(|_| Ok(Arc::new(DataType::Utf8))),
+            fun: Arc::new(move |input| match get_nth_scalar_value(input, 0) {
+                Some(value) => {
+                    let mut hasher = FnvHasher::default();
+
+                    value.hash(&mut hasher);
+
+                    Ok(ColumnarValue::Scalar(ScalarValue::UInt64(Some(
+                        hasher.finish(),
+                    ))))
+                }
+                None => Err(datafusion::error::DataFusionError::Execution(
+                    "must have exactly one value to hash".to_string(),
+                )),
+            }),
+        };
+        Expr::ScalarUDF(datafusion::logical_expr::expr::ScalarUDF::new(
+            Arc::new(udf),
+            args,
+        ))
+    }
+}

crates/sqlbuiltins/src/functions/scalars/mod.rs

@@ -1,24 +1,22 @@
 pub mod df_scalars;
+pub mod hashing;
 pub mod kdl;
 pub mod postgres;
-use crate::{
-    document,
-    functions::{BuiltinFunction, BuiltinScalarUDF, ConstBuiltinFunction},
-};
-use datafusion::logical_expr::BuiltinScalarFunction;
-
-use protogen::metastore::types::catalog::FunctionType;
 
 use std::sync::Arc;
 
-use datafusion::{
-    arrow::datatypes::{DataType, Field},
-    logical_expr::{Expr, ScalarUDF, Signature, TypeSignature, Volatility},
-    physical_plan::ColumnarValue,
-    scalar::ScalarValue,
-};
+use crate::document;
+use crate::functions::{BuiltinFunction, BuiltinScalarUDF, ConstBuiltinFunction};
+use datafusion::logical_expr::BuiltinScalarFunction;
+use protogen::metastore::types::catalog::FunctionType;
+
+use datafusion::arrow::datatypes::{DataType, Field};
+use datafusion::logical_expr::{Expr, ScalarUDF, Signature, TypeSignature, Volatility};
+use datafusion::physical_plan::ColumnarValue;
+use datafusion::scalar::ScalarValue;
 
 pub struct ConnectionId;
+
 impl ConstBuiltinFunction for ConnectionId {
     const NAME: &'static str = "connection_id";
     const DESCRIPTION: &'static str = "Returns the connection id of the current session";
@@ -34,7 +32,9 @@ impl BuiltinScalarUDF for ConnectionId {
         session_var("connection_id")
     }
 }
+
 pub struct Version;
+
 impl ConstBuiltinFunction for Version {
     const NAME: &'static str = "version";
     const DESCRIPTION: &'static str = "Returns the version of the database";

testdata/sqllogictests/functions/hashing.slt

@@ -0,0 +1,108 @@
+statement error
+select siphash(1, 2, 3);
+
+statement error
+select siphash(1, 2);
+
+statement ok
+select siphash(1);
+
+statement ok
+select siphash('000');
+
+statement ok
+select siphash(9001);
+
+statement ok
+select siphash(true);
+
+
+statement error
+select fnv(1, 2, 3);
+
+statement error
+select fnv(1, 2);
+
+statement ok
+select fnv(1);
+
+statement ok
+select fnv('000');
+
+statement ok
+select fnv(9001);
+
+statement ok
+select fnv(true);
+
+query I
+select siphash();
+----
+13715208377448023093
+
+query I
+select siphash(42);
+----
+8315904219845249920
+
+query I
+select siphash(3000);
+----
+14490819164275330428
+
+query I
+select siphash('42');
+----
+8771948186893062792
+
+query I
+select siphash([0, 100, 100, 300, 500, 800]);
+----
+3773389192103504674
+
+query I
+select fnv();
+----
+12478008331234465636
+
+query I
+select fnv(42);
+----
+10346157209210711374
+
+query I
+select fnv(3000);
+----
+4500112066730064389
+
+query I
+select fnv('42');
+----
+16857446072837519227
+
+query I
+select fnv([0, 100, 100, 300, 500, 800]);
+----
+16366770854503053831
+
+# rerun some earlier cases to ensure we're not accidentally stateful
+
+query I
+select fnv();
+----
+12478008331234465636
+
+query I
+select fnv('42');
+----
+16857446072837519227
+
+query I
+select siphash();
+----
+13715208377448023093
+
+query I
+select siphash('42');
+----
+8771948186893062792