[CORE] Enable bit_length Spark function (#5069)

apache · Mar 22, 2024 · f203fd8 · f203fd8
1 parent 9d859d8
commit f203fd8
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 5 deletions.
diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala
@@ -177,6 +177,7 @@ object CHExpressionUtil {
     DATE_FROM_UNIX_DATE -> DefaultValidator(),
     MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
     SPARK_PARTITION_ID -> DefaultValidator(),
-    SKEWNESS -> DefaultValidator()
+    SKEWNESS -> DefaultValidator(),
+    BIT_LENGTH -> DefaultValidator()
   )
 }
diff --git a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala
@@ -521,27 +521,35 @@ class VeloxFunctionsValidateSuite extends VeloxWholeStageTransformerSuite {
 
   test("regexp_replace") {
     runQueryAndCompare(
-      "SELECT regexp_replace(l_partkey, '\\w', 'something') FROM lineitem limit 100") {
+      "SELECT regexp_replace(c_comment, '\\w', 'something') FROM customer limit 50") {
       checkOperatorMatch[ProjectExecTransformer]
     }
     runQueryAndCompare(
-      "SELECT regexp_replace(l_partkey, '\\w', 'something', 3) FROM lineitem limit 100") {
+      "SELECT regexp_replace(c_comment, '\\w', 'something', 3) FROM customer limit 50") {
       checkOperatorMatch[ProjectExecTransformer]
     }
   }
 
   test("lag/lead window function with negative input offset") {
     runQueryAndCompare(
       "select lag(l_orderkey, -2) over" +
-        " (partition by l_suppkey order by l_orderkey) from lineitem ") {
+        " (partition by l_suppkey order by l_orderkey) from lineitem") {
       checkOperatorMatch[WindowExecTransformer]
     }
 
     runQueryAndCompare(
       "select lead(l_orderkey, -2) over" +
-        " (partition by l_suppkey order by l_orderkey) from lineitem ") {
+        " (partition by l_suppkey order by l_orderkey) from lineitem") {
       checkOperatorMatch[WindowExecTransformer]
     }
   }
 
+  test("bit_length") {
+    runQueryAndCompare(
+      "select bit_length(c_comment), bit_length(cast(c_comment as binary))" +
+        " from customer limit 50") {
+      checkOperatorMatch[ProjectExecTransformer]
+    }
+  }
+
 }
diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala
@@ -96,6 +96,7 @@ object ExpressionMappings {
     Sig[StringDecode](DECODE),
     Sig[Encode](ENCODE),
     Sig[Uuid](UUID),
+    Sig[BitLength](BIT_LENGTH),
 
     // URL functions
     Sig[ParseUrl](PARSE_URL),

diff --git a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala
@@ -113,6 +113,7 @@ object ExpressionNames {
   final val DECODE = "decode"
   final val ENCODE = "encode"
   final val UUID = "uuid"
+  final val BIT_LENGTH = "bit_length"
 
   // URL functions
   final val PARSE_URL = "parse_url"