From 6c084b949a08d8258a43a9eb0c14c7e17ec743e1 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 21 Mar 2024 13:18:08 +0800 Subject: [PATCH 1/4] Initial commit --- .../scala/io/glutenproject/expression/ExpressionMappings.scala | 1 + .../main/scala/io/glutenproject/expression/ExpressionNames.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala index e85f8d4e2d63..a286ed556f80 100644 --- a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionMappings.scala @@ -96,6 +96,7 @@ object ExpressionMappings { Sig[StringDecode](DECODE), Sig[Encode](ENCODE), Sig[Uuid](UUID), + Sig[BitLength](BIT_LENGTH), // URL functions Sig[ParseUrl](PARSE_URL), diff --git a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala index 5dbb926bfd98..cb9e1ab71cd7 100644 --- a/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/io/glutenproject/expression/ExpressionNames.scala @@ -113,6 +113,7 @@ object ExpressionNames { final val DECODE = "decode" final val ENCODE = "encode" final val UUID = "uuid" + final val BIT_LENGTH = "bit_length" // URL functions final val PARSE_URL = "parse_url" From 8c0f3502f29bab4b2b77393db2928b8a67bc010a Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 21 Mar 2024 13:23:25 +0800 Subject: [PATCH 2/4] Add a test --- .../execution/VeloxFunctionsValidateSuite.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala index e5bee97611a2..82673567944b 100644 --- a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala @@ -527,13 +527,20 @@ class VeloxFunctionsValidateSuite extends VeloxWholeStageTransformerSuite { test("lag/lead window function with negative input offset") { runQueryAndCompare( "select lag(l_orderkey, -2) over" + - " (partition by l_suppkey order by l_orderkey) from lineitem ") { + " (partition by l_suppkey order by l_orderkey) from lineitem") { checkOperatorMatch[WindowExecTransformer] } runQueryAndCompare( "select lead(l_orderkey, -2) over" + - " (partition by l_suppkey order by l_orderkey) from lineitem ") { + " (partition by l_suppkey order by l_orderkey) from lineitem") { + checkOperatorMatch[WindowExecTransformer] + } + } + + test("bit_length") { + runQueryAndCompare( + "select bit_length(l_orderkey), bit_length(cast(l_orderkey as binary)) from lineitem") { checkOperatorMatch[WindowExecTransformer] } } From 5e26856494bc1bcfe37defc1cffd545bf530f306 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 21 Mar 2024 13:26:21 +0800 Subject: [PATCH 3/4] Add to blacklist for CH --- .../main/scala/io/glutenproject/utils/CHExpressionUtil.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala index 86fd425f438d..24555c05c1a0 100644 --- a/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/io/glutenproject/utils/CHExpressionUtil.scala @@ -177,6 +177,7 @@ object CHExpressionUtil { DATE_FROM_UNIX_DATE -> DefaultValidator(), MONOTONICALLY_INCREASING_ID -> DefaultValidator(), SPARK_PARTITION_ID -> DefaultValidator(), - SKEWNESS -> DefaultValidator() + SKEWNESS -> DefaultValidator(), + BIT_LENGTH -> DefaultValidator() ) } From c2696d4ccfafcff9b655294795830297e9111449 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 21 Mar 2024 16:29:52 +0800 Subject: [PATCH 4/4] Fix test failure --- .../execution/VeloxFunctionsValidateSuite.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala index 82673567944b..eca535ce5591 100644 --- a/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/io/glutenproject/execution/VeloxFunctionsValidateSuite.scala @@ -515,11 +515,11 @@ class VeloxFunctionsValidateSuite extends VeloxWholeStageTransformerSuite { test("regexp_replace") { runQueryAndCompare( - "SELECT regexp_replace(l_partkey, '\\w', 'something') FROM lineitem limit 100") { + "SELECT regexp_replace(c_comment, '\\w', 'something') FROM customer limit 50") { checkOperatorMatch[ProjectExecTransformer] } runQueryAndCompare( - "SELECT regexp_replace(l_partkey, '\\w', 'something', 3) FROM lineitem limit 100") { + "SELECT regexp_replace(c_comment, '\\w', 'something', 3) FROM customer limit 50") { checkOperatorMatch[ProjectExecTransformer] } } @@ -540,8 +540,9 @@ class VeloxFunctionsValidateSuite extends VeloxWholeStageTransformerSuite { test("bit_length") { runQueryAndCompare( - "select bit_length(l_orderkey), bit_length(cast(l_orderkey as binary)) from lineitem") { - checkOperatorMatch[WindowExecTransformer] + "select bit_length(c_comment), bit_length(cast(c_comment as binary))" + + " from customer limit 50") { + checkOperatorMatch[ProjectExecTransformer] } }