tests: Add SQL file-based test framework

.github/workflows/pr_build_linux.yml

@@ -250,6 +250,7 @@ jobs:
           - name: "expressions"
             value: |
               org.apache.comet.CometExpressionSuite
+              org.apache.comet.CometSqlFileTestSuite
               org.apache.comet.CometExpressionCoverageSuite
               org.apache.comet.CometHashExpressionSuite
               org.apache.comet.CometTemporalExpressionSuite

.github/workflows/pr_build_macos.yml

@@ -193,6 +193,7 @@ jobs:
           - name: "expressions"
             value: |
               org.apache.comet.CometExpressionSuite
+              org.apache.comet.CometSqlFileTestSuite
               org.apache.comet.CometExpressionCoverageSuite
               org.apache.comet.CometHashExpressionSuite
               org.apache.comet.CometTemporalExpressionSuite

spark/src/test/resources/sql-tests/expressions/arithmetic.sql

@@ -0,0 +1,38 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+-- negative
+statement
+CREATE TABLE test_neg(col1 int) USING parquet
+
+statement
+INSERT INTO test_neg VALUES(1), (2), (3), (3)
+
+query
+SELECT negative(col1), -(col1) FROM test_neg
+
+-- integral division overflow
+statement
+CREATE TABLE test_div(c1 long, c2 short) USING parquet
+
+statement
+INSERT INTO test_div VALUES(-9223372036854775808, -1)
+
+query
+SELECT c1 div c2 FROM test_div ORDER BY c1

spark/src/test/resources/sql-tests/expressions/bitwise.sql

@@ -0,0 +1,50 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+-- Setup
+statement
+CREATE TABLE test(col1 int, col2 int) USING parquet
+
+statement
+INSERT INTO test VALUES(1111, 2)
+
+statement
+INSERT INTO test VALUES(1111, 2)
+
+statement
+INSERT INTO test VALUES(3333, 4)
+
+statement
+INSERT INTO test VALUES(5555, 6)
+
+-- Queries
+query
+SELECT col1 & col2, col1 | col2, col1 ^ col2 FROM test
+
+query
+SELECT col1 & 1234, col1 | 1234, col1 ^ 1234 FROM test
+
+query
+SELECT shiftright(col1, 2), shiftright(col1, col2) FROM test
+
+query
+SELECT shiftleft(col1, 2), shiftleft(col1, col2) FROM test
+
+query
+SELECT ~(11), ~col1, ~col2 FROM test

spark/src/test/resources/sql-tests/expressions/boolean.sql

@@ -0,0 +1,41 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+-- compare true/false to negative zero
+statement
+CREATE TABLE test(col1 boolean, col2 float) USING parquet
+
+statement
+INSERT INTO test VALUES(true, -0.0)
+
+statement
+INSERT INTO test VALUES(false, -0.0)
+
+query
+SELECT col1, negative(col2), cast(col1 as float), col1 = negative(col2) FROM test
+
+-- not
+statement
+CREATE TABLE test_not(col1 int, col2 boolean) USING parquet
+
+statement
+INSERT INTO test_not VALUES(1, false), (2, true), (3, true), (3, false)
+
+query
+SELECT col1, col2, NOT(col2), !(col2) FROM test_not

spark/src/test/resources/sql-tests/expressions/datetime.sql

@@ -0,0 +1,28 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+-- DatePart functions
+statement
+CREATE TABLE test_dt(col timestamp) USING parquet
+
+statement
+INSERT INTO test_dt VALUES (timestamp('2024-06-15 10:30:00')), (timestamp('1900-01-01')), (null)
+
+query
+SELECT col, year(col), month(col), day(col), weekday(col), dayofweek(col), dayofyear(col), weekofyear(col), quarter(col) FROM test_dt

spark/src/test/resources/sql-tests/expressions/hash.sql

@@ -0,0 +1,28 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+-- hash functions
+statement
+CREATE TABLE test(col string, a int, b float) USING parquet
+
+statement
+INSERT INTO test VALUES ('Spark SQL  ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999), ('Spark SQL  ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999)
+
+query
+SELECT md5(col), md5(cast(a as string)), md5(cast(b as string)), hash(col), hash(col, 1), hash(col, 0), hash(col, a, b), hash(b, a, col), xxhash64(col), xxhash64(col, 1), xxhash64(col, 0), xxhash64(col, a, b), xxhash64(b, a, col), sha2(col, 0), sha2(col, 256), sha2(col, 224), sha2(col, 384), sha2(col, 512), sha2(col, 128), sha2(col, -1), sha1(col), sha1(cast(a as string)), sha1(cast(b as string)) FROM test

spark/src/test/resources/sql-tests/expressions/in_set.sql

@@ -0,0 +1,38 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: spark.sql.optimizer.inSetConversionThreshold=100,0
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+-- test in(set)/not in(set)
+statement
+CREATE TABLE names(id int, name varchar(20)) USING parquet
+
+statement
+INSERT INTO names VALUES(1, 'James'), (1, 'Jones'), (2, 'Smith'), (3, 'Smith'), (NULL, 'Jones'), (4, NULL)
+
+query
+SELECT * FROM names WHERE id in (1, 2, 4, NULL)
+
+query
+SELECT * FROM names WHERE name in ('Smith', 'Brown', NULL)
+
+query
+SELECT * FROM names WHERE id not in (1)
+
+query spark_answer_only
+SELECT * FROM names WHERE name not in ('Smith', 'Brown', NULL)

spark/src/test/resources/sql-tests/expressions/parquet_default_values.sql

@@ -0,0 +1,29 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- parquet default values
+statement
+CREATE TABLE t1(col1 boolean) USING parquet
+
+statement
+INSERT INTO t1 VALUES(true)
+
+statement
+ALTER TABLE t1 ADD COLUMN col2 string DEFAULT 'hello'
+
+query
+SELECT * FROM t1

spark/src/test/resources/sql-tests/expressions/string.sql

@@ -0,0 +1,49 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- substring with start < 1
+statement
+CREATE TABLE t(col string) USING parquet
+
+statement
+INSERT INTO t VALUES('123456')
+
+query
+SELECT substring(col, 0) FROM t
+
+query
+SELECT substring(col, -1) FROM t
+
+-- md5
+statement
+CREATE TABLE test_md5(col String) USING parquet
+
+statement
+INSERT INTO test_md5 VALUES ('test1'), ('test1'), ('test2'), ('test2'), (NULL), ('')
+
+query
+SELECT md5(col) FROM test_md5
+
+-- unhex
+statement
+CREATE TABLE unhex_table(col string) USING parquet
+
+statement
+INSERT INTO unhex_table VALUES ('537061726B2053514C'), ('737472696E67'), ('\0'), (''), ('###'), ('G123'), ('hello'), ('A1B'), ('0A1B')
+
+query
+SELECT unhex(col) FROM unhex_table