Skip to content

[SPARK-28391][SQL][PYTHON][TESTS] Convert and port 'pgSQL/select_implicit.sql' into UDF test base #25233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Udbhav30 wants to merge 2 commits into from
Closed

[SPARK-28391][SQL][PYTHON][TESTS] Convert and port 'pgSQL/select_implicit.sql' into UDF test base #25233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6870764
[SPARK-28391][SQL][PYTHON][TESTS] Convert and port 'pgSQL/select_impl…
Udbhav30 Jul 23, 2019
6a983f8
added comment in code as per review comments
Udbhav30 Jul 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 167 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/udf/pgSQL/udf-select_implicit.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
--
-- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
--
--
-- SELECT_IMPLICIT
-- Test cases for queries with ordering terms missing from the target list.
-- This used to be called "junkfilter.sql".
-- The parser uses the term "resjunk" to handle these cases.
-- - thomas 1998-07-09
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/select_implicit.sql
--
-- This test file was converted from pgSQL/select_implicit.sql
-- [SPARK-28445] Inconsistency between Scala and Python/Panda udfs when groupby with udf() is used
-- TODO: We should add UDFs in GROUP BY clause when [SPARK-28445] is resolved.

-- load test data
CREATE TABLE test_missing_target (a int, b int, c string, d string) using parquet;
INSERT INTO test_missing_target VALUES (0, 1, 'XXXX', 'A');
INSERT INTO test_missing_target VALUES (1, 2, 'ABAB', 'b');
INSERT INTO test_missing_target VALUES (2, 2, 'ABAB', 'c');
INSERT INTO test_missing_target VALUES (3, 3, 'BBBB', 'D');
INSERT INTO test_missing_target VALUES (4, 3, 'BBBB', 'e');
INSERT INTO test_missing_target VALUES (5, 3, 'bbbb', 'F');
INSERT INTO test_missing_target VALUES (6, 4, 'cccc', 'g');
INSERT INTO test_missing_target VALUES (7, 4, 'cccc', 'h');
INSERT INTO test_missing_target VALUES (8, 4, 'CCCC', 'I');
INSERT INTO test_missing_target VALUES (9, 4, 'CCCC', 'j');


-- w/ existing GROUP BY target
SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY
test_missing_target.c
ORDER BY udf(c);

-- w/o existing GROUP BY target using a relation name in GROUP BY clause
SELECT udf(count(*)) FROM test_missing_target GROUP BY test_missing_target.c
ORDER BY udf(c);

-- w/o existing GROUP BY target and w/o existing a different ORDER BY target
-- failure expected
SELECT udf(count(*)) FROM test_missing_target GROUP BY a ORDER BY udf(b);

-- w/o existing GROUP BY target and w/o existing same ORDER BY target
SELECT udf(count(*)) FROM test_missing_target GROUP BY b ORDER BY udf(b);

-- w/ existing GROUP BY target using a relation name in target
SELECT udf(test_missing_target.b), udf(count(*))
FROM test_missing_target GROUP BY b ORDER BY udf(b);

-- w/o existing GROUP BY target
SELECT udf(c) FROM test_missing_target ORDER BY udf(a);

-- w/o existing ORDER BY target
SELECT udf(count(*)) FROM test_missing_target GROUP BY b ORDER BY udf(b) desc;

-- group using reference number
SELECT udf(count(*)) FROM test_missing_target ORDER BY udf(1) desc;

-- order using reference number
SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY 1 ORDER BY 1;

-- group using reference number out of range
-- failure expected
SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY 3;

-- group w/o existing GROUP BY and ORDER BY target under ambiguous condition
-- failure expected
SELECT udf(count(*)) FROM test_missing_target x, test_missing_target y
WHERE udf(x.a) = udf(y.a)
GROUP BY b ORDER BY udf(b);

-- order w/ target under ambiguous condition
-- failure NOT expected
SELECT udf(a), udf(a) FROM test_missing_target
ORDER BY udf(a);

-- order expression w/ target under ambiguous condition
-- failure NOT expected
SELECT udf(udf(a)/2), udf(udf(a)/2) FROM test_missing_target
ORDER BY udf(udf(a)/2);

-- group expression w/ target under ambiguous condition
-- failure NOT expected
SELECT udf(a/2), udf(a/2) FROM test_missing_target
GROUP BY a/2 ORDER BY udf(a/2);

-- group w/ existing GROUP BY target under ambiguous condition
SELECT udf(x.b), udf(count(*)) FROM test_missing_target x, test_missing_target y
WHERE udf(x.a) = udf(y.a)
GROUP BY x.b ORDER BY udf(x.b);

-- group w/o existing GROUP BY target under ambiguous condition
SELECT udf(count(*)) FROM test_missing_target x, test_missing_target y
WHERE udf(x.a) = udf(y.a)
GROUP BY x.b ORDER BY udf(x.b);

-- [SPARK-28329] SELECT INTO syntax
-- group w/o existing GROUP BY target under ambiguous condition
-- into a table
-- SELECT count(*) INTO TABLE test_missing_target2
-- FROM test_missing_target x, test_missing_target y
-- WHERE x.a = y.a
-- GROUP BY x.b ORDER BY x.b;
-- SELECT * FROM test_missing_target2;


-- Functions and expressions

-- w/ existing GROUP BY target
SELECT a%2, udf(count(udf(b))) FROM test_missing_target
GROUP BY test_missing_target.a%2
ORDER BY udf(test_missing_target.a%2);

-- w/o existing GROUP BY target using a relation name in GROUP BY clause
SELECT udf(count(c)) FROM test_missing_target
GROUP BY lower(test_missing_target.c)
ORDER BY udf(lower(test_missing_target.c));

-- w/o existing GROUP BY target and w/o existing a different ORDER BY target
-- failure expected
SELECT udf(count(udf(a))) FROM test_missing_target GROUP BY a ORDER BY udf(b);

-- w/o existing GROUP BY target and w/o existing same ORDER BY target
SELECT udf(count(b)) FROM test_missing_target GROUP BY b/2 ORDER BY udf(b/2);

-- w/ existing GROUP BY target using a relation name in target
SELECT udf(lower(test_missing_target.c)), udf(count(udf(c)))
FROM test_missing_target GROUP BY lower(c) ORDER BY udf(lower(c));

-- w/o existing GROUP BY target
SELECT udf(a) FROM test_missing_target ORDER BY udf(upper(udf(d)));

-- w/o existing ORDER BY target
SELECT udf(count(b)) FROM test_missing_target
GROUP BY (b + 1) / 2 ORDER BY udf((b + 1) / 2) desc;

-- group w/o existing GROUP BY and ORDER BY target under ambiguous condition
-- failure expected
SELECT udf(count(udf(x.a))) FROM test_missing_target x, test_missing_target y
WHERE udf(x.a) = udf(y.a)
GROUP BY b/2 ORDER BY udf(b/2);

-- group w/ existing GROUP BY target under ambiguous condition
SELECT udf(x.b/2), udf(count(udf(x.b))) FROM test_missing_target x,
test_missing_target y
WHERE udf(x.a) = udf(y.a)
GROUP BY x.b/2 ORDER BY udf(x.b/2);

-- group w/o existing GROUP BY target under ambiguous condition
-- failure expected due to ambiguous b in count(b)
SELECT udf(count(udf(b))) FROM test_missing_target x, test_missing_target y
WHERE udf(x.a) = udf(y.a)
GROUP BY x.b/2;

-- [SPARK-28329] SELECT INTO syntax
-- group w/o existing GROUP BY target under ambiguous condition
-- into a table
-- SELECT count(x.b) INTO TABLE test_missing_target3
-- FROM test_missing_target x, test_missing_target y
-- WHERE x.a = y.a
-- GROUP BY x.b/2 ORDER BY x.b/2;
-- SELECT * FROM test_missing_target3;

-- Cleanup
DROP TABLE test_missing_target;
-- DROP TABLE test_missing_target2;
-- DROP TABLE test_missing_target3;
Loading