Skip to content

HIVE-23807 Wrong results with vectorization enabled #1234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vineetgarg02 merged 1 commit into from
Jul 14, 2020
Merged

HIVE-23807 Wrong results with vectorization enabled #1234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions itests/src/test/resources/testconfiguration.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ minillap.query.files=acid_bucket_pruning.q,\
bucket5.q,\
bucket6.q,\
except_distinct.q,\
vectorization_cast_string_to_date.q,\
explainuser_2.q,\
empty_dir_in_table.q,\
intersect_all.q,\
Expand Down
14 changes: 9 additions & 5 deletions ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToDate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hive.common.util.DateParser;

import java.sql.Date;

import java.nio.charset.StandardCharsets;

/**
Expand Down Expand Up @@ -116,14 +118,16 @@ public void evaluate(VectorizedRowBatch batch) {

private void evaluate(LongColumnVector outV, BytesColumnVector inV, int i) {
String dateString = new String(inV.vector[i], inV.start[i], inV.length[i], StandardCharsets.UTF_8);
if (dateParser.parseDate(dateString, sqlDate)) {
try {
Date utilDate = Date.valueOf(dateString);
sqlDate.setTime(utilDate.getTime());
outV.vector[i] = DateWritable.dateToDays(sqlDate);
return;
} catch (IllegalArgumentException e) {
outV.vector[i] = 1;
outV.isNull[i] = true;
outV.noNulls = false;
}

outV.vector[i] = 1;
outV.isNull[i] = true;
outV.noNulls = false;
}

@Override
Expand Down
33 changes: 33 additions & 0 deletions ql/src/test/queries/clientpositive/vectorization_cast_string_to_date.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
CREATE TABLE `test13`(
`portfolio_valuation_date` string,
`price_cut_off_datetime` string,
`portfolio_id_valuation_source` string,
`contributor_full_path` string,
`position_market_value` double,
`mandate_name` string)
STORED AS ORC;

INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, -0.26, "foo");

INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.33, "foo");

INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, -0.03, "foo");

INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.16, "foo");

INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.08, "foo");

set hive.fetch.task.conversion=none;
set hive.explain.user=false;

set hive.vectorized.execution.enabled=false;
select Cast(`test13`.`price_cut_off_datetime` AS date) from test13;


set hive.vectorized.execution.enabled=true;
select Cast(`test13`.`price_cut_off_datetime` AS date) from test13;
118 changes: 118 additions & 0 deletions ql/src/test/results/clientpositive/llap/vectorization_cast_string_to_date.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
PREHOOK: query: CREATE TABLE `test13`(
`portfolio_valuation_date` string,
`price_cut_off_datetime` string,
`portfolio_id_valuation_source` string,
`contributor_full_path` string,
`position_market_value` double,
`mandate_name` string)
STORED AS ORC
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@test13
POSTHOOK: query: CREATE TABLE `test13`(
`portfolio_valuation_date` string,
`price_cut_off_datetime` string,
`portfolio_id_valuation_source` string,
`contributor_full_path` string,
`position_market_value` double,
`mandate_name` string)
STORED AS ORC
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@test13
PREHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, -0.26, "foo")
PREHOOK: type: QUERY
PREHOOK: Output: default@test13
POSTHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, -0.26, "foo")
POSTHOOK: type: QUERY
POSTHOOK: Output: default@test13
POSTHOOK: Lineage: test13.contributor_full_path SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
POSTHOOK: Lineage: test13.mandate_name SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_id_valuation_source SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_valuation_date SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
POSTHOOK: Lineage: test13.position_market_value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ]
POSTHOOK: Lineage: test13.price_cut_off_datetime SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
PREHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.33, "foo")
PREHOOK: type: QUERY
PREHOOK: Output: default@test13
POSTHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.33, "foo")
POSTHOOK: type: QUERY
POSTHOOK: Output: default@test13
POSTHOOK: Lineage: test13.contributor_full_path SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
POSTHOOK: Lineage: test13.mandate_name SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col6, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_id_valuation_source SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_valuation_date SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
POSTHOOK: Lineage: test13.position_market_value EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col5, type:string, comment:), ]
POSTHOOK: Lineage: test13.price_cut_off_datetime SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
PREHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, -0.03, "foo")
PREHOOK: type: QUERY
PREHOOK: Output: default@test13
POSTHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, -0.03, "foo")
POSTHOOK: type: QUERY
POSTHOOK: Output: default@test13
POSTHOOK: Lineage: test13.contributor_full_path SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
POSTHOOK: Lineage: test13.mandate_name SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col6, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_id_valuation_source SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_valuation_date SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
POSTHOOK: Lineage: test13.position_market_value EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col5, type:string, comment:), ]
POSTHOOK: Lineage: test13.price_cut_off_datetime SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
PREHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.16, "foo")
PREHOOK: type: QUERY
PREHOOK: Output: default@test13
POSTHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.16, "foo")
POSTHOOK: type: QUERY
POSTHOOK: Output: default@test13
POSTHOOK: Lineage: test13.contributor_full_path SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
POSTHOOK: Lineage: test13.mandate_name SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col6, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_id_valuation_source SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_valuation_date SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
POSTHOOK: Lineage: test13.position_market_value EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col5, type:string, comment:), ]
POSTHOOK: Lineage: test13.price_cut_off_datetime SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
PREHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.08, "foo")
PREHOOK: type: QUERY
PREHOOK: Output: default@test13
POSTHOOK: query: INSERT INTO test13 values (
"2020-01-31", "2020-02-07T03:14:48.007Z", "37", NULL, 0.08, "foo")
POSTHOOK: type: QUERY
POSTHOOK: Output: default@test13
POSTHOOK: Lineage: test13.contributor_full_path SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
POSTHOOK: Lineage: test13.mandate_name SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col6, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_id_valuation_source SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
POSTHOOK: Lineage: test13.portfolio_valuation_date SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
POSTHOOK: Lineage: test13.position_market_value EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col5, type:string, comment:), ]
POSTHOOK: Lineage: test13.price_cut_off_datetime SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
PREHOOK: query: select Cast(`test13`.`price_cut_off_datetime` AS date) from test13
PREHOOK: type: QUERY
PREHOOK: Input: default@test13
#### A masked pattern was here ####
POSTHOOK: query: select Cast(`test13`.`price_cut_off_datetime` AS date) from test13
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test13
#### A masked pattern was here ####
NULL
NULL
NULL
NULL
NULL
PREHOOK: query: select Cast(`test13`.`price_cut_off_datetime` AS date) from test13
PREHOOK: type: QUERY
PREHOOK: Input: default@test13
#### A masked pattern was here ####
POSTHOOK: query: select Cast(`test13`.`price_cut_off_datetime` AS date) from test13
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test13
#### A masked pattern was here ####
NULL
NULL
NULL
NULL
NULL