Fix bug for str_to_date nullable input column (#1984)

* Fix bug for str_to_date nullable input column Signed-off-by: JaySon-Huang <jayson.hjs@gmail.com> * make test case more clear Signed-off-by: JaySon-Huang <jayson.hjs@gmail.com> * Remove unreasonable include Signed-off-by: JaySon-Huang <jayson.hjs@gmail.com>
pingcap · May 24, 2021 · 254fb4b · 254fb4b
1 parent b466172
commit 254fb4b
Show file tree

Hide file tree

Showing 8 changed files with 34 additions and 17 deletions.
diff --git a/dbms/src/Functions/FunctionsConversion.h b/dbms/src/Functions/FunctionsConversion.h
@@ -1677,8 +1677,13 @@ class FunctionStrToDate : public IFunction
             {
                 if (input_column->isColumnNullable())
                 {
-                    null_res[i] = input_column->isNullAt(i);
-                    continue;
+                    // For null input, just set the result as null
+                    if (bool is_null = input_column->isNullAt(i); is_null)
+                    {
+                        null_res[i] = is_null;
+                        continue;
+                    }
+                    // else fallthrough to parsing
                 }
 
                 const auto str_ref = col_from->getDataAt(i);

diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.cpp
@@ -1,3 +1,4 @@
+#include <Functions/FunctionsConversion.h>
 #include <Storages/DeltaMerge/DeltaMergeHelpers.h>
 
 namespace DB
@@ -101,4 +102,4 @@ void appendIntoHandleColumn(ColumnVector<Handle>::Container & handle_column, con
 }
 
 } // namespace DM
-} // namespace DB
+} // namespace DB
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h
@@ -7,9 +7,7 @@
 #include <Core/Block.h>
 #include <Core/SortDescription.h>
 #include <DataStreams/IBlockInputStream.h>
-#include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionHelpers.h>
-#include <Functions/FunctionsConversion.h>
 #include <IO/WriteHelpers.h>
 #include <Interpreters/sortBlock.h>
 #include <Storages/ColumnsDescription.h>

diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
@@ -3,6 +3,7 @@
 #include <Common/TiFlashMetrics.h>
 #include <Common/typeid_cast.h>
 #include <Core/SortDescription.h>
+#include <Functions/FunctionsConversion.h>
 #include <Interpreters/sortBlock.h>
 #include <Storages/DeltaMerge/DMContext.h>
 #include <Storages/DeltaMerge/DMSegmentThreadInputStream.h>

diff --git a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp
@@ -2,6 +2,8 @@
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeEnum.h>
 #include <DataTypes/DataTypeFixedString.h>
+#include <DataTypes/DataTypeMyDate.h>
+#include <DataTypes/DataTypeMyDateTime.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeTuple.h>
 #include <DataTypes/DataTypeUUID.h>

diff --git a/dbms/src/Storages/DeltaMerge/RowKeyRange.h b/dbms/src/Storages/DeltaMerge/RowKeyRange.h
@@ -10,6 +10,7 @@
 #include <Storages/Transaction/TiKVKeyValue.h>
 #include <Storages/Transaction/TiKVRecordFormat.h>
 #include <Storages/Transaction/Types.h>
+#include <Columns/ColumnString.h>
 
 namespace DB::DM
 {

diff --git a/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp
@@ -1,6 +1,7 @@
 #include <Columns/ColumnNullable.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeDecimal.h>
+#include <DataTypes/DataTypeEnum.h>
 #include <DataTypes/DataTypeMyDate.h>
 #include <DataTypes/DataTypeMyDateTime.h>
 #include <DataTypes/DataTypeString.h>
@@ -12,6 +13,11 @@
 
 namespace DB
 {
+namespace ErrorCodes
+{
+extern const int BAD_ARGUMENTS;
+}
+
 namespace DM
 {
 

diff --git a/tests/fullstack-test/expr/str_to_date.test b/tests/fullstack-test/expr/str_to_date.test
@@ -11,30 +11,33 @@ func> wait_table test t
 # Note that we need to put `str_to_date` in group by to make sure it is pushed down
 
 ## Test suite 1 - Allow zero day
-#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('0000/00/00', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by date order by a
-mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('0000/00/00', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by date order by a
+#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('00/00/0000', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by a,date order by a
+mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('00/00/0000', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by a,date order by a
 +------------+------------+-----+
 | a          | date       | cnt |
 +------------+------------+-----+
 | 0/0/2012   | 2012-00-00 |   1 |
-| 00/00/0000 | 0000-00-00 |   2 |
+| 00/00/0000 | 0000-00-00 |   1 |
 | 13/05/2019 | 2019-05-13 |   1 |
+| abc        | 0000-00-00 |   1 |
 +------------+------------+-----+
 
 ## Test suite 1 - Disallow zero day
-#mysql> set sql_mode='NO_ZERO_IN_DATE,NO_ZERO_DATE'; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('0000/00/00', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by date order by a
-mysql> set sql_mode='NO_ZERO_IN_DATE,NO_ZERO_DATE'; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('0000/00/00', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by date order by a
+# The sql_mode does not effect the result set
+#mysql> set sql_mode='NO_ZERO_IN_DATE,NO_ZERO_DATE'; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('00/00/0000', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by a,date order by a
+mysql> set sql_mode='NO_ZERO_IN_DATE,NO_ZERO_DATE'; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, ifnull(str_to_date(a, '%d/%m/%Y'),str_to_date('00/00/0000', '%d/%m/%Y')) as date, count(*) as cnt from test.t where suite = 1 group by a,date order by a
 +------------+------------+-----+
 | a          | date       | cnt |
 +------------+------------+-----+
 | 0/0/2012   | NULL       |   1 |
-| 00/00/0000 | NULL       |   2 |
+| 00/00/0000 | NULL       |   1 |
 | 13/05/2019 | 2019-05-13 |   1 |
+| abc        | NULL       |   1 |
 +------------+------------+-----+
 
 ## Test suite 2 - showing datetime with fractions
-#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, str_to_date(a, '%d/%b/%Y  %H:%i:%S.%f') as date from test.t where suite = 2 group by date order by a
-mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select /*+ agg_to_cop() */ a, str_to_date(a, '%d/%b/%Y  %H:%i:%S.%f') as date from test.t where suite = 2 group by date order by a
+#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, str_to_date(a, '%d/%b/%Y  %H:%i:%S.%f') as date from test.t where suite = 2 group by a,date order by a
+mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, str_to_date(a, '%d/%b/%Y  %H:%i:%S.%f') as date from test.t where suite = 2 group by a,date order by a
 +-----------------------------+----------------------------+
 | a                           | date                       |
 +-----------------------------+----------------------------+
@@ -44,8 +47,8 @@ mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='t
 +-----------------------------+----------------------------+
 
 ## Test suite 3 - showing datetime without fractions
-#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, str_to_date(a, ' %d/%b/%Y  %H:%i:%S') as date from test.t where suite = 3 group by date order by a
-mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, str_to_date(a, ' %d/%b/%Y  %H:%i:%S') as date from test.t where suite = 3 group by date order by a
+#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, str_to_date(a, ' %d/%b/%Y  %H:%i:%S') as date from test.t where suite = 3 group by a,date order by a
+mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, str_to_date(a, ' %d/%b/%Y  %H:%i:%S') as date from test.t where suite = 3 group by a,date order by a
 +-------------------------+---------------------+
 | a                       | date                |
 +-------------------------+---------------------+
@@ -54,8 +57,8 @@ mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='t
 +-------------------------+---------------------+
 
 ## Test suite 4 - showing date
-#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, str_to_date(a, '%d/%b/%Y ') as date from test.t where suite = 4 group by date order by a
-mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, str_to_date(a, '%d/%b/%Y ') as date from test.t where suite = 4 group by date order by a
+#mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; explain select a, str_to_date(a, '%d/%b/%Y ') as date from test.t where suite = 4 group by a,date order by a
+mysql> set sql_mode=''; set tidb_allow_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, str_to_date(a, '%d/%b/%Y ') as date from test.t where suite = 4 group by a,date order by a
 +---------------+------------+
 | a             | date       |
 +---------------+------------+