From 10346a7c18cd620fcb5be64d47bf25125b17f9ff Mon Sep 17 00:00:00 2001 From: yangzhg Date: Thu, 11 Jun 2020 18:44:33 +0800 Subject: [PATCH 1/5] first --- be/src/exec/text_converter.hpp | 2 ++ be/src/exprs/cast_functions.cpp | 11 +++++++++++ be/src/exprs/cast_functions.h | 2 +- be/src/olap/delete_handler.cpp | 2 ++ be/src/olap/utils.cpp | 10 ++++++++++ be/src/olap/utils.h | 2 ++ .../java/org/apache/doris/analysis/BoolLiteral.java | 4 ++-- .../main/java/org/apache/doris/analysis/CastExpr.java | 6 +++--- .../java/org/apache/doris/load/DeleteHandler.java | 10 ++++++++++ 9 files changed, 43 insertions(+), 6 deletions(-) diff --git a/be/src/exec/text_converter.hpp b/be/src/exec/text_converter.hpp index 1f2b1dce8ca447..e60aa6da300d01 100644 --- a/be/src/exec/text_converter.hpp +++ b/be/src/exec/text_converter.hpp @@ -83,8 +83,10 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc, } case TYPE_BOOLEAN: + LOG(INFO) << data << ":" << len; *reinterpret_cast(slot) = StringParser::string_to_bool(data, len, &parse_result); + LOG(INFO) << "data:" << data << " length:" << len << " convert to bool is " << parse_result ; break; case TYPE_TINYINT: diff --git a/be/src/exprs/cast_functions.cpp b/be/src/exprs/cast_functions.cpp index 9c89f524856b7b..7a79354057ed31 100644 --- a/be/src/exprs/cast_functions.cpp +++ b/be/src/exprs/cast_functions.cpp @@ -229,6 +229,17 @@ StringVal CastFunctions::cast_to_string_val(FunctionContext* ctx, const StringVa return sv; } +BooleanVal CastFunctions::cast_to_boolean_val(FunctionContext* ctx, const StringVal& val) { + if (val.is_null) return BooleanVal::null(); + StringParser::ParseResult result; + BooleanVal ret; + ret.val = StringParser::string_to_bool(reinterpret_cast(val.ptr), val.len, &result); + if (UNLIKELY(result != StringParser::PARSE_SUCCESS || std::isnan(ret.val) || std::isinf(ret.val))) { + return BooleanVal::null(); + } + return ret; +} + #if 0 StringVal CastFunctions::CastToChar(FunctionContext* ctx, const StringVal& val) { if (val.is_null) return StringVal::null(); diff --git a/be/src/exprs/cast_functions.h b/be/src/exprs/cast_functions.h index 8a3b612b24e7db..26eb13088f01ba 100644 --- a/be/src/exprs/cast_functions.h +++ b/be/src/exprs/cast_functions.h @@ -33,7 +33,7 @@ class CastFunctions { static BooleanVal cast_to_boolean_val(FunctionContext* context, const LargeIntVal& val); static BooleanVal cast_to_boolean_val(FunctionContext* context, const FloatVal& val); static BooleanVal cast_to_boolean_val(FunctionContext* context, const DoubleVal& val); - // static BooleanVal cast_to_boolean_val(FunctionContext* context, const StringVal& val); + static BooleanVal cast_to_boolean_val(FunctionContext* context, const StringVal& val); static BooleanVal cast_to_boolean_val(FunctionContext* context, const DateTimeVal& val); static TinyIntVal cast_to_tiny_int_val(FunctionContext* context, const BooleanVal& val); diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index 3435dc3da0de2e..369db7a4cb8aea 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -155,6 +155,8 @@ OLAPStatus DeleteConditionHandler::check_condition_valid( } } else if (field_type == OLAP_FIELD_TYPE_DATE || field_type == OLAP_FIELD_TYPE_DATETIME) { valid_condition = valid_datetime(value_str); + } else if (field_type == OLAP_FIELD_TYPE_BOOL) { + valid_condition = valid_bool(value_str); } else { OLAP_LOG_WARNING("unknown field type. [type=%d]", field_type); } diff --git a/be/src/olap/utils.cpp b/be/src/olap/utils.cpp index 93a61e3e77402a..f34108f28839ba 100644 --- a/be/src/olap/utils.cpp +++ b/be/src/olap/utils.cpp @@ -51,6 +51,7 @@ #include "olap/olap_define.h" #include "util/errno.h" #include "util/mutex.h" +#include "util/string_parser.hpp" using std::string; using std::set; @@ -1264,6 +1265,15 @@ bool valid_datetime(const string &value_str) { } } +bool valid_bool(const std::string& value_str) { + if (value_str == "0" || value_str == "1") { + return true; + } + StringParser::ParseResult result; + StringParser::string_to_bool(value_str.c_str(), value_str.length(), &result); + return result == StringParser::PARSE_SUCCESS; +} + void write_log_info(char *buf, size_t buf_len, const char *fmt, ...) { va_list args; va_start(args, fmt); diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h index 09cbf8c3f65362..feba5266f4c2ba 100644 --- a/be/src/olap/utils.h +++ b/be/src/olap/utils.h @@ -321,6 +321,8 @@ bool valid_decimal(const std::string& value_str, const uint32_t precision, const // 粗略检查date或者datetime类型是否正确 bool valid_datetime(const std::string& value_str); +bool valid_bool(const std::string& value_str); + #define OLAP_LOG_WRITE(level, fmt, arg...) \ do { \ char buf[10240] = {0}; \ diff --git a/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java index 6e22b75885de7e..832f9103fe21c3 100644 --- a/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -40,9 +40,9 @@ public BoolLiteral(boolean value) { public BoolLiteral(String value) throws AnalysisException { this.type = Type.BOOLEAN; - if (value.toLowerCase().equals("true")) { + if (value.toLowerCase().equals("true") || value.equals("1")) { this.value = true; - } else if (value.toLowerCase().equals("false")) { + } else if (value.toLowerCase().equals("false") || value.equals("0")) { this.value = false; } else { throw new AnalysisException("Invalid BOOLEAN literal: " + value); diff --git a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java index f8b4177721b086..7416089a1c125d 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -100,9 +100,9 @@ public static void initBuiltins(FunctionSet functionSet) { continue; } // Disable casting from string to boolean - if (fromType.isStringType() && toType.isBoolean()) { - continue; - } + // if (fromType.isStringType() && toType.isBoolean()) { + // continue; + // } // Disable casting from boolean to decimal or datetime or date if (fromType.isBoolean() && (toType.equals(Type.DECIMAL) || toType.equals(Type.DECIMALV2) || diff --git a/fe/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/src/main/java/org/apache/doris/load/DeleteHandler.java index bba013aea5d159..c12813a5ec239a 100644 --- a/fe/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -33,6 +33,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; @@ -493,7 +494,16 @@ private void checkDeleteV2(OlapTable table, Partition partition, List String value = null; try { BinaryPredicate binaryPredicate = (BinaryPredicate) condition; + // if a bool cond passed to be, be's zone_map cannot handle bool correctly, + // change it to a tinyint type here; value = ((LiteralExpr) binaryPredicate.getChild(1)).getStringValue(); + if (column.getDataType() == PrimitiveType.BOOLEAN ) { + if (value.toLowerCase().equals("true")) { + binaryPredicate.setChild(1, LiteralExpr.create("1", Type.TINYINT)); + } else if (value.toLowerCase().equals("false")) { + binaryPredicate.setChild(1, LiteralExpr.create("0", Type.TINYINT)); + } + } LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType())); } catch (AnalysisException e) { // ErrorReport.reportDdlException(ErrorCode.ERR_INVALID_VALUE, value); From df1953eb0f3c98f04c7536ea149b88cc0853145c Mon Sep 17 00:00:00 2001 From: yangzhg Date: Thu, 11 Jun 2020 18:51:43 +0800 Subject: [PATCH 2/5] first --- be/src/exec/text_converter.hpp | 2 -- fe/src/main/java/org/apache/doris/analysis/CastExpr.java | 4 ---- 2 files changed, 6 deletions(-) diff --git a/be/src/exec/text_converter.hpp b/be/src/exec/text_converter.hpp index e60aa6da300d01..1f2b1dce8ca447 100644 --- a/be/src/exec/text_converter.hpp +++ b/be/src/exec/text_converter.hpp @@ -83,10 +83,8 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc, } case TYPE_BOOLEAN: - LOG(INFO) << data << ":" << len; *reinterpret_cast(slot) = StringParser::string_to_bool(data, len, &parse_result); - LOG(INFO) << "data:" << data << " length:" << len << " convert to bool is " << parse_result ; break; case TYPE_TINYINT: diff --git a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java index 7416089a1c125d..bcf65c0ed4a255 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -99,10 +99,6 @@ public static void initBuiltins(FunctionSet functionSet) { if (toType.isNull()) { continue; } - // Disable casting from string to boolean - // if (fromType.isStringType() && toType.isBoolean()) { - // continue; - // } // Disable casting from boolean to decimal or datetime or date if (fromType.isBoolean() && (toType.equals(Type.DECIMAL) || toType.equals(Type.DECIMALV2) || From 0e30e1ca31a91397a848df241697ad949aa25cf2 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Thu, 18 Jun 2020 19:07:13 +0800 Subject: [PATCH 3/5] fix code style --- be/src/exprs/cast_functions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/exprs/cast_functions.cpp b/be/src/exprs/cast_functions.cpp index 7a79354057ed31..d68bb2a9967f25 100644 --- a/be/src/exprs/cast_functions.cpp +++ b/be/src/exprs/cast_functions.cpp @@ -236,7 +236,7 @@ BooleanVal CastFunctions::cast_to_boolean_val(FunctionContext* ctx, const String ret.val = StringParser::string_to_bool(reinterpret_cast(val.ptr), val.len, &result); if (UNLIKELY(result != StringParser::PARSE_SUCCESS || std::isnan(ret.val) || std::isinf(ret.val))) { return BooleanVal::null(); - } + } return ret; } From 3f0e5971620a2be3517415f81b77a225e0dd9132 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Fri, 19 Jun 2020 20:20:30 +0800 Subject: [PATCH 4/5] xx --- be/src/exprs/cast_functions.cpp | 18 ++++++++++-------- .../org/apache/doris/analysis/BoolLiteral.java | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/be/src/exprs/cast_functions.cpp b/be/src/exprs/cast_functions.cpp index d68bb2a9967f25..09c13c79a893ce 100644 --- a/be/src/exprs/cast_functions.cpp +++ b/be/src/exprs/cast_functions.cpp @@ -230,14 +230,16 @@ StringVal CastFunctions::cast_to_string_val(FunctionContext* ctx, const StringVa } BooleanVal CastFunctions::cast_to_boolean_val(FunctionContext* ctx, const StringVal& val) { - if (val.is_null) return BooleanVal::null(); - StringParser::ParseResult result; - BooleanVal ret; - ret.val = StringParser::string_to_bool(reinterpret_cast(val.ptr), val.len, &result); - if (UNLIKELY(result != StringParser::PARSE_SUCCESS || std::isnan(ret.val) || std::isinf(ret.val))) { - return BooleanVal::null(); - } - return ret; + if (val.is_null) { + return BooleanVal::null(); + } + StringParser::ParseResult result; + BooleanVal ret; + ret.val = StringParser::string_to_bool(reinterpret_cast(val.ptr), val.len, &result); + if (UNLIKELY(result != StringParser::PARSE_SUCCESS)) { + return BooleanVal::null(); + } + return ret; } #if 0 diff --git a/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java index 832f9103fe21c3..ea493c3d8e613f 100644 --- a/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -40,9 +40,9 @@ public BoolLiteral(boolean value) { public BoolLiteral(String value) throws AnalysisException { this.type = Type.BOOLEAN; - if (value.toLowerCase().equals("true") || value.equals("1")) { + if (value.trim().toLowerCase().equals("true") || value.trim().equals("1")) { this.value = true; - } else if (value.toLowerCase().equals("false") || value.equals("0")) { + } else if (value.trim().toLowerCase().equals("false") || value.trim().equals("0")) { this.value = false; } else { throw new AnalysisException("Invalid BOOLEAN literal: " + value); From 3294e897eefdfebb202bb245ad4f54978cbee4a0 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Sun, 28 Jun 2020 16:58:04 +0800 Subject: [PATCH 5/5] make insert into and load consistent --- be/src/exprs/cast_functions.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/be/src/exprs/cast_functions.cpp b/be/src/exprs/cast_functions.cpp index 09c13c79a893ce..115eb42787b93d 100644 --- a/be/src/exprs/cast_functions.cpp +++ b/be/src/exprs/cast_functions.cpp @@ -235,9 +235,16 @@ BooleanVal CastFunctions::cast_to_boolean_val(FunctionContext* ctx, const String } StringParser::ParseResult result; BooleanVal ret; - ret.val = StringParser::string_to_bool(reinterpret_cast(val.ptr), val.len, &result); - if (UNLIKELY(result != StringParser::PARSE_SUCCESS)) { - return BooleanVal::null(); + IntVal int_val = cast_to_int_val(ctx, val); + if (!int_val.is_null && int_val.val == 0) { + ret.val = false; + } else if (!int_val.is_null && int_val.val == 1) { + ret.val = true; + } else { + ret.val = StringParser::string_to_bool(reinterpret_cast(val.ptr), val.len, &result); + if (UNLIKELY(result != StringParser::PARSE_SUCCESS)) { + return BooleanVal::null(); + } } return ret; }