[Branch-2.0](Serde) Fix content displayed by complex types in MySQL C…

…lient (#26880) backport #25946 and #26301
apache · Nov 15, 2023 · fececab · fececab
1 parent 9450a59
commit fececab
Show file tree

Hide file tree

Showing 115 changed files with 2,773 additions and 2,706 deletions.
diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
@@ -86,7 +86,7 @@ class IDataType : private boost::noncopyable {
  virtual Status from_string(ReadBuffer& rb, IColumn* column) const;
 
  // get specific serializer or deserializer
- virtual DataTypeSerDeSPtr get_serde() const = 0;
+ virtual DataTypeSerDeSPtr get_serde(int nesting_level = 1) const = 0;
 
 protected:
  virtual String do_get_name() const;

diff --git a/be/src/vec/data_types/data_type_agg_state.h b/be/src/vec/data_types/data_type_agg_state.h
@@ -126,7 +126,9 @@ class DataTypeAggState : public DataTypeString {
  return _agg_function->create_serialize_column();
  }
 
- DataTypeSerDeSPtr get_serde() const override { return _agg_serialized_type->get_serde(); };
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return _agg_serialized_type->get_serde(nesting_level);
+ };
 
  DataTypePtr get_serialized_type() const { return _agg_serialized_type; }
 

diff --git a/be/src/vec/data_types/data_type_array.h b/be/src/vec/data_types/data_type_array.h
@@ -111,8 +111,9 @@ class DataTypeArray final : public IDataType {
  void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
  Status from_string(ReadBuffer& rb, IColumn* column) const override;
 
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeArraySerDe>(nested->get_serde());
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeArraySerDe>(nested->get_serde(nesting_level + 1),
+ nesting_level);
  };
 };
 

diff --git a/be/src/vec/data_types/data_type_bitmap.h b/be/src/vec/data_types/data_type_bitmap.h
@@ -113,8 +113,8 @@ class DataTypeBitMap : public IDataType {
 
  static void deserialize_as_stream(BitmapValue& value, BufferReadable& buf);
 
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeBitMapSerDe>();
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeBitMapSerDe>(nesting_level);
  };
 };
 

diff --git a/be/src/vec/data_types/data_type_date.h b/be/src/vec/data_types/data_type_date.h
@@ -77,7 +77,9 @@ class DataTypeDate final : public DataTypeNumberBase<Int64> {
 
  MutableColumnPtr create_column() const override;
 
- DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeDate64SerDe>(); }
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeDate64SerDe>(nesting_level);
+ }
 };
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_date_time.h b/be/src/vec/data_types/data_type_date_time.h
@@ -86,8 +86,8 @@ class DataTypeDateTime final : public DataTypeNumberBase<Int64> {
 
  std::string to_string(const IColumn& column, size_t row_num) const override;
 
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeDateTimeSerDe>();
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeDateTimeSerDe>(nesting_level);
  }
 
  Field get_field(const TExprNode& node) const override {

diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h
@@ -254,8 +254,8 @@ class DataTypeDecimal final : public IDataType {
  std::string to_string(const IColumn& column, size_t row_num) const override;
  void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
  Status from_string(ReadBuffer& rb, IColumn* column) const override;
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeDecimalSerDe<T>>(scale, precision);
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeDecimalSerDe<T>>(scale, precision, nesting_level);
  };
 
  /// Decimal specific

diff --git a/be/src/vec/data_types/data_type_fixed_length_object.h b/be/src/vec/data_types/data_type_fixed_length_object.h
@@ -83,8 +83,8 @@ class DataTypeFixedLengthObject final : public IDataType {
  bool have_subtypes() const override { return false; }
 
  bool can_be_inside_low_cardinality() const override { return false; }
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeFixedLengthObjectSerDe>();
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeFixedLengthObjectSerDe>(nesting_level);
  };
 };
 

diff --git a/be/src/vec/data_types/data_type_hll.h b/be/src/vec/data_types/data_type_hll.h
@@ -101,7 +101,9 @@ class DataTypeHLL : public IDataType {
 
  static void deserialize_as_stream(HyperLogLog& value, BufferReadable& buf);
 
- DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeHLLSerDe>(); };
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeHLLSerDe>(nesting_level);
+ };
 };
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_jsonb.h b/be/src/vec/data_types/data_type_jsonb.h
@@ -93,7 +93,9 @@ class DataTypeJsonb final : public IDataType {
  std::string to_string(const IColumn& column, size_t row_num) const override;
  void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
  Status from_string(ReadBuffer& rb, IColumn* column) const override;
- DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeJsonbSerDe>(); };
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeJsonbSerDe>(nesting_level);
+ };
 
 private:
  DataTypeString data_type_string;

diff --git a/be/src/vec/data_types/data_type_map.h b/be/src/vec/data_types/data_type_map.h
@@ -104,8 +104,10 @@ class DataTypeMap final : public IDataType {
  std::string to_string(const IColumn& column, size_t row_num) const override;
  void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
  Status from_string(ReadBuffer& rb, IColumn* column) const override;
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeMapSerDe>(key_type->get_serde(), value_type->get_serde());
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeMapSerDe>(key_type->get_serde(nesting_level + 1),
+ value_type->get_serde(nesting_level + 1),
+ nesting_level);
  };
 };
 

diff --git a/be/src/vec/data_types/data_type_nothing.h b/be/src/vec/data_types/data_type_nothing.h
@@ -90,7 +90,7 @@ class DataTypeNothing final : public IDataType {
  }
 
  bool have_subtypes() const override { return false; }
- DataTypeSerDeSPtr get_serde() const override {
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
  LOG(FATAL) << get_name() << " not support serde";
  };
 };

diff --git a/be/src/vec/data_types/data_type_nullable.h b/be/src/vec/data_types/data_type_nullable.h
@@ -123,8 +123,9 @@ class DataTypeNullable final : public IDataType {
  const DataTypePtr& get_nested_type() const { return nested_data_type; }
  bool is_null_literal() const override { return nested_data_type->is_null_literal(); }
 
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeNullableSerDe>(nested_data_type->get_serde());
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeNullableSerDe>(nested_data_type->get_serde(nesting_level),
+ nesting_level);
  }
 
 private:

diff --git a/be/src/vec/data_types/data_type_number_base.h b/be/src/vec/data_types/data_type_number_base.h
@@ -154,8 +154,8 @@ class DataTypeNumberBase : public IDataType {
  Status from_string(ReadBuffer& rb, IColumn* column) const override;
  bool is_null_literal() const override { return _is_null_literal; }
  void set_null_literal(bool flag) { _is_null_literal = flag; }
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeNumberSerDe<T>>();
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeNumberSerDe<T>>(nesting_level);
  };
 
 private:

diff --git a/be/src/vec/data_types/data_type_object.h b/be/src/vec/data_types/data_type_object.h
@@ -83,8 +83,8 @@ class DataTypeObject : public IDataType {
  LOG(FATAL) << "Unimplemented get_field for object";
  }
 
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeObjectSerDe>();
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeObjectSerDe>(nesting_level);
  };
 };
 } // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_quantilestate.h b/be/src/vec/data_types/data_type_quantilestate.h
@@ -104,8 +104,8 @@ class DataTypeQuantileState : public IDataType {
  static void serialize_as_stream(const QuantileState<T>& value, BufferWritable& buf);
 
  static void deserialize_as_stream(QuantileState<T>& value, BufferReadable& buf);
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeQuantileStateSerDe<T>>();
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeQuantileStateSerDe<T>>(nesting_level);
  };
 };
 using DataTypeQuantileStateDouble = DataTypeQuantileState<double>;

diff --git a/be/src/vec/data_types/data_type_string.h b/be/src/vec/data_types/data_type_string.h
@@ -91,8 +91,8 @@ class DataTypeString : public IDataType {
  std::string to_string(const IColumn& column, size_t row_num) const override;
  void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
  Status from_string(ReadBuffer& rb, IColumn* column) const override;
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeStringSerDe>();
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeStringSerDe>(nesting_level);
  };
 };
 

diff --git a/be/src/vec/data_types/data_type_struct.h b/be/src/vec/data_types/data_type_struct.h
@@ -129,12 +129,12 @@ class DataTypeStruct final : public IDataType {
  std::string to_string(const IColumn& column, size_t row_num) const override;
  void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
  bool get_have_explicit_names() const { return have_explicit_names; }
- DataTypeSerDeSPtr get_serde() const override {
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
  DataTypeSerDeSPtrs ptrs;
  for (auto iter = elems.begin(); iter < elems.end(); ++iter) {
- ptrs.push_back((*iter)->get_serde());
+ ptrs.push_back((*iter)->get_serde(nesting_level + 1));
  }
- return std::make_shared<DataTypeStructSerDe>(ptrs, names);
+ return std::make_shared<DataTypeStructSerDe>(ptrs, names, nesting_level);
  };
 };
 

diff --git a/be/src/vec/data_types/data_type_time.h b/be/src/vec/data_types/data_type_time.h
@@ -69,7 +69,9 @@ class DataTypeTime final : public DataTypeNumberBase<Float64> {
  bool can_be_used_in_boolean_context() const override { return true; }
  bool can_be_inside_nullable() const override { return true; }
 
- DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeTimeSerDe>(); };
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeTimeSerDe>(nesting_level);
+ };
  TypeIndex get_type_id() const override { return TypeIndex::Time; }
  const char* get_family_name() const override { return "time"; }
 };
@@ -104,8 +106,8 @@ class DataTypeTimeV2 final : public DataTypeNumberBase<Float64> {
  bool can_be_inside_nullable() const override { return true; }
 
  void to_pb_column_meta(PColumnMeta* col_meta) const override;
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeTimeV2SerDe>(_scale);
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeTimeV2SerDe>(_scale, nesting_level);
  };
  TypeIndex get_type_id() const override { return TypeIndex::TimeV2; }
  const char* get_family_name() const override { return "timev2"; }

diff --git a/be/src/vec/data_types/data_type_time_v2.h b/be/src/vec/data_types/data_type_time_v2.h
@@ -69,7 +69,9 @@ class DataTypeDateV2 final : public DataTypeNumberBase<UInt32> {
 
  bool can_be_inside_nullable() const override { return true; }
 
- DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeDateV2SerDe>(); }
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeDateV2SerDe>(nesting_level);
+ }
 
  Field get_field(const TExprNode& node) const override {
  DateV2Value<DateV2ValueType> value;
@@ -127,8 +129,8 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase<UInt64> {
  std::string to_string(const IColumn& column, size_t row_num) const override;
  void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
  Status from_string(ReadBuffer& rb, IColumn* column) const override;
- DataTypeSerDeSPtr get_serde() const override {
- return std::make_shared<DataTypeDateTimeV2SerDe>(_scale);
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+ return std::make_shared<DataTypeDateTimeV2SerDe>(_scale, nesting_level);
  };
 
  Field get_field(const TExprNode& node) const override {

diff --git a/be/src/vec/data_types/serde/data_type_array_serde.cpp b/be/src/vec/data_types/serde/data_type_array_serde.cpp
@@ -65,15 +65,13 @@ void DataTypeArraySerDe::serialize_one_cell_to_json(const IColumn& column, int r
 Status DataTypeArraySerDe::deserialize_column_from_json_vector(IColumn& column,
  std::vector<Slice>& slices,
  int* num_deserialized,
- const FormatOptions& options,
- int nesting_level) const {
+ const FormatOptions& options) const {
  DESERIALIZE_COLUMN_FROM_JSON_VECTOR();
  return Status::OK();
 }
 
 Status DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice,
- const FormatOptions& options,
- int nesting_level) const {
+ const FormatOptions& options) const {
  if (slice.empty()) {
  return Status::InvalidArgument("slice is empty!");
  }
@@ -142,15 +140,15 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice
  }
 
  int elem_deserialized = 0;
- Status st = nested_serde->deserialize_column_from_json_vector(
- nested_column, slices, &elem_deserialized, options, nesting_level + 1);
+ Status st = nested_serde->deserialize_column_from_json_vector(nested_column, slices,
+  &elem_deserialized, options);
  offsets.emplace_back(offsets.back() + elem_deserialized);
  return st;
 }
 
-Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(IColumn& column, Slice& slice,
-   const FormatOptions& options,
-  int nesting_level) const {
+Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(
+ IColumn& column, Slice& slice, const FormatOptions& options,
+ int hive_text_complex_type_delimiter_level) const {
  if (slice.empty()) {
  return Status::InvalidArgument("slice is empty!");
  }
@@ -159,7 +157,8 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(IColumn& column,
  IColumn& nested_column = array_column.get_data();
  DCHECK(nested_column.is_nullable());
 
- char collection_delimiter = options.get_collection_delimiter(nesting_level);
+ char collection_delimiter =
+ options.get_collection_delimiter(hive_text_complex_type_delimiter_level);
 
  std::vector<Slice> slices;
  for (int idx = 0, start = 0; idx <= slice.size; idx++) {
@@ -172,23 +171,22 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(IColumn& column,
 
  int elem_deserialized = 0;
  Status status = nested_serde->deserialize_column_from_hive_text_vector(
- nested_column, slices, &elem_deserialized, options, nesting_level + 1);
+ nested_column, slices, &elem_deserialized, options,
+ hive_text_complex_type_delimiter_level + 1);
  offsets.emplace_back(offsets.back() + elem_deserialized);
  return status;
 }
 
-Status DataTypeArraySerDe::deserialize_column_from_hive_text_vector(IColumn& column,
- std::vector<Slice>& slices,
- int* num_deserialized,
- const FormatOptions& options,
- int nesting_level) const {
+Status DataTypeArraySerDe::deserialize_column_from_hive_text_vector(
+ IColumn& column, std::vector<Slice>& slices, int* num_deserialized,
+ const FormatOptions& options, int hive_text_complex_type_delimiter_level) const {
  DESERIALIZE_COLUMN_FROM_HIVE_TEXT_VECTOR();
  return Status::OK();
 }
 
-void DataTypeArraySerDe::serialize_one_cell_to_hive_text(const IColumn& column, int row_num,
-   BufferWritable& bw, FormatOptions& options,
-  int nesting_level) const {
+void DataTypeArraySerDe::serialize_one_cell_to_hive_text(
+ const IColumn& column, int row_num, BufferWritable& bw, FormatOptions& options,
+ int hive_text_complex_type_delimiter_level) const {
  auto result = check_column_const_set_readability(column, row_num);
  ColumnPtr ptr = result.first;
  row_num = result.second;
@@ -201,13 +199,13 @@ void DataTypeArraySerDe::serialize_one_cell_to_hive_text(const IColumn& column,
 
  const IColumn& nested_column = data_column.get_data();
 
- char delimiter = options.get_collection_delimiter(nesting_level);
+ char delimiter = options.get_collection_delimiter(hive_text_complex_type_delimiter_level);
  for (size_t i = start; i < end; ++i) {
  if (i != start) {
  bw.write(delimiter);
  }
  nested_serde->serialize_one_cell_to_hive_text(nested_column, i, bw, options,
- nesting_level + 1);
+ hive_text_complex_type_delimiter_level + 1);
  }
 }
 
@@ -288,7 +286,8 @@ Status DataTypeArraySerDe::_write_column_to_mysql(const IColumn& column,
  }
  }
  if (data.is_null_at(j)) {
- if (0 != result.push_string("NULL", strlen("NULL"))) {
+ if (0 != result.push_string(NULL_IN_COMPLEX_TYPE.c_str(),
+ strlen(NULL_IN_COMPLEX_TYPE.c_str()))) {
  return Status::InternalError("pack mysql buffer failed.");
  }
  } else {