diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp index 49f335e2f7ad9b..c60e35665ac3ea 100644 --- a/be/src/exec/broker_scanner.cpp +++ b/be/src/exec/broker_scanner.cpp @@ -37,38 +37,6 @@ namespace doris { -class Slice { -public: - Slice(const uint8_t* data, int size) : _data(data), _size(size) { } - Slice(const std::string& str) : _data((const uint8_t*)str.data()), _size(str.size()) { } - - ~Slice() { - // No need to delete _begin, because it only record the index in a std::string. - // The c-string will be released along with the std::string object. - } - - int size() const { - return _size; - } - - const uint8_t* data() const { - return _data; - } - - const uint8_t* end() const { - return _data + _size; - } -private: - friend std::ostream& operator<<(std::ostream& os, const Slice& slice); - const uint8_t* _data; - int _size; -}; - -std::ostream& operator<<(std::ostream& os, const Slice& slice) { - os << std::string((const char*)slice._data, slice._size); - return os; -} - BrokerScanner::BrokerScanner(RuntimeState* state, RuntimeProfile* profile, const TBrokerScanRangeParams& params, @@ -81,8 +49,8 @@ BrokerScanner::BrokerScanner(RuntimeState* state, _ranges(ranges), _broker_addresses(broker_addresses), // _splittable(params.splittable), - _value_separator(params.column_separator), - _line_delimiter(params.line_delimiter), + _value_separator(static_cast(params.column_separator)), + _line_delimiter(static_cast(params.line_delimiter)), _cur_file_reader(nullptr), _cur_line_reader(nullptr), _cur_decompressor(nullptr), @@ -397,9 +365,9 @@ void BrokerScanner::close() { void BrokerScanner::split_line( const Slice& line, std::vector* values) { // line-begin char and line-end char are considered to be 'delimeter' - const uint8_t* value = line.data(); - const uint8_t* ptr = line.data(); - for (size_t i = 0; i < line.size(); ++i, ++ptr) { + const char* value = line.data; + const char* ptr = line.data; + for (size_t i = 0; i < line.size; ++i, ++ptr) { if (*ptr == _value_separator) { values->emplace_back(value, ptr - value); value = ptr + 1; @@ -411,12 +379,12 @@ void BrokerScanner::split_line( void BrokerScanner::fill_fix_length_string( const Slice& value, MemPool* pool, char** new_value_p, const int new_value_length) { - if (new_value_length != 0 && value.size() < new_value_length) { + if (new_value_length != 0 && value.size < new_value_length) { *new_value_p = reinterpret_cast(pool->allocate(new_value_length)); // 'value' is guaranteed not to be nullptr - memcpy(*new_value_p, value.data(), value.size()); - for (int i = value.size(); i < new_value_length; ++i) { + memcpy(*new_value_p, value.data, value.size); + for (int i = value.size; i < new_value_length; ++i) { (*new_value_p)[i] = '\0'; } } @@ -430,13 +398,13 @@ bool BrokerScanner::check_decimal_input( const Slice& slice, int precision, int scale, std::stringstream* error_msg) { - const uint8_t* value = slice.data(); - int value_length = slice.size(); + const char* value = slice.data; + size_t value_length = slice.size; if (value_length > (precision + 2)) { (*error_msg) << "the length of decimal value is overflow. " << "precision in schema: (" << precision << ", " << scale << "); " - << "value: [" << slice << "]; " + << "value: [" << slice.to_string() << "]; " << "str actual length: " << value_length << ";"; return false; } @@ -479,21 +447,21 @@ bool BrokerScanner::check_decimal_input( if (value_int_len > (precision - scale)) { (*error_msg) << "the int part length longer than schema precision [" << precision << "]. " - << "value [" << slice << "]. "; + << "value [" << slice.to_string() << "]. "; return false; } else if (value_frac_len > scale) { (*error_msg) << "the frac part length longer than schema scale [" << scale << "]. " - << "value [" << slice << "]. "; + << "value [" << slice.to_string() << "]. "; return false; } return true; } bool is_null(const Slice& slice) { - return slice.size() == 2 && - slice.data()[0] == '\\' && - slice.data()[1] == 'N'; + return slice.size == 2 && + slice.data[0] == '\\' && + slice.data[1] == 'N'; } // Writes a slot in _tuple from an value containing text data. @@ -503,29 +471,29 @@ bool BrokerScanner::write_slot( Tuple* tuple, MemPool* tuple_pool, std::stringstream* error_msg) { - if (value.size() == 0 && !slot->type().is_string_type()) { + if (value.size == 0 && !slot->type().is_string_type()) { (*error_msg) << "the length of input should not be 0. " << "column_name: " << column_name << "; " << "type: " << slot->type(); return false; } - char* value_to_convert = (char*)value.data(); - int value_to_convert_length = value.size(); + char* value_to_convert = value.data; + size_t value_to_convert_length = value.size; // Fill all the spaces if it is 'TYPE_CHAR' type if (slot->type().is_string_type()) { int char_len = column_type.len; - if (value.size() > char_len) { + if (value.size > char_len) { (*error_msg) << "the length of input is too long than schema. " << "column_name: " << column_name << "; " - << "input_str: [" << value << "] " + << "input_str: [" << value.to_string() << "] " << "type: " << slot->type() << "; " << "schema length: " << char_len << "; " - << "actual length: " << value.size() << "; "; + << "actual length: " << value.size << "; "; return false; } - if (slot->type().type == TYPE_CHAR && value.size() < char_len) { + if (slot->type().type == TYPE_CHAR && value.size < char_len) { if (!is_null(value)) { fill_fix_length_string( value, tuple_pool, @@ -547,7 +515,7 @@ bool BrokerScanner::write_slot( (*error_msg) << "convert csv string to " << slot->type() << " failed. " << "column_name: " << column_name << "; " - << "input_str: [" << value << "]; "; + << "input_str: [" << value.to_string() << "]; "; return false; } @@ -576,7 +544,7 @@ bool BrokerScanner::line_to_src_tuple(const Slice& line) { error_msg << "actual column number is less than schema column number. " << "actual number: " << values.size() << " sep: " << _value_separator << ", " << "schema number: " << _src_slot_descs.size() << "; "; - _state->append_error_msg_to_file(std::string((const char*)line.data(), line.size()), + _state->append_error_msg_to_file(std::string(line.data, line.size), error_msg.str()); _counter->num_rows_filtered++; return false; @@ -585,7 +553,7 @@ bool BrokerScanner::line_to_src_tuple(const Slice& line) { error_msg << "actual column number is more than schema column number. " << "actual number: " << values.size() << " sep: " << _value_separator << ", " << "schema number: " << _src_slot_descs.size() << "; "; - _state->append_error_msg_to_file(std::string((const char*)line.data(), line.size()), + _state->append_error_msg_to_file(std::string(line.data, line.size), error_msg.str()); _counter->num_rows_filtered++; return false; @@ -601,8 +569,8 @@ bool BrokerScanner::line_to_src_tuple(const Slice& line) { _src_tuple->set_not_null(slot_desc->null_indicator_offset()); void* slot = _src_tuple->get_slot(slot_desc->tuple_offset()); StringValue* str_slot = reinterpret_cast(slot); - str_slot->ptr = (char*)value.data(); - str_slot->len = value.size(); + str_slot->ptr = value.data; + str_slot->len = value.size; } return true; @@ -624,7 +592,7 @@ bool BrokerScanner::fill_dest_tuple(const Slice& line, Tuple* dest_tuple, MemPoo std::stringstream error_msg; error_msg << "column(" << slot_desc->col_name() << ") value is null"; _state->append_error_msg_to_file( - std::string((const char*)line.data(), line.size()), error_msg.str()); + std::string(line.data, line.size), error_msg.str()); _counter->num_rows_filtered++; return false; } diff --git a/be/src/exec/broker_scanner.h b/be/src/exec/broker_scanner.h index 0f14c58ec52616..26a84c7ca3a54f 100644 --- a/be/src/exec/broker_scanner.h +++ b/be/src/exec/broker_scanner.h @@ -27,6 +27,7 @@ #include "gen_cpp/PlanNodes_types.h" #include "gen_cpp/Types_types.h" #include "runtime/mem_pool.h" +#include "util/slice.h" #include "util/runtime_profile.h" namespace doris { @@ -121,8 +122,8 @@ class BrokerScanner { std::unique_ptr _text_converter; - uint8_t _value_separator; - uint8_t _line_delimiter; + char _value_separator; + char _line_delimiter; // Reader FileReader* _cur_file_reader; diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 31b92f9b4e7009..e3576b0dacd297 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -361,7 +361,7 @@ void OlapScanner::_convert_row_to_tuple(Tuple* tuple) { size_t len = field->size(); switch (slot_desc->type().type) { case TYPE_CHAR: { - StringSlice* slice = reinterpret_cast(ptr); + Slice* slice = reinterpret_cast(ptr); StringValue *slot = tuple->get_string_slot(slot_desc->tuple_offset()); slot->ptr = slice->data; slot->len = strnlen(slot->ptr, slice->size); @@ -369,7 +369,7 @@ void OlapScanner::_convert_row_to_tuple(Tuple* tuple) { } case TYPE_VARCHAR: case TYPE_HLL: { - StringSlice* slice = reinterpret_cast(ptr); + Slice* slice = reinterpret_cast(ptr); StringValue *slot = tuple->get_string_slot(slot_desc->tuple_offset()); slot->ptr = slice->data; slot->len = slice->size; diff --git a/be/src/olap/aggregate_func.h b/be/src/olap/aggregate_func.h index 0d1f6e249b4bf5..2c627952395451 100644 --- a/be/src/olap/aggregate_func.h +++ b/be/src/olap/aggregate_func.h @@ -198,8 +198,8 @@ struct AggregateFuncTraits bool r_null = *reinterpret_cast(right); *reinterpret_cast(left) = r_null; if (!r_null) { - StringSlice* l_slice = reinterpret_cast(left + 1); - const StringSlice* r_slice = reinterpret_cast(right + 1); + Slice* l_slice = reinterpret_cast(left + 1); + const Slice* r_slice = reinterpret_cast(right + 1); if (arena == nullptr || l_slice->size >= r_slice->size) { memory_copy(l_slice->data, r_slice->data, r_slice->size); l_slice->size = r_slice->size; @@ -224,13 +224,13 @@ struct AggregateFuncTraits struct AggregateFuncTraits { static void aggregate(char* left, const char* right, Arena* arena) { - StringSlice* l_slice = reinterpret_cast(left + 1); + Slice* l_slice = reinterpret_cast(left + 1); size_t hll_ptr = *(size_t*)(l_slice->data - sizeof(HllContext*)); HllContext* context = (reinterpret_cast(hll_ptr)); HllSetHelper::fill_set(right + 1, context); } static void finalize(char* data) { - StringSlice* slice = reinterpret_cast(data); + Slice* slice = reinterpret_cast(data); size_t hll_ptr = *(size_t*)(slice->data - sizeof(HllContext*)); HllContext* context = (reinterpret_cast(hll_ptr)); std::map index_to_value; diff --git a/be/src/olap/column_reader.cpp b/be/src/olap/column_reader.cpp index d5dd60708d175e..38e9cd35c2149c 100644 --- a/be/src/olap/column_reader.cpp +++ b/be/src/olap/column_reader.cpp @@ -106,7 +106,7 @@ OLAPStatus StringColumnDirectReader::init( return OLAP_ERR_COLUMN_STREAM_NOT_EXIST; } - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(StringSlice))); + _values = reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); ReadOnlyFileStream* length_stream = extract_stream(_column_unique_id, StreamInfoMessage::LENGTH, @@ -436,7 +436,7 @@ OLAPStatus StringColumnDictionaryReader::init( } */ - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(StringSlice))); + _values = reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); int64_t read_buffer_size = 1024; char* _read_buffer = new(std::nothrow) char[read_buffer_size]; diff --git a/be/src/olap/column_reader.h b/be/src/olap/column_reader.h index 3deb8900f834f4..566dde1dc6ee0e 100644 --- a/be/src/olap/column_reader.h +++ b/be/src/olap/column_reader.h @@ -123,7 +123,7 @@ class StringColumnDirectReader { private: bool _eof; uint32_t _column_unique_id; - StringSlice* _values; + Slice* _values; ReadOnlyFileStream* _data_stream; RunLengthIntegerReader* _length_reader; }; @@ -158,7 +158,7 @@ class StringColumnDictionaryReader { bool _eof; uint32_t _dictionary_size; uint32_t _column_unique_id; - StringSlice* _values; + Slice* _values; char* _read_buffer; //uint64_t _dictionary_size; //uint64_t* _offset_dictionary; // 用来查找响应数据的数字对应的offset @@ -335,27 +335,27 @@ class DefaultValueReader : public ColumnReader { } case OLAP_FIELD_TYPE_CHAR: { _values = - reinterpret_cast(mem_pool->allocate(size * sizeof(StringSlice))); + reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); int32_t length = _length; char* string_buffer = reinterpret_cast(mem_pool->allocate(size * length)); memset(string_buffer, 0, size * length); for (int i = 0; i < size; ++i) { memory_copy(string_buffer, _default_value.c_str(), _default_value.length()); - ((StringSlice*)_values)[i].size = length; - ((StringSlice*)_values)[i].data = string_buffer; + ((Slice*)_values)[i].size = length; + ((Slice*)_values)[i].data = string_buffer; string_buffer += length; } break; } case OLAP_FIELD_TYPE_VARCHAR: { _values = - reinterpret_cast(mem_pool->allocate(size * sizeof(StringSlice))); + reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); int32_t length = _default_value.length(); char* string_buffer = reinterpret_cast(mem_pool->allocate(size * length)); for (int i = 0; i < size; ++i) { memory_copy(string_buffer, _default_value.c_str(), length); - ((StringSlice*)_values)[i].size = length; - ((StringSlice*)_values)[i].data = string_buffer; + ((Slice*)_values)[i].size = length; + ((Slice*)_values)[i].data = string_buffer; string_buffer += length; } break; diff --git a/be/src/olap/column_writer.cpp b/be/src/olap/column_writer.cpp index a95f5390af0f77..91ed92919d815a 100755 --- a/be/src/olap/column_writer.cpp +++ b/be/src/olap/column_writer.cpp @@ -266,7 +266,7 @@ OLAPStatus ColumnWriter::write(RowCursor* row_cursor) { _field_info.type == OLAP_FIELD_TYPE_VARCHAR || _field_info.type == OLAP_FIELD_TYPE_HLL) { - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); _bf->add_bytes(slice->data, slice->size); } else { _bf->add_bytes(buf, field->size()); diff --git a/be/src/olap/column_writer.h b/be/src/olap/column_writer.h index 82b364125e727c..4ec0e53816bcde 100644 --- a/be/src/olap/column_writer.h +++ b/be/src/olap/column_writer.h @@ -438,7 +438,7 @@ class VarStringColumnWriter : public ColumnWriter { bool is_null = field->is_null(cursor->get_buf()); if (!is_null) { char* buf = field->get_ptr(cursor->get_buf()); - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); res = write(slice->data, slice->size); if (res != OLAP_SUCCESS) { LOG(WARNING) << "fail to write varchar, res=" << res; @@ -521,7 +521,7 @@ class FixLengthStringColumnWriter : public VarStringColumnWriter { if (!is_null) { //const char* str = reinterpret_cast(buf); - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); res = VarStringColumnWriter::write(slice->data, slice->size); if (res != OLAP_SUCCESS) { LOG(WARNING) << "fail to write fix-length string, res=" << res; diff --git a/be/src/olap/field.cpp b/be/src/olap/field.cpp index a18cd665b3e3a9..42562e71cb0d97 100644 --- a/be/src/olap/field.cpp +++ b/be/src/olap/field.cpp @@ -58,7 +58,7 @@ Field::Field(const FieldInfo& field_info) _type_info = get_type_info(field_info.type); if (_type == OLAP_FIELD_TYPE_CHAR || _type == OLAP_FIELD_TYPE_VARCHAR || _type == OLAP_FIELD_TYPE_HLL) { - _size = sizeof(StringSlice); + _size = sizeof(Slice); } else { /* * the field_info.size and field_info.index_length is equal to zero, diff --git a/be/src/olap/field.h b/be/src/olap/field.h index dddd431237c526..1688a328e5dcb1 100644 --- a/be/src/olap/field.h +++ b/be/src/olap/field.h @@ -24,12 +24,12 @@ #include "olap/field_info.h" #include "olap/olap_common.h" #include "olap/olap_define.h" -#include "olap/string_slice.h" #include "olap/types.h" #include "olap/utils.h" #include "runtime/mem_pool.h" #include "util/hash_util.hpp" #include "util/mem_util.hpp" +#include "util/slice.h" namespace doris { @@ -151,8 +151,8 @@ inline int Field::index_cmp(char* left, char* right) const { int32_t res = 0; if (_type == OLAP_FIELD_TYPE_VARCHAR) { - StringSlice* l_slice = reinterpret_cast(left + 1); - StringSlice* r_slice = reinterpret_cast(right + 1); + Slice* l_slice = reinterpret_cast(left + 1); + Slice* r_slice = reinterpret_cast(right + 1); if (r_slice->size + OLAP_STRING_MAX_BYTES > _index_size) { // 如果field的实际长度比short key长,则仅比较前缀,确保相同short key的所有block都被扫描, @@ -225,7 +225,7 @@ inline void Field::agg_init(char* dest, const char* src) { } else { bool is_null = *reinterpret_cast(src); *reinterpret_cast(dest) = is_null; - StringSlice* slice = reinterpret_cast(dest + 1); + Slice* slice = reinterpret_cast(dest + 1); size_t hll_ptr = *(size_t*)(slice->data - sizeof(HllContext*)); HllContext* context = (reinterpret_cast(hll_ptr)); HllSetHelper::init_context(context); @@ -244,7 +244,7 @@ inline void Field::to_index(char* dest, const char* src) { if (_type == OLAP_FIELD_TYPE_VARCHAR) { // 先清零,再拷贝 memset(dest + 1, 0, _index_size); - const StringSlice* slice = reinterpret_cast(src + 1); + const Slice* slice = reinterpret_cast(src + 1); size_t copy_size = slice->size < _index_size - OLAP_STRING_MAX_BYTES ? slice->size : _index_size - OLAP_STRING_MAX_BYTES; *reinterpret_cast(dest + 1) = copy_size; @@ -252,7 +252,7 @@ inline void Field::to_index(char* dest, const char* src) { } else if (_type == OLAP_FIELD_TYPE_CHAR) { // 先清零,再拷贝 memset(dest + 1, 0, _index_size); - const StringSlice* slice = reinterpret_cast(src + 1); + const Slice* slice = reinterpret_cast(src + 1); memory_copy(dest + 1, slice->data, _index_size); } else { memory_copy(dest + 1, src + 1, size()); diff --git a/be/src/olap/hll.cpp b/be/src/olap/hll.cpp index c26f0ac116c100..39e534189a6c5c 100644 --- a/be/src/olap/hll.cpp +++ b/be/src/olap/hll.cpp @@ -22,7 +22,7 @@ #include #include -#include "olap/string_slice.h" +#include "util/slice.h" using std::map; using std::nothrow; @@ -209,7 +209,7 @@ void HllSetHelper::set_max_register(char* registers, int registers_len, void HllSetHelper::fill_set(const char* data, HllContext* context) { HllSetResolver resolver; - const StringSlice* slice = reinterpret_cast(data); + const Slice* slice = reinterpret_cast(data); resolver.init(slice->data, slice->size); resolver.parse(); if (resolver.get_hll_data_type() == HLL_DATA_EXPLICIT) { diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 0f62e05ef70aa8..f145ef028d26c3 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -71,7 +71,7 @@ void MemTable::insert(Tuple* tuple) { switch (type.type) { case TYPE_CHAR: { const StringValue* src = tuple->get_string_slot(slot->tuple_offset()); - StringSlice* dest = (StringSlice*)(_tuple_buf + offset); + Slice* dest = (Slice*)(_tuple_buf + offset); dest->size = (*_field_infos)[i].length; dest->data = _arena.Allocate(dest->size); memcpy(dest->data, src->ptr, src->len); @@ -80,7 +80,7 @@ void MemTable::insert(Tuple* tuple) { } case TYPE_VARCHAR: { const StringValue* src = tuple->get_string_slot(slot->tuple_offset()); - StringSlice* dest = (StringSlice*)(_tuple_buf + offset); + Slice* dest = (Slice*)(_tuple_buf + offset); dest->size = src->len; dest->data = _arena.Allocate(dest->size); memcpy(dest->data, src->ptr, dest->size); @@ -88,7 +88,7 @@ void MemTable::insert(Tuple* tuple) { } case TYPE_HLL: { const StringValue* src = tuple->get_string_slot(slot->tuple_offset()); - StringSlice* dest = (StringSlice*)(_tuple_buf + offset); + Slice* dest = (Slice*)(_tuple_buf + offset); dest->size = src->len; bool exist = _skip_list->Contains(_tuple_buf); if (exist) { diff --git a/be/src/olap/new_status.cpp b/be/src/olap/new_status.cpp index ef9e79daf95919..3fcb19544ce061 100644 --- a/be/src/olap/new_status.cpp +++ b/be/src/olap/new_status.cpp @@ -39,7 +39,7 @@ NewStatus IOError(const std::string& context, int err) { return NewStatus::IOError(context, ErrnoToString(err), err); } -NewStatus::NewStatus(Code code, const StringSlice& msg, const StringSlice& msg2, int32_t posix_code) +NewStatus::NewStatus(Code code, const Slice& msg, const Slice& msg2, int32_t posix_code) : _code(code), _posix_code(posix_code) { DCHECK(code != kOk); const uint32_t len1 = msg.size; diff --git a/be/src/olap/new_status.h b/be/src/olap/new_status.h index 8d694815c1bcc8..73e1cad699a656 100644 --- a/be/src/olap/new_status.h +++ b/be/src/olap/new_status.h @@ -6,7 +6,7 @@ #define DORIS_BE_SRC_OLAP_STATUS_H #include -#include "olap/string_slice.h" +#include "util/slice.h" namespace doris { @@ -30,51 +30,51 @@ class NewStatus { static NewStatus OK() { return NewStatus(); } // Return error status of an appropriate type. - static NewStatus NotFound(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus NotFound(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kNotFound, msg, msg2, posix_code); } - static NewStatus Corruption(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus Corruption(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kCorruption, msg, msg2, posix_code); } - static NewStatus NotSupported(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus NotSupported(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kNotSupported, msg, msg2, posix_code); } - static NewStatus InvalidArgument(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus InvalidArgument(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kInvalidArgument, msg, msg2, posix_code); } - static NewStatus AlreadyExist(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus AlreadyExist(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kAlreadyExist, msg, msg2, posix_code); } - static NewStatus NoSpace(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus NoSpace(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kNoSpace, msg, msg2, posix_code); } - static NewStatus EndOfFile(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus EndOfFile(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kEndOfFile, msg, msg2, posix_code); } - static NewStatus DiskFailure(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus DiskFailure(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kDiskFailure, msg, msg2, posix_code); } - static NewStatus IOError(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus IOError(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kIOError, msg, msg2, posix_code); } - static NewStatus TimedOut(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus TimedOut(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kTimedOut, msg, msg2, posix_code); } - static NewStatus MemoryLimitExceeded(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus MemoryLimitExceeded(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kMemoryLimitExceeded, msg, msg2, posix_code); } - static NewStatus DeadLock(const StringSlice& msg, const StringSlice& msg2 = StringSlice(), + static NewStatus DeadLock(const Slice& msg, const Slice& msg2 = Slice(), int32_t posix_code = -1) { return NewStatus(kDeadLock, msg, msg2, posix_code); } @@ -159,7 +159,7 @@ class NewStatus { // _state[4..] == message const char* _state; - NewStatus(Code code, const StringSlice& msg, const StringSlice& msg2, int32_t posix_code); + NewStatus(Code code, const Slice& msg, const Slice& msg2, int32_t posix_code); static const char* CopyState(const char* s); }; diff --git a/be/src/olap/olap_cond.cpp b/be/src/olap/olap_cond.cpp index b11e4c1c94d473..0b94a0a6340fbb 100644 --- a/be/src/olap/olap_cond.cpp +++ b/be/src/olap/olap_cond.cpp @@ -431,7 +431,7 @@ bool Cond::eval(const BloomFilter& bf) const { case OP_EQ: { bool existed = false; if (operand_field->is_string_type()) { - StringSlice* slice = (StringSlice*)(operand_field->ptr()); + Slice* slice = (Slice*)(operand_field->ptr()); existed = bf.test_bytes(slice->data, slice->size); } else { existed = bf.test_bytes(operand_field->ptr(), operand_field->size()); @@ -443,7 +443,7 @@ bool Cond::eval(const BloomFilter& bf) const { for (; it != operand_set.end(); ++it) { bool existed = false; if ((*it)->is_string_type()) { - StringSlice* slice = (StringSlice*)((*it)->ptr()); + Slice* slice = (Slice*)((*it)->ptr()); existed = bf.test_bytes(slice->data, slice->size); } else { existed = bf.test_bytes((*it)->ptr(), (*it)->size()); diff --git a/be/src/olap/olap_index.cpp b/be/src/olap/olap_index.cpp index bbc55e638cf01b..5fbc81cf7315b2 100644 --- a/be/src/olap/olap_index.cpp +++ b/be/src/olap/olap_index.cpp @@ -191,7 +191,7 @@ OLAPStatus MemIndex::load_segment(const char* file, size_t *current_num_rows_per storage_field_offset += (*_fields)[i].index_length + null_byte; mem_ptr = mem_buf + mem_field_offset; if ((*_fields)[i].type == OLAP_FIELD_TYPE_VARCHAR) { - mem_field_offset += sizeof(StringSlice) + 1; + mem_field_offset += sizeof(Slice) + 1; for (size_t j = 0; j < num_entries; ++j) { /* * Varchar is null_byte|length|content in OlapIndex storage @@ -207,7 +207,7 @@ OLAPStatus MemIndex::load_segment(const char* file, size_t *current_num_rows_per // 2. copy length and content size_t storage_field_bytes = *reinterpret_cast(storage_ptr + null_byte); - StringSlice* slice = reinterpret_cast(mem_ptr + 1); + Slice* slice = reinterpret_cast(mem_ptr + 1); char* data = reinterpret_cast(_mem_pool->allocate(storage_field_bytes)); memory_copy(data, storage_ptr + sizeof(StringLengthType) + null_byte, storage_field_bytes); slice->data = data; @@ -217,7 +217,7 @@ OLAPStatus MemIndex::load_segment(const char* file, size_t *current_num_rows_per storage_ptr += storage_row_bytes; } } else if ((*_fields)[i].type == OLAP_FIELD_TYPE_CHAR) { - mem_field_offset += sizeof(StringSlice) + 1; + mem_field_offset += sizeof(Slice) + 1; size_t storage_field_bytes = (*_fields)[i].index_length; for (size_t j = 0; j < num_entries; ++j) { /* @@ -232,7 +232,7 @@ OLAPStatus MemIndex::load_segment(const char* file, size_t *current_num_rows_per memory_copy(mem_ptr, storage_ptr, null_byte); // 2. copy length and content - StringSlice* slice = reinterpret_cast(mem_ptr + 1); + Slice* slice = reinterpret_cast(mem_ptr + 1); char* data = reinterpret_cast(_mem_pool->allocate(storage_field_bytes)); memory_copy(data, storage_ptr + null_byte, storage_field_bytes); slice->data = data; diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index ce89abc42b795c..a51c7950f66758 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -1060,7 +1060,7 @@ OLAPStatus BinaryReader::next(RowCursor* row, MemPool* mem_pool) { if (schema[i].type == OLAP_FIELD_TYPE_CHAR || schema[i].type == OLAP_FIELD_TYPE_VARCHAR || schema[i].type == OLAP_FIELD_TYPE_HLL) { - StringSlice slice(_row_buf + offset, field_size); + Slice slice(_row_buf + offset, field_size); row->set_field_content(i, reinterpret_cast(&slice), mem_pool); } else { row->set_field_content(i, _row_buf + offset, mem_pool); @@ -1183,7 +1183,7 @@ OLAPStatus LzoBinaryReader::next(RowCursor* row, MemPool* mem_pool) { if (schema[i].type == OLAP_FIELD_TYPE_CHAR || schema[i].type == OLAP_FIELD_TYPE_VARCHAR || schema[i].type == OLAP_FIELD_TYPE_HLL) { - StringSlice slice(_row_buf + _next_row_start + offset, field_size); + Slice slice(_row_buf + _next_row_start + offset, field_size); row->set_field_content(i, reinterpret_cast(&slice), mem_pool); } else { row->set_field_content(i, _row_buf + _next_row_start + offset, mem_pool); diff --git a/be/src/olap/row_block.cpp b/be/src/olap/row_block.cpp index 614041b60c2ca4..eddef5e136424b 100644 --- a/be/src/olap/row_block.cpp +++ b/be/src/olap/row_block.cpp @@ -141,7 +141,7 @@ void RowBlock::_convert_storage_to_memory() { size_t storage_field_bytes = *(StringLengthType*)(value_ptr + null_byte); value_ptr += sizeof(StringLengthType); - StringSlice* slice = reinterpret_cast(memory_ptr + 1); + Slice* slice = reinterpret_cast(memory_ptr + 1); slice->data = value_ptr + null_byte; slice->size = storage_field_bytes; @@ -163,7 +163,7 @@ void RowBlock::_convert_storage_to_memory() { memory_copy(memory_ptr, storage_ptr, null_byte); // 2. copy length and content - StringSlice* slice = reinterpret_cast(memory_ptr + 1); + Slice* slice = reinterpret_cast(memory_ptr + 1); slice->data = storage_ptr + null_byte; slice->size = storage_field_bytes; @@ -215,7 +215,7 @@ void RowBlock::_convert_memory_to_storage(uint32_t num_rows) { storage_variable_ptr += null_byte; // 3. copy length and content - StringSlice* slice = reinterpret_cast(memory_ptr + 1); + Slice* slice = reinterpret_cast(memory_ptr + 1); *reinterpret_cast(storage_variable_ptr) = slice->size; storage_variable_ptr += sizeof(StringLengthType); memory_copy(storage_variable_ptr, slice->data, slice->size); @@ -238,7 +238,7 @@ void RowBlock::_convert_memory_to_storage(uint32_t num_rows) { memory_copy(storage_ptr, memory_ptr, null_byte); // 2. copy content - StringSlice* slice = reinterpret_cast(memory_ptr + 1); + Slice* slice = reinterpret_cast(memory_ptr + 1); memory_copy(storage_ptr + null_byte, slice->data, slice->size); memory_ptr += _mem_row_bytes; storage_ptr += storage_field_bytes + null_byte; @@ -328,14 +328,14 @@ void RowBlock::_compute_layout() { if (has_nullbyte()) { storage_variable_bytes += sizeof(char); } - memory_size += sizeof(StringSlice) + sizeof(char); + memory_size += sizeof(Slice) + sizeof(char); } else { storage_fixed_bytes += field.length; if (has_nullbyte()) { storage_fixed_bytes += sizeof(char); } if (field.type == OLAP_FIELD_TYPE_CHAR) { - memory_size += sizeof(StringSlice) + sizeof(char); + memory_size += sizeof(Slice) + sizeof(char); } else { memory_size += field.length + sizeof(char); } diff --git a/be/src/olap/row_cursor.cpp b/be/src/olap/row_cursor.cpp index 5d395fb0cd3aca..71d7c5c862cd28 100644 --- a/be/src/olap/row_cursor.cpp +++ b/be/src/olap/row_cursor.cpp @@ -60,7 +60,7 @@ OLAPStatus RowCursor::_init(const std::vector& tablet_schema, if (type == OLAP_FIELD_TYPE_CHAR || type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_HLL) { - field_buf_lens.push_back(sizeof(StringSlice)); + field_buf_lens.push_back(sizeof(Slice)); } else { field_buf_lens.push_back(tablet_schema[i].length); } @@ -185,12 +185,12 @@ OLAPStatus RowCursor::init_scan_key(const std::vector& tablet_schema, fixed_ptr = _fixed_buf + _field_array[cid]->get_offset(); FieldType type = tablet_schema[cid].type; if (type == OLAP_FIELD_TYPE_VARCHAR) { - StringSlice* slice = reinterpret_cast(fixed_ptr + 1); + Slice* slice = reinterpret_cast(fixed_ptr + 1); slice->data = variable_ptr; slice->size = scan_keys[cid].length(); variable_ptr += scan_keys[cid].length(); } else if (type == OLAP_FIELD_TYPE_CHAR) { - StringSlice* slice = reinterpret_cast(fixed_ptr + 1); + Slice* slice = reinterpret_cast(fixed_ptr + 1); slice->data = variable_ptr; slice->size = std::max(scan_keys[cid].length(), (size_t)(tablet_schema[cid].length)); variable_ptr += slice->size; @@ -228,17 +228,17 @@ OLAPStatus RowCursor::allocate_memory_for_string_type( fixed_ptr = _fixed_buf + _field_array[cid]->get_offset(); FieldType type = tablet_schema[cid].type; if (type == OLAP_FIELD_TYPE_VARCHAR) { - StringSlice* slice = reinterpret_cast(fixed_ptr + 1); + Slice* slice = reinterpret_cast(fixed_ptr + 1); slice->data = variable_ptr; slice->size = tablet_schema[cid].length - OLAP_STRING_MAX_BYTES; variable_ptr += slice->size; } else if (type == OLAP_FIELD_TYPE_CHAR) { - StringSlice* slice = reinterpret_cast(fixed_ptr + 1); + Slice* slice = reinterpret_cast(fixed_ptr + 1); slice->data = variable_ptr; slice->size = tablet_schema[cid].length; variable_ptr += slice->size; } else if (type == OLAP_FIELD_TYPE_HLL) { - StringSlice* slice = reinterpret_cast(fixed_ptr + 1); + Slice* slice = reinterpret_cast(fixed_ptr + 1); HllContext* context = nullptr; if (mem_pool != nullptr) { char* mem = reinterpret_cast(mem_pool->allocate(sizeof(HllContext))); diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 45c8744c44f936..89ebc6e443cd12 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -289,7 +289,7 @@ bool RowBlockChanger::change_row_block( write_helper.set_not_null(i); int p = ref_block.tablet_schema()[ref_column].length - 1; - StringSlice* slice = reinterpret_cast(field_to_read->get_ptr(read_helper.get_buf())); + Slice* slice = reinterpret_cast(field_to_read->get_ptr(read_helper.get_buf())); char* buf = slice->data; while (p >= 0 && buf[p] == '\0') { p--; diff --git a/be/src/olap/segment_group.cpp b/be/src/olap/segment_group.cpp index 07ec48d71be155..a5c24c0fd7fca7 100644 --- a/be/src/olap/segment_group.cpp +++ b/be/src/olap/segment_group.cpp @@ -87,7 +87,7 @@ SegmentGroup::SegmentGroup(OLAPTable* table, Version version, VersionHash versio _short_key_length += tablet_schema[i].index_length + 1;// 1 for null byte if (tablet_schema[i].type == OLAP_FIELD_TYPE_CHAR || tablet_schema[i].type == OLAP_FIELD_TYPE_VARCHAR) { - _new_short_key_length += sizeof(StringSlice) + 1; + _new_short_key_length += sizeof(Slice) + 1; } else { _new_short_key_length += tablet_schema[i].index_length + 1; } @@ -121,7 +121,7 @@ SegmentGroup::SegmentGroup(OLAPTable* table, bool delete_flag, _short_key_length += tablet_schema[i].index_length + 1;// 1 for null byte if (tablet_schema[i].type == OLAP_FIELD_TYPE_CHAR || tablet_schema[i].type == OLAP_FIELD_TYPE_VARCHAR) { - _new_short_key_length += sizeof(StringSlice) + 1; + _new_short_key_length += sizeof(Slice) + 1; } else { _new_short_key_length += tablet_schema[i].index_length + 1; } diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 66231a8d4a01da..fe234d93c54a4e 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -28,10 +28,10 @@ #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/field_info.h" -#include "olap/string_slice.h" #include "runtime/mem_pool.h" #include "util/hash_util.hpp" #include "util/mem_util.hpp" +#include "util/slice.h" #include "util/types.h" namespace doris { @@ -708,18 +708,18 @@ struct FieldTypeTraits { template<> struct FieldTypeTraits { - typedef StringSlice CppType; + typedef Slice CppType; static const char* name() { return "char"; } static int equal(const void* left, const void* right) { - const StringSlice* l_slice = reinterpret_cast(left); - const StringSlice* r_slice = reinterpret_cast(right); + const Slice* l_slice = reinterpret_cast(left); + const Slice* r_slice = reinterpret_cast(right); return *l_slice == *r_slice; } static int cmp(const void* left, const void* right) { - const StringSlice* l_slice = reinterpret_cast(left); - const StringSlice* r_slice = reinterpret_cast(right); + const Slice* l_slice = reinterpret_cast(left); + const Slice* r_slice = reinterpret_cast(right); return l_slice->compare(*r_slice); } static OLAPStatus from_string(char* buf, const std::string& scan_key) { @@ -730,7 +730,7 @@ struct FieldTypeTraits { return OLAP_ERR_INPUT_PARAMETER_ERROR; } - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); memory_copy(slice->data, scan_key.c_str(), value_len); if (slice->size < value_len) { /* @@ -743,34 +743,34 @@ struct FieldTypeTraits { return OLAP_SUCCESS; } static std::string to_string(char* src) { - StringSlice* slice = reinterpret_cast(src); + Slice* slice = reinterpret_cast(src); return slice->to_string(); } static void copy_with_pool(char* dest, const char* src, MemPool* mem_pool) { - StringSlice* l_slice = reinterpret_cast(dest); - const StringSlice* r_slice = reinterpret_cast(src); + Slice* l_slice = reinterpret_cast(dest); + const Slice* r_slice = reinterpret_cast(src); l_slice->data = reinterpret_cast(mem_pool->allocate(r_slice->size)); memory_copy(l_slice->data, r_slice->data, r_slice->size); l_slice->size = r_slice->size; } static void copy_without_pool(char* dest, const char* src) { - StringSlice* l_slice = reinterpret_cast(dest); - const StringSlice* r_slice = reinterpret_cast(src); + Slice* l_slice = reinterpret_cast(dest); + const Slice* r_slice = reinterpret_cast(src); memory_copy(l_slice->data, r_slice->data, r_slice->size); l_slice->size = r_slice->size; } static void set_to_max(char* buf) { // this function is used by scan key, // the size may be greater than length in schema. - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); memset(slice->data, 0xff, slice->size); } static void set_to_min(char* buf) { - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); memset(slice->data, 0, slice->size); } static bool is_min(char* buf) { - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); size_t i = 0; while (i < slice->size) { if (slice->data[i] != '\0') { @@ -781,25 +781,25 @@ struct FieldTypeTraits { return true; } static uint32_t hash_code(char* data, uint32_t seed) { - StringSlice* slice = reinterpret_cast(data); + Slice* slice = reinterpret_cast(data); return HashUtil::hash(slice->data, slice->size, seed); } }; template<> struct FieldTypeTraits { - typedef StringSlice CppType; + typedef Slice CppType; static const char* name() { return "varchar"; } static int equal(const void* left, const void* right) { - const StringSlice* l_slice = reinterpret_cast(left); - const StringSlice* r_slice = reinterpret_cast(right); + const Slice* l_slice = reinterpret_cast(left); + const Slice* r_slice = reinterpret_cast(right); return *l_slice == *r_slice; } static int cmp(const void* left, const void* right) { - const StringSlice* l_slice = reinterpret_cast(left); - const StringSlice* r_slice = reinterpret_cast(right); + const Slice* l_slice = reinterpret_cast(left); + const Slice* r_slice = reinterpret_cast(right); return l_slice->compare(*r_slice); } static OLAPStatus from_string(char* buf, const std::string& scan_key) { @@ -810,44 +810,44 @@ struct FieldTypeTraits { return OLAP_ERR_INPUT_PARAMETER_ERROR; } - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); memory_copy(slice->data, scan_key.c_str(), value_len); slice->size = value_len; return OLAP_SUCCESS; } static std::string to_string(char* src) { - StringSlice* slice = reinterpret_cast(src); + Slice* slice = reinterpret_cast(src); return slice->to_string(); } static void copy_with_pool(char* dest, const char* src, MemPool* mem_pool) { - StringSlice* l_slice = reinterpret_cast(dest); - const StringSlice* r_slice = reinterpret_cast(src); + Slice* l_slice = reinterpret_cast(dest); + const Slice* r_slice = reinterpret_cast(src); l_slice->data = reinterpret_cast(mem_pool->allocate(r_slice->size)); memory_copy(l_slice->data, r_slice->data, r_slice->size); l_slice->size = r_slice->size; } static void copy_without_pool(char* dest, const char* src) { - StringSlice* l_slice = reinterpret_cast(dest); - const StringSlice* r_slice = reinterpret_cast(src); + Slice* l_slice = reinterpret_cast(dest); + const Slice* r_slice = reinterpret_cast(src); memory_copy(l_slice->data, r_slice->data, r_slice->size); l_slice->size = r_slice->size; } static void set_to_max(char* buf) { - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); slice->size = 1; memset(slice->data, 0xFF, 1); } static void set_to_min(char* buf) { - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); slice->size = 0; } static bool is_min(char* buf) { - StringSlice* slice = reinterpret_cast(buf); + Slice* slice = reinterpret_cast(buf); return (slice->size == 0); } static uint32_t hash_code(char* data, uint32_t seed) { - StringSlice* slice = reinterpret_cast(data); + Slice* slice = reinterpret_cast(data); return HashUtil::hash(slice->data, slice->size, seed); } }; diff --git a/be/src/olap/wrapper_field.cpp b/be/src/olap/wrapper_field.cpp index 559dd6d34fabd7..02ab4363dd82d5 100644 --- a/be/src/olap/wrapper_field.cpp +++ b/be/src/olap/wrapper_field.cpp @@ -68,7 +68,7 @@ WrapperField::WrapperField(Field* rep, size_t variable_len, bool is_string_type) _buf = _field_buf + 1; if (_is_string_type) { - StringSlice* slice = reinterpret_cast(_buf); + Slice* slice = reinterpret_cast(_buf); slice->size = variable_len; slice->data = _buf + fixed_len; } diff --git a/be/src/runtime/vectorized_row_batch.cpp b/be/src/runtime/vectorized_row_batch.cpp index f38009d3db55bc..2631795cf89b9a 100644 --- a/be/src/runtime/vectorized_row_batch.cpp +++ b/be/src/runtime/vectorized_row_batch.cpp @@ -54,7 +54,7 @@ void VectorizedRowBatch::dump_to_row_block(RowBlock* row_block) { if (field_info.type == OLAP_FIELD_TYPE_CHAR || field_info.type == OLAP_FIELD_TYPE_VARCHAR || field_info.type == OLAP_FIELD_TYPE_HLL) { - field_size = sizeof(StringSlice); + field_size = sizeof(Slice); } else { field_size = field_info.length; } @@ -98,7 +98,7 @@ void VectorizedRowBatch::dump_to_row_block(RowBlock* row_block) { if (field_info.type == OLAP_FIELD_TYPE_CHAR || field_info.type == OLAP_FIELD_TYPE_VARCHAR || field_info.type == OLAP_FIELD_TYPE_HLL) { - field_size = sizeof(StringSlice); + field_size = sizeof(Slice); } else { field_size = field_info.length; } diff --git a/be/src/olap/string_slice.h b/be/src/util/slice.h similarity index 76% rename from be/src/olap/string_slice.h rename to be/src/util/slice.h index 2907fcf5e154a7..85176f62d297e8 100644 --- a/be/src/olap/string_slice.h +++ b/be/src/util/slice.h @@ -31,35 +31,44 @@ namespace doris { /// @brief A wrapper around externally allocated data. /// -/// StringSlice is a simple structure containing a pointer into some external -/// storage and a size. The user of a StringSlice must ensure that the slice +/// Slice is a simple structure containing a pointer into some external +/// storage and a size. The user of a Slice must ensure that the slice /// is not used after the corresponding external storage has been /// deallocated. /// -/// Multiple threads can invoke const methods on a StringSlice without +/// Multiple threads can invoke const methods on a Slice without /// external synchronization, but if any of the threads may call a -/// non-const method, all threads accessing the same StringSlice must use +/// non-const method, all threads accessing the same Slice must use /// external synchronization. -struct StringSlice { +struct Slice { public: char* data; size_t size; // Intentionally copyable /// Create an empty slice. - StringSlice() : data(const_cast("")), size(0) { } + Slice() : data(const_cast("")), size(0) { } /// Create a slice that refers to a @c char byte array. - StringSlice(const char* d, size_t n) : + Slice(const char* d, size_t n) : data(const_cast(d)), size(n) { } + + // Create a slice that refers to a @c uint8_t byte array. + // + // @param [in] d + // The input array. + // @param [in] n + // Number of bytes in the array. + Slice(const uint8_t* s, size_t n) : + data(const_cast(reinterpret_cast(s))), size(n) { } /// Create a slice that refers to the contents of the given string. - StringSlice(const std::string& s) : // NOLINT(runtime/explicit) + Slice(const std::string& s) : // NOLINT(runtime/explicit) data(const_cast(s.data())), size(s.size()) { } /// Create a slice that refers to a C-string s[0,strlen(s)-1]. - StringSlice(const char* s) : // NOLINT(runtime/explicit) + Slice(const char* s) : // NOLINT(runtime/explicit) data(const_cast(s)), size(strlen(s)) { } /* @@ -121,24 +130,24 @@ struct StringSlice { std::string to_string() const { return std::string(data, size); } /// Do a three-way comparison of the slice's data. - int compare(const StringSlice& b) const; + int compare(const Slice& b) const; /// Check whether the slice starts with the given prefix. - bool starts_with(const StringSlice& x) const { + bool starts_with(const Slice& x) const { return ((size >= x.size) && (mem_equal(data, x.data, x.size))); } /// @brief Comparator struct, useful for ordered collections (like STL maps). struct Comparator { - /// Compare two slices using StringSlice::compare() + /// Compare two slices using Slice::compare() /// /// @param [in] a - /// The slice to call StringSlice::compare() at. + /// The slice to call Slice::compare() at. /// @param [in] b - /// The slice to use as a parameter for StringSlice::compare(). - /// @return @c true iff @c a is less than @c b by StringSlice::compare(). - bool operator()(const StringSlice& a, const StringSlice& b) const { + /// The slice to use as a parameter for Slice::compare(). + /// @return @c true iff @c a is less than @c b by Slice::compare(). + bool operator()(const Slice& a, const Slice& b) const { return a.compare(b) < 0; } }; @@ -156,7 +165,7 @@ struct StringSlice { } } - friend bool operator==(const StringSlice& x, const StringSlice& y); + friend bool operator==(const Slice& x, const Slice& y); static bool mem_equal(const void* a, const void* b, size_t n) { return memcmp(a, b, n) == 0; @@ -169,17 +178,17 @@ struct StringSlice { }; /// Check whether two slices are identical. -inline bool operator==(const StringSlice& x, const StringSlice& y) { +inline bool operator==(const Slice& x, const Slice& y) { return ((x.size == y.size) && - (StringSlice::mem_equal(x.data, y.data, x.size))); + (Slice::mem_equal(x.data, y.data, x.size))); } /// Check whether two slices are not identical. -inline bool operator!=(const StringSlice& x, const StringSlice& y) { +inline bool operator!=(const Slice& x, const Slice& y) { return !(x == y); } -inline int StringSlice::compare(const StringSlice& b) const { +inline int Slice::compare(const Slice& b) const { const int min_len = (size < b.size) ? size : b.size; int r = mem_compare(data, b.data, min_len); if (r == 0) { @@ -189,11 +198,11 @@ inline int StringSlice::compare(const StringSlice& b) const { return r; } -/// @brief STL map whose keys are StringSlices. +/// @brief STL map whose keys are Slices. /// /// An example of usage: /// @code -/// typedef StringSliceMap::type MySliceMap; +/// typedef SliceMap::type MySliceMap; /// /// MySliceMap my_map; /// my_map.insert(MySliceMap::value_type(a, 1)); @@ -205,9 +214,9 @@ inline int StringSlice::compare(const StringSlice& b) const { /// } /// @endcode template -struct StringSliceMap { +struct SliceMap { /// A handy typedef for the slice map with appropriate comparison operator. - typedef std::map type; + typedef std::map type; }; } // namespace doris diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp index 88fa0b08056df3..fb99929cdfa630 100644 --- a/be/test/olap/column_reader_test.cpp +++ b/be/test/olap/column_reader_test.cpp @@ -2479,7 +2479,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 5, _mem_pool.get()), OLAP_SUCCESS); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); for (uint32_t i = 0; i < 2; i++) { value++; @@ -2548,7 +2548,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { ASSERT_EQ(is_null[0], true); ASSERT_EQ(is_null[1], false); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); value++; ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); } @@ -2609,7 +2609,7 @@ TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); } @@ -2684,14 +2684,14 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); - value = reinterpret_cast(_col_vector->col_data()); + value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); } @@ -2766,14 +2766,14 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); ASSERT_EQ(_column_reader->seek(&position1), OLAP_SUCCESS); _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); - value = reinterpret_cast(_col_vector->col_data()); + value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); } @@ -2836,7 +2836,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 5, _mem_pool.get()), OLAP_SUCCESS); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "abcde", value->size) == 0); for (uint32_t i = 0; i < 2; i++) { @@ -2904,7 +2904,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithPresent) { ASSERT_EQ(is_null[0], true); ASSERT_EQ(is_null[1], false); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); value++; ASSERT_TRUE(strncmp(value->data, "abcde", value->size) == 0); } @@ -2986,7 +2986,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 5, _mem_pool.get()), OLAP_SUCCESS); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); ASSERT_TRUE(strncmp(value->data, "abcde", value->size) == 0); @@ -3058,7 +3058,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { _col_vector.reset(new ColumnVector()); ASSERT_EQ(_column_reader->next_vector( _col_vector.get(), 3, _mem_pool.get()), OLAP_SUCCESS); - StringSlice* value = reinterpret_cast(_col_vector->col_data()); + Slice* value = reinterpret_cast(_col_vector->col_data()); for (uint32_t i = 0; i < 65533; i++) { ASSERT_TRUE(strncmp(value->data + i, "a", 1) == 0); diff --git a/be/test/olap/row_block_test.cpp b/be/test/olap/row_block_test.cpp index 04961dbcd59655..15d3316e1657b1 100644 --- a/be/test/olap/row_block_test.cpp +++ b/be/test/olap/row_block_test.cpp @@ -163,7 +163,7 @@ TEST_F(TestRowBlock, write_and_read) { { char buf[10]; memset(buf, 'a' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_not_null(1); row.set_field_content(1, (const char*)&val, block.mem_pool()); } @@ -171,7 +171,7 @@ TEST_F(TestRowBlock, write_and_read) { { char buf[10]; memset(buf, '0' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_not_null(2); row.set_field_content(2, (const char*)&val, block.mem_pool()); } @@ -203,7 +203,7 @@ TEST_F(TestRowBlock, write_and_read) { } { ASSERT_FALSE(row.is_null(1)); - StringSlice* slice = (StringSlice*)row.get_field_content_ptr(1); + Slice* slice = (Slice*)row.get_field_content_ptr(1); char buf[10]; memset(buf, 'a' + i, 10); ASSERT_EQ(10, slice->size); @@ -211,7 +211,7 @@ TEST_F(TestRowBlock, write_and_read) { } { ASSERT_FALSE(row.is_null(2)); - StringSlice* slice = (StringSlice*)row.get_field_content_ptr(2); + Slice* slice = (Slice*)row.get_field_content_ptr(2); char buf[20]; memset(buf, '0' + i, 10); ASSERT_EQ(10, slice->size); @@ -279,7 +279,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { { char buf[10]; memset(buf, 'a' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_not_null(1); row.set_field_content(1, (const char*)&val, block.mem_pool()); } @@ -287,7 +287,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { { char buf[10]; memset(buf, '0' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_not_null(2); row.set_field_content(2, (const char*)&val, block.mem_pool()); } @@ -319,7 +319,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { } { ASSERT_FALSE(row.is_null(1)); - StringSlice* slice = (StringSlice*)row.get_field_content_ptr(1); + Slice* slice = (Slice*)row.get_field_content_ptr(1); char buf[10]; memset(buf, 'a' + i, 10); ASSERT_EQ(10, slice->size); @@ -327,7 +327,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { } { ASSERT_FALSE(row.is_null(2)); - StringSlice* slice = (StringSlice*)row.get_field_content_ptr(2); + Slice* slice = (Slice*)row.get_field_content_ptr(2); char buf[20]; memset(buf, '0' + i, 10); ASSERT_EQ(10, slice->size); @@ -394,14 +394,14 @@ TEST_F(TestRowBlock, compress_failed) { { char buf[10]; memset(buf, 'a' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_field_content(1, (const char*)&val, block.mem_pool()); } // varchar { char buf[10]; memset(buf, '0' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_field_content(2, (const char*)&val, block.mem_pool()); } } @@ -471,14 +471,14 @@ TEST_F(TestRowBlock, decompress_failed) { { char buf[10]; memset(buf, 'a' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_field_content(1, (const char*)&val, block.mem_pool()); } // varchar { char buf[10]; memset(buf, '0' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_field_content(2, (const char*)&val, block.mem_pool()); } } @@ -572,7 +572,7 @@ TEST_F(TestRowBlock, find_row) { { char buf[10]; memset(buf, 'a' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_not_null(1); row.set_field_content(1, (const char*)&val, block.mem_pool()); } @@ -580,7 +580,7 @@ TEST_F(TestRowBlock, find_row) { { char buf[10]; memset(buf, '0' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); row.set_not_null(2); row.set_field_content(2, (const char*)&val, block.mem_pool()); } @@ -602,7 +602,7 @@ TEST_F(TestRowBlock, find_row) { { char buf[10]; memset(buf, 'a' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); find_row.set_not_null(1); find_row.set_field_content(1, (const char*)&val, block.mem_pool()); } @@ -610,7 +610,7 @@ TEST_F(TestRowBlock, find_row) { { char buf[10]; memset(buf, '0' + i, 10); - StringSlice val(buf, 10); + Slice val(buf, 10); find_row.set_not_null(2); find_row.set_field_content(2, (const char*)&val, block.mem_pool()); } @@ -633,14 +633,14 @@ TEST_F(TestRowBlock, find_row) { { char buf[10]; memset(buf, 'c', 9); - StringSlice val(buf, 9); + Slice val(buf, 9); find_row.set_field_content(1, (const char*)&val, block.mem_pool()); } // varchar { char buf[10]; memset(buf, '0', 10); - StringSlice val(buf, 10); + Slice val(buf, 10); find_row.set_field_content(2, (const char*)&val, block.mem_pool()); } uint32_t row_index; @@ -658,14 +658,14 @@ TEST_F(TestRowBlock, find_row) { { char buf[10]; memset(buf, 'c', 9); - StringSlice val(buf, 9); + Slice val(buf, 9); find_row.set_field_content(1, (const char*)&val, block.mem_pool()); } // varchar { char buf[10]; memset(buf, '0', 10); - StringSlice val(buf, 10); + Slice val(buf, 10); find_row.set_field_content(2, (const char*)&val, block.mem_pool()); } uint32_t row_index; diff --git a/be/test/olap/row_cursor_test.cpp b/be/test/olap/row_cursor_test.cpp index 62e0bb941e5503..604126d0f0cd05 100644 --- a/be/test/olap/row_cursor_test.cpp +++ b/be/test/olap/row_cursor_test.cpp @@ -341,7 +341,7 @@ TEST_F(TestRowCursor, EqualAndCompare) { ASSERT_EQ(left.get_fixed_len(), 78); ASSERT_EQ(left.get_variable_len(), 20); - StringSlice l_char("well"); + Slice l_char("well"); int32_t l_int = 10; left.set_field_content(0, reinterpret_cast(&l_char), _mem_pool.get()); left.set_field_content(1, reinterpret_cast(&l_int), _mem_pool.get()); @@ -382,14 +382,14 @@ TEST_F(TestRowCursor, IndexCmp) { ASSERT_EQ(left.get_fixed_len(), 78); ASSERT_EQ(left.get_variable_len(), 20); - StringSlice l_char("well"); + Slice l_char("well"); int32_t l_int = 10; left.set_field_content(0, reinterpret_cast(&l_char), _mem_pool.get()); left.set_field_content(1, reinterpret_cast(&l_int), _mem_pool.get()); RowCursor right_eq; res = right_eq.init(tablet_schema); - StringSlice r_char_eq("well"); + Slice r_char_eq("well"); int32_t r_int_eq = 10; right_eq.set_field_content(0, reinterpret_cast(&r_char_eq), _mem_pool.get()); right_eq.set_field_content(1, reinterpret_cast(&r_int_eq), _mem_pool.get()); @@ -398,7 +398,7 @@ TEST_F(TestRowCursor, IndexCmp) { RowCursor right_lt; res = right_lt.init(tablet_schema); - StringSlice r_char_lt("well"); + Slice r_char_lt("well"); int32_t r_int_lt = 11; right_lt.set_field_content(0, reinterpret_cast(&r_char_lt), _mem_pool.get()); right_lt.set_field_content(1, reinterpret_cast(&r_int_lt), _mem_pool.get()); @@ -406,7 +406,7 @@ TEST_F(TestRowCursor, IndexCmp) { RowCursor right_gt; res = right_gt.init(tablet_schema); - StringSlice r_char_gt("good"); + Slice r_char_gt("good"); int32_t r_int_gt = 10; right_gt.set_field_content(0, reinterpret_cast(&r_char_gt), _mem_pool.get()); right_gt.set_field_content(1, reinterpret_cast(&r_int_gt), _mem_pool.get()); @@ -423,14 +423,14 @@ TEST_F(TestRowCursor, FullKeyCmp) { ASSERT_EQ(left.get_fixed_len(), 78); ASSERT_EQ(left.get_variable_len(), 20); - StringSlice l_char("well"); + Slice l_char("well"); int32_t l_int = 10; left.set_field_content(0, reinterpret_cast(&l_char), _mem_pool.get()); left.set_field_content(1, reinterpret_cast(&l_int), _mem_pool.get()); RowCursor right_eq; res = right_eq.init(tablet_schema); - StringSlice r_char_eq("well"); + Slice r_char_eq("well"); int32_t r_int_eq = 10; right_eq.set_field_content(0, reinterpret_cast(&r_char_eq), _mem_pool.get()); right_eq.set_field_content(1, reinterpret_cast(&r_int_eq), _mem_pool.get()); @@ -438,7 +438,7 @@ TEST_F(TestRowCursor, FullKeyCmp) { RowCursor right_lt; res = right_lt.init(tablet_schema); - StringSlice r_char_lt("well"); + Slice r_char_lt("well"); int32_t r_int_lt = 11; right_lt.set_field_content(0, reinterpret_cast(&r_char_lt), _mem_pool.get()); right_lt.set_field_content(1, reinterpret_cast(&r_int_lt), _mem_pool.get()); @@ -446,7 +446,7 @@ TEST_F(TestRowCursor, FullKeyCmp) { RowCursor right_gt; res = right_gt.init(tablet_schema); - StringSlice r_char_gt("good"); + Slice r_char_gt("good"); int32_t r_int_gt = 10; right_gt.set_field_content(0, reinterpret_cast(&r_char_gt), _mem_pool.get()); right_gt.set_field_content(1, reinterpret_cast(&r_int_gt), _mem_pool.get()); @@ -467,12 +467,12 @@ TEST_F(TestRowCursor, AggregateWithoutNull) { RowCursor left; res = left.init(tablet_schema); - StringSlice l_char("well"); + Slice l_char("well"); int32_t l_int = 10; int128_t l_largeint = (int128_t)(1) << 100; double l_double = 8.8; decimal12_t l_decimal(11, 22); - StringSlice l_varchar("beijing"); + Slice l_varchar("beijing"); left.set_field_content(0, reinterpret_cast(&l_char), _mem_pool.get()); left.set_field_content(1, reinterpret_cast(&l_int), _mem_pool.get()); left.set_field_content(2, reinterpret_cast(&l_largeint), _mem_pool.get()); @@ -485,12 +485,12 @@ TEST_F(TestRowCursor, AggregateWithoutNull) { RowCursor right; res = right.init(tablet_schema); - StringSlice r_char("well"); + Slice r_char("well"); int32_t r_int = 10; int128_t r_largeint = (int128_t)(1) << 100; double r_double = 5.5; decimal12_t r_decimal(22, 22); - StringSlice r_varchar("shenzhen"); + Slice r_varchar("shenzhen"); right.set_field_content(0, reinterpret_cast(&r_char), _mem_pool.get()); right.set_field_content(1, reinterpret_cast(&r_int), _mem_pool.get()); right.set_field_content(2, reinterpret_cast(&r_largeint), _mem_pool.get()); @@ -509,7 +509,7 @@ TEST_F(TestRowCursor, AggregateWithoutNull) { decimal12_t agg_decimal = *reinterpret_cast(row.get_field_content_ptr(4)); ASSERT_TRUE(agg_decimal == r_decimal); - StringSlice* agg_varchar = reinterpret_cast(row.get_field_content_ptr(5)); + Slice* agg_varchar = reinterpret_cast(row.get_field_content_ptr(5)); ASSERT_EQ(agg_varchar->compare(r_varchar), 0); } @@ -527,10 +527,10 @@ TEST_F(TestRowCursor, AggregateWithNull) { RowCursor left; res = left.init(tablet_schema); - StringSlice l_char("well"); + Slice l_char("well"); int32_t l_int = 10; int128_t l_largeint = (int128_t)(1) << 100; - StringSlice l_varchar("beijing"); + Slice l_varchar("beijing"); left.set_field_content(0, reinterpret_cast(&l_char), _mem_pool.get()); left.set_field_content(1, reinterpret_cast(&l_int), _mem_pool.get()); left.set_field_content(2, reinterpret_cast(&l_largeint), _mem_pool.get()); @@ -543,7 +543,7 @@ TEST_F(TestRowCursor, AggregateWithNull) { RowCursor right; res = right.init(tablet_schema); - StringSlice r_char("well"); + Slice r_char("well"); int32_t r_int = 10; int128_t r_largeint = (int128_t)(1) << 100; double r_double = 5.5;