Optimize agg with collation & Optimize mem utils

CMakeLists.txt

@@ -511,6 +511,11 @@ if (ARCH_AMD64)
  else()
  add_definitions(-DTIFLASH_COMPILER_VPCLMULQDQ_SUPPORT=0)
  endif()
+
+ check_cxx_compiler_flag("-mmovbe" TIFLASH_COMPILER_MOVBE_SUPPORT)
+ if (TIFLASH_COMPILER_MOVBE_SUPPORT)
+ set(COMPILER_MOVBE_FLAG "-mmovbe")
+ endif()
 else()
  add_definitions(-DTIFLASH_COMPILER_VPCLMULQDQ_SUPPORT=0)
 endif()

dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h

@@ -193,36 +193,36 @@ struct SingleValueDataString
 
  Int32 size = -1; /// -1 indicates that there is no value.
  Int32 capacity = 0; /// power of two or zero
- char * large_data;
- TiDB::TiDBCollatorPtr collator = nullptr;
+ char * large_data{};
+ TiDB::TiDBCollatorPtr collator{};
 
  bool less(const StringRef & a, const StringRef & b) const
  {
- if (collator == nullptr)
+ if (unlikely(collator == nullptr))
  return a < b;
- return collator->compare(a.data, a.size, b.data, b.size) < 0;
+ return collator->compareFastPath(a.data, a.size, b.data, b.size) < 0;
  }
 
  bool greater(const StringRef & a, const StringRef & b) const
  {
- if (collator == nullptr)
+ if (unlikely(collator == nullptr))
  return a > b;
- return collator->compare(a.data, a.size, b.data, b.size) > 0;
+ return collator->compareFastPath(a.data, a.size, b.data, b.size) > 0;
  }
 
  bool equalTo(const StringRef & a, const StringRef & b) const
  {
- if (collator == nullptr)
+ if (unlikely(collator == nullptr))
  return a == b;
- return collator->compare(a.data, a.size, b.data, b.size) == 0;
+ return collator->compareFastPath(a.data, a.size, b.data, b.size) == 0;
  }
 
 public:
  static constexpr Int32 AUTOMATIC_STORAGE_SIZE = 64;
  static constexpr Int32 MAX_SMALL_STRING_SIZE = AUTOMATIC_STORAGE_SIZE - sizeof(size) - sizeof(capacity) - sizeof(large_data) - sizeof(collator);
 
 private:
- char small_data[MAX_SMALL_STRING_SIZE]; /// Including the terminating zero.
+ char small_data[MAX_SMALL_STRING_SIZE]{}; /// Including the terminating zero.
 
 public:
  bool has() const

dbms/src/AggregateFunctions/IAggregateFunction.h

@@ -439,7 +439,7 @@ struct AggregationCollatorsWrapper<true>
  if (likely(collators.size() > column_index))
  {
  if (collators[column_index] != nullptr)
- return collators[column_index]->sortKey(in.data, in.size, sort_key_containers[column_index]);
+ return collators[column_index]->sortKeyFastPath(in.data, in.size, sort_key_containers[column_index]);
  return in;
  }
  else if (collators.empty())

dbms/src/Columns/ColumnString.cpp

@@ -18,6 +18,7 @@
 #include <Common/HashTable/Hash.h>
 #include <DataStreams/ColumnGathererStream.h>
 #include <Storages/Transaction/CollatorUtils.h>
+#include <common/memcpy.h>
 #include <fmt/core.h>
 
 
@@ -94,7 +95,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len
 
  size_t old_chars_size = chars.size();
  chars.resize(old_chars_size + nested_length);
- memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset], nested_length);
+ inline_memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset], nested_length);
 
  if (start == 0 && offsets.empty())
  {
@@ -315,12 +316,10 @@ int ColumnString::compareAtWithCollationImpl(size_t n, size_t m, const IColumn &
 {
  const auto & rhs = static_cast<const ColumnString &>(rhs_);
 
- return collator.compare(
- reinterpret_cast<const char *>(&chars[offsetAt(n)]),
- sizeAt(n) - 1, // Skip last zero byte.
- reinterpret_cast<const char *>(&rhs.chars[rhs.offsetAt(m)]),
- rhs.sizeAt(m) - 1 // Skip last zero byte.
- );
+ auto a = getDataAt(n);
+ auto b = rhs.getDataAt(m);
+
+ return collator.compare(a.data, a.size, b.data, b.size);
 }
 
 // Derived must implement function `int compare(const char *, size_t, const char *, size_t)`.

dbms/src/Columns/ColumnString.h

@@ -211,10 +211,10 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
 
  StringRef res;
 
- if (collator != nullptr)
+ if (likely(collator != nullptr))
  {
  // Skip last zero byte.
- auto sort_key = collator->sortKey(reinterpret_cast<const char *>(src), string_size - 1, sort_key_container);
+ auto sort_key = collator->sortKeyFastPath(reinterpret_cast<const char *>(src), string_size - 1, sort_key_container);
  string_size = sort_key.size;
  src = sort_key.data;
  }
@@ -244,10 +244,10 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
  {
  size_t string_size = sizeAt(n);
  size_t offset = offsetAt(n);
- if (collator != nullptr)
+ if (likely(collator != nullptr))
  {
  // Skip last zero byte.
- auto sort_key = collator->sortKey(reinterpret_cast<const char *>(&chars[offset]), string_size - 1, sort_key_container);
+ auto sort_key = collator->sortKeyFastPath(reinterpret_cast<const char *>(&chars[offset]), string_size - 1, sort_key_container);
  string_size = sort_key.size;
  hash.update(reinterpret_cast<const char *>(&string_size), sizeof(string_size));
  hash.update(sort_key.data, sort_key.size);
@@ -278,16 +278,7 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
  int compareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override
  {
  const auto & rhs = static_cast<const ColumnString &>(rhs_);
-
- const size_t size = sizeAt(n);
- const size_t rhs_size = rhs.sizeAt(m);
-
- int cmp = memcmp(&chars[offsetAt(n)], &rhs.chars[rhs.offsetAt(m)], std::min(size, rhs_size));
-
- if (cmp != 0)
- return cmp;
- else
- return size > rhs_size ? 1 : (size < rhs_size ? -1 : 0);
+ return getDataAtWithTerminatingZero(n).compare(rhs.getDataAtWithTerminatingZero(m));
  }
 
  int compareAt(size_t n, size_t m, const IColumn & rhs_, int, const ICollator & collator) const override

dbms/src/Columns/ColumnsCommon.cpp

@@ -12,13 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#if __SSE2__
-#include <emmintrin.h>
-#endif
-
 #include <Columns/ColumnsCommon.h>
 #include <Columns/IColumn.h>
-
+#include <common/memcpy.h>
 
 namespace DB
 {
@@ -146,7 +142,7 @@ struct ResultOffsetsBuilder
  {
  const auto offsets_size_old = res_offsets.size();
  res_offsets.resize(offsets_size_old + SIMD_BYTES);
- memcpy(&res_offsets[offsets_size_old], src_offsets_pos, SIMD_BYTES * sizeof(IColumn::Offset));
+ inline_memcpy(&res_offsets[offsets_size_old], src_offsets_pos, SIMD_BYTES * sizeof(IColumn::Offset));
 
  if (!first)
  {
@@ -194,7 +190,7 @@ void filterArraysImplGeneric(
 {
  const size_t size = src_offsets.size();
  if (size != filt.size())
- throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
+ throw Exception(fmt::format("size of filter {} doesn't match size of column {}", filt.size(), size), ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
  ResultOffsetsBuilder result_offsets_builder(res_offsets);
 
@@ -223,7 +219,7 @@ void filterArraysImplGeneric(
 
  const auto elems_size_old = res_elems.size();
  res_elems.resize(elems_size_old + size);
- memcpy(&res_elems[elems_size_old], &src_elems[offset], size * sizeof(T));
+ inline_memcpy(&res_elems[elems_size_old], &src_elems[offset], size * sizeof(T));
  };
 
 #if __SSE2__
@@ -233,7 +229,7 @@ void filterArraysImplGeneric(
 
  while (filt_pos < filt_end_aligned)
  {
- const auto mask = _mm_movemask_epi8(_mm_cmpgt_epi8(
+ uint32_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(
  _mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)),
  zero_vec));
 
@@ -254,13 +250,16 @@ void filterArraysImplGeneric(
  /// copy elements for SIMD_BYTES arrays at once
  const auto elems_size_old = res_elems.size();
  res_elems.resize(elems_size_old + chunk_size);
- memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T));
+ inline_memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T));
  }
  else
  {
- for (size_t i = 0; i < SIMD_BYTES; ++i)
- if (filt_pos[i])
- copy_array(offsets_pos + i);
+ while (mask)
+ {
+ size_t index = __builtin_ctz(mask);
+ copy_array(offsets_pos + index);
+ mask = mask & (mask - 1);
+ }
  }
 
  filt_pos += SIMD_BYTES;