Merge branch 'master' into support-lightweight-algo

.github/licenserc.yml

@@ -39,6 +39,7 @@ header:
  - 'Cargo.lock'
  - 'Cargo.toml'
  - 'rust-toolchain'
+ - 'rust-toolchain.toml'
  - '.devcontainer/'
  - '**/OWNERS'
  - 'OWNERS_ALIASES'

contrib/tiflash-proxy-cmake/CMakeLists.txt

@@ -87,7 +87,7 @@ add_custom_command(OUTPUT ${_TIFLASH_PROXY_LIBRARY}
  WORKING_DIRECTORY ${_TIFLASH_PROXY_SOURCE_DIR}
  DEPENDS "${_TIFLASH_PROXY_SRCS}"
  "${_TIFLASH_PROXY_SOURCE_DIR}/Cargo.lock"
- "${TiFlash_SOURCE_DIR}/rust-toolchain")
+ "${TiFlash_SOURCE_DIR}/rust-toolchain.toml")
 
 add_custom_target(tiflash_proxy ALL DEPENDS ${_TIFLASH_PROXY_LIBRARY})
 add_library(libtiflash_proxy SHARED IMPORTED GLOBAL)

dbms/src/Common/ArrayCache.h

@@ -378,8 +378,6 @@ class ArrayCache : private boost::noncopyable
  {
  --left_it;
 
- //std::cerr << "left_it->isFree(): " << left_it->isFree() << "\n";
-
  if (left_it->chunk == region.chunk && left_it->isFree())
  {
  region.size += left_it->size;
@@ -392,8 +390,6 @@ class ArrayCache : private boost::noncopyable
  ++right_it;
  if (right_it != adjacency_list.end())
  {
- //std::cerr << "right_it->isFree(): " << right_it->isFree() << "\n";
-
  if (right_it->chunk == region.chunk && right_it->isFree())
  {
  region.size += right_it->size;
@@ -402,8 +398,6 @@ class ArrayCache : private boost::noncopyable
  }
  }
 
- //std::cerr << "size is enlarged: " << was_size << " -> " << region.size << "\n";
-
  size_multimap.insert(region);
  }
 
@@ -433,10 +427,6 @@ class ArrayCache : private boost::noncopyable
  if (lru_list.empty())
  return nullptr;
 
- /*for (const auto & elem : adjacency_list)
- std::cerr << (!elem.SizeMultimapHook::is_linked() ? "\033[1m" : "") << elem.size << (!elem.SizeMultimapHook::is_linked() ? "\033[0m " : " ");
- std::cerr << '\n';*/
-
  auto it = adjacency_list.iterator_to(lru_list.front());
 
  while (true)
@@ -544,8 +534,6 @@ class ArrayCache : private boost::noncopyable
  return allocateFromFreeRegion(*free_region, size);
  }
 
- // std::cerr << "Requested size: " << size << "\n";
-
  /// Evict something from cache and continue.
  while (true)
  {

dbms/src/Common/FieldVisitors.cpp

@@ -221,57 +221,121 @@ String FieldVisitorToString::operator()(const Tuple & x_def) const
 
 String FieldVisitorToDebugString::operator()(const Null &) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return "NULL";
+ case RedactMode::Disable:
+ return "NULL";
+ case RedactMode::Marker:
+ return Redact::toMarkerString("NULL", /*ignore_escape*/ true);
+ }
 }
 String FieldVisitorToDebugString::operator()(const UInt64 & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatQuoted(x);
+ case RedactMode::Disable:
+ return formatQuoted(x);
+ case RedactMode::Marker:
+ return Redact::toMarkerString(formatQuoted(x), /*ignore_escape*/ true);
+ }
 }
 String FieldVisitorToDebugString::operator()(const Int64 & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatQuoted(x);
+ case RedactMode::Disable:
+ return formatQuoted(x);
+ case RedactMode::Marker:
+ return Redact::toMarkerString(formatQuoted(x), /*ignore_escape*/ true);
+ }
 }
 String FieldVisitorToDebugString::operator()(const Float64 & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatFloat(x);
+ case RedactMode::Disable:
+ return formatFloat(x);
+ case RedactMode::Marker:
+ return Redact::toMarkerString(formatFloat(x), /*ignore_escape*/ true);
+ }
 }
 String FieldVisitorToDebugString::operator()(const String & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatQuoted(x);
+ case RedactMode::Disable:
+ return formatQuoted(x);
+ case RedactMode::Marker:
+ // The string may contains utf-8 char that need to be escaped
+ return Redact::toMarkerString(formatQuoted(x), /*ignore_escape*/ false);
+ }
 }
 String FieldVisitorToDebugString::operator()(const DecimalField<Decimal32> & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatQuoted(x);
+ case RedactMode::Disable:
+ return formatQuoted(x);
+ case RedactMode::Marker:
+ return Redact::toMarkerString(formatQuoted(x), /*ignore_escape*/ true);
+ }
 }
 String FieldVisitorToDebugString::operator()(const DecimalField<Decimal64> & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatQuoted(x);
+ case RedactMode::Disable:
+ return formatQuoted(x);
+ case RedactMode::Marker:
+ return Redact::toMarkerString(formatQuoted(x), /*ignore_escape*/ true);
+ }
 }
 String FieldVisitorToDebugString::operator()(const DecimalField<Decimal128> & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatQuoted(x);
+ case RedactMode::Disable:
+ return formatQuoted(x);
+ case RedactMode::Marker:
+ return Redact::toMarkerString(formatQuoted(x), /*ignore_escape*/ true);
+ }
 }
 String FieldVisitorToDebugString::operator()(const DecimalField<Decimal256> & x) const
 {
- if (Redact::REDACT_LOG.load(std::memory_order_relaxed))
+ const auto v = Redact::REDACT_LOG.load(std::memory_order_relaxed);
+ switch (v)
+ {
+ case RedactMode::Enable:
  return "?";
- return formatQuoted(x);
+ case RedactMode::Disable:
+ return formatQuoted(x);
+ case RedactMode::Marker:
+ return Redact::toMarkerString(formatQuoted(x), /*ignore_escape*/ true);
+ }
 }
 
 String FieldVisitorToDebugString::operator()(const Array & x) const

dbms/src/Common/HashTable/HashTable.h

@@ -34,13 +34,6 @@
 #include <utility>
 
 
-#ifdef DBMS_HASH_MAP_DEBUG_RESIZES
-#include <Common/Stopwatch.h>
-
-#include <iomanip>
-#include <iostream>
-#endif
-
 /** NOTE HashTable could only be used for memmoveable (position independent) types.
  * Example: std::string is not position independent in libstdc++ with C++11 ABI or in libc++.
  * Also, key in hash table must be of type, that zero bytes is compared equals to zero key.
@@ -487,9 +480,6 @@ class HashTable
  if unlikely (!resize_callback())
  throw DB::ResizeException("Error in hash table resize");
  }
-#ifdef DBMS_HASH_MAP_DEBUG_RESIZES
- Stopwatch watch;
-#endif
 
  size_t old_size = grower.bufSize();
 
@@ -568,12 +558,6 @@ class HashTable
  if (&buf[i] != &buf[updated_place_value])
  Cell::move(&buf[i], &buf[updated_place_value]);
  }
-
-#ifdef DBMS_HASH_MAP_DEBUG_RESIZES
- watch.stop();
- std::cerr << std::fixed << std::setprecision(3) << "Resize from " << old_size << " to " << grower.bufSize()
- << " took " << watch.elapsedSeconds() << " sec." << std::endl;
-#endif
  }
 
 

dbms/src/Common/OptimizedRegularExpression.inl.h

@@ -277,13 +277,6 @@ void OptimizedRegularExpressionImpl<thread_safe>::analyze(
  required_substring = trivial_substrings.front().first;
  required_substring_is_prefix = trivial_substrings.front().second == 0;
  }
-
- /* std::cerr
- << "regexp: " << regexp
- << ", is_trivial: " << is_trivial
- << ", required_substring: " << required_substring
- << ", required_substring_is_prefix: " << required_substring_is_prefix
- << std::endl;*/
 }