ARROW-2653: [C++] Refactor hash table support

LICENSE.txt

@@ -261,42 +261,6 @@ comments, complaints, performance data, etc to dl@cs.oswego.edu
 
 --------------------------------------------------------------------------------
 
-src/plasma/thirdparty/xxhash: BSD 2-Clause License
-
-xxHash - Fast Hash algorithm
-Copyright (C) 2012-2016, Yann Collet
-
-BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-You can contact the author at :
-- xxHash homepage: http://www.xxhash.com
-- xxHash source repository : https://github.com/Cyan4973/xxHash
-
---------------------------------------------------------------------------------
-
 src/plasma/common.cc (some portions)
 
 Copyright (c) Austin Appleby (aappleby (AT) gmail)
@@ -797,3 +761,37 @@ SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/util/xxhash/ have the following license
+(BSD 2-Clause License)
+
+xxHash Library
+Copyright (c) 2012-2014, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash homepage: http://www.xxhash.com
+- xxHash source repository : https://github.com/Cyan4973/xxHash

cpp/build-support/lint_cpp_cli.py

@@ -71,9 +71,13 @@ def lint_file(path):
 EXCLUSIONS = [
  'arrow/python/iterators.h',
  'arrow/util/date.h',
+ 'arrow/util/hashing.h',
  'arrow/util/macros.h',
  'arrow/util/parallel.h',
  'arrow/util/string_view/string_view.hpp',
+ 'arrow/util/xxhash/xxhash.c',
+ 'arrow/util/xxhash/xxhash.h',
+ 'arrow/visitor_inline.h',
  'gandiva/cache.h',
  'gandiva/jni',
  'test',

cpp/cmake_modules/SetupCxxFlags.cmake

@@ -19,7 +19,7 @@
 # instruction sets that would boost performance.
 include(CheckCXXCompilerFlag)
 # x86/amd64 compiler flags
-CHECK_CXX_COMPILER_FLAG("-msse3" CXX_SUPPORTS_SSE3)
+CHECK_CXX_COMPILER_FLAG("-msse4.2" CXX_SUPPORTS_SSE4_2)
 # power compiler flags
 CHECK_CXX_COMPILER_FLAG("-maltivec" CXX_SUPPORTS_ALTIVEC)
 
@@ -212,8 +212,8 @@ if (BUILD_WARNING_FLAGS)
 endif(BUILD_WARNING_FLAGS)
 
 # Only enable additional instruction sets if they are supported
-if (CXX_SUPPORTS_SSE3 AND ARROW_SSE3)
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse3")
+if (CXX_SUPPORTS_SSE4_2)
+ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse4.2")
 endif()
 
 if (CXX_SUPPORTS_ALTIVEC AND ARROW_ALTIVEC)

cpp/src/arrow/CMakeLists.txt

@@ -48,7 +48,6 @@ set(ARROW_SRCS
  util/compression.cc
  util/cpu-info.cc
  util/decimal.cc
- util/hash.cc
  util/io-util.cc
  util/logging.cc
  util/key_value_metadata.cc

cpp/src/arrow/array.h

@@ -168,6 +168,26 @@ struct ARROW_EXPORT ArrayData {
 
  std::shared_ptr<ArrayData> Copy() const { return std::make_shared<ArrayData>(*this); }
 
+ // Access a buffer's data as a typed C pointer
+ template <typename T>
+ inline const T* GetValues(int i) const {
+ if (buffers[i]) {
+ return reinterpret_cast<const T*>(buffers[i]->data()) + offset;
+ } else {
+ return NULLPTR;
+ }
+ }
+
+ // Access a buffer's data as a typed C pointer
+ template <typename T>
+ inline T* GetMutableValues(int i) {
+ if (buffers[i]) {
+ return reinterpret_cast<T*>(buffers[i]->mutable_data()) + offset;
+ } else {
+ return NULLPTR;
+ }
+ }
+
  std::shared_ptr<DataType> type;
  int64_t length;
  int64_t null_count;

cpp/src/arrow/builder-benchmark.cc

@@ -15,11 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
+#include <cstdint>
+#include <limits>
+#include <random>
+#include <string>
+#include <vector>
+
 #include "benchmark/benchmark.h"
 
 #include "arrow/builder.h"
 #include "arrow/memory_pool.h"
 #include "arrow/test-util.h"
+#include "arrow/util/bit-util.h"
 
 namespace arrow {
 
@@ -148,7 +156,6 @@ static void BM_BuildBinaryArray(benchmark::State& state) { // NOLINT non-const
  std::shared_ptr<Array> out;
  ABORT_NOT_OK(builder.Finish(&out));
  }
- // Assuming a string here needs on average 2 bytes
  state.SetBytesProcessed(state.iterations() * iterations * value.size());
 }
 
@@ -171,18 +178,195 @@ static void BM_BuildFixedSizeBinaryArray(
  state.SetBytesProcessed(state.iterations() * iterations * width);
 }
 
-BENCHMARK(BM_BuildPrimitiveArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildVectorNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+// ----------------------------------------------------------------------
+// DictionaryBuilder benchmarks
+
+// Testing with different distributions of integer values helps stress
+// the hash table's robustness.
+
+// Make a vector out of `n_distinct` sequential int values
+template <class Integer>
+static std::vector<Integer> MakeSequentialIntDictFodder(int32_t n_values,
+ int32_t n_distinct) {
+ std::default_random_engine gen(42);
+ std::vector<Integer> values(n_values);
+ {
+ std::uniform_int_distribution<Integer> values_dist(0, n_distinct - 1);
+ std::generate(values.begin(), values.end(), [&]() { return values_dist(gen); });
+ }
+ return values;
+}
+
+// Make a vector out of `n_distinct` int values with potentially colliding hash
+// entries as only their highest bits differ.
+template <class Integer>
+static std::vector<Integer> MakeSimilarIntDictFodder(int32_t n_values,
+ int32_t n_distinct) {
+ std::default_random_engine gen(42);
+ std::vector<Integer> values(n_values);
+ {
+ std::uniform_int_distribution<Integer> values_dist(0, n_distinct - 1);
+ auto max_int = std::numeric_limits<Integer>::max();
+ auto multiplier = static_cast<Integer>(BitUtil::NextPower2(max_int / n_distinct / 2));
+ std::generate(values.begin(), values.end(),
+ [&]() { return multiplier * values_dist(gen); });
+ }
+ return values;
+}
+
+// Make a vector out of `n_distinct` random int values
+template <class Integer>
+static std::vector<Integer> MakeRandomIntDictFodder(int32_t n_values,
+ int32_t n_distinct) {
+ std::default_random_engine gen(42);
+ std::vector<Integer> values_dict(n_distinct);
+ std::vector<Integer> values(n_values);
+
+ {
+ std::uniform_int_distribution<Integer> values_dist(
+ 0, std::numeric_limits<Integer>::max());
+ std::generate(values_dict.begin(), values_dict.end(),
+ [&]() { return static_cast<Integer>(values_dist(gen)); });
+ }
+ {
+ std::uniform_int_distribution<int32_t> indices_dist(0, n_distinct - 1);
+ std::generate(values.begin(), values.end(),
+ [&]() { return values_dict[indices_dist(gen)]; });
+ }
+ return values;
+}
+
+// Make a vector out of `n_distinct` string values
+static std::vector<std::string> MakeStringDictFodder(int32_t n_values,
+ int32_t n_distinct) {
+ std::default_random_engine gen(42);
+ std::vector<std::string> values_dict(n_distinct);
+ std::vector<std::string> values(n_values);
 
-BENCHMARK(BM_BuildBooleanArrayNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+ {
+ auto it = values_dict.begin();
+ // Add empty string
+ *it++ = "";
+ // Add a few similar strings
+ *it++ = "abc";
+ *it++ = "abcdef";
+ *it++ = "abcfgh";
+ // Add random strings
+ std::uniform_int_distribution<int32_t> length_dist(2, 20);
+ std::independent_bits_engine<std::default_random_engine, 8, uint8_t> bytes_gen(42);
 
-BENCHMARK(BM_BuildAdaptiveIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+ std::generate(it, values_dict.end(), [&]() {
+ auto length = length_dist(gen);
+ std::string s(length, 'X');
+ for (int32_t i = 0; i < length; ++i) {
+ s[i] = bytes_gen();
+ }
+ return s;
+ });
+ }
+ {
+ std::uniform_int_distribution<int32_t> indices_dist(0, n_distinct - 1);
+ std::generate(values.begin(), values.end(),
+ [&]() { return values_dict[indices_dist(gen)]; });
+ }
+ return values;
+}
+
+template <class DictionaryBuilderType, class Scalar>
+static void BenchmarkScalarDictionaryArray(
+ benchmark::State& state, // NOLINT non-const reference
+ const std::vector<Scalar>& fodder) {
+ while (state.KeepRunning()) {
+ DictionaryBuilder<Int64Type> builder(default_memory_pool());
+ for (const auto value : fodder) {
+ ABORT_NOT_OK(builder.Append(value));
+ }
+ std::shared_ptr<Array> out;
+ ABORT_NOT_OK(builder.Finish(&out));
+ }
+ state.SetBytesProcessed(state.iterations() * fodder.size() * sizeof(Scalar));
+}
+
+static void BM_BuildInt64DictionaryArrayRandom(
+ benchmark::State& state) { // NOLINT non-const reference
+ const auto fodder = MakeRandomIntDictFodder<int64_t>(10000, 100);
+ BenchmarkScalarDictionaryArray<DictionaryBuilder<Int64Type>>(state, fodder);
+}
+
+static void BM_BuildInt64DictionaryArraySequential(
+ benchmark::State& state) { // NOLINT non-const reference
+ const auto fodder = MakeSequentialIntDictFodder<int64_t>(10000, 100);
+ BenchmarkScalarDictionaryArray<DictionaryBuilder<Int64Type>>(state, fodder);
+}
+
+static void BM_BuildInt64DictionaryArraySimilar(
+ benchmark::State& state) { // NOLINT non-const reference
+ const auto fodder = MakeSimilarIntDictFodder<int64_t>(10000, 100);
+ BenchmarkScalarDictionaryArray<DictionaryBuilder<Int64Type>>(state, fodder);
+}
+
+static void BM_BuildStringDictionaryArray(
+ benchmark::State& state) { // NOLINT non-const reference
+ const auto fodder = MakeStringDictFodder(10000, 100);
+ auto type = binary();
+ auto fodder_size =
+ std::accumulate(fodder.begin(), fodder.end(), 0,
+ [&](size_t acc, const std::string& s) { return acc + s.size(); });
+
+ while (state.KeepRunning()) {
+ BinaryDictionaryBuilder builder(default_memory_pool());
+ for (const auto& value : fodder) {
+ ABORT_NOT_OK(builder.Append(value));
+ }
+ std::shared_ptr<Array> out;
+ ABORT_NOT_OK(builder.Finish(&out));
+ }
+ state.SetBytesProcessed(state.iterations() * fodder_size);
+}
+
+// ----------------------------------------------------------------------
+// Benchmark declarations
+
+static constexpr int32_t kRepetitions = 2;
+
+BENCHMARK(BM_BuildPrimitiveArrayNoNulls)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildVectorNoNulls)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
+
+BENCHMARK(BM_BuildBooleanArrayNoNulls)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
+
+BENCHMARK(BM_BuildAdaptiveIntNoNulls)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
 BENCHMARK(BM_BuildAdaptiveIntNoNullsScalarAppend)
  ->Repetitions(3)
  ->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildAdaptiveUIntNoNulls)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
 
-BENCHMARK(BM_BuildBinaryArray)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildFixedSizeBinaryArray)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildBinaryArray)->Repetitions(kRepetitions)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildFixedSizeBinaryArray)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
+
+BENCHMARK(BM_BuildInt64DictionaryArrayRandom)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildInt64DictionaryArraySequential)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildInt64DictionaryArraySimilar)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
+
+BENCHMARK(BM_BuildStringDictionaryArray)
+ ->Repetitions(kRepetitions)
+ ->Unit(benchmark::kMicrosecond);
 
 } // namespace arrow