ARROW-28: Adding google's benchmark library to the toolchain

This isn't yet complete, but before I go further I think its worth asking some questions on peoples' preferences:

1.  It seems that the build third-party script is setting up an install directory that it is not making use of.   Do we want to keep this functionality and start adding new libraries to be placed there?  The gtest component of the tool-chain assumes it is in its own location, and this how I patterned google benchmark integration.

2.  Do we want to couple unit test builds with benchmark builds?  I am currently aiming for having them decoupled and having benchmarks off by default.

3.  I am not familiar with the Darwin/mac build environment and it is not clear if the CXX flags are required universally.  (I need to fix it anyways to move -DGTEST_USE_OWN_TR1_TUPLE=1 back to be gtest only).  Travis-ci might provide the answer.

4.  Any other basic features in the benchmark toolchain people would like to see as part of this PR?  Wes mentioned starting to create benchmarking tools lib, but I think that likely belongs in a separate PR.

Author: Micah Kornfield <emkornfield@gmail.com>

Closes #29 from emkornfield/emk_add_benchmark and squashes the following commits:

dbd4e71 [Micah Kornfield] only run unittests is travis
ab21150 [Micah Kornfield] Enable benchmarks in cpp toolchain
40847ee [Micah Kornfield] WIP-Adding google's benchmark library to the toolchain

ci/travis_before_script_cpp.sh

@@ -19,7 +19,7 @@ echo $GTEST_HOME
 
 : ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
 
-cmake -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
+cmake -DARROW_BUILD_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
 make -j4
 make install
 

ci/travis_script_cpp.sh

@@ -9,9 +9,9 @@ pushd $CPP_BUILD_DIR
 make lint
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
- valgrind --tool=memcheck --leak-check=yes --error-exitcode=1 ctest
+ valgrind --tool=memcheck --leak-check=yes --error-exitcode=1 ctest -L unittest
 else
- ctest
+ ctest -L unittest
 fi
 
 popd

cpp/CMakeLists.txt

@@ -55,12 +55,21 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
  option(ARROW_BUILD_TESTS
  "Build the Arrow googletest unit tests"
  ON)
+
+ option(ARROW_BUILD_BENCHMARKS
+ "Build the Arrow micro benchmarks"
+ OFF)
+
 endif()
 
 if(NOT ARROW_BUILD_TESTS)
  set(NO_TESTS 1)
 endif()
 
+if(NOT ARROW_BUILD_BENCHMARKS)
+ set(NO_BENCHMARKS 1)
+endif()
+
 
 ############################################################
 # Compiler flags
@@ -251,16 +260,73 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
 include_directories(src)
 
 ############################################################
-# Testing
+# Benchmarking 
 ############################################################
+# Add a new micro benchmark, with or without an executable that should be built.
+# If benchmarks are enabled then they will be run along side unit tests with ctest.
+# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests, 
+# respectively.
+#
+# REL_BENCHMARK_NAME is the name of the benchmark app. It may be a single component
+# (e.g. monotime-benchmark) or contain additional components (e.g.
+# net/net_util-benchmark). Either way, the last component must be a globally
+# unique name. 
+
+# The benchmark will registered as unit test with ctest with a label
+# of 'benchmark'.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+function(ADD_ARROW_BENCHMARK REL_BENCHMARK_NAME)
+ if(NO_BENCHMARKS)
+ return()
+ endif()
+ get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
+
+ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
+ # This benchmark has a corresponding .cc file, set it up as an executable.
+ set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
+ add_executable(${BENCHMARK_NAME} "${REL_BENCHMARK_NAME}.cc")
+ target_link_libraries(${BENCHMARK_NAME} ${ARROW_BENCHMARK_LINK_LIBS})
+ add_dependencies(runbenchmark ${BENCHMARK_NAME})
+ set(NO_COLOR "--color_print=false")
+ else()
+ # No executable, just invoke the benchmark (probably a script) directly.
+ set(BENCHMARK_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME})
+ set(NO_COLOR "")
+ endif()
+
+ add_test(${BENCHMARK_NAME}
+ ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} benchmark ${BENCHMARK_PATH} ${NO_COLOR})
+ set_tests_properties(${BENCHMARK_NAME} PROPERTIES LABELS "benchmark")
+ if(ARGN)
+ set_tests_properties(${BENCHMARK_NAME} PROPERTIES ${ARGN})
+ endif()
+endfunction()
+
+# A wrapper for add_dependencies() that is compatible with NO_BENCHMARKS.
+function(ADD_ARROW_BENCHMARK_DEPENDENCIES REL_BENCHMARK_NAME)
+ if(NO_BENCHMARKS)
+ return()
+ endif()
+ get_filename_component(BENCMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
 
+ add_dependencies(${BENCHMARK_NAME} ${ARGN})
+endfunction()
+
+
+############################################################
+# Testing
+############################################################
 # Add a new test case, with or without an executable that should be built.
 #
 # REL_TEST_NAME is the name of the test. It may be a single component
 # (e.g. monotime-test) or contain additional components (e.g.
 # net/net_util-test). Either way, the last component must be a globally
 # unique name.
 #
+# The unit test is added with a label of "unittest" to support filtering with
+# ctest.
+#
 # Arguments after the test name will be passed to set_tests_properties().
 function(ADD_ARROW_TEST REL_TEST_NAME)
  if(NO_TESTS)
@@ -273,13 +339,15 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
  set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
  add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
  target_link_libraries(${TEST_NAME} ${ARROW_TEST_LINK_LIBS})
+ add_dependencies(unittest ${TEST_NAME})
  else()
  # No executable, just invoke the test (probably a script) directly.
  set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
  endif()
 
  add_test(${TEST_NAME}
- ${BUILD_SUPPORT_DIR}/run-test.sh ${TEST_PATH})
+ ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
+ set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
  if(ARGN)
  set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
  endif()
@@ -335,13 +403,28 @@ if ("$ENV{GTEST_HOME}" STREQUAL "")
  set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0)
 endif()
 
+## Google Benchmark 
+if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
+ set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
+endif()
+
+
 if(ARROW_BUILD_TESTS)
+ add_custom_target(unittest ctest -L unittest)
  find_package(GTest REQUIRED)
  include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
  ADD_THIRDPARTY_LIB(gtest
  STATIC_LIB ${GTEST_STATIC_LIB})
 endif()
 
+if(ARROW_BUILD_BENCHMARKS)
+ add_custom_target(runbenchmark ctest -L benchmark)
+ find_package(GBenchmark REQUIRED)
+ include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR})
+ ADD_THIRDPARTY_LIB(benchmark
+ STATIC_LIB ${GBENCHMARK_STATIC_LIB})
+endif()
+
 ## Google PerfTools
 ##
 ## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment
@@ -366,6 +449,7 @@ endif()
 ############################################################
 set(ARROW_MIN_TEST_LIBS arrow arrow_test_main ${ARROW_BASE_LIBS})
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
+set(ARROW_BENCHMARK_LINK_LIBS arrow arrow_benchmark_main ${ARROW_BASE_LIBS})
 
 ############################################################
 # "make ctags" target

cpp/README.md

@@ -29,20 +29,35 @@ Simple debug build:
  mkdir debug
  cd debug
  cmake ..
- make
- ctest
+ make unittest
 
 Simple release build:
 
  mkdir release
  cd release
  cmake .. -DCMAKE_BUILD_TYPE=Release
- make
- ctest
+ make unittest
+
+Detailed unit test logs will be placed in the build directory under `build/test-logs`.
+
+### Building/Running benchmarks
+
+Follow the directions for simple build except run cmake 
+with the `--ARROW_BUILD_BENCHMARKS` parameter set correctly:
+
+ cmake -DARROW_BUILD_BENCHMARKS=ON ..
+
+and instead of make unittest run either `make; ctest` to run both unit tests 
+and benchmarks or `make runbenchmark` to run only the benchmark tests.
+
+Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
+
 
 ### Third-party environment variables
 
 To set up your own specific build toolchain, here are the relevant environment
 variables
 
 * Googletest: `GTEST_HOME` (only required to build the unit tests)
+* Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks)
+