Bundle grpc/protobuf in tiflash

.gitmodules

@@ -65,3 +65,14 @@
 [submodule "contrib/benchmark"]
 	path = contrib/benchmark
 	url = https://github.com/google/benchmark
+[submodule "contrib/protobuf"]
+	path = contrib/protobuf
+	url = https://github.com/protocolbuffers/protobuf
+  branch = v3.8.0
+[submodule "contrib/abseil-cpp"]
+	path = contrib/abseil-cpp
+	url = https://github.com/abseil/abseil-cpp
+[submodule "contrib/grpc"]
+	path = contrib/grpc
+	url = https://github.com/grpc/grpc
+ 	branch = v1.26.0

CMakeLists.txt

@@ -318,6 +318,7 @@ include (cmake/find_execinfo.cmake)
 include (cmake/find_readline_edit.cmake)
 include (cmake/find_re2.cmake)
 include (cmake/find_llvm.cmake)
+include (cmake/find_protobuf.cmake)
 include (cmake/find_grpc.cmake)
 include (cmake/find_kvproto.cmake)
 include (cmake/find_tipb.cmake)

cmake/find_grpc.cmake

@@ -1,21 +1,41 @@
-# TODO We should make a bundled gPRC and protoobuf repository, instance of rely on system library.
+# Normally we use the internal gRPC framework.
+# You can set USE_INTERNAL_GRPC_LIBRARY to OFF to force using the external gRPC framework, which should be installed in the system in this case.
+# The external gRPC framework can be installed in the system by running
+# sudo apt-get install libgrpc++-dev protobuf-compiler-grpc
+option(USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk)" ${NOT_UNBUNDLED})
 
-# gRPC and relateds
-if (GRPC_ROOT_DIR)
-    list(PREPEND CMAKE_SYSTEM_PREFIX_PATH ${GRPC_ROOT_DIR})
-    message(STATUS "Add ${GRPC_ROOT_DIR} to search path for protobuf && grpc")
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/CMakeLists.txt")
+    if(USE_INTERNAL_GRPC_LIBRARY)
+        message(WARNING "submodule contrib/grpc is missing. to fix try run: \n git submodule update --init")
+        message(WARNING "Can't use internal grpc")
+        set(USE_INTERNAL_GRPC_LIBRARY 0)
+    endif()
+    set(MISSING_INTERNAL_GRPC_LIBRARY 1)
 endif()
 
-find_package(Protobuf REQUIRED)
-message(STATUS "Using protobuf: ${Protobuf_VERSION} : ${Protobuf_INCLUDE_DIRS}, ${Protobuf_LIBRARIES}")
+if(NOT USE_INTERNAL_GRPC_LIBRARY)
+    find_package(gRPC)
+    if(NOT gRPC_INCLUDE_DIRS OR NOT gRPC_LIBRARIES)
+        message(WARNING "Can't find system gRPC library")
+        set(EXTERNAL_GRPC_LIBRARY_FOUND 0)
+    elseif(NOT gRPC_CPP_PLUGIN)
+        message(WARNING "Can't find system grpc_cpp_plugin")
+        set(EXTERNAL_GRPC_LIBRARY_FOUND 0)
+    else()
+        set(EXTERNAL_GRPC_LIBRARY_FOUND 1)
+    endif()
+endif()
 
-find_package(c-ares REQUIRED)
-message(STATUS "Using c-ares: ${c-ares_INCLUDE_DIR}, ${c-ares_LIBRARY}")
+if(NOT EXTERNAL_GRPC_LIBRARY_FOUND AND NOT MISSING_INTERNAL_GRPC_LIBRARY)
+    set(gRPC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/grpc/include")
+    set(gRPC_LIBRARIES grpc grpc++)
+    set(gRPC_CPP_PLUGIN $<TARGET_FILE:grpc_cpp_plugin>)
 
-find_package(ZLIB REQUIRED)
-message(STATUS "Using ZLIB: ${ZLIB_INCLUDE_DIRS}, ${ZLIB_LIBRARIES}")
+    include("${ClickHouse_SOURCE_DIR}/contrib/grpc-cmake/protobuf_generate_grpc.cmake")
 
-find_package(gRPC CONFIG REQUIRED)
-message(STATUS "Using gRPC: ${gRPC_VERSION}")
+    set(USE_INTERNAL_GRPC_LIBRARY 1)
+endif()
 
 set(gRPC_FOUND TRUE)
+
+message(STATUS "Using gRPC: ${gRPC_VERSION} : ${gRPC_INCLUDE_DIRS}, ${gRPC_LIBRARIES}, ${gRPC_CPP_PLUGIN}")

cmake/find_poco.cmake

@@ -24,6 +24,7 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
     set (ENABLE_DATA_SQLITE 0 CACHE BOOL "")
     set (ENABLE_DATA_MYSQL 0 CACHE BOOL "")
     set (ENABLE_DATA_POSTGRESQL 0 CACHE BOOL "")
+    set (ENABLE_DATA_ODBC 0 CACHE BOOL "")
     # new after 2.0.0:
     set (POCO_ENABLE_ZIP 0 CACHE BOOL "")
     set (POCO_ENABLE_PAGECOMPILER 0 CACHE BOOL "")

cmake/find_protobuf.cmake

@@ -0,0 +1,50 @@
+# Normally we use the internal protobuf library.
+# You can set USE_INTERNAL_PROTOBUF_LIBRARY to OFF to force using the external protobuf library, which should be installed in the system in this case.
+# The external protobuf library can be installed in the system by running
+# sudo apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev
+option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled. (Experimental. Set to OFF on your own risk)" ${NOT_UNBUNDLED})
+
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt")
+  if(USE_INTERNAL_PROTOBUF_LIBRARY)
+    message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init")
+    message(WARNING "Can't use internal protobuf")
+    set(USE_INTERNAL_PROTOBUF_LIBRARY 0)
+  endif()
+  set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1)
+endif()
+
+if(NOT USE_INTERNAL_PROTOBUF_LIBRARY)
+  find_package(Protobuf)
+  if(NOT Protobuf_INCLUDE_DIR OR NOT Protobuf_LIBRARY)
+    message(WARNING "Can't find system protobuf library")
+    set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0)
+  elseif(NOT Protobuf_PROTOC_EXECUTABLE)
+    message(WARNING "Can't find system protobuf compiler")
+    set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0)
+  else()
+    set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 1)
+  endif()
+endif()
+
+if(NOT EXTERNAL_PROTOBUF_LIBRARY_FOUND AND NOT MISSING_INTERNAL_PROTOBUF_LIBRARY)
+  set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src")
+  set(Protobuf_LIBRARY libprotobuf)
+  set(Protobuf_PROTOC_EXECUTABLE "$<TARGET_FILE:protoc>")
+  set(Protobuf_PROTOC_LIBRARY libprotoc)
+
+  include("${ClickHouse_SOURCE_DIR}/contrib/protobuf-cmake/protobuf_generate.cmake")
+
+  set(USE_INTERNAL_PROTOBUF_LIBRARY 1)
+endif()
+
+if(OS_FREEBSD AND SANITIZE STREQUAL "address")
+  # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found
+  # #include <sanitizer/asan_interface.h>
+  if(LLVM_INCLUDE_DIRS)
+    set(Protobuf_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}" ${LLVM_INCLUDE_DIRS})
+  else()
+    message(WARNING "Can't use protobuf on FreeBSD with address sanitizer without LLVM")
+  endif()
+endif()
+
+message(STATUS "Using protobuf: ${Protobuf_VERSION} : ${Protobuf_INCLUDE_DIR}, ${Protobuf_LIBRARY}, ${Protobuf_PROTOC_EXECUTABLE}")

cmake/find_re2.cmake

@@ -1,18 +1,41 @@
-option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ${NOT_UNBUNDLED})
+option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ON)
+
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/re2")
+    if(USE_INTERNAL_RE2_LIBRARY)
+        message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init")
+        message (WARNING "Can't find internal re2 library")
+    endif()
+    set(USE_INTERNAL_RE2_LIBRARY 0)
+    set(MISSING_INTERNAL_RE2_LIBRARY 1)
+endif()
 
 if (NOT USE_INTERNAL_RE2_LIBRARY)
     find_library (RE2_LIBRARY re2)
     find_path (RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_INCLUDE_PATHS})
+    if (NOT RE2_LIBRARY OR NOT RE2_INCLUDE_DIR)
+        message (WARNING "Can't find system re2 library")
+    endif ()
 endif ()
 
+string(FIND ${CMAKE_CURRENT_BINARY_DIR} " " _have_space)
+if(_have_space GREATER 0)
+    message(WARNING "Using spaces in build path [${CMAKE_CURRENT_BINARY_DIR}] highly not recommended. Library re2st will be disabled.")
+    set (MISSING_INTERNAL_RE2_ST_LIBRARY 1)
+endif()
+
 if (RE2_LIBRARY AND RE2_INCLUDE_DIR)
     set (RE2_ST_LIBRARY ${RE2_LIBRARY})
-else ()
+elseif (NOT MISSING_INTERNAL_RE2_LIBRARY)
     set (USE_INTERNAL_RE2_LIBRARY 1)
     set (RE2_LIBRARY re2)
-    set (RE2_ST_LIBRARY re2_st)
-    set (USE_RE2_ST 1)
+    set (RE2_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/re2")
+    if (NOT MISSING_INTERNAL_RE2_ST_LIBRARY)
+        set (RE2_ST_LIBRARY re2_st)
+        set (USE_RE2_ST 1)
+    else ()
+        set (RE2_ST_LIBRARY ${RE2_LIBRARY})
+        message (WARNING "Using internal re2 library instead of re2_st")
+    endif ()
 endif ()
 
-message (STATUS "Using re2: ${RE2_INCLUDE_DIR} : ${RE2_LIBRARY}")
-message (STATUS "Using re2_st: ${USE_RE2_ST}; ${RE2_ST_INCLUDE_DIR}; ${RE2_ST_LIBRARY}")
+message (STATUS "Using re2: ${RE2_INCLUDE_DIR} : ${RE2_LIBRARY}; ${RE2_ST_INCLUDE_DIR} : ${RE2_ST_LIBRARY}")

contrib/CMakeLists.txt

@@ -1,14 +1,3 @@
-add_subdirectory (kvproto/cpp)
-# -----
-set (_save ${ENABLE_TESTS})
-set (ENABLE_TESTS 0)
-# disable ENABLE_TESTS for client-c
-# GTest package conflicts with CH's GTest, so we have to disable test compilation in ch.
-add_subdirectory (client-c)
-set (ENABLE_TESTS ${_save})
-# -------
-add_subdirectory (tipb/cpp)
-
 if (NOT MSVC)
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast")
 endif ()
@@ -26,9 +15,7 @@ if (USE_INTERNAL_ZSTD_LIBRARY)
 endif ()
 
 if (USE_INTERNAL_RE2_LIBRARY)
-    set(RE2_BUILD_TESTING 0 CACHE INTERNAL "")
-    add_subdirectory (re2)
-    add_subdirectory (re2_st)
+    add_subdirectory (re2-cmake)
 endif ()
 
 if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
@@ -82,6 +69,26 @@ if (USE_INTERNAL_CCTZ_LIBRARY)
     add_subdirectory (cctz-cmake)
 endif ()
 
+if (USE_INTERNAL_PROTOBUF_LIBRARY)
+    add_subdirectory(protobuf-cmake)
+endif ()
+
+if (USE_INTERNAL_GRPC_LIBRARY)
+    add_subdirectory(abseil-cpp-cmake)
+    add_subdirectory(grpc-cmake)
+endif ()
+
+add_subdirectory (kvproto/cpp)
+# -----
+set (_save ${ENABLE_TESTS})
+set (ENABLE_TESTS 0)
+# disable ENABLE_TESTS for client-c
+# GTest package conflicts with CH's GTest, so we have to disable test compilation in ch.
+add_subdirectory (client-c)
+set (ENABLE_TESTS ${_save})
+# -------
+add_subdirectory (tipb/cpp)
+
 # TODO: remove tcmalloc
 if (ENABLE_TCMALLOC AND USE_INTERNAL_GPERFTOOLS_LIBRARY)
     add_subdirectory (libtcmalloc)

contrib/abseil-cpp-cmake/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
+if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt")
+  message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive")
+endif()
+set(BUILD_TESTING OFF)
+set(ABSL_PROPAGATE_CXX_STD ON)
+add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp")
+
+add_library(abseil_swiss_tables INTERFACE)
+
+target_link_libraries(abseil_swiss_tables INTERFACE
+  absl::flat_hash_map
+  absl::flat_hash_set
+)
+
+get_target_property(FLAT_HASH_MAP_INCLUDE_DIR absl::flat_hash_map INTERFACE_INCLUDE_DIRECTORIES)
+target_include_directories (abseil_swiss_tables SYSTEM BEFORE INTERFACE ${FLAT_HASH_MAP_INCLUDE_DIR})
+
+get_target_property(FLAT_HASH_SET_INCLUDE_DIR absl::flat_hash_set INTERFACE_INCLUDE_DIRECTORIES)
+target_include_directories (abseil_swiss_tables SYSTEM BEFORE INTERFACE ${FLAT_HASH_SET_INCLUDE_DIR})

contrib/grpc-cmake/CMakeLists.txt

@@ -0,0 +1,79 @@
+set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc")
+set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc")
+
+# Use re2 from ClickHouse contrib, not from gRPC third_party.
+if(NOT RE2_INCLUDE_DIR)
+  message(FATAL_ERROR " grpc: The location of the \"re2\" library is unknown")
+endif()
+set(gRPC_RE2_PROVIDER "clickhouse" CACHE STRING "" FORCE)
+set(_gRPC_RE2_INCLUDE_DIR "${RE2_INCLUDE_DIR}")
+set(_gRPC_RE2_LIBRARIES "${RE2_LIBRARY}")
+
+# Use zlib from ClickHouse contrib, not from gRPC third_party.
+if(NOT ZLIB_INCLUDE_DIRS)
+  message(FATAL_ERROR " grpc: The location of the \"zlib\" library is unknown")
+endif()
+set(gRPC_ZLIB_PROVIDER "clickhouse" CACHE STRING "" FORCE)
+set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_INCLUDE_DIRS}")
+set(_gRPC_ZLIB_LIBRARIES "${ZLIB_LIBRARIES}")
+
+# Use protobuf from ClickHouse contrib, not from gRPC third_party.
+if(NOT Protobuf_INCLUDE_DIR OR NOT Protobuf_LIBRARY)
+  message(FATAL_ERROR " grpc: The location of the \"protobuf\" library is unknown")
+elseif (NOT Protobuf_PROTOC_EXECUTABLE)
+  message(FATAL_ERROR " grpc: The location of the protobuf compiler is unknown")
+elseif (NOT Protobuf_PROTOC_LIBRARY)
+  message(FATAL_ERROR " grpc: The location of the protobuf compiler's library is unknown")
+endif()
+set(gRPC_PROTOBUF_PROVIDER "clickhouse" CACHE STRING "" FORCE)
+set(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}")
+set(_gRPC_PROTOBUF_LIBRARIES "${Protobuf_LIBRARY}")
+set(_gRPC_PROTOBUF_PROTOC "protoc")
+set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_PROTOC_EXECUTABLE}")
+set(_gRPC_PROTOBUF_PROTOC_LIBRARIES "${Protobuf_PROTOC_LIBRARY}")
+
+# Use OpenSSL from ClickHouse contrib, not from gRPC third_party.
+set(gRPC_SSL_PROVIDER "clickhouse" CACHE STRING "" FORCE)
+set(_gRPC_SSL_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR})
+set(_gRPC_SSL_LIBRARIES ${OPENSSL_LIBRARIES})
+
+# Use abseil-cpp from ClickHouse contrib, not from gRPC third_party.
+set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE)
+
+# Choose to build static or shared library for c-ares.
+if (MAKE_STATIC_LIBRARIES)
+  set(CARES_STATIC ON CACHE BOOL "" FORCE)
+  set(CARES_SHARED OFF CACHE BOOL "" FORCE)
+else ()
+  set(CARES_STATIC OFF CACHE BOOL "" FORCE)
+  set(CARES_SHARED ON CACHE BOOL "" FORCE)
+endif ()
+
+# Disable looking for libnsl on a platforms that has gethostbyname in glibc
+#
+# c-ares searching for gethostbyname in the libnsl library, however in the
+# version that shipped with gRPC it doing it wrong [1], since it uses
+# CHECK_LIBRARY_EXISTS(), which will return TRUE even if the function exists in
+# another dependent library. The upstream already contains correct macro [2],
+# but it is not included in gRPC (even upstream gRPC, not the one that is
+# shipped with clickhousee).
+#
+#   [1]: https://github.com/c-ares/c-ares/blob/e982924acee7f7313b4baa4ee5ec000c5e373c30/CMakeLists.txt#L125
+#   [2]: https://github.com/c-ares/c-ares/blob/44fbc813685a1fa8aa3f27fcd7544faf612d376a/CMakeLists.txt#L146
+#
+# And because if you by some reason have libnsl [3] installed, clickhouse will
+# reject to start w/o it. While this is completelly different library.
+#
+#   [3]: https://packages.debian.org/bullseye/libnsl2
+if (NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+  set(HAVE_LIBNSL OFF CACHE BOOL "" FORCE)
+endif()
+
+# We don't want to build C# extensions.
+set(gRPC_BUILD_CSHARP_EXT OFF)
+
+add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}")
+
+# The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes,
+# so we need to redefine it back.
+include("${ClickHouse_SOURCE_DIR}/contrib/grpc-cmake/protobuf_generate_grpc.cmake")