diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index 75f8a63368d8..ea8eef40933d 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -9,6 +9,18 @@ on: # yamllint disable-line rule:truthy
branches:
- 'backport/**'
jobs:
+ PythonUnitTests:
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ - name: Python unit tests
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 -m unittest discover -s . -p '*_test.py'
DockerHubPushAarch64:
runs-on: [self-hosted, style-checker-aarch64]
steps:
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 5816a58081d8..737a61735a5a 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -911,6 +911,34 @@ jobs:
docker rm -f "$(docker ps -a -q)" ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
+##################################### Docker images #######################################
+############################################################################################
+ DockerServerImages:
+ needs:
+ - BuilderDebRelease
+ - BuilderDebAarch64
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type head
+ python3 docker_server.py --release-type head --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"
+############################################################################################
##################################### BUILD REPORTER #######################################
############################################################################################
BuilderReport:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 4a3880543c4d..6c55b80090a0 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -4,7 +4,7 @@ env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
-on: # yamllint disable-line rule:truthy
+on: # yamllint disable-line rule:truthy
pull_request:
types:
- synchronize
@@ -956,6 +956,34 @@ jobs:
docker rm -f "$(docker ps -a -q)" ||:
sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
+##################################### Docker images #######################################
+############################################################################################
+ DockerServerImages:
+ needs:
+ - BuilderDebRelease
+ - BuilderDebAarch64
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type head --no-push
+ python3 docker_server.py --release-type head --no-push --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"
+############################################################################################
##################################### BUILD REPORTER #######################################
############################################################################################
BuilderReport:
@@ -3016,6 +3044,7 @@ jobs:
needs:
- StyleCheck
- DockerHubPush
+ - DockerServerImages
- CheckLabels
- BuilderReport
- FastTest
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0fe025080dd5..ea2e1ed33fb5 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -32,7 +32,32 @@ jobs:
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
- file: ${{runner.temp}}/release_packages/*
+ file: ${{runner.temp}}/push_to_artifactory/*
overwrite: true
tag: ${{ github.ref }}
file_glob: true
+ ############################################################################################
+ ##################################### Docker images #######################################
+ ############################################################################################
+ DockerServerImages:
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type auto --version "${{ github.ref }}"
+ python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"
diff --git a/.gitignore b/.gitignore
index 35ede89817a5..3845572c773f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -153,3 +153,5 @@ website/package-lock.json
/programs/server/metadata
/programs/server/store
+
+utils/local-engine/tests/testConfig.h
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9649fc32d746..3157d3a1ec49 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -187,6 +187,9 @@ if (COMPILER_CLANG)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
endif()
endif()
+
+ # set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
+ # set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
endif ()
# If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
@@ -261,13 +264,16 @@ endif ()
# Add a section with the hash of the compiled machine code for integrity checks.
# Only for official builds, because adding a section can be time consuming (rewrite of several GB).
# And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
-if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
- set (USE_BINARY_HASH 1)
+if (OBJCOPY_PATH AND CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE OR CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64.cmake$"))
+ set (USE_BINARY_HASH 1 CACHE STRING "Calculate binary hash and store it in the separate section")
endif ()
# Allows to build stripped binary in a separate directory
-if (OBJCOPY_PATH AND READELF_PATH)
- set(BUILD_STRIPPED_BINARIES_PREFIX "" CACHE STRING "Build stripped binaries with debug info in separate directory")
+if (OBJCOPY_PATH AND STRIP_PATH)
+ option(INSTALL_STRIPPED_BINARIES "Build stripped binaries with debug info in separate directory" OFF)
+ if (INSTALL_STRIPPED_BINARIES)
+ set(STRIPPED_BINARIES_OUTPUT "stripped" CACHE STRING "A separate directory for stripped information")
+ endif()
endif()
cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
@@ -517,8 +523,10 @@ macro (add_executable target)
if (${type} STREQUAL EXECUTABLE)
# disabled for TSAN and gcc since libtsan.a provides overrides too
if (TARGET clickhouse_new_delete)
- # operator::new/delete for executables (MemoryTracker stuff)
- target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+ if (NOT ${target} STREQUAL Git::Git)
+ # operator::new/delete for executables (MemoryTracker stuff)
+ target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+ endif()
endif()
# In case of static jemalloc, because zone_register() is located in zone.c and
diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt
index ddec09121e1f..ef7ec6d7fc03 100644
--- a/base/glibc-compatibility/CMakeLists.txt
+++ b/base/glibc-compatibility/CMakeLists.txt
@@ -51,6 +51,6 @@ if (GLIBC_COMPATIBILITY)
message (STATUS "Some symbols from glibc will be replaced for compatibility")
-elseif (YANDEX_OFFICIAL_BUILD)
+elseif (CLICKHOUSE_OFFICIAL_BUILD)
message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.")
endif ()
diff --git a/benchmark/greenplum/result_parser.py b/benchmark/greenplum/result_parser.py
index 8af20d265a02..4ed1aa5c4a56 100755
--- a/benchmark/greenplum/result_parser.py
+++ b/benchmark/greenplum/result_parser.py
@@ -4,11 +4,12 @@
import sys
import json
+
def parse_block(block=[], options=[]):
- #print('block is here', block)
- #show_query = False
- #show_query = options.show_query
+ # print('block is here', block)
+ # show_query = False
+ # show_query = options.show_query
result = []
query = block[0].strip()
if len(block) > 4:
@@ -20,9 +21,9 @@ def parse_block(block=[], options=[]):
timing2 = block[2].strip().split()[1]
timing3 = block[3].strip().split()[1]
if options.show_queries:
- result.append( query )
+ result.append(query)
if not options.show_first_timings:
- result += [ timing1 , timing2, timing3 ]
+ result += [timing1, timing2, timing3]
else:
result.append(timing1)
return result
@@ -37,12 +38,12 @@ def read_stats_file(options, fname):
for line in f.readlines():
- if 'SELECT' in line:
+ if "SELECT" in line:
if len(block) > 1:
- result.append( parse_block(block, options) )
- block = [ line ]
- elif 'Time:' in line:
- block.append( line )
+ result.append(parse_block(block, options))
+ block = [line]
+ elif "Time:" in line:
+ block.append(line)
return result
@@ -50,7 +51,7 @@ def read_stats_file(options, fname):
def compare_stats_files(options, arguments):
result = []
file_output = []
- pyplot_colors = ['y', 'b', 'g', 'r']
+ pyplot_colors = ["y", "b", "g", "r"]
for fname in arguments[1:]:
file_output.append((read_stats_file(options, fname)))
if len(file_output[0]) > 0:
@@ -58,65 +59,92 @@ def compare_stats_files(options, arguments):
for idx, data_set in enumerate(file_output):
int_result = []
for timing in data_set:
- int_result.append(float(timing[0])) #y values
- result.append([[x for x in range(0, len(int_result)) ], int_result,
-pyplot_colors[idx] + '^' ] )
-# result.append([x for x in range(1, len(int_result)) ]) #x values
-# result.append( pyplot_colors[idx] + '^' )
+ int_result.append(float(timing[0])) # y values
+ result.append(
+ [
+ [x for x in range(0, len(int_result))],
+ int_result,
+ pyplot_colors[idx] + "^",
+ ]
+ )
+ # result.append([x for x in range(1, len(int_result)) ]) #x values
+ # result.append( pyplot_colors[idx] + '^' )
return result
+
def parse_args():
from optparse import OptionParser
- parser = OptionParser(usage='usage: %prog [options] [result_file_path]..')
- parser.add_option("-q", "--show-queries", help="Show statements along with timings", action="store_true", dest="show_queries")
- parser.add_option("-f", "--show-first-timings", help="Show only first tries timings", action="store_true", dest="show_first_timings")
- parser.add_option("-c", "--compare-mode", help="Prepare output for pyplot comparing result files.", action="store", dest="compare_mode")
+
+ parser = OptionParser(usage="usage: %prog [options] [result_file_path]..")
+ parser.add_option(
+ "-q",
+ "--show-queries",
+ help="Show statements along with timings",
+ action="store_true",
+ dest="show_queries",
+ )
+ parser.add_option(
+ "-f",
+ "--show-first-timings",
+ help="Show only first tries timings",
+ action="store_true",
+ dest="show_first_timings",
+ )
+ parser.add_option(
+ "-c",
+ "--compare-mode",
+ help="Prepare output for pyplot comparing result files.",
+ action="store",
+ dest="compare_mode",
+ )
(options, arguments) = parser.parse_args(sys.argv)
if len(arguments) < 2:
parser.print_usage()
sys.exit(1)
- return ( options, arguments )
+ return (options, arguments)
+
def gen_pyplot_code(options, arguments):
- result = ''
+ result = ""
data_sets = compare_stats_files(options, arguments)
for idx, data_set in enumerate(data_sets, start=0):
x_values, y_values, line_style = data_set
- result += '\nplt.plot('
- result += '%s, %s, \'%s\'' % ( x_values, y_values, line_style )
- result += ', label=\'%s try\')' % idx
- print('import matplotlib.pyplot as plt')
+ result += "\nplt.plot("
+ result += "%s, %s, '%s'" % (x_values, y_values, line_style)
+ result += ", label='%s try')" % idx
+ print("import matplotlib.pyplot as plt")
print(result)
- print( 'plt.xlabel(\'Try number\')' )
- print( 'plt.ylabel(\'Timing\')' )
- print( 'plt.title(\'Benchmark query timings\')' )
- print('plt.legend()')
- print('plt.show()')
+ print("plt.xlabel('Try number')")
+ print("plt.ylabel('Timing')")
+ print("plt.title('Benchmark query timings')")
+ print("plt.legend()")
+ print("plt.show()")
def gen_html_json(options, arguments):
tuples = read_stats_file(options, arguments[1])
- print('{')
+ print("{")
print('"system: GreenPlum(x2),')
- print(('"version": "%s",' % '4.3.9.1'))
+ print(('"version": "%s",' % "4.3.9.1"))
print('"data_size": 10000000,')
print('"time": "",')
print('"comments": "",')
print('"result":')
- print('[')
+ print("[")
for s in tuples:
print(s)
- print(']')
- print('}')
+ print("]")
+ print("}")
def main():
- ( options, arguments ) = parse_args()
+ (options, arguments) = parse_args()
if len(arguments) > 2:
gen_pyplot_code(options, arguments)
else:
gen_html_json(options, arguments)
-if __name__ == '__main__':
+
+if __name__ == "__main__":
main()
diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 8de2de1e3bbf..288ed03be0b5 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -5,8 +5,8 @@
SET(VERSION_REVISION 54460)
SET(VERSION_MAJOR 22)
SET(VERSION_MINOR 3)
-SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 75366fc95e510b7ac76759ef670702ae5f488a51)
-SET(VERSION_DESCRIBE v22.3.1.1-testing)
-SET(VERSION_STRING 22.3.1.1)
+SET(VERSION_PATCH 4)
+SET(VERSION_GITHASH abb756d3ca261a4a2c853adb6938d2195b8cbff2)
+SET(VERSION_DESCRIBE v22.3.4.44-lts)
+SET(VERSION_STRING 22.3.4.44)
# end of autochange
diff --git a/cmake/strip.sh b/cmake/strip.sh
deleted file mode 100755
index de5968871593..000000000000
--- a/cmake/strip.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-BINARY_PATH=$1
-BINARY_NAME=$(basename $BINARY_PATH)
-DESTINATION_STRIPPED_DIR=$2
-OBJCOPY_PATH=${3:objcopy}
-READELF_PATH=${4:readelf}
-
-BUILD_ID=$($READELF_PATH -n $1 | sed -n '/Build ID/ { s/.*: //p; q; }')
-BUILD_ID_PREFIX=${BUILD_ID:0:2}
-BUILD_ID_SUFFIX=${BUILD_ID:2}
-TEMP_BINARY_PATH="${BINARY_PATH}_temp"
-
-DESTINATION_DEBUG_INFO_DIR="$DESTINATION_STRIPPED_DIR/lib/debug/.build-id"
-DESTINATION_STRIP_BINARY_DIR="$DESTINATION_STRIPPED_DIR/bin"
-
-mkdir -p "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX"
-mkdir -p "$DESTINATION_STRIP_BINARY_DIR"
-
-$OBJCOPY_PATH --only-keep-debug "$BINARY_PATH" "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX/$BUILD_ID_SUFFIX.debug"
-
-touch "$TEMP_BINARY_PATH"
-$OBJCOPY_PATH --add-gnu-debuglink "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX/$BUILD_ID_SUFFIX.debug" "$BINARY_PATH" "$TEMP_BINARY_PATH"
-$OBJCOPY_PATH --strip-all "$TEMP_BINARY_PATH" "$DESTINATION_STRIP_BINARY_DIR/$BINARY_NAME"
-rm -f "$TEMP_BINARY_PATH"
diff --git a/cmake/strip_binary.cmake b/cmake/strip_binary.cmake
index e430807772dd..1f24790a1590 100644
--- a/cmake/strip_binary.cmake
+++ b/cmake/strip_binary.cmake
@@ -11,16 +11,43 @@ macro(clickhouse_strip_binary)
message(FATAL_ERROR "A binary path name must be provided for stripping binary")
endif()
-
if (NOT DEFINED STRIP_DESTINATION_DIR)
message(FATAL_ERROR "Destination directory for stripped binary must be provided")
endif()
add_custom_command(TARGET ${STRIP_TARGET} POST_BUILD
- COMMAND bash ${ClickHouse_SOURCE_DIR}/cmake/strip.sh ${STRIP_BINARY_PATH} ${STRIP_DESTINATION_DIR} ${OBJCOPY_PATH} ${READELF_PATH}
- COMMENT "Stripping clickhouse binary" VERBATIM
+ COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/lib/debug/bin"
+ COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/bin"
+ COMMAND cp "${STRIP_BINARY_PATH}" "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
+ COMMAND "${OBJCOPY_PATH}" --only-keep-debug --compress-debug-sections "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}" "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug"
+ COMMAND chmod 0644 "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug"
+ COMMAND "${STRIP_PATH}" --remove-section=.comment --remove-section=.note "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
+ COMMAND "${OBJCOPY_PATH}" --add-gnu-debuglink "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug" "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
+ COMMENT "Stripping clickhouse binary" VERBATIM
)
install(PROGRAMS ${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET} DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
- install(DIRECTORY ${STRIP_DESTINATION_DIR}/lib/debug DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
+ install(FILES ${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug DESTINATION ${CMAKE_INSTALL_LIBDIR}/debug/${CMAKE_INSTALL_FULL_BINDIR}/${STRIP_TARGET}.debug COMPONENT clickhouse)
+endmacro()
+
+
+macro(clickhouse_make_empty_debug_info_for_nfpm)
+ set(oneValueArgs TARGET DESTINATION_DIR)
+ cmake_parse_arguments(EMPTY_DEBUG "" "${oneValueArgs}" "" ${ARGN})
+
+ if (NOT DEFINED EMPTY_DEBUG_TARGET)
+ message(FATAL_ERROR "A target name must be provided for stripping binary")
+ endif()
+
+ if (NOT DEFINED EMPTY_DEBUG_DESTINATION_DIR)
+ message(FATAL_ERROR "Destination directory for empty debug must be provided")
+ endif()
+
+ add_custom_command(TARGET ${EMPTY_DEBUG_TARGET} POST_BUILD
+ COMMAND mkdir -p "${EMPTY_DEBUG_DESTINATION_DIR}/lib/debug"
+ COMMAND touch "${EMPTY_DEBUG_DESTINATION_DIR}/lib/debug/${EMPTY_DEBUG_TARGET}.debug"
+ COMMENT "Addiding empty debug info for NFPM" VERBATIM
+ )
+
+ install(FILES "${EMPTY_DEBUG_DESTINATION_DIR}/lib/debug/${EMPTY_DEBUG_TARGET}.debug" DESTINATION "${CMAKE_INSTALL_LIBDIR}/debug/${CMAKE_INSTALL_FULL_BINDIR}" COMPONENT clickhouse)
endmacro()
diff --git a/cmake/tools.cmake b/cmake/tools.cmake
index d6fddd0509e8..d571a46ad26d 100644
--- a/cmake/tools.cmake
+++ b/cmake/tools.cmake
@@ -170,32 +170,32 @@ else ()
message (FATAL_ERROR "Cannot find objcopy.")
endif ()
-# Readelf (FIXME copypaste)
+# Strip (FIXME copypaste)
if (COMPILER_GCC)
- find_program (READELF_PATH NAMES "llvm-readelf" "llvm-readelf-13" "llvm-readelf-12" "llvm-readelf-11" "readelf")
+ find_program (STRIP_PATH NAMES "llvm-strip" "llvm-strip-13" "llvm-strip-12" "llvm-strip-11" "strip")
else ()
- find_program (READELF_PATH NAMES "llvm-readelf-${COMPILER_VERSION_MAJOR}" "llvm-readelf" "readelf")
+ find_program (STRIP_PATH NAMES "llvm-strip-${COMPILER_VERSION_MAJOR}" "llvm-strip" "strip")
endif ()
-if (NOT READELF_PATH AND OS_DARWIN)
+if (NOT STRIP_PATH AND OS_DARWIN)
find_program (BREW_PATH NAMES "brew")
if (BREW_PATH)
execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
if (LLVM_PREFIX)
- find_program (READELF_PATH NAMES "llvm-readelf" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
+ find_program (STRIP_PATH NAMES "llvm-strip" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
endif ()
- if (NOT READELF_PATH)
+ if (NOT STRIP_PATH)
execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
if (BINUTILS_PREFIX)
- find_program (READELF_PATH NAMES "readelf" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
+ find_program (STRIP_PATH NAMES "strip" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
endif ()
endif ()
endif ()
endif ()
-if (READELF_PATH)
- message (STATUS "Using readelf: ${READELF_PATH}")
+if (STRIP_PATH)
+ message (STATUS "Using strip: ${STRIP_PATH}")
else ()
- message (FATAL_ERROR "Cannot find readelf.")
+ message (FATAL_ERROR "Cannot find strip.")
endif ()
diff --git a/cmake/version.cmake b/cmake/version.cmake
index 963f291c0f30..acaa772ff2ff 100644
--- a/cmake/version.cmake
+++ b/cmake/version.cmake
@@ -18,6 +18,6 @@ set (VERSION_STRING_SHORT "${VERSION_MAJOR}.${VERSION_MINOR}")
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")
-if(YANDEX_OFFICIAL_BUILD)
+if(CLICKHOUSE_OFFICIAL_BUILD)
set(VERSION_OFFICIAL " (official build)")
endif()
diff --git a/contrib/arrow b/contrib/arrow
index 1d9cc51daa4e..86bc582fbc19 160000
--- a/contrib/arrow
+++ b/contrib/arrow
@@ -1 +1 @@
-Subproject commit 1d9cc51daa4e7e9fc6926320ef73759818bd736e
+Subproject commit 86bc582fbc19cba14ab2fd635a6d2c159cb91519
diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt
index 2e4059efc170..ef1e52c409a9 100644
--- a/contrib/arrow-cmake/CMakeLists.txt
+++ b/contrib/arrow-cmake/CMakeLists.txt
@@ -464,6 +464,16 @@ foreach (TOOL ${PARQUET_TOOLS})
target_link_libraries(${TOOL} PRIVATE _parquet)
endforeach ()
-# The library is large - avoid bloat.
-target_compile_options (_arrow PRIVATE -g0)
-target_compile_options (_parquet PRIVATE -g0)
+
+
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+ # otherwise we cannot debug arrow code
+ message(STATUS "build arrow in debug profile")
+ target_compile_options (_arrow PRIVATE)
+ target_compile_options (_parquet PRIVATE)
+else()
+ # The library is large - avoid bloat.
+ message(STATUS "build arrow in non debug profile")
+ target_compile_options (_arrow PRIVATE -g0)
+ target_compile_options (_parquet PRIVATE -g0)
+endif()
\ No newline at end of file
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
index 99ab2d53ca9c..0003f89510f4 100644
--- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -138,7 +138,8 @@
/* #undef JEMALLOC_MUTEX_INIT_CB */
/* Non-empty if the tls_model attribute is supported. */
-#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+/*#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))*/
+#define JEMALLOC_TLS_MODEL
/*
* JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
diff --git a/contrib/libhdfs3 b/contrib/libhdfs3
index 9194af445886..f2e2601b10b4 160000
--- a/contrib/libhdfs3
+++ b/contrib/libhdfs3
@@ -1 +1 @@
-Subproject commit 9194af44588633c1b2dae44bf945804401ff883e
+Subproject commit f2e2601b10b4eb0fadff549b5b63319a965993df
diff --git a/contrib/llvm-cmake/CMakeLists.txt b/contrib/llvm-cmake/CMakeLists.txt
index 6ff07f0e0167..cd21b35a4737 100644
--- a/contrib/llvm-cmake/CMakeLists.txt
+++ b/contrib/llvm-cmake/CMakeLists.txt
@@ -1,12 +1,9 @@
-# During cross-compilation in our CI we have to use llvm-tblgen and other building tools
-# tools to be build for host architecture and everything else for target architecture (e.g. AArch64)
-# Possible workaround is to use llvm-tblgen from some package...
-# But lets just enable LLVM for native builds
-if (CMAKE_CROSSCOMPILING OR SANITIZE STREQUAL "undefined")
- set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF)
+if (APPLE OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined")
+ set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF)
else()
- set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON)
+ set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON)
endif()
+
option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT})
if (NOT ENABLE_EMBEDDED_COMPILER)
@@ -79,9 +76,9 @@ message (STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}")
message (STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}")
# ld: unknown option: --color-diagnostics
-if (APPLE)
+#if (APPLE)
set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
-endif ()
+#endif ()
# Do not adjust RPATH in llvm, since then it will not be able to find libcxx/libcxxabi/libunwind
set (CMAKE_INSTALL_RPATH "ON")
diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile
new file mode 100644
index 000000000000..207dddce1bbe
--- /dev/null
+++ b/docker/keeper/Dockerfile
@@ -0,0 +1,72 @@
+FROM ubuntu:20.04 AS glibc-donor
+
+ARG TARGETARCH
+RUN arch=${TARGETARCH:-amd64} \
+ && case $arch in \
+ amd64) rarch=x86_64 ;; \
+ arm64) rarch=aarch64 ;; \
+ esac \
+ && ln -s "${rarch}-linux-gnu" /lib/linux-gnu
+
+
+FROM alpine
+
+ENV LANG=en_US.UTF-8 \
+ LANGUAGE=en_US:en \
+ LC_ALL=en_US.UTF-8 \
+ TZ=UTC \
+ CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml
+
+COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/
+COPY --from=glibc-donor /etc/nsswitch.conf /etc/
+COPY entrypoint.sh /entrypoint.sh
+RUN arch=${TARGETARCH:-amd64} \
+ && case $arch in \
+ amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.31.so /lib64/ld-linux-x86-64.so.2 ;; \
+ arm64) ln -sf /lib/ld-2.31.so /lib/ld-linux-aarch64.so.1 ;; \
+ esac
+
+ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release"
+ARG VERSION="22.4.1.917"
+ARG PACKAGES="clickhouse-keeper"
+
+# user/group precreated explicitly with fixed uid/gid on purpose.
+# It is especially important for rootless containers: in that case entrypoint
+# can't do chown and owners of mounted volumes should be configured externally.
+# We do that in advance at the begining of Dockerfile before any packages will be
+# installed to prevent picking those uid / gid by some unrelated software.
+# The same uid / gid (101) is used both for alpine and ubuntu.
+
+
+ARG TARGETARCH
+RUN arch=${TARGETARCH:-amd64} \
+ && for package in ${PACKAGES}; do \
+ { \
+ { echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
+ && wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" -O "/tmp/${package}-${VERSION}-${arch}.tgz" \
+ && tar xvzf "/tmp/${package}-${VERSION}-${arch}.tgz" --strip-components=1 -C / ; \
+ } || \
+ { echo "Fallback to ${REPOSITORY}/${package}-${VERSION}.tgz" \
+ && wget -c -q "${REPOSITORY}/${package}-${VERSION}.tgz" -O "/tmp/${package}-${VERSION}.tgz" \
+ && tar xvzf "/tmp/${package}-${VERSION}.tgz" --strip-components=2 -C / ; \
+ } ; \
+ } || exit 1 \
+ ; done \
+ && rm /tmp/*.tgz /install -r \
+ && addgroup -S -g 101 clickhouse \
+ && adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse keeper" -u 101 clickhouse \
+ && mkdir -p /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper \
+ && chown clickhouse:clickhouse /var/lib/clickhouse \
+ && chown root:clickhouse /var/log/clickhouse-keeper \
+ && chmod +x /entrypoint.sh \
+ && apk add --no-cache su-exec bash tzdata \
+ && cp /usr/share/zoneinfo/UTC /etc/localtime \
+ && echo "UTC" > /etc/timezone \
+ && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper
+
+
+EXPOSE 2181 10181 44444
+
+VOLUME /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/keeper/Dockerfile.alpine b/docker/keeper/Dockerfile.alpine
new file mode 120000
index 000000000000..1d1fe94df492
--- /dev/null
+++ b/docker/keeper/Dockerfile.alpine
@@ -0,0 +1 @@
+Dockerfile
\ No newline at end of file
diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh
new file mode 100644
index 000000000000..3aacf655c281
--- /dev/null
+++ b/docker/keeper/entrypoint.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set +x
+set -eo pipefail
+shopt -s nullglob
+
+DO_CHOWN=1
+if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
+ DO_CHOWN=0
+fi
+
+CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
+CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+
+# support --user
+if [ "$(id -u)" = "0" ]; then
+ USER=$CLICKHOUSE_UID
+ GROUP=$CLICKHOUSE_GID
+ if command -v gosu &> /dev/null; then
+ gosu="gosu $USER:$GROUP"
+ elif command -v su-exec &> /dev/null; then
+ gosu="su-exec $USER:$GROUP"
+ else
+ echo "No gosu/su-exec detected!"
+ exit 1
+ fi
+else
+ USER="$(id -u)"
+ GROUP="$(id -g)"
+ gosu=""
+ DO_CHOWN=0
+fi
+
+KEEPER_CONFIG="${KEEPER_CONFIG:-/etc/clickhouse-keeper/config.yaml}"
+
+if [ -f "$KEEPER_CONFIG" ] && ! $gosu test -f "$KEEPER_CONFIG" -a -r "$KEEPER_CONFIG"; then
+ echo "Configuration file '$KEEPER_CONFIG' isn't readable by user with id '$USER'"
+ exit 1
+fi
+
+DATA_DIR="${CLICKHOUSE_DATA_DIR:-/var/lib/clickhouse}"
+LOG_DIR="${LOG_DIR:-/var/log/clickhouse-keeper}"
+LOG_PATH="${LOG_DIR}/clickhouse-keeper.log"
+ERROR_LOG_PATH="${LOG_DIR}/clickhouse-keeper.err.log"
+COORDINATION_LOG_DIR="${DATA_DIR}/coordination/log"
+COORDINATION_SNAPSHOT_DIR="${DATA_DIR}/coordination/snapshots"
+CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0}
+
+for dir in "$DATA_DIR" \
+ "$LOG_DIR" \
+ "$TMP_DIR" \
+ "$COORDINATION_LOG_DIR" \
+ "$COORDINATION_SNAPSHOT_DIR"
+do
+ # check if variable not empty
+ [ -z "$dir" ] && continue
+ # ensure directories exist
+ if ! mkdir -p "$dir"; then
+ echo "Couldn't create necessary directory: $dir"
+ exit 1
+ fi
+
+ if [ "$DO_CHOWN" = "1" ]; then
+ # ensure proper directories permissions
+ # but skip it for if directory already has proper premissions, cause recursive chown may be slow
+ if [ "$(stat -c %u "$dir")" != "$USER" ] || [ "$(stat -c %g "$dir")" != "$GROUP" ]; then
+ chown -R "$USER:$GROUP" "$dir"
+ fi
+ elif ! $gosu test -d "$dir" -a -w "$dir" -a -r "$dir"; then
+ echo "Necessary directory '$dir' isn't accessible by user with id '$USER'"
+ exit 1
+ fi
+done
+
+# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments
+if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
+ # Watchdog is launched by default, but does not send SIGINT to the main process,
+ # so the container can't be finished by ctrl+c
+ export CLICKHOUSE_WATCHDOG_ENABLE
+
+ cd /var/lib/clickhouse
+
+ # There is a config file. It is already tested with gosu (if it is readably by keeper user)
+ if [ -f "$KEEPER_CONFIG" ]; then
+ exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
+ fi
+
+ # There is no config file. Will use embedded one
+ exec $gosu /usr/bin/clickhouse-keeper --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
+fi
+
+# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image
+exec "$@"
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index e3e2e689b175..a57a734e3dff 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -95,6 +95,14 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
&& apt-get install gcc-11 g++-11 --yes \
&& apt-get clean
+# Architecture of the image when BuildKit/buildx is used
+ARG TARGETARCH
+ARG NFPM_VERSION=2.15.0
+
+RUN arch=${TARGETARCH:-amd64} \
+ && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
+ && dpkg -i /tmp/nfpm.deb \
+ && rm /tmp/nfpm.deb
COPY build.sh /
-CMD ["bash", "-c", "/build.sh 2>&1 | ts"]
+CMD ["bash", "-c", "/build.sh 2>&1"]
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index 2f18b07ffe19..31416e1a0ee7 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -1,7 +1,13 @@
#!/usr/bin/env bash
+exec &> >(ts)
set -x -e
+cache_status () {
+ ccache --show-config ||:
+ ccache --show-stats ||:
+}
+
mkdir -p build/cmake/toolchain/darwin-x86_64
tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
@@ -19,15 +25,23 @@ read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
env
cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
-ccache --show-config ||:
-ccache --show-stats ||:
+cache_status
+# clear cache stats
ccache --zero-stats ||:
-# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
+# No quotes because I want it to expand to nothing if empty.
+# shellcheck disable=SC2086
ninja $NINJA_FLAGS clickhouse-bundle
-ccache --show-config ||:
-ccache --show-stats ||:
+cache_status
+
+if [ -n "$MAKE_DEB" ]; then
+ rm -rf /build/packages/root
+ # No quotes because I want it to expand to nothing if empty.
+ # shellcheck disable=SC2086
+ DESTDIR=/build/packages/root ninja $NINJA_FLAGS install
+ bash -x /build/packages/build
+fi
mv ./programs/clickhouse* /output
mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds
@@ -84,8 +98,7 @@ fi
# ../docker/packager/other/fuzzer.sh
# fi
-ccache --show-config ||:
-ccache --show-stats ||:
+cache_status
if [ "${CCACHE_DEBUG:-}" == "1" ]
then
diff --git a/docker/packager/packager b/docker/packager/packager
index 05b2e02df969..f82d402d613a 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
import subprocess
import os
import argparse
@@ -8,36 +8,39 @@ import sys
SCRIPT_PATH = os.path.realpath(__file__)
-IMAGE_MAP = {
- "deb": "clickhouse/deb-builder",
- "binary": "clickhouse/binary-builder",
-}
def check_image_exists_locally(image_name):
try:
- output = subprocess.check_output("docker images -q {} 2> /dev/null".format(image_name), shell=True)
+ output = subprocess.check_output(
+ f"docker images -q {image_name} 2> /dev/null", shell=True
+ )
return output != ""
- except subprocess.CalledProcessError as ex:
+ except subprocess.CalledProcessError:
return False
+
def pull_image(image_name):
try:
- subprocess.check_call("docker pull {}".format(image_name), shell=True)
+ subprocess.check_call(f"docker pull {image_name}", shell=True)
return True
- except subprocess.CalledProcessError as ex:
- logging.info("Cannot pull image {}".format(image_name))
+ except subprocess.CalledProcessError:
+ logging.info(f"Cannot pull image {image_name}".format())
return False
+
def build_image(image_name, filepath):
context = os.path.dirname(filepath)
- build_cmd = "docker build --network=host -t {} -f {} {}".format(image_name, filepath, context)
- logging.info("Will build image with cmd: '{}'".format(build_cmd))
+ build_cmd = f"docker build --network=host -t {image_name} -f {filepath} {context}"
+ logging.info("Will build image with cmd: '%s'", build_cmd)
subprocess.check_call(
build_cmd,
shell=True,
)
-def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache_dir, docker_image_version):
+
+def run_docker_image_with_env(
+ image_name, output, env_variables, ch_root, ccache_dir, docker_image_version
+):
env_part = " -e ".join(env_variables)
if env_part:
env_part = " -e " + env_part
@@ -47,28 +50,52 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache
else:
interactive = ""
- cmd = "docker run --network=host --rm --volume={output_path}:/output --volume={ch_root}:/build --volume={ccache_dir}:/ccache {env} {interactive} {img_name}".format(
- output_path=output,
- ch_root=ch_root,
- ccache_dir=ccache_dir,
- env=env_part,
- img_name=image_name + ":" + docker_image_version,
- interactive=interactive
+ cmd = (
+ f"docker run --network=host --rm --volume={output}:/output "
+ f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} "
+ f"{interactive} {image_name}:{docker_image_version}"
)
- logging.info("Will build ClickHouse pkg with cmd: '{}'".format(cmd))
+ logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd)
subprocess.check_call(cmd, shell=True)
-def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries):
+
+def is_release_build(build_type, package_type, sanitizer, split_binary):
+ return (
+ build_type == ""
+ and package_type == "deb"
+ and sanitizer == ""
+ and not split_binary
+ )
+
+
+def parse_env_variables(
+ build_type,
+ compiler,
+ sanitizer,
+ package_type,
+ image_type,
+ cache,
+ distcc_hosts,
+ split_binary,
+ clang_tidy,
+ version,
+ author,
+ official,
+ additional_pkgs,
+ with_coverage,
+ with_binaries,
+):
DARWIN_SUFFIX = "-darwin"
DARWIN_ARM_SUFFIX = "-darwin-aarch64"
ARM_SUFFIX = "-aarch64"
FREEBSD_SUFFIX = "-freebsd"
- PPC_SUFFIX = '-ppc64le'
+ PPC_SUFFIX = "-ppc64le"
result = []
- cmake_flags = ['$CMAKE_FLAGS']
+ result.append("OUTPUT_DIR=/output")
+ cmake_flags = ["$CMAKE_FLAGS"]
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
@@ -77,46 +104,73 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
if is_cross_darwin:
- cc = compiler[:-len(DARWIN_SUFFIX)]
+ cc = compiler[: -len(DARWIN_SUFFIX)]
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
- cmake_flags.append("-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/x86_64-apple-darwin-install_name_tool")
- cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib")
+ cmake_flags.append(
+ "-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
+ "x86_64-apple-darwin-install_name_tool"
+ )
+ cmake_flags.append(
+ "-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib"
+ )
cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
- cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake")
+ cmake_flags.append(
+ "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake"
+ )
elif is_cross_darwin_arm:
- cc = compiler[:-len(DARWIN_ARM_SUFFIX)]
+ cc = compiler[: -len(DARWIN_ARM_SUFFIX)]
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar")
- cmake_flags.append("-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/aarch64-apple-darwin-install_name_tool")
- cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib")
+ cmake_flags.append(
+ "-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/"
+ "aarch64-apple-darwin-install_name_tool"
+ )
+ cmake_flags.append(
+ "-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib"
+ )
cmake_flags.append("-DLINKER_NAME=/cctools/bin/aarch64-apple-darwin-ld")
- cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake")
+ cmake_flags.append(
+ "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake"
+ )
elif is_cross_arm:
- cc = compiler[:-len(ARM_SUFFIX)]
- cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
- result.append("DEB_ARCH_FLAG=-aarm64")
+ cc = compiler[: -len(ARM_SUFFIX)]
+ cmake_flags.append(
+ "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake"
+ )
+ result.append("DEB_ARCH=arm64")
elif is_cross_freebsd:
- cc = compiler[:-len(FREEBSD_SUFFIX)]
- cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake")
+ cc = compiler[: -len(FREEBSD_SUFFIX)]
+ cmake_flags.append(
+ "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake"
+ )
elif is_cross_ppc:
- cc = compiler[:-len(PPC_SUFFIX)]
- cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake")
+ cc = compiler[: -len(PPC_SUFFIX)]
+ cmake_flags.append(
+ "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake"
+ )
else:
cc = compiler
- result.append("DEB_ARCH_FLAG=-aamd64")
+ result.append("DEB_ARCH=amd64")
- cxx = cc.replace('gcc', 'g++').replace('clang', 'clang++')
+ cxx = cc.replace("gcc", "g++").replace("clang", "clang++")
if image_type == "deb":
- result.append("DEB_CC={}".format(cc))
- result.append("DEB_CXX={}".format(cxx))
- # For building fuzzers
- result.append("CC={}".format(cc))
- result.append("CXX={}".format(cxx))
- elif image_type == "binary":
- result.append("CC={}".format(cc))
- result.append("CXX={}".format(cxx))
- cmake_flags.append('-DCMAKE_C_COMPILER=`which {}`'.format(cc))
- cmake_flags.append('-DCMAKE_CXX_COMPILER=`which {}`'.format(cxx))
+ result.append("MAKE_DEB=true")
+ cmake_flags.append("-DENABLE_TESTS=0")
+ cmake_flags.append("-DENABLE_UTILS=0")
+ cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON")
+ cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON")
+ cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON")
+ cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr")
+ cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc")
+ cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var")
+ cmake_flags.append("-DBUILD_STANDALONE_KEEPER=ON")
+ if is_release_build(build_type, package_type, sanitizer, split_binary):
+ cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON")
+
+ result.append(f"CC={cc}")
+ result.append(f"CXX={cxx}")
+ cmake_flags.append(f"-DCMAKE_C_COMPILER={cc}")
+ cmake_flags.append(f"-DCMAKE_CXX_COMPILER={cxx}")
# Create combined output archive for split build and for performance tests.
if package_type == "performance":
@@ -126,12 +180,14 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
result.append("COMBINED_OUTPUT=shared_build")
if sanitizer:
- result.append("SANITIZER={}".format(sanitizer))
+ result.append(f"SANITIZER={sanitizer}")
if build_type:
- result.append("BUILD_TYPE={}".format(build_type))
+ result.append(f"BUILD_TYPE={build_type.capitalize()}")
+ else:
+ result.append("BUILD_TYPE=None")
- if cache == 'distcc':
- result.append("CCACHE_PREFIX={}".format(cache))
+ if cache == "distcc":
+ result.append(f"CCACHE_PREFIX={cache}")
if cache:
result.append("CCACHE_DIR=/ccache")
@@ -142,109 +198,188 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
# result.append("CCACHE_UMASK=777")
if distcc_hosts:
- hosts_with_params = ["{}/24,lzo".format(host) for host in distcc_hosts] + ["localhost/`nproc`"]
- result.append('DISTCC_HOSTS="{}"'.format(" ".join(hosts_with_params)))
+ hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [
+ "localhost/`nproc`"
+ ]
+ result.append('DISTCC_HOSTS="' + " ".join(hosts_with_params) + '"')
elif cache == "distcc":
- result.append('DISTCC_HOSTS="{}"'.format("localhost/`nproc`"))
+ result.append('DISTCC_HOSTS="localhost/`nproc`"')
- if alien_pkgs:
- result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'")
+ if additional_pkgs:
+ result.append("MAKE_APK=true")
+ result.append("MAKE_RPM=true")
+ result.append("MAKE_TGZ=true")
if with_binaries == "programs":
- result.append('BINARY_OUTPUT=programs')
+ result.append("BINARY_OUTPUT=programs")
elif with_binaries == "tests":
- result.append('ENABLE_TESTS=1')
- result.append('BINARY_OUTPUT=tests')
- cmake_flags.append('-DENABLE_TESTS=1')
+ result.append("ENABLE_TESTS=1")
+ result.append("BINARY_OUTPUT=tests")
+ cmake_flags.append("-DENABLE_TESTS=1")
if split_binary:
- cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1')
+ cmake_flags.append(
+ "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 "
+ "-DCLICKHOUSE_SPLIT_BINARY=1"
+ )
# We can't always build utils because it requires too much space, but
# we have to build them at least in some way in CI. The split build is
# probably the least heavy disk-wise.
- cmake_flags.append('-DENABLE_UTILS=1')
+ cmake_flags.append("-DENABLE_UTILS=1")
if clang_tidy:
- cmake_flags.append('-DENABLE_CLANG_TIDY=1')
- cmake_flags.append('-DENABLE_UTILS=1')
- cmake_flags.append('-DENABLE_TESTS=1')
- cmake_flags.append('-DENABLE_EXAMPLES=1')
+ cmake_flags.append("-DENABLE_CLANG_TIDY=1")
+ cmake_flags.append("-DENABLE_UTILS=1")
+ cmake_flags.append("-DENABLE_TESTS=1")
+ cmake_flags.append("-DENABLE_EXAMPLES=1")
# Don't stop on first error to find more clang-tidy errors in one run.
- result.append('NINJA_FLAGS=-k0')
+ result.append("NINJA_FLAGS=-k0")
if with_coverage:
- cmake_flags.append('-DWITH_COVERAGE=1')
+ cmake_flags.append("-DWITH_COVERAGE=1")
if version:
- result.append("VERSION_STRING='{}'".format(version))
+ result.append(f"VERSION_STRING='{version}'")
if author:
- result.append("AUTHOR='{}'".format(author))
+ result.append(f"AUTHOR='{author}'")
if official:
- cmake_flags.append('-DYANDEX_OFFICIAL_BUILD=1')
+ cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1")
- result.append('CMAKE_FLAGS="' + ' '.join(cmake_flags) + '"')
+ result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"')
return result
+
if __name__ == "__main__":
- logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
- parser = argparse.ArgumentParser(description="ClickHouse building script using prebuilt Docker image")
- # 'performance' creates a combined .tgz with server and configs to be used for performance test.
- parser.add_argument("--package-type", choices=['deb', 'binary', 'performance'], required=True)
- parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ description="ClickHouse building script using prebuilt Docker image",
+ )
+ # 'performance' creates a combined .tgz with server
+ # and configs to be used for performance test.
+ parser.add_argument(
+ "--package-type",
+ choices=("deb", "binary", "performance"),
+ required=True,
+ help="a build type",
+ )
+ parser.add_argument(
+ "--clickhouse-repo-path",
+ default=os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir
+ ),
+ help="ClickHouse git repository",
+ )
parser.add_argument("--output-dir", required=True)
parser.add_argument("--build-type", choices=("debug", ""), default="")
- parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64",
- "clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64",
- "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", "clang-13-ppc64le",
- "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13")
- parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
+ parser.add_argument(
+ "--compiler",
+ choices=(
+ "clang-11",
+ "clang-11-darwin",
+ "clang-11-darwin-aarch64",
+ "clang-11-aarch64",
+ "clang-12",
+ "clang-12-darwin",
+ "clang-12-darwin-aarch64",
+ "clang-12-aarch64",
+ "clang-13",
+ "clang-13-darwin",
+ "clang-13-darwin-aarch64",
+ "clang-13-aarch64",
+ "clang-13-ppc64le",
+ "clang-11-freebsd",
+ "clang-12-freebsd",
+ "clang-13-freebsd",
+ "gcc-11",
+ ),
+ default="clang-13",
+ help="a compiler to use",
+ )
+ parser.add_argument(
+ "--sanitizer",
+ choices=("address", "thread", "memory", "undefined", ""),
+ default="",
+ )
parser.add_argument("--split-binary", action="store_true")
parser.add_argument("--clang-tidy", action="store_true")
- parser.add_argument("--cache", choices=("", "ccache", "distcc"), default="")
- parser.add_argument("--ccache_dir", default= os.getenv("HOME", "") + '/.ccache')
+ parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
+ parser.add_argument(
+ "--ccache_dir",
+ default=os.getenv("HOME", "") + "/.ccache",
+ help="a directory with ccache",
+ )
parser.add_argument("--distcc-hosts", nargs="+")
parser.add_argument("--force-build-image", action="store_true")
parser.add_argument("--version")
- parser.add_argument("--author", default="clickhouse")
+ parser.add_argument("--author", default="clickhouse", help="a package author")
parser.add_argument("--official", action="store_true")
- parser.add_argument("--alien-pkgs", nargs='+', default=[])
+ parser.add_argument("--additional-pkgs", action="store_true")
parser.add_argument("--with-coverage", action="store_true")
- parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="")
- parser.add_argument("--docker-image-version", default="latest")
+ parser.add_argument(
+ "--with-binaries", choices=("programs", "tests", ""), default=""
+ )
+ parser.add_argument(
+ "--docker-image-version", default="latest", help="docker image tag to use"
+ )
args = parser.parse_args()
if not os.path.isabs(args.output_dir):
args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
- image_type = 'binary' if args.package_type == 'performance' else args.package_type
- image_name = IMAGE_MAP[image_type]
+ image_type = "binary" if args.package_type == "performance" else args.package_type
+ image_name = "clickhouse/binary-builder"
if not os.path.isabs(args.clickhouse_repo_path):
ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path))
else:
ch_root = args.clickhouse_repo_path
- if args.alien_pkgs and not image_type == "deb":
- raise Exception("Can add alien packages only in deb build")
+ if args.additional_pkgs and image_type != "deb":
+ raise Exception("Can build additional packages only in deb build")
- if args.with_binaries != "" and not image_type == "deb":
+ if args.with_binaries != "" and image_type != "deb":
raise Exception("Can add additional binaries only in deb build")
if args.with_binaries != "" and image_type == "deb":
- logging.info("Should place {} to output".format(args.with_binaries))
+ logging.info("Should place %s to output", args.with_binaries)
dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile")
image_with_version = image_name + ":" + args.docker_image_version
- if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image:
+ if (
+ image_type != "freebsd"
+ and not check_image_exists_locally(image_name)
+ or args.force_build_image
+ ):
if not pull_image(image_with_version) or args.force_build_image:
build_image(image_with_version, dockerfile)
env_prepared = parse_env_variables(
- args.build_type, args.compiler, args.sanitizer, args.package_type, image_type,
- args.cache, args.distcc_hosts, args.split_binary, args.clang_tidy,
- args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries)
+ args.build_type,
+ args.compiler,
+ args.sanitizer,
+ args.package_type,
+ image_type,
+ args.cache,
+ args.distcc_hosts,
+ args.split_binary,
+ args.clang_tidy,
+ args.version,
+ args.author,
+ args.official,
+ args.additional_pkgs,
+ args.with_coverage,
+ args.with_binaries,
+ )
- run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir, args.docker_image_version)
- logging.info("Output placed into {}".format(args.output_dir))
+ run_docker_image_with_env(
+ image_name,
+ args.output_dir,
+ env_prepared,
+ ch_root,
+ args.ccache_dir,
+ args.docker_image_version,
+ )
+ logging.info("Output placed into %s", args.output_dir)
diff --git a/docker/server/.gitignore b/docker/server/.gitignore
deleted file mode 100644
index 692758d55aa1..000000000000
--- a/docker/server/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-alpine-root/*
-tgz-packages/*
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
deleted file mode 100644
index 5b7990ab0300..000000000000
--- a/docker/server/Dockerfile
+++ /dev/null
@@ -1,122 +0,0 @@
-FROM ubuntu:20.04
-
-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ARG repository="deb https://packages.clickhouse.com/deb stable main"
-ARG version=22.1.1.*
-
-# set non-empty deb_location_url url to create a docker image
-# from debs created by CI build, for example:
-# docker build . --network host --build-arg version="21.4.1.6282" --build-arg deb_location_url="https://clickhouse-builds.s3.yandex.net/21852/069cfbff388b3d478d1a16dc7060b48073f5d522/clickhouse_build_check/clang-11_relwithdebuginfo_none_bundled_unsplitted_disable_False_deb/" -t filimonovq/clickhouse-server:pr21852
-ARG deb_location_url=""
-
-# set non-empty single_binary_location_url to create docker image
-# from a single binary url (useful for non-standard builds - with sanitizers, for arm64).
-# for example (run on aarch64 server):
-# docker build . --network host --build-arg single_binary_location_url="https://builds.clickhouse.com/master/aarch64/clickhouse" -t altinity/clickhouse-server:master-testing-arm
-# note: clickhouse-odbc-bridge is not supported there.
-ARG single_binary_location_url=""
-
-# see https://github.com/moby/moby/issues/4032#issuecomment-192327844
-ARG DEBIAN_FRONTEND=noninteractive
-
-# user/group precreated explicitly with fixed uid/gid on purpose.
-# It is especially important for rootless containers: in that case entrypoint
-# can't do chown and owners of mounted volumes should be configured externally.
-# We do that in advance at the begining of Dockerfile before any packages will be
-# installed to prevent picking those uid / gid by some unrelated software.
-# The same uid / gid (101) is used both for alpine and ubuntu.
-
-# To drop privileges, we need 'su' command, that simply changes uid and gid.
-# In fact, the 'su' command from Linux is not so simple, due to inherent vulnerability in Linux:
-# https://ruderich.org/simon/notes/su-sudo-from-root-tty-hijacking
-# It has to mitigate this drawback of Linux, and to do this, 'su' command is creating it's own pseudo-terminal
-# and forwarding commands. Due to some ridiculous curcumstances, it does not work in Docker (or it does)
-# and for these reasons people are using alternatives to the 'su' command in Docker,
-# that don't mess with the terminal, don't care about closing the opened files, etc...
-# but can only be safe to drop privileges inside Docker.
-# The question - what implementation of 'su' command to use.
-# It should be a simple script doing about just two syscalls.
-# Some people tend to use 'gosu' tool that is written in Go.
-# It is not used for several reasons:
-# 1. Dependency on some foreign code in yet another programming language - does not sound alright.
-# 2. Anselmo D. Adams suggested not to use it due to false positive alarms in some undisclosed security scanners.
-
-COPY su-exec.c /su-exec.c
-
-RUN groupadd -r clickhouse --gid=101 \
- && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \
- && apt-get update \
- && apt-get install --yes --no-install-recommends \
- apt-transport-https \
- ca-certificates \
- dirmngr \
- gnupg \
- locales \
- wget \
- tzdata \
- && mkdir -p /etc/apt/sources.list.d \
- && apt-key adv --keyserver keyserver.ubuntu.com --recv 8919F6BD2B48D754 \
- && echo $repository > /etc/apt/sources.list.d/clickhouse.list \
- && if [ -n "$deb_location_url" ]; then \
- echo "installing from custom url with deb packages: $deb_location_url" \
- rm -rf /tmp/clickhouse_debs \
- && mkdir -p /tmp/clickhouse_debs \
- && wget --progress=bar:force:noscroll "${deb_location_url}/clickhouse-common-static_${version}_amd64.deb" -P /tmp/clickhouse_debs \
- && wget --progress=bar:force:noscroll "${deb_location_url}/clickhouse-client_${version}_all.deb" -P /tmp/clickhouse_debs \
- && wget --progress=bar:force:noscroll "${deb_location_url}/clickhouse-server_${version}_all.deb" -P /tmp/clickhouse_debs \
- && dpkg -i /tmp/clickhouse_debs/*.deb ; \
- elif [ -n "$single_binary_location_url" ]; then \
- echo "installing from single binary url: $single_binary_location_url" \
- && rm -rf /tmp/clickhouse_binary \
- && mkdir -p /tmp/clickhouse_binary \
- && wget --progress=bar:force:noscroll "$single_binary_location_url" -O /tmp/clickhouse_binary/clickhouse \
- && chmod +x /tmp/clickhouse_binary/clickhouse \
- && /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" ; \
- else \
- echo "installing from repository: $repository" \
- && apt-get update \
- && apt-get --yes -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade \
- && apt-get install --allow-unauthenticated --yes --no-install-recommends \
- clickhouse-common-static=$version \
- clickhouse-client=$version \
- clickhouse-server=$version ; \
- fi \
- && apt-get install -y --no-install-recommends tcc libc-dev && \
- tcc /su-exec.c -o /bin/su-exec && \
- chown root:root /bin/su-exec && \
- chmod 0755 /bin/su-exec && \
- rm /su-exec.c && \
- apt-get purge -y --auto-remove tcc libc-dev libc-dev-bin libc6-dev linux-libc-dev \
- && clickhouse-local -q 'SELECT * FROM system.build_options' \
- && rm -rf \
- /var/lib/apt/lists/* \
- /var/cache/debconf \
- /tmp/* \
- && apt-get clean \
- && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client \
- && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client
-
-# we need to allow "others" access to clickhouse folder, because docker container
-# can be started with arbitrary uid (openshift usecase)
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-ENV TZ UTC
-
-RUN mkdir /docker-entrypoint-initdb.d
-
-COPY docker_related_config.xml /etc/clickhouse-server/config.d/
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
-
-EXPOSE 9000 8123 9009
-VOLUME /var/lib/clickhouse
-
-ENV CLICKHOUSE_CONFIG /etc/clickhouse-server/config.xml
-
-ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
new file mode 120000
index 000000000000..fd45f0f7c7cd
--- /dev/null
+++ b/docker/server/Dockerfile
@@ -0,0 +1 @@
+Dockerfile.ubuntu
\ No newline at end of file
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index cd192c0c9da4..5aaf5dd55115 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -1,3 +1,14 @@
+FROM ubuntu:20.04 AS glibc-donor
+ARG TARGETARCH
+
+RUN arch=${TARGETARCH:-amd64} \
+ && case $arch in \
+ amd64) rarch=x86_64 ;; \
+ arm64) rarch=aarch64 ;; \
+ esac \
+ && ln -s "${rarch}-linux-gnu" /lib/linux-gnu
+
+
FROM alpine
ENV LANG=en_US.UTF-8 \
@@ -6,7 +17,24 @@ ENV LANG=en_US.UTF-8 \
TZ=UTC \
CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml
-COPY alpine-root/ /
+COPY --from=glibc-donor /lib/linux-gnu/libc.so.6 /lib/linux-gnu/libdl.so.2 /lib/linux-gnu/libm.so.6 /lib/linux-gnu/libpthread.so.0 /lib/linux-gnu/librt.so.1 /lib/linux-gnu/libnss_dns.so.2 /lib/linux-gnu/libnss_files.so.2 /lib/linux-gnu/libresolv.so.2 /lib/linux-gnu/ld-2.31.so /lib/
+COPY --from=glibc-donor /etc/nsswitch.conf /etc/
+COPY docker_related_config.xml /etc/clickhouse-server/config.d/
+COPY entrypoint.sh /entrypoint.sh
+
+ARG TARGETARCH
+
+RUN arch=${TARGETARCH:-amd64} \
+ && case $arch in \
+ amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.31.so /lib64/ld-linux-x86-64.so.2 ;; \
+ arm64) ln -sf /lib/ld-2.31.so /lib/ld-linux-aarch64.so.1 ;; \
+ esac
+
+# lts / testing / prestable / etc
+ARG REPO_CHANNEL="stable"
+ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
+ARG VERSION="20.9.3.45"
+ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# user/group precreated explicitly with fixed uid/gid on purpose.
# It is especially important for rootless containers: in that case entrypoint
@@ -15,9 +43,23 @@ COPY alpine-root/ /
# installed to prevent picking those uid / gid by some unrelated software.
# The same uid / gid (101) is used both for alpine and ubuntu.
-RUN addgroup -S -g 101 clickhouse \
+RUN arch=${TARGETARCH:-amd64} \
+ && for package in ${PACKAGES}; do \
+ { \
+ { echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
+ && wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" -O "/tmp/${package}-${VERSION}-${arch}.tgz" \
+ && tar xvzf "/tmp/${package}-${VERSION}-${arch}.tgz" --strip-components=1 -C / ; \
+ } || \
+ { echo "Fallback to ${REPOSITORY}/${package}-${VERSION}.tgz" \
+ && wget -c -q "${REPOSITORY}/${package}-${VERSION}.tgz" -O "/tmp/${package}-${VERSION}.tgz" \
+ && tar xvzf "/tmp/${package}-${VERSION}.tgz" --strip-components=2 -C / ; \
+ } ; \
+ } || exit 1 \
+ ; done \
+ && rm /tmp/*.tgz /install -r \
+ && addgroup -S -g 101 clickhouse \
&& adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse server" -u 101 clickhouse \
- && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client \
+ && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server/config.d /etc/clickhouse-server/users.d /etc/clickhouse-client /docker-entrypoint-initdb.d \
&& chown clickhouse:clickhouse /var/lib/clickhouse \
&& chown root:clickhouse /var/log/clickhouse-server \
&& chmod +x /entrypoint.sh \
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
new file mode 100644
index 000000000000..cc198772251a
--- /dev/null
+++ b/docker/server/Dockerfile.ubuntu
@@ -0,0 +1,128 @@
+FROM ubuntu:20.04
+
+# see https://github.com/moby/moby/issues/4032#issuecomment-192327844
+ARG DEBIAN_FRONTEND=noninteractive
+
+COPY su-exec.c /su-exec.c
+
+# ARG for quick switch to a given ubuntu mirror
+ARG apt_archive="http://archive.ubuntu.com"
+RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list \
+ && groupadd -r clickhouse --gid=101 \
+ && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \
+ && apt-get update \
+ && apt-get install --yes --no-install-recommends \
+ apt-transport-https \
+ ca-certificates \
+ dirmngr \
+ gnupg \
+ locales \
+ wget \
+ tzdata \
+ && apt-get install -y --no-install-recommends tcc libc-dev && \
+ tcc /su-exec.c -o /bin/su-exec && \
+ chown root:root /bin/su-exec && \
+ chmod 0755 /bin/su-exec && \
+ rm /su-exec.c && \
+ apt-get purge -y --auto-remove tcc libc-dev libc-dev-bin libc6-dev linux-libc-dev \
+ && apt-get clean
+
+ARG REPO_CHANNEL="stable"
+ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
+ARG VERSION=22.1.1.*
+ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
+
+# set non-empty deb_location_url url to create a docker image
+# from debs created by CI build, for example:
+# docker build . --network host --build-arg version="21.4.1.6282" --build-arg deb_location_url="https://clickhouse-builds.s3.yandex.net/21852/069cfbff388b3d478d1a16dc7060b48073f5d522/clickhouse_build_check/clang-11_relwithdebuginfo_none_bundled_unsplitted_disable_False_deb/" -t filimonovq/clickhouse-server:pr21852
+ARG deb_location_url=""
+
+# set non-empty single_binary_location_url to create docker image
+# from a single binary url (useful for non-standard builds - with sanitizers, for arm64).
+# for example (run on aarch64 server):
+# docker build . --network host --build-arg single_binary_location_url="https://builds.clickhouse.com/master/aarch64/clickhouse" -t altinity/clickhouse-server:master-testing-arm
+# note: clickhouse-odbc-bridge is not supported there.
+ARG single_binary_location_url=""
+
+# user/group precreated explicitly with fixed uid/gid on purpose.
+# It is especially important for rootless containers: in that case entrypoint
+# can't do chown and owners of mounted volumes should be configured externally.
+# We do that in advance at the begining of Dockerfile before any packages will be
+# installed to prevent picking those uid / gid by some unrelated software.
+# The same uid / gid (101) is used both for alpine and ubuntu.
+
+# To drop privileges, we need 'su' command, that simply changes uid and gid.
+# In fact, the 'su' command from Linux is not so simple, due to inherent vulnerability in Linux:
+# https://ruderich.org/simon/notes/su-sudo-from-root-tty-hijacking
+# It has to mitigate this drawback of Linux, and to do this, 'su' command is creating it's own pseudo-terminal
+# and forwarding commands. Due to some ridiculous curcumstances, it does not work in Docker (or it does)
+# and for these reasons people are using alternatives to the 'su' command in Docker,
+# that don't mess with the terminal, don't care about closing the opened files, etc...
+# but can only be safe to drop privileges inside Docker.
+# The question - what implementation of 'su' command to use.
+# It should be a simple script doing about just two syscalls.
+# Some people tend to use 'gosu' tool that is written in Go.
+# It is not used for several reasons:
+# 1. Dependency on some foreign code in yet another programming language - does not sound alright.
+# 2. Anselmo D. Adams suggested not to use it due to false positive alarms in some undisclosed security scanners.
+
+ARG TARGETARCH
+
+RUN arch=${TARGETARCH:-amd64} \
+ && if [ -n "${deb_location_url}" ]; then \
+ echo "installing from custom url with deb packages: ${deb_location_url}" \
+ rm -rf /tmp/clickhouse_debs \
+ && mkdir -p /tmp/clickhouse_debs \
+ && for package in ${PACKAGES}; do \
+ { wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_${arch}.deb" -P /tmp/clickhouse_debs || \
+ wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_all.deb" -P /tmp/clickhouse_debs ; } \
+ || exit 1 \
+ ; done \
+ && dpkg -i /tmp/clickhouse_debs/*.deb ; \
+ elif [ -n "${single_binary_location_url}" ]; then \
+ echo "installing from single binary url: ${single_binary_location_url}" \
+ && rm -rf /tmp/clickhouse_binary \
+ && mkdir -p /tmp/clickhouse_binary \
+ && wget --progress=bar:force:noscroll "${single_binary_location_url}" -O /tmp/clickhouse_binary/clickhouse \
+ && chmod +x /tmp/clickhouse_binary/clickhouse \
+ && /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" ; \
+ else \
+ mkdir -p /etc/apt/sources.list.d \
+ && apt-key adv --keyserver keyserver.ubuntu.com --recv 8919F6BD2B48D754 \
+ && echo ${REPOSITORY} > /etc/apt/sources.list.d/clickhouse.list \
+ && echo "installing from repository: ${REPOSITORY}" \
+ && apt-get update \
+ && apt-get --yes -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade \
+ && for package in ${PACKAGES}; do \
+ apt-get install --allow-unauthenticated --yes --no-install-recommends "${package}=${VERSION}" || exit 1 \
+ ; done \
+ ; fi \
+ && clickhouse-local -q 'SELECT * FROM system.build_options' \
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /var/cache/debconf \
+ /tmp/* \
+ && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client \
+ && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client
+
+# we need to allow "others" access to clickhouse folder, because docker container
+# can be started with arbitrary uid (openshift usecase)
+
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+ENV TZ UTC
+
+RUN mkdir /docker-entrypoint-initdb.d
+
+COPY docker_related_config.xml /etc/clickhouse-server/config.d/
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 9000 8123 9009
+VOLUME /var/lib/clickhouse
+
+ENV CLICKHOUSE_CONFIG /etc/clickhouse-server/config.xml
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/server/alpine-build.sh b/docker/server/alpine-build.sh
deleted file mode 100755
index 1b448c61fbbd..000000000000
--- a/docker/server/alpine-build.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/bash
-set -x
-
-REPO_CHANNEL="${REPO_CHANNEL:-stable}" # lts / testing / prestable / etc
-REPO_URL="${REPO_URL:-"https://repo.yandex.ru/clickhouse/tgz/${REPO_CHANNEL}"}"
-VERSION="${VERSION:-20.9.3.45}"
-DOCKER_IMAGE="${DOCKER_IMAGE:-clickhouse/clickhouse-server}"
-
-# where original files live
-DOCKER_BUILD_FOLDER="${BASH_SOURCE%/*}"
-
-# we will create root for our image here
-CONTAINER_ROOT_FOLDER="${DOCKER_BUILD_FOLDER}/alpine-root"
-
-# clean up the root from old runs, it's reconstructed each time
-rm -rf "$CONTAINER_ROOT_FOLDER"
-mkdir -p "$CONTAINER_ROOT_FOLDER"
-
-# where to put downloaded tgz
-TGZ_PACKAGES_FOLDER="${DOCKER_BUILD_FOLDER}/tgz-packages"
-mkdir -p "$TGZ_PACKAGES_FOLDER"
-
-PACKAGES=( "clickhouse-client" "clickhouse-server" "clickhouse-common-static" )
-
-# download tars from the repo
-for package in "${PACKAGES[@]}"
-do
- wget -c -q --show-progress "${REPO_URL}/${package}-${VERSION}.tgz" -O "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz"
-done
-
-# unpack tars
-for package in "${PACKAGES[@]}"
-do
- tar xvzf "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" --strip-components=2 -C "$CONTAINER_ROOT_FOLDER"
-done
-
-# prepare few more folders
-mkdir -p "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/users.d" \
- "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d" \
- "${CONTAINER_ROOT_FOLDER}/var/log/clickhouse-server" \
- "${CONTAINER_ROOT_FOLDER}/var/lib/clickhouse" \
- "${CONTAINER_ROOT_FOLDER}/docker-entrypoint-initdb.d" \
- "${CONTAINER_ROOT_FOLDER}/lib64"
-
-cp "${DOCKER_BUILD_FOLDER}/docker_related_config.xml" "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d/"
-cp "${DOCKER_BUILD_FOLDER}/entrypoint.sh" "${CONTAINER_ROOT_FOLDER}/entrypoint.sh"
-
-## get glibc components from ubuntu 20.04 and put them to expected place
-docker pull ubuntu:20.04
-ubuntu20image=$(docker create --rm ubuntu:20.04)
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libc.so.6 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libdl.so.2 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libm.so.6 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libpthread.so.0 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/librt.so.1 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libnss_files.so.2 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib"
-docker cp -L "${ubuntu20image}":/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64"
-docker cp -L "${ubuntu20image}":/etc/nsswitch.conf "${CONTAINER_ROOT_FOLDER}/etc"
-
-docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "${DOCKER_IMAGE}:${VERSION}-alpine" --pull
-rm -rf "$CONTAINER_ROOT_FOLDER"
diff --git a/docker/server/local.Dockerfile b/docker/server/local.Dockerfile
deleted file mode 100644
index 0d86c9ce45a3..000000000000
--- a/docker/server/local.Dockerfile
+++ /dev/null
@@ -1,47 +0,0 @@
-# Since right now we can't set volumes to the docker during build, we split building container in stages:
-# 1. build base container
-# 2. run base conatiner with mounted volumes
-# 3. commit container as image
-# 4. build final container atop that image
-# Middle steps are performed by the bash script.
-
-FROM ubuntu:18.04 as clickhouse-server-base
-ARG gosu_ver=1.14
-
-VOLUME /packages/
-
-# update to allow installing dependencies of clickhouse automatically
-RUN apt update; \
- DEBIAN_FRONTEND=noninteractive \
- apt install -y locales;
-
-ADD https://github.com/tianon/gosu/releases/download/${gosu_ver}/gosu-amd64 /bin/gosu
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-
-# installing via apt to simulate real-world scenario, where user installs deb package and all it's dependecies automatically.
-CMD DEBIAN_FRONTEND=noninteractive \
- apt install -y \
- /packages/clickhouse-common-static_*.deb \
- /packages/clickhouse-server_*.deb ;
-
-FROM clickhouse-server-base:postinstall as clickhouse-server
-
-RUN mkdir /docker-entrypoint-initdb.d
-
-COPY docker_related_config.xml /etc/clickhouse-server/config.d/
-COPY entrypoint.sh /entrypoint.sh
-
-RUN chmod +x \
- /entrypoint.sh \
- /bin/gosu
-
-EXPOSE 9000 8123 9009
-VOLUME /var/lib/clickhouse
-
-ENV CLICKHOUSE_CONFIG /etc/clickhouse-server/config.xml
-
-ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/test/fuzzer/generate-test-j2.py b/docker/test/fuzzer/generate-test-j2.py
index bcc1bf6bc845..11525163ed8e 100755
--- a/docker/test/fuzzer/generate-test-j2.py
+++ b/docker/test/fuzzer/generate-test-j2.py
@@ -11,7 +11,7 @@ def removesuffix(text, suffix):
https://www.python.org/dev/peps/pep-0616/
"""
if suffix and text.endswith(suffix):
- return text[:-len(suffix)]
+ return text[: -len(suffix)]
else:
return text[:]
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index e18c07bf2c13..74711f476f83 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -13,7 +13,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
echo "$script_dir"
repo_dir=ch
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-13_debug_none_bundled_unsplitted_disable_False_binary"}
-BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
+BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
function clone
{
diff --git a/docker/test/integration/hive_server/http_api_server.py b/docker/test/integration/hive_server/http_api_server.py
index 4818b785c896..8a9d3da48463 100644
--- a/docker/test/integration/hive_server/http_api_server.py
+++ b/docker/test/integration/hive_server/http_api_server.py
@@ -3,55 +3,55 @@
import datetime
from flask import Flask, flash, request, redirect, url_for
+
def run_command(command, wait=False):
print("{} - execute shell command:{}".format(datetime.datetime.now(), command))
lines = []
- p = subprocess.Popen(command,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- shell=True)
+ p = subprocess.Popen(
+ command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True
+ )
if wait:
- for l in iter(p.stdout.readline, b''):
+ for l in iter(p.stdout.readline, b""):
lines.append(l)
p.poll()
return (lines, p.returncode)
else:
- return(iter(p.stdout.readline, b''), 0)
+ return (iter(p.stdout.readline, b""), 0)
-UPLOAD_FOLDER = './'
-ALLOWED_EXTENSIONS = {'txt', 'sh'}
+UPLOAD_FOLDER = "./"
+ALLOWED_EXTENSIONS = {"txt", "sh"}
app = Flask(__name__)
-app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
+app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER
+
-@app.route('/')
+@app.route("/")
def hello_world():
- return 'Hello World'
+ return "Hello World"
def allowed_file(filename):
- return '.' in filename and \
- filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+ return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
-@app.route('/upload', methods=['GET', 'POST'])
+@app.route("/upload", methods=["GET", "POST"])
def upload_file():
- if request.method == 'POST':
+ if request.method == "POST":
# check if the post request has the file part
- if 'file' not in request.files:
- flash('No file part')
+ if "file" not in request.files:
+ flash("No file part")
return redirect(request.url)
- file = request.files['file']
+ file = request.files["file"]
# If the user does not select a file, the browser submits an
# empty file without a filename.
- if file.filename == '':
- flash('No selected file')
+ if file.filename == "":
+ flash("No selected file")
return redirect(request.url)
if file and allowed_file(file.filename):
filename = file.filename
- file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
- return redirect(url_for('upload_file', name=filename))
- return '''
+ file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename))
+ return redirect(url_for("upload_file", name=filename))
+ return """
Upload new File
Upload new File
@@ -59,12 +59,15 @@ def upload_file():
- '''
-@app.route('/run', methods=['GET', 'POST'])
+ """
+
+
+@app.route("/run", methods=["GET", "POST"])
def parse_request():
data = request.data # data is empty
run_command(data, wait=True)
- return 'Ok'
+ return "Ok"
+
-if __name__ == '__main__':
- app.run(port=5011)
+if __name__ == "__main__":
+ app.run(port=5011)
diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh
index d7534270e2c5..4dec82234bcf 100644
--- a/docker/test/keeper-jepsen/run.sh
+++ b/docker/test/keeper-jepsen/run.sh
@@ -2,7 +2,7 @@
set -euo pipefail
-CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-13_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
+CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-13_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
@@ -10,7 +10,7 @@ if [ -z "$CLICKHOUSE_REPO_PATH" ]; then
CLICKHOUSE_REPO_PATH=ch
rm -rf ch ||:
mkdir ch ||:
- wget -nv -nd -c "https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz"
+ wget -nv -nd -c "https://clickhouse-test-reports.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz"
tar -C ch --strip-components=1 -xf clickhouse_no_subs.tar.gz
ls -lath ||:
fi
diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh
index 16ac304d7fb8..54f71ce05bb4 100755
--- a/docker/test/performance-comparison/compare.sh
+++ b/docker/test/performance-comparison/compare.sh
@@ -1294,15 +1294,15 @@ create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
select '' test_name,
'$(sed -n 's/.*/\1/p' report.html)' test_status,
0 test_duration_ms,
- 'https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/performance_comparison/report.html#fail1' report_url
+ 'https://clickhouse-test-reports.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/performance_comparison/report.html#fail1' report_url
union all
select test || ' #' || toString(query_index), 'slower' test_status, 0 test_duration_ms,
- 'https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/performance_comparison/report.html#changes-in-performance.'
+ 'https://clickhouse-test-reports.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/performance_comparison/report.html#changes-in-performance.'
|| test || '.' || toString(query_index) report_url
from queries where changed_fail != 0 and diff > 0
union all
select test || ' #' || toString(query_index), 'unstable' test_status, 0 test_duration_ms,
- 'https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/performance_comparison/report.html#unstable-queries.'
+ 'https://clickhouse-test-reports.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/performance_comparison/report.html#unstable-queries.'
|| test || '.' || toString(query_index) report_url
from queries where unstable_fail != 0
)
diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh
index 8fa6eb5ec835..ae9e677713fd 100755
--- a/docker/test/performance-comparison/download.sh
+++ b/docker/test/performance-comparison/download.sh
@@ -16,26 +16,17 @@ right_sha=$4
datasets=${CHPC_DATASETS-"hits1 hits10 hits100 values"}
declare -A dataset_paths
-if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
- dataset_paths["hits10"]="https://clickhouse-private-datasets.s3.amazonaws.com/hits_10m_single/partitions/hits_10m_single.tar"
- dataset_paths["hits100"]="https://clickhouse-private-datasets.s3.amazonaws.com/hits_100m_single/partitions/hits_100m_single.tar"
- dataset_paths["hits1"]="https://clickhouse-datasets.s3.amazonaws.com/hits/partitions/hits_v1.tar"
- dataset_paths["values"]="https://clickhouse-datasets.s3.amazonaws.com/values_with_expressions/partitions/test_values.tar"
-else
- dataset_paths["hits10"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar"
- dataset_paths["hits100"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_100m_single/partitions/hits_100m_single.tar"
- dataset_paths["hits1"]="https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar"
- dataset_paths["values"]="https://clickhouse-datasets.s3.yandex.net/values_with_expressions/partitions/test_values.tar"
-fi
+dataset_paths["hits10"]="https://clickhouse-private-datasets.s3.amazonaws.com/hits_10m_single/partitions/hits_10m_single.tar"
+dataset_paths["hits100"]="https://clickhouse-private-datasets.s3.amazonaws.com/hits_100m_single/partitions/hits_100m_single.tar"
+dataset_paths["hits1"]="https://clickhouse-datasets.s3.amazonaws.com/hits/partitions/hits_v1.tar"
+dataset_paths["values"]="https://clickhouse-datasets.s3.amazonaws.com/values_with_expressions/partitions/test_values.tar"
function download
{
# Historically there were various paths for the performance test package.
# Test all of them.
- declare -a urls_to_try=("https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/performance/performance.tgz"
- "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/clickhouse_build_check/performance/performance.tgz"
- )
+ declare -a urls_to_try=("https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/performance/performance.tgz")
for path in "${urls_to_try[@]}"
do
diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh
index 3d37a6c0e92e..767807d008b7 100755
--- a/docker/test/performance-comparison/entrypoint.sh
+++ b/docker/test/performance-comparison/entrypoint.sh
@@ -4,7 +4,7 @@ set -ex
CHPC_CHECK_START_TIMESTAMP="$(date +%s)"
export CHPC_CHECK_START_TIMESTAMP
-S3_URL=${S3_URL:="https://clickhouse-builds.s3.yandex.net"}
+S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
COMMON_BUILD_PREFIX="/clickhouse_build_check"
if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
@@ -64,9 +64,7 @@ function find_reference_sha
# Historically there were various path for the performance test package,
# test all of them.
unset found
- declare -a urls_to_try=("https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/performance/performance.tgz"
- "https://clickhouse-builds.s3.yandex.net/0/$REF_SHA/clickhouse_build_check/performance/performance.tgz"
- )
+ declare -a urls_to_try=("https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/performance/performance.tgz")
for path in "${urls_to_try[@]}"
do
if curl_with_retry "$path"
diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py
index 61987d342993..2266641397be 100755
--- a/docker/test/performance-comparison/perf.py
+++ b/docker/test/performance-comparison/perf.py
@@ -19,58 +19,126 @@
from threading import Thread
from scipy import stats
-logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', level='WARNING')
+logging.basicConfig(
+ format="%(asctime)s: %(levelname)s: %(module)s: %(message)s", level="WARNING"
+)
total_start_seconds = time.perf_counter()
stage_start_seconds = total_start_seconds
+
def reportStageEnd(stage):
global stage_start_seconds, total_start_seconds
current = time.perf_counter()
- print(f'stage\t{stage}\t{current - stage_start_seconds:.3f}\t{current - total_start_seconds:.3f}')
+ print(
+ f"stage\t{stage}\t{current - stage_start_seconds:.3f}\t{current - total_start_seconds:.3f}"
+ )
stage_start_seconds = current
def tsv_escape(s):
- return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','')
+ return (
+ s.replace("\\", "\\\\")
+ .replace("\t", "\\t")
+ .replace("\n", "\\n")
+ .replace("\r", "")
+ )
-parser = argparse.ArgumentParser(description='Run performance test.')
+parser = argparse.ArgumentParser(description="Run performance test.")
# Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set.
-parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file')
-parser.add_argument('--host', nargs='*', default=['localhost'], help="Space-separated list of server hostname(s). Corresponds to '--port' options.")
-parser.add_argument('--port', nargs='*', default=[9000], help="Space-separated list of server port(s). Corresponds to '--host' options.")
-parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.')
-parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.')
-parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.')
-parser.add_argument('--max-query-seconds', type=int, default=15, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.')
-parser.add_argument('--prewarm-max-query-seconds', type=int, default=180, help='For how many seconds at most a prewarm (cold storage) query is allowed to run. The script finishes with error if this time is exceeded.')
-parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.')
-parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.')
-parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.')
-parser.add_argument('--print-settings', action='store_true', help='Print test settings and exit.')
-parser.add_argument('--keep-created-tables', action='store_true', help="Don't drop the created tables after the test.")
-parser.add_argument('--use-existing-tables', action='store_true', help="Don't create or drop the tables, use the existing ones instead.")
+parser.add_argument(
+ "file",
+ metavar="FILE",
+ type=argparse.FileType("r", encoding="utf-8"),
+ nargs=1,
+ help="test description file",
+)
+parser.add_argument(
+ "--host",
+ nargs="*",
+ default=["localhost"],
+ help="Space-separated list of server hostname(s). Corresponds to '--port' options.",
+)
+parser.add_argument(
+ "--port",
+ nargs="*",
+ default=[9000],
+ help="Space-separated list of server port(s). Corresponds to '--host' options.",
+)
+parser.add_argument(
+ "--runs", type=int, default=1, help="Number of query runs per server."
+)
+parser.add_argument(
+ "--max-queries",
+ type=int,
+ default=None,
+ help="Test no more than this number of queries, chosen at random.",
+)
+parser.add_argument(
+ "--queries-to-run",
+ nargs="*",
+ type=int,
+ default=None,
+ help="Space-separated list of indexes of queries to test.",
+)
+parser.add_argument(
+ "--max-query-seconds",
+ type=int,
+ default=15,
+ help="For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.",
+)
+parser.add_argument(
+ "--prewarm-max-query-seconds",
+ type=int,
+ default=180,
+ help="For how many seconds at most a prewarm (cold storage) query is allowed to run. The script finishes with error if this time is exceeded.",
+)
+parser.add_argument(
+ "--profile-seconds",
+ type=int,
+ default=0,
+ help="For how many seconds to profile a query for which the performance has changed.",
+)
+parser.add_argument(
+ "--long", action="store_true", help="Do not skip the tests tagged as long."
+)
+parser.add_argument(
+ "--print-queries", action="store_true", help="Print test queries and exit."
+)
+parser.add_argument(
+ "--print-settings", action="store_true", help="Print test settings and exit."
+)
+parser.add_argument(
+ "--keep-created-tables",
+ action="store_true",
+ help="Don't drop the created tables after the test.",
+)
+parser.add_argument(
+ "--use-existing-tables",
+ action="store_true",
+ help="Don't create or drop the tables, use the existing ones instead.",
+)
args = parser.parse_args()
-reportStageEnd('start')
+reportStageEnd("start")
test_name = os.path.splitext(os.path.basename(args.file[0].name))[0]
tree = et.parse(args.file[0])
root = tree.getroot()
-reportStageEnd('parse')
+reportStageEnd("parse")
# Process query parameters
-subst_elems = root.findall('substitutions/substitution')
-available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... }
+subst_elems = root.findall("substitutions/substitution")
+available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... }
for e in subst_elems:
- name = e.find('name').text
- values = [v.text for v in e.findall('values/value')]
+ name = e.find("name").text
+ values = [v.text for v in e.findall("values/value")]
if not values:
- raise Exception(f'No values given for substitution {{{name}}}')
+ raise Exception(f"No values given for substitution {{{name}}}")
available_parameters[name] = values
@@ -78,7 +146,7 @@ def tsv_escape(s):
# parameters. The set of parameters is determined based on the first list.
# Note: keep the order of queries -- sometimes we have DROP IF EXISTS
# followed by CREATE in create queries section, so the order matters.
-def substitute_parameters(query_templates, other_templates = []):
+def substitute_parameters(query_templates, other_templates=[]):
query_results = []
other_results = [[]] * (len(other_templates))
for i, q in enumerate(query_templates):
@@ -103,17 +171,21 @@ def substitute_parameters(query_templates, other_templates = []):
# and reporting the queries marked as short.
test_queries = []
is_short = []
-for e in root.findall('query'):
- new_queries, [new_is_short] = substitute_parameters([e.text], [[e.attrib.get('short', '0')]])
+for e in root.findall("query"):
+ new_queries, [new_is_short] = substitute_parameters(
+ [e.text], [[e.attrib.get("short", "0")]]
+ )
test_queries += new_queries
is_short += [eval(s) for s in new_is_short]
-assert(len(test_queries) == len(is_short))
+assert len(test_queries) == len(is_short)
# If we're given a list of queries to run, check that it makes sense.
for i in args.queries_to_run or []:
if i < 0 or i >= len(test_queries):
- print(f'There is no query no. {i} in this test, only [{0}-{len(test_queries) - 1}] are present')
+ print(
+ f"There is no query no. {i} in this test, only [{0}-{len(test_queries) - 1}] are present"
+ )
exit(1)
# If we're only asked to print the queries, do that and exit.
@@ -125,60 +197,65 @@ def substitute_parameters(query_templates, other_templates = []):
# Print short queries
for i, s in enumerate(is_short):
if s:
- print(f'short\t{i}')
+ print(f"short\t{i}")
# If we're only asked to print the settings, do that and exit. These are settings
# for clickhouse-benchmark, so we print them as command line arguments, e.g.
# '--max_memory_usage=10000000'.
if args.print_settings:
- for s in root.findall('settings/*'):
- print(f'--{s.tag}={s.text}')
+ for s in root.findall("settings/*"):
+ print(f"--{s.tag}={s.text}")
exit(0)
# Skip long tests
if not args.long:
- for tag in root.findall('.//tag'):
- if tag.text == 'long':
- print('skipped\tTest is tagged as long.')
+ for tag in root.findall(".//tag"):
+ if tag.text == "long":
+ print("skipped\tTest is tagged as long.")
sys.exit(0)
# Print report threshold for the test if it is set.
ignored_relative_change = 0.05
-if 'max_ignored_relative_change' in root.attrib:
+if "max_ignored_relative_change" in root.attrib:
ignored_relative_change = float(root.attrib["max_ignored_relative_change"])
- print(f'report-threshold\t{ignored_relative_change}')
+ print(f"report-threshold\t{ignored_relative_change}")
-reportStageEnd('before-connect')
+reportStageEnd("before-connect")
# Open connections
-servers = [{'host': host or args.host[0], 'port': port or args.port[0]} for (host, port) in itertools.zip_longest(args.host, args.port)]
+servers = [
+ {"host": host or args.host[0], "port": port or args.port[0]}
+ for (host, port) in itertools.zip_longest(args.host, args.port)
+]
# Force settings_is_important to fail queries on unknown settings.
-all_connections = [clickhouse_driver.Client(**server, settings_is_important=True) for server in servers]
+all_connections = [
+ clickhouse_driver.Client(**server, settings_is_important=True) for server in servers
+]
for i, s in enumerate(servers):
print(f'server\t{i}\t{s["host"]}\t{s["port"]}')
-reportStageEnd('connect')
+reportStageEnd("connect")
if not args.use_existing_tables:
# Run drop queries, ignoring errors. Do this before all other activity,
# because clickhouse_driver disconnects on error (this is not configurable),
# and the new connection loses the changes in settings.
- drop_query_templates = [q.text for q in root.findall('drop_query')]
+ drop_query_templates = [q.text for q in root.findall("drop_query")]
drop_queries = substitute_parameters(drop_query_templates)
for conn_index, c in enumerate(all_connections):
for q in drop_queries:
try:
c.execute(q)
- print(f'drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
+ print(f"drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}")
except:
pass
- reportStageEnd('drop-1')
+ reportStageEnd("drop-1")
# Apply settings.
-settings = root.findall('settings/*')
+settings = root.findall("settings/*")
for conn_index, c in enumerate(all_connections):
for s in settings:
# requires clickhouse-driver >= 1.1.5 to accept arbitrary new settings
@@ -189,48 +266,52 @@ def substitute_parameters(query_templates, other_templates = []):
# the test, which is wrong.
c.execute("select 1")
-reportStageEnd('settings')
+reportStageEnd("settings")
# Check tables that should exist. If they don't exist, just skip this test.
-tables = [e.text for e in root.findall('preconditions/table_exists')]
+tables = [e.text for e in root.findall("preconditions/table_exists")]
for t in tables:
for c in all_connections:
try:
res = c.execute("select 1 from {} limit 1".format(t))
except:
exception_message = traceback.format_exception_only(*sys.exc_info()[:2])[-1]
- skipped_message = ' '.join(exception_message.split('\n')[:2])
- print(f'skipped\t{tsv_escape(skipped_message)}')
+ skipped_message = " ".join(exception_message.split("\n")[:2])
+ print(f"skipped\t{tsv_escape(skipped_message)}")
sys.exit(0)
-reportStageEnd('preconditions')
+reportStageEnd("preconditions")
if not args.use_existing_tables:
# Run create and fill queries. We will run them simultaneously for both
# servers, to save time. The weird XML search + filter is because we want to
# keep the relative order of elements, and etree doesn't support the
# appropriate xpath query.
- create_query_templates = [q.text for q in root.findall('./*')
- if q.tag in ('create_query', 'fill_query')]
+ create_query_templates = [
+ q.text for q in root.findall("./*") if q.tag in ("create_query", "fill_query")
+ ]
create_queries = substitute_parameters(create_query_templates)
# Disallow temporary tables, because the clickhouse_driver reconnects on
# errors, and temporary tables are destroyed. We want to be able to continue
# after some errors.
for q in create_queries:
- if re.search('create temporary table', q, flags=re.IGNORECASE):
- print(f"Temporary tables are not allowed in performance tests: '{q}'",
- file = sys.stderr)
+ if re.search("create temporary table", q, flags=re.IGNORECASE):
+ print(
+ f"Temporary tables are not allowed in performance tests: '{q}'",
+ file=sys.stderr,
+ )
sys.exit(1)
def do_create(connection, index, queries):
for q in queries:
connection.execute(q)
- print(f'create\t{index}\t{connection.last_query.elapsed}\t{tsv_escape(q)}')
+ print(f"create\t{index}\t{connection.last_query.elapsed}\t{tsv_escape(q)}")
threads = [
- Thread(target = do_create, args = (connection, index, create_queries))
- for index, connection in enumerate(all_connections)]
+ Thread(target=do_create, args=(connection, index, create_queries))
+ for index, connection in enumerate(all_connections)
+ ]
for t in threads:
t.start()
@@ -238,14 +319,16 @@ def do_create(connection, index, queries):
for t in threads:
t.join()
- reportStageEnd('create')
+ reportStageEnd("create")
# By default, test all queries.
queries_to_run = range(0, len(test_queries))
if args.max_queries:
# If specified, test a limited number of queries chosen at random.
- queries_to_run = random.sample(range(0, len(test_queries)), min(len(test_queries), args.max_queries))
+ queries_to_run = random.sample(
+ range(0, len(test_queries)), min(len(test_queries), args.max_queries)
+ )
if args.queries_to_run:
# Run the specified queries.
@@ -255,16 +338,16 @@ def do_create(connection, index, queries):
profile_total_seconds = 0
for query_index in queries_to_run:
q = test_queries[query_index]
- query_prefix = f'{test_name}.query{query_index}'
+ query_prefix = f"{test_name}.query{query_index}"
# We have some crazy long queries (about 100kB), so trim them to a sane
# length. This means we can't use query text as an identifier and have to
# use the test name + the test-wide query index.
query_display_name = q
if len(query_display_name) > 1000:
- query_display_name = f'{query_display_name[:1000]}...({query_index})'
+ query_display_name = f"{query_display_name[:1000]}...({query_index})"
- print(f'display-name\t{query_index}\t{tsv_escape(query_display_name)}')
+ print(f"display-name\t{query_index}\t{tsv_escape(query_display_name)}")
# Prewarm: run once on both servers. Helps to bring the data into memory,
# precompile the queries, etc.
@@ -272,10 +355,10 @@ def do_create(connection, index, queries):
# new one. We want to run them on the new server only, so that the PR author
# can ensure that the test works properly. Remember the errors we had on
# each server.
- query_error_on_connection = [None] * len(all_connections);
+ query_error_on_connection = [None] * len(all_connections)
for conn_index, c in enumerate(all_connections):
try:
- prewarm_id = f'{query_prefix}.prewarm0'
+ prewarm_id = f"{query_prefix}.prewarm0"
try:
# During the warmup runs, we will also:
@@ -283,25 +366,30 @@ def do_create(connection, index, queries):
# * collect profiler traces, which might be helpful for analyzing
# test coverage. We disable profiler for normal runs because
# it makes the results unstable.
- res = c.execute(q, query_id = prewarm_id,
- settings = {
- 'max_execution_time': args.prewarm_max_query_seconds,
- 'query_profiler_real_time_period_ns': 10000000,
- 'memory_profiler_step': '4Mi',
- })
+ res = c.execute(
+ q,
+ query_id=prewarm_id,
+ settings={
+ "max_execution_time": args.prewarm_max_query_seconds,
+ "query_profiler_real_time_period_ns": 10000000,
+ "memory_profiler_step": "4Mi",
+ },
+ )
except clickhouse_driver.errors.Error as e:
# Add query id to the exception to make debugging easier.
e.args = (prewarm_id, *e.args)
- e.message = prewarm_id + ': ' + e.message
+ e.message = prewarm_id + ": " + e.message
raise
- print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}')
+ print(
+ f"prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}"
+ )
except KeyboardInterrupt:
raise
except:
# FIXME the driver reconnects on error and we lose settings, so this
# might lead to further errors or unexpected behavior.
- query_error_on_connection[conn_index] = traceback.format_exc();
+ query_error_on_connection[conn_index] = traceback.format_exc()
continue
# Report all errors that ocurred during prewarm and decide what to do next.
@@ -311,14 +399,14 @@ def do_create(connection, index, queries):
no_errors = []
for i, e in enumerate(query_error_on_connection):
if e:
- print(e, file = sys.stderr)
+ print(e, file=sys.stderr)
else:
no_errors.append(i)
if len(no_errors) == 0:
continue
elif len(no_errors) < len(all_connections):
- print(f'partial\t{query_index}\t{no_errors}')
+ print(f"partial\t{query_index}\t{no_errors}")
this_query_connections = [all_connections[index] for index in no_errors]
@@ -337,27 +425,34 @@ def do_create(connection, index, queries):
all_server_times.append([])
while True:
- run_id = f'{query_prefix}.run{run}'
+ run_id = f"{query_prefix}.run{run}"
for conn_index, c in enumerate(this_query_connections):
try:
- res = c.execute(q, query_id = run_id, settings = {'max_execution_time': args.max_query_seconds})
+ res = c.execute(
+ q,
+ query_id=run_id,
+ settings={"max_execution_time": args.max_query_seconds},
+ )
except clickhouse_driver.errors.Error as e:
# Add query id to the exception to make debugging easier.
e.args = (run_id, *e.args)
- e.message = run_id + ': ' + e.message
+ e.message = run_id + ": " + e.message
raise
elapsed = c.last_query.elapsed
all_server_times[conn_index].append(elapsed)
server_seconds += elapsed
- print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{elapsed}')
+ print(f"query\t{query_index}\t{run_id}\t{conn_index}\t{elapsed}")
if elapsed > args.max_query_seconds:
# Do not stop processing pathologically slow queries,
# since this may hide errors in other queries.
- print(f'The query no. {query_index} is taking too long to run ({elapsed} s)', file=sys.stderr)
+ print(
+ f"The query no. {query_index} is taking too long to run ({elapsed} s)",
+ file=sys.stderr,
+ )
# Be careful with the counter, after this line it's the next iteration
# already.
@@ -386,7 +481,7 @@ def do_create(connection, index, queries):
break
client_seconds = time.perf_counter() - start_seconds
- print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}')
+ print(f"client-time\t{query_index}\t{client_seconds}\t{server_seconds}")
# Run additional profiling queries to collect profile data, but only if test times appeared to be different.
# We have to do it after normal runs because otherwise it will affect test statistics too much
@@ -397,13 +492,15 @@ def do_create(connection, index, queries):
# Don't fail if for some reason there are not enough measurements.
continue
- pvalue = stats.ttest_ind(all_server_times[0], all_server_times[1], equal_var = False).pvalue
+ pvalue = stats.ttest_ind(
+ all_server_times[0], all_server_times[1], equal_var=False
+ ).pvalue
median = [statistics.median(t) for t in all_server_times]
# Keep this consistent with the value used in report. Should eventually move
# to (median[1] - median[0]) / min(median), which is compatible with "times"
# difference we use in report (max(median) / min(median)).
relative_diff = (median[1] - median[0]) / median[0]
- print(f'diff\t{query_index}\t{median[0]}\t{median[1]}\t{relative_diff}\t{pvalue}')
+ print(f"diff\t{query_index}\t{median[0]}\t{median[1]}\t{relative_diff}\t{pvalue}")
if abs(relative_diff) < ignored_relative_change or pvalue > 0.05:
continue
@@ -412,25 +509,31 @@ def do_create(connection, index, queries):
profile_start_seconds = time.perf_counter()
run = 0
while time.perf_counter() - profile_start_seconds < args.profile_seconds:
- run_id = f'{query_prefix}.profile{run}'
+ run_id = f"{query_prefix}.profile{run}"
for conn_index, c in enumerate(this_query_connections):
try:
- res = c.execute(q, query_id = run_id, settings = {'query_profiler_real_time_period_ns': 10000000})
- print(f'profile\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}')
+ res = c.execute(
+ q,
+ query_id=run_id,
+ settings={"query_profiler_real_time_period_ns": 10000000},
+ )
+ print(
+ f"profile\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}"
+ )
except clickhouse_driver.errors.Error as e:
# Add query id to the exception to make debugging easier.
e.args = (run_id, *e.args)
- e.message = run_id + ': ' + e.message
+ e.message = run_id + ": " + e.message
raise
run += 1
profile_total_seconds += time.perf_counter() - profile_start_seconds
-print(f'profile-total\t{profile_total_seconds}')
+print(f"profile-total\t{profile_total_seconds}")
-reportStageEnd('run')
+reportStageEnd("run")
# Run drop queries
if not args.keep_created_tables and not args.use_existing_tables:
@@ -438,6 +541,6 @@ def do_create(connection, index, queries):
for conn_index, c in enumerate(all_connections):
for q in drop_queries:
c.execute(q)
- print(f'drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
+ print(f"drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}")
- reportStageEnd('drop-2')
+ reportStageEnd("drop-2")
diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py
index 4cff6b419496..0cb8481ee6e7 100755
--- a/docker/test/performance-comparison/report.py
+++ b/docker/test/performance-comparison/report.py
@@ -12,9 +12,13 @@
import sys
import traceback
-parser = argparse.ArgumentParser(description='Create performance test report')
-parser.add_argument('--report', default='main', choices=['main', 'all-queries'],
- help='Which report to build')
+parser = argparse.ArgumentParser(description="Create performance test report")
+parser.add_argument(
+ "--report",
+ default="main",
+ choices=["main", "all-queries"],
+ help="Which report to build",
+)
args = parser.parse_args()
tables = []
@@ -31,8 +35,8 @@
# max seconds to run one query by itself, not counting preparation
allowed_single_run_time = 2
-color_bad='#ffb0c0'
-color_good='#b0d050'
+color_bad = "#ffb0c0"
+color_good = "#b0d050"
header_template = """
@@ -151,24 +155,29 @@
table_anchor = 0
row_anchor = 0
+
def currentTableAnchor():
global table_anchor
- return f'{table_anchor}'
+ return f"{table_anchor}"
+
def newTableAnchor():
global table_anchor
table_anchor += 1
return currentTableAnchor()
+
def currentRowAnchor():
global row_anchor
global table_anchor
- return f'{table_anchor}.{row_anchor}'
+ return f"{table_anchor}.{row_anchor}"
+
def nextRowAnchor():
global row_anchor
global table_anchor
- return f'{table_anchor}.{row_anchor + 1}'
+ return f"{table_anchor}.{row_anchor + 1}"
+
def advanceRowAnchor():
global row_anchor
@@ -178,43 +187,58 @@ def advanceRowAnchor():
def tr(x, anchor=None):
- #return '
{x}
'.format(a=a, x=str(x))
+ # return '
{x}
'.format(a=a, x=str(x))
anchor = anchor if anchor else advanceRowAnchor()
- return f'
".format(
+ cell_attributes=cell_attributes, value=value
+ )
+
+
+def tableRow(cell_values, cell_attributes=[], anchor=None):
return tr(
- ''.join([td(v, a)
- for v, a in itertools.zip_longest(
- cell_values, cell_attributes,
- fillvalue = '')
- if a is not None and v is not None]),
- anchor)
-
-def tableHeader(cell_values, cell_attributes = []):
+ "".join(
+ [
+ td(v, a)
+ for v, a in itertools.zip_longest(
+ cell_values, cell_attributes, fillvalue=""
+ )
+ if a is not None and v is not None
+ ]
+ ),
+ anchor,
+ )
+
+
+def tableHeader(cell_values, cell_attributes=[]):
return tr(
- ''.join([th(v, a)
- for v, a in itertools.zip_longest(
- cell_values, cell_attributes,
- fillvalue = '')
- if a is not None and v is not None]))
+ "".join(
+ [
+ th(v, a)
+ for v, a in itertools.zip_longest(
+ cell_values, cell_attributes, fillvalue=""
+ )
+ if a is not None and v is not None
+ ]
+ )
+ )
+
def tableStart(title):
- cls = '-'.join(title.lower().split(' ')[:3]);
+ cls = "-".join(title.lower().split(" ")[:3])
global table_anchor
table_anchor = cls
anchor = currentTableAnchor()
- help_anchor = '-'.join(title.lower().split(' '));
+ help_anchor = "-".join(title.lower().split(" "))
return f"""