[CH-178]Support cache on locals for remote hdfs files

.github/workflows/backport_branches.yml

@@ -9,6 +9,18 @@ on: # yamllint disable-line rule:truthy
  branches:
  - 'backport/**'
 jobs:
+ PythonUnitTests:
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ - name: Python unit tests
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 -m unittest discover -s . -p '*_test.py'
  DockerHubPushAarch64:
  runs-on: [self-hosted, style-checker-aarch64]
  steps:

.github/workflows/master.yml

@@ -911,6 +911,34 @@ jobs:
  docker rm -f "$(docker ps -a -q)" ||:
  sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
+##################################### Docker images #######################################
+############################################################################################
+ DockerServerImages:
+ needs:
+ - BuilderDebRelease
+ - BuilderDebAarch64
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type head
+ python3 docker_server.py --release-type head --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
  BuilderReport:

.github/workflows/pull_request.yml

@@ -4,7 +4,7 @@ env:
  # Force the stdout and stderr streams to be unbuffered
  PYTHONUNBUFFERED: 1
 
-on: # yamllint disable-line rule:truthy
+on:  # yamllint disable-line rule:truthy
  pull_request:
  types:
  - synchronize
@@ -956,6 +956,34 @@ jobs:
  docker rm -f "$(docker ps -a -q)" ||:
  sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
+##################################### Docker images #######################################
+############################################################################################
+ DockerServerImages:
+ needs:
+ - BuilderDebRelease
+ - BuilderDebAarch64
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type head --no-push
+ python3 docker_server.py --release-type head --no-push --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
  BuilderReport:
@@ -3016,6 +3044,7 @@ jobs:
  needs:
  - StyleCheck
  - DockerHubPush
+ - DockerServerImages
  - CheckLabels
  - BuilderReport
  - FastTest

.github/workflows/release.yml

@@ -32,7 +32,32 @@ jobs:
  uses: svenstaro/upload-release-action@v2
  with:
  repo_token: ${{ secrets.GITHUB_TOKEN }}
- file: ${{runner.temp}}/release_packages/*
+ file: ${{runner.temp}}/push_to_artifactory/*
  overwrite: true
  tag: ${{ github.ref }}
  file_glob: true
+ ############################################################################################
+ ##################################### Docker images #######################################
+ ############################################################################################
+ DockerServerImages:
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type auto --version "${{ github.ref }}"
+ python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"

.gitignore

@@ -153,3 +153,5 @@ website/package-lock.json
 /programs/server/metadata
 /programs/server/store
 
+
+utils/local-engine/tests/testConfig.h

CMakeLists.txt

@@ -187,6 +187,9 @@ if (COMPILER_CLANG)
  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
  endif()
  endif()
+
+ # set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
+ # set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
 endif ()
 
 # If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
@@ -261,13 +264,16 @@ endif ()
 # Add a section with the hash of the compiled machine code for integrity checks.
 # Only for official builds, because adding a section can be time consuming (rewrite of several GB).
 # And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
-if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
- set (USE_BINARY_HASH 1)
+if (OBJCOPY_PATH AND CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE OR CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64.cmake$"))
+ set (USE_BINARY_HASH 1 CACHE STRING "Calculate binary hash and store it in the separate section")
 endif ()
 
 # Allows to build stripped binary in a separate directory
-if (OBJCOPY_PATH AND READELF_PATH)
- set(BUILD_STRIPPED_BINARIES_PREFIX "" CACHE STRING "Build stripped binaries with debug info in separate directory")
+if (OBJCOPY_PATH AND STRIP_PATH)
+ option(INSTALL_STRIPPED_BINARIES "Build stripped binaries with debug info in separate directory" OFF)
+ if (INSTALL_STRIPPED_BINARIES)
+ set(STRIPPED_BINARIES_OUTPUT "stripped" CACHE STRING "A separate directory for stripped information")
+ endif()
 endif()
 
 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
@@ -517,8 +523,10 @@ macro (add_executable target)
  if (${type} STREQUAL EXECUTABLE)
  # disabled for TSAN and gcc since libtsan.a provides overrides too
  if (TARGET clickhouse_new_delete)
- # operator::new/delete for executables (MemoryTracker stuff)
- target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+ if (NOT ${target} STREQUAL Git::Git)
+ # operator::new/delete for executables (MemoryTracker stuff)
+ target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+ endif()
  endif()
 
  # In case of static jemalloc, because zone_register() is located in zone.c and

base/glibc-compatibility/CMakeLists.txt

@@ -51,6 +51,6 @@ if (GLIBC_COMPATIBILITY)
 
  message (STATUS "Some symbols from glibc will be replaced for compatibility")
 
-elseif (YANDEX_OFFICIAL_BUILD)
+elseif (CLICKHOUSE_OFFICIAL_BUILD)
  message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.")
 endif ()

benchmark/greenplum/result_parser.py

@@ -4,11 +4,12 @@
 import sys
 import json
 
+
 def parse_block(block=[], options=[]):
 
- #print('block is here', block)
- #show_query = False
- #show_query = options.show_query
+ # print('block is here', block)
+ # show_query = False
+ # show_query = options.show_query
  result = []
  query = block[0].strip()
  if len(block) > 4:
@@ -20,9 +21,9 @@ def parse_block(block=[], options=[]):
  timing2 = block[2].strip().split()[1]
  timing3 = block[3].strip().split()[1]
  if options.show_queries:
- result.append( query )
+ result.append(query)
  if not options.show_first_timings:
- result += [ timing1 , timing2, timing3 ]
+ result += [timing1, timing2, timing3]
  else:
  result.append(timing1)
  return result
@@ -37,86 +38,113 @@ def read_stats_file(options, fname):
 
  for line in f.readlines():
 
- if 'SELECT' in line:
+ if "SELECT" in line:
  if len(block) > 1:
- result.append( parse_block(block, options) )
- block = [ line ]
- elif 'Time:' in line:
- block.append( line )
+ result.append(parse_block(block, options))
+ block = [line]
+ elif "Time:" in line:
+ block.append(line)
 
  return result
 
 
 def compare_stats_files(options, arguments):
  result = []
  file_output = []
- pyplot_colors = ['y', 'b', 'g', 'r']
+ pyplot_colors = ["y", "b", "g", "r"]
  for fname in arguments[1:]:
  file_output.append((read_stats_file(options, fname)))
  if len(file_output[0]) > 0:
  timings_count = len(file_output[0])
  for idx, data_set in enumerate(file_output):
  int_result = []
  for timing in data_set:
- int_result.append(float(timing[0])) #y values
- result.append([[x for x in range(0, len(int_result)) ], int_result,
-pyplot_colors[idx] + '^' ] )
-# result.append([x for x in range(1, len(int_result)) ]) #x values
-# result.append( pyplot_colors[idx] + '^' )
+ int_result.append(float(timing[0])) # y values
+ result.append(
+ [
+ [x for x in range(0, len(int_result))],
+ int_result,
+ pyplot_colors[idx] + "^",
+ ]
+ )
+ # result.append([x for x in range(1, len(int_result)) ]) #x values
+ # result.append( pyplot_colors[idx] + '^' )
 
  return result
 
+
 def parse_args():
  from optparse import OptionParser
- parser = OptionParser(usage='usage: %prog [options] [result_file_path]..')
- parser.add_option("-q", "--show-queries", help="Show statements along with timings", action="store_true", dest="show_queries")
- parser.add_option("-f", "--show-first-timings", help="Show only first tries timings", action="store_true", dest="show_first_timings")
- parser.add_option("-c", "--compare-mode", help="Prepare output for pyplot comparing result files.", action="store", dest="compare_mode")
+
+ parser = OptionParser(usage="usage: %prog [options] [result_file_path]..")
+ parser.add_option(
+ "-q",
+ "--show-queries",
+ help="Show statements along with timings",
+ action="store_true",
+ dest="show_queries",
+ )
+ parser.add_option(
+ "-f",
+ "--show-first-timings",
+ help="Show only first tries timings",
+ action="store_true",
+ dest="show_first_timings",
+ )
+ parser.add_option(
+ "-c",
+ "--compare-mode",
+ help="Prepare output for pyplot comparing result files.",
+ action="store",
+ dest="compare_mode",
+ )
  (options, arguments) = parser.parse_args(sys.argv)
  if len(arguments) < 2:
  parser.print_usage()
  sys.exit(1)
- return ( options, arguments )
+ return (options, arguments)
+
 
 def gen_pyplot_code(options, arguments):
- result = ''
+ result = ""
  data_sets = compare_stats_files(options, arguments)
  for idx, data_set in enumerate(data_sets, start=0):
  x_values, y_values, line_style = data_set
- result += '\nplt.plot('
- result += '%s, %s, \'%s\'' % ( x_values, y_values, line_style )
- result += ', label=\'%s try\')' % idx
- print('import matplotlib.pyplot as plt')
+ result += "\nplt.plot("
+ result += "%s, %s, '%s'" % (x_values, y_values, line_style)
+ result += ", label='%s try')" % idx
+ print("import matplotlib.pyplot as plt")
  print(result)
- print( 'plt.xlabel(\'Try number\')' )
- print( 'plt.ylabel(\'Timing\')' )
- print( 'plt.title(\'Benchmark query timings\')' )
- print('plt.legend()')
- print('plt.show()')
+ print("plt.xlabel('Try number')")
+ print("plt.ylabel('Timing')")
+ print("plt.title('Benchmark query timings')")
+ print("plt.legend()")
+ print("plt.show()")
 
 
 def gen_html_json(options, arguments):
  tuples = read_stats_file(options, arguments[1])
- print('{')
+ print("{")
  print('"system: GreenPlum(x2),')
- print(('"version": "%s",' % '4.3.9.1'))
+ print(('"version": "%s",' % "4.3.9.1"))
  print('"data_size": 10000000,')
  print('"time": "",')
  print('"comments": "",')
  print('"result":')
- print('[')
+ print("[")
  for s in tuples:
  print(s)
- print(']')
- print('}')
+ print("]")
+ print("}")
 
 
 def main():
- ( options, arguments ) = parse_args()
+ (options, arguments) = parse_args()
  if len(arguments) > 2:
  gen_pyplot_code(options, arguments)
  else:
  gen_html_json(options, arguments)
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
  main()

cmake/autogenerated_versions.txt

@@ -5,8 +5,8 @@
 SET(VERSION_REVISION 54460)
 SET(VERSION_MAJOR 22)
 SET(VERSION_MINOR 3)
-SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 75366fc95e510b7ac76759ef670702ae5f488a51)
-SET(VERSION_DESCRIBE v22.3.1.1-testing)
-SET(VERSION_STRING 22.3.1.1)
+SET(VERSION_PATCH 4)
+SET(VERSION_GITHASH abb756d3ca261a4a2c853adb6938d2195b8cbff2)
+SET(VERSION_DESCRIBE v22.3.4.44-lts)
+SET(VERSION_STRING 22.3.4.44)
 # end of autochange