[CH-367] fix CH will continue to execute after JNI call fail

.github/workflows/master.yml

@@ -839,6 +839,34 @@ jobs:
  docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
  sudo rm -fr "$TEMP_PATH"
 ############################################################################################
+##################################### Docker images #######################################
+############################################################################################
+ DockerServerImages:
+ needs:
+ - BuilderDebRelease
+ - BuilderDebAarch64
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type head
+ python3 docker_server.py --release-type head --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
  BuilderReport:

.github/workflows/pull_request.yml

@@ -894,6 +894,34 @@ jobs:
  docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
  sudo rm -fr "$TEMP_PATH"
 ############################################################################################
+##################################### Docker images #######################################
+############################################################################################
+ DockerServerImages:
+ needs:
+ - BuilderDebRelease
+ - BuilderDebAarch64
+ runs-on: [self-hosted, style-checker]
+ steps:
+ - name: Clear repository
+ run: |
+ sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+ - name: Check out repository code
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # otherwise we will have no version info
+ - name: Check docker clickhouse/clickhouse-server building
+ run: |
+ cd "$GITHUB_WORKSPACE/tests/ci"
+ python3 docker_server.py --release-type head --no-push
+ python3 docker_server.py --release-type head --no-push --no-ubuntu \
+ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+ - name: Cleanup
+ if: always()
+ run: |
+ docker kill "$(docker ps -q)" ||:
+ docker rm -f "$(docker ps -a -q)" ||:
+ sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
  BuilderReport:

.github/workflows/release.yml

@@ -14,7 +14,9 @@ jobs:
  steps:
  - name: Deploy packages and assets
  run: |
- curl '${{ secrets.PACKAGES_RELEASE_URL }}/release/${{ github.ref }}?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true' -d ''
+ GITHUB_TAG="${GITHUB_REF#refs/tags/}"
+ curl --silent --data '' \
+ '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true'
  ############################################################################################
  ##################################### Docker images #######################################
  ############################################################################################

.gitignore

@@ -166,3 +166,5 @@ tests/queries/0_stateless/*.generated-expect
 /rust/**/target
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
+
+utils/local-engine/tests/testConfig.h

CMakeLists.txt

@@ -178,7 +178,7 @@ else ()
  set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
-option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
+option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." OFF)
 if (OS_DARWIN)
  # Disable the curl, azure, senry build on MacOS
  set (ENABLE_CURL_BUILD OFF)
@@ -422,6 +422,8 @@ endif ()
 
 include(cmake/dbms_glob_sources.cmake)
 
+set (CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 add_library(global-group INTERFACE)
 if (OS_LINUX OR OS_ANDROID)
  include(cmake/linux/default_libs.cmake)
@@ -452,7 +454,6 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
 if (OS_LINUX AND NOT ARCH_AARCH64)
  # Slightly more efficient code can be generated
  # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
@@ -510,8 +511,10 @@ macro (clickhouse_add_executable target)
  if (${type} STREQUAL EXECUTABLE)
  # disabled for TSAN and gcc since libtsan.a provides overrides too
  if (TARGET clickhouse_new_delete)
- # operator::new/delete for executables (MemoryTracker stuff)
- target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+ if (NOT ${target} STREQUAL Git::Git)
+ # operator::new/delete for executables (MemoryTracker stuff)
+ target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+ endif()
  endif()
 
  # In case of static jemalloc, because zone_register() is located in zone.c and

cmake/autogenerated_versions.txt

@@ -5,8 +5,8 @@
 SET(VERSION_REVISION 54470)
 SET(VERSION_MAJOR 23)
 SET(VERSION_MINOR 1)
-SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 688e488e930c83eefeac4f87c4cc029cc5b231e3)
-SET(VERSION_DESCRIBE v23.1.1.1-testing)
-SET(VERSION_STRING 23.1.1.1)
+SET(VERSION_PATCH 3)
+SET(VERSION_GITHASH 8dfb1700858195fa704221e360fa0798ac6ee9ed)
+SET(VERSION_DESCRIBE v23.1.3.1-stable)
+SET(VERSION_STRING 23.1.3.1)
 # end of autochange

contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -139,7 +139,8 @@
 /* #undef JEMALLOC_MUTEX_INIT_CB */
 
 /* Non-empty if the tls_model attribute is supported. */
-#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+/*#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))*/
+#define JEMALLOC_TLS_MODEL
 
 /*
  * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables

docs/en/sql-reference/functions/json-functions.md

@@ -200,6 +200,7 @@ Examples:
 ``` sql
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(String, Array(Float64))') = ('hello',[-100,200,300])
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(b Array(Float64), a String)') = ([-100,200,300],'hello')
+SELECT JSONExtract('{"a": "hello", "b": "world"}', 'Map(String, String)') = map('a', 'hello', 'b', 'world');
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Nullable(Int8))') = [-100, NULL, NULL]
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable(Int64)') = NULL
 SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8') = 1

docs/en/sql-reference/functions/tuple-map-functions.md

@@ -66,6 +66,46 @@ Result:
 
 - [Map(key, value)](../../sql-reference/data-types/map.md) data type
 
+## mapFromArrays
+
+Merges an [Array](../../sql-reference/data-types/array.md) of keys and an [Array](../../sql-reference/data-types/array.md) of values into a [Map(key, value)](../../sql-reference/data-types/map.md). Notice that the second argument could also be a [Map](../../sql-reference/data-types/map.md), thus it is casted to an Array when executing.
+
+The function is a more convenient alternative to `CAST((key_array, value_array_or_map), 'Map(key_type, value_type)')`. For example, instead of writing `CAST((['aa', 'bb'], [4, 5]), 'Map(String, UInt32)')`, you can write `mapFromArrays(['aa', 'bb'], [4, 5])`.
+
+**Syntax**
+
+```sql
+mapFromArrays(keys, values)
+``` 
+
+Alias: `MAP_FROM_ARRAYS(keys, values)`
+
+**Parameters**
+- `keys` — Given key array to create a map from. The nested type of array must be: [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UUID](../../sql-reference/data-types/uuid.md), [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md), [Date32](../../sql-reference/data-types/date32.md), [Enum](../../sql-reference/data-types/enum.md)
+- `values` - Given value array or map to create a map from.
+
+**Returned value**
+
+- A map whose keys and values are constructed from the key array and value array/map.
+
+**Example**
+
+Query:
+
+```sql
+select mapFromArrays(['a', 'b', 'c'], [1, 2, 3])
+
+┌─mapFromArrays(['a', 'b', 'c'], [1, 2, 3])─┐
+│ {'a':1,'b':2,'c':3} │
+└───────────────────────────────────────────┘
+
+SELECT mapFromArrays([1, 2, 3], map('a', 1, 'b', 2, 'c', 3))
+
+┌─mapFromArrays([1, 2, 3], map('a', 1, 'b', 2, 'c', 3))─┐
+│ {1:('a',1),2:('b',2),3:('c',3)} │
+└───────────────────────────────────────────────────────┘
+```
+
 ## mapAdd
 
 Collect all the keys and sum corresponding values.
@@ -429,6 +469,8 @@ Result:
 │ {} │
 └────────────────────────────┘
 ``` 
+
+
 
 ## mapApply 
 

programs/benchmark/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_BENCHMARK_SOURCES Benchmark.cpp)
+set (CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp)
 
 set (CLICKHOUSE_BENCHMARK_LINK
  PRIVATE

programs/compressor/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Also in utils
 
-set (CLICKHOUSE_COMPRESSOR_SOURCES Compressor.cpp)
+set (CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp)
 
 set (CLICKHOUSE_COMPRESSOR_LINK
  PRIVATE

programs/extract-from-config/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ExtractFromConfig.cpp)
+set (CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp)
 
 set (CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK
  PRIVATE

programs/format/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_FORMAT_SOURCES Format.cpp)
+set (CLICKHOUSE_FORMAT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Format.cpp)
 
 set (CLICKHOUSE_FORMAT_LINK
  PRIVATE

programs/git-import/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_GIT_IMPORT_SOURCES git-import.cpp)
+set (CLICKHOUSE_GIT_IMPORT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/git-import.cpp)
 
 set (CLICKHOUSE_GIT_IMPORT_LINK
  PRIVATE

programs/install/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_INSTALL_SOURCES Install.cpp)
+set (CLICKHOUSE_INSTALL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Install.cpp)
 
 set (CLICKHOUSE_INSTALL_LINK
  PRIVATE

programs/keeper-converter/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES KeeperConverter.cpp)
+set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/KeeperConverter.cpp)
 
 set (CLICKHOUSE_KEEPER_CONVERTER_LINK
  PRIVATE

programs/local/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_LOCAL_SOURCES LocalServer.cpp)
+set (CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp)
 
 set (CLICKHOUSE_LOCAL_LINK
  PRIVATE

programs/obfuscator/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_OBFUSCATOR_SOURCES Obfuscator.cpp)
+set (CLICKHOUSE_OBFUSCATOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Obfuscator.cpp)
 
 set (CLICKHOUSE_OBFUSCATOR_LINK
  PRIVATE

programs/server/CMakeLists.txt

@@ -1,8 +1,8 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake)
 
 set(CLICKHOUSE_SERVER_SOURCES
- MetricsTransmitter.cpp
- Server.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
 )
 
 set (LINK_RESOURCE_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $<TARGET_FILE:clickhouse_server_configs> -Wl,${NO_WHOLE_ARCHIVE}")
@@ -36,4 +36,6 @@ clickhouse_embed_binaries(
  TARGET clickhouse_server_configs
  RESOURCES config.xml users.xml embedded.xml play.html dashboard.html js/uplot.js
 )
-add_dependencies(clickhouse-server-lib clickhouse_server_configs)
+if(NOT CLICKHOUSE_ONE_SHARED)
+ add_dependencies(clickhouse-server-lib clickhouse_server_configs)
+endif()

src/AggregateFunctions/AggregateFunctionArray.h

@@ -13,7 +13,7 @@ struct Settings;
 
 namespace ErrorCodes
 {
- extern const int SIZES_OF_ARRAYS_DOESNT_MATCH;
+ extern const int SIZES_OF_ARRAYS_DONT_MATCH;
  extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
@@ -129,7 +129,7 @@ class AggregateFunctionArray final : public IAggregateFunctionHelper<AggregateFu
  const IColumn::Offsets & ith_offsets = ith_column.getOffsets();
 
  if (ith_offsets[row_num] != end || (row_num != 0 && ith_offsets[row_num - 1] != begin))
- throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
+ throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
  }
 
  for (size_t i = begin; i < end; ++i)

src/AggregateFunctions/AggregateFunctionForEach.h

@@ -19,7 +19,7 @@ namespace ErrorCodes
 {
  extern const int ILLEGAL_TYPE_OF_ARGUMENT;
  extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
- extern const int SIZES_OF_ARRAYS_DOESNT_MATCH;
+ extern const int SIZES_OF_ARRAYS_DONT_MATCH;
 }
 
 
@@ -197,7 +197,7 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper<Aggre
  const IColumn::Offsets & ith_offsets = ith_column.getOffsets();
 
  if (ith_offsets[row_num] != end || (row_num != 0 && ith_offsets[row_num - 1] != begin))
- throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
+ throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
  }
 
  AggregateFunctionForEachData & state = ensureAggregateData(place, end - begin, *arena);

src/AggregateFunctions/AggregateFunctionGroupArray.cpp

@@ -121,7 +121,7 @@ AggregateFunctionPtr createAggregateFunctionGroupArraySample(
 
 void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
 {
- AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
+ AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = true };
 
  factory.registerFunction("groupArray", { createAggregateFunctionGroupArray<false>, properties });
  factory.registerFunction("groupArraySample", { createAggregateFunctionGroupArraySample, properties });

src/Columns/ColumnAggregateFunction.cpp

@@ -525,6 +525,35 @@ void ColumnAggregateFunction::insertDefault()
  pushBackAndCreateState(data, arena, func.get());
 }
 
+void ColumnAggregateFunction::insertRangeSelective(
+ const IColumn & from, const IColumn::Selector & selector, size_t selector_start, size_t length)
+{
+ const ColumnAggregateFunction & from_concrete = static_cast<const ColumnAggregateFunction &>(from);
+ const auto & from_data = from_concrete.data;
+ if (!empty() && src.get() != &from_concrete)
+ {
+ ensureOwnership();
+ Arena & arena = createOrGetArena();
+ Arena * arena_ptr = &arena;
+ data.reserve(size() + length);
+ for (size_t i = 0; i < length; ++i)
+ {
+ pushBackAndCreateState(data, arena, func.get());
+ func->merge(data.back(), from_data[selector[selector_start + i]], arena_ptr);
+ }
+ return;
+ }
+ /// Keep shared ownership of aggregation states.
+ src = from_concrete.getPtr();
+
+ size_t old_size = data.size();
+ data.resize(old_size + length);
+ auto * data_start = data.data();
+ size_t element_size = sizeof(data[0]);
+ for (size_t i = 0; i < length; ++i)
+ memcpy(data_start + old_size + i, &from_concrete.data[selector[selector_start + i]], element_size);
+}
+
 StringRef ColumnAggregateFunction::serializeValueIntoArena(size_t n, Arena & arena, const char *& begin) const
 {
  WriteBufferFromArena out(arena, begin);

src/Columns/ColumnAggregateFunction.h

@@ -184,6 +184,8 @@ class ColumnAggregateFunction final : public COWHelper<IColumn, ColumnAggregateF
 
  void insertRangeFrom(const IColumn & from, size_t start, size_t length) override;
 
+ void insertRangeSelective(const IColumn & src, const IColumn::Selector & selector, size_t selector_start, size_t length) override;
+
  void popBack(size_t n) override;
 
  ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;