diff --git a/.clang-tidy b/.clang-tidy
index afc2ca04e2ef..3903911a277a 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -28,7 +28,6 @@ Checks: '*,
-bugprone-not-null-terminated-result,
-bugprone-reserved-identifier, # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
-bugprone-unchecked-optional-access,
- -bugprone-*, -- category temporarily disabled because some check(s) in it are slow
-cert-dcl16-c,
-cert-dcl37-c,
@@ -39,40 +38,9 @@ Checks: '*,
-cert-oop54-cpp,
-cert-oop57-cpp,
- -clang-analyzer-optin.performance.Padding,
- -clang-analyzer-optin.portability.UnixAPI,
- -clang-analyzer-security.insecureAPI.bzero,
- -clang-analyzer-security.insecureAPI.strcpy,
- -clang-analyzer-*, -- category temporarily disabled because some check(s) in it are slow
-
- -cppcoreguidelines-avoid-c-arrays,
- -cppcoreguidelines-avoid-const-or-ref-data-members,
- -cppcoreguidelines-avoid-do-while,
- -cppcoreguidelines-avoid-goto,
- -cppcoreguidelines-avoid-magic-numbers,
- -cppcoreguidelines-avoid-non-const-global-variables,
- -cppcoreguidelines-explicit-virtual-functions,
- -cppcoreguidelines-init-variables,
- -cppcoreguidelines-interfaces-global-init,
- -cppcoreguidelines-macro-usage,
- -cppcoreguidelines-narrowing-conversions,
- -cppcoreguidelines-no-malloc,
- -cppcoreguidelines-non-private-member-variables-in-classes,
- -cppcoreguidelines-owning-memory,
- -cppcoreguidelines-prefer-member-initializer,
- -cppcoreguidelines-pro-bounds-array-to-pointer-decay,
- -cppcoreguidelines-pro-bounds-constant-array-index,
- -cppcoreguidelines-pro-bounds-pointer-arithmetic,
- -cppcoreguidelines-pro-type-const-cast,
- -cppcoreguidelines-pro-type-cstyle-cast,
- -cppcoreguidelines-pro-type-member-init,
- -cppcoreguidelines-pro-type-reinterpret-cast,
- -cppcoreguidelines-pro-type-static-cast-downcast,
- -cppcoreguidelines-pro-type-union-access,
- -cppcoreguidelines-pro-type-vararg,
- -cppcoreguidelines-slicing,
- -cppcoreguidelines-special-member-functions,
- -cppcoreguidelines-*, -- category temporarily disabled because some check(s) in it are slow
+ -clang-analyzer-unix.Malloc,
+
+ -cppcoreguidelines-*, # impractical in a codebase as large as ClickHouse, also slow
-darwin-*,
@@ -84,7 +52,6 @@ Checks: '*,
-google-readability-function-size,
-google-readability-namespace-comments,
-google-readability-todo,
- -google-upgrade-googletest-case,
-hicpp-avoid-c-arrays,
-hicpp-avoid-goto,
diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index a9503136b1ae..c20255579e2d 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -399,6 +399,13 @@ jobs:
clear-repository: true
submodules: true
fetch-depth: 0 # otherwise we will have no info about contributors
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
@@ -441,6 +448,13 @@ jobs:
clear-repository: true
submodules: true
fetch-depth: 0 # otherwise we will have no info about contributors
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
diff --git a/.github/workflows/docs_check.yml b/.github/workflows/docs_check.yml
index f41bc23bc22c..78724a4a924b 100644
--- a/.github/workflows/docs_check.yml
+++ b/.github/workflows/docs_check.yml
@@ -17,6 +17,7 @@ on: # yamllint disable-line rule:truthy
- 'docker/docs/**'
- 'docs/**'
- 'utils/check-style/aspell-ignore/**'
+ - 'tests/ci/docs_check.py'
jobs:
CheckLabels:
runs-on: [self-hosted, style-checker]
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 0a21fb93beb5..4771e5842666 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -581,6 +581,13 @@ jobs:
clear-repository: true
submodules: true
fetch-depth: 0 # otherwise we will have no info about contributors
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
@@ -707,6 +714,13 @@ jobs:
clear-repository: true
submodules: true
fetch-depth: 0 # otherwise we will have no info about contributors
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index bbc58e17f9b6..676fb993ca89 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -17,6 +17,7 @@ on: # yamllint disable-line rule:truthy
- 'docker/docs/**'
- 'docs/**'
- 'utils/check-style/aspell-ignore/**'
+ - 'tests/ci/docs_check.py'
##########################################################################################
##################################### SMALL CHECKS #######################################
##########################################################################################
@@ -646,6 +647,13 @@ jobs:
with:
clear-repository: true
submodules: true
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
@@ -769,6 +777,13 @@ jobs:
with:
clear-repository: true
submodules: true
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index 6d999d1bee76..fba56339d166 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -456,6 +456,13 @@ jobs:
clear-repository: true
submodules: true
fetch-depth: 0 # otherwise we will have no info about contributors
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
@@ -498,6 +505,13 @@ jobs:
clear-repository: true
submodules: true
fetch-depth: 0 # otherwise we will have no info about contributors
+ - name: Apply sparse checkout for contrib # in order to check that it doesn't break build
+ run: |
+ rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed'
+ git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored'
+ "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK'
+ du -hs "$GITHUB_WORKSPACE/contrib" ||:
+ find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||:
- name: Build
run: |
sudo rm -fr "$TEMP_PATH"
diff --git a/.gitmodules b/.gitmodules
index caed96b03208..f790e0f8d5a7 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -257,6 +257,9 @@
[submodule "contrib/corrosion"]
path = contrib/corrosion
url = https://github.com/corrosion-rs/corrosion
+[submodule "contrib/libssh"]
+ path = contrib/libssh
+ url = https://github.com/ClickHouse/libssh.git
[submodule "contrib/morton-nd"]
path = contrib/morton-nd
url = https://github.com/morton-nd/morton-nd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4b0c8e61a0d7..e95daca2a466 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
### Table of Contents
+**[ClickHouse release v23.9, 2023-09-28](#239)**
**[ClickHouse release v23.8 LTS, 2023-08-31](#238)**
**[ClickHouse release v23.7, 2023-07-27](#237)**
**[ClickHouse release v23.6, 2023-06-30](#236)**
@@ -11,6 +12,173 @@
# 2023 Changelog
+### ClickHouse release 23.9, 2023-09-28
+
+#### Backward Incompatible Change
+* Remove the `status_info` configuration option and dictionaries status from the default Prometheus handler. [#54090](https://github.com/ClickHouse/ClickHouse/pull/54090) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* The experimental parts metadata cache is removed from the codebase. [#54215](https://github.com/ClickHouse/ClickHouse/pull/54215) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Disable setting `input_format_json_try_infer_numbers_from_strings` by default, so we don't try to infer numbers from strings in JSON formats by default to avoid possible parsing errors when sample data contains strings that looks like a number. [#55099](https://github.com/ClickHouse/ClickHouse/pull/55099) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### New Feature
+* Improve schema inference from JSON formats: 1) Now it's possible to infer named Tuples from JSON objects without experimantal JSON type under a setting `input_format_json_try_infer_named_tuples_from_objects` in JSON formats. Previously without experimantal type JSON we could only infer JSON objects as Strings or Maps, now we can infer named Tuple. Resulting Tuple type will conain all keys of objects that were read in data sample during schema inference. It can be useful for reading structured JSON data without sparse objects. The setting is enabled by default. 2) Allow parsing JSON array into a column with type String under setting `input_format_json_read_arrays_as_strings`. It can help reading arrays with values with different types. 3) Allow to use type String for JSON keys with unkown types (`null`/`[]`/`{}`) in sample data under setting `input_format_json_infer_incomplete_types_as_strings`. Now in JSON formats we can read any value into String column and we can avoid getting error `Cannot determine type for column 'column_name' by first 25000 rows of data, most likely this column contains only Nulls or empty Arrays/Maps` during schema inference by using type String for unknown types, so the data will be read successfully. [#54427](https://github.com/ClickHouse/ClickHouse/pull/54427) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added IO scheduling support for remote disks. Storage configuration for disk types `s3`, `s3_plain`, `hdfs` and `azure_blob_storage` can now contain `read_resource` and `write_resource` elements holding resource names. Scheduling policies for these resources can be configured in a separate server configuration section `resources`. Queries can be marked using setting `workload` and classified using server configuration section `workload_classifiers` to achieve diverse resource scheduling goals. More details in [the docs](https://clickhouse.com/docs/en/operations/workload-scheduling). [#47009](https://github.com/ClickHouse/ClickHouse/pull/47009) ([Sergei Trifonov](https://github.com/serxa)). Added "bandwidth_limit" IO scheduling node type. It allows you to specify `max_speed` and `max_burst` constraints on traffic passing though this node. [#54618](https://github.com/ClickHouse/ClickHouse/pull/54618) ([Sergei Trifonov](https://github.com/serxa)).
+* Added new type of authentication based on SSH keys. It works only for the native TCP protocol. [#41109](https://github.com/ClickHouse/ClickHouse/pull/41109) ([George Gamezardashvili](https://github.com/InfJoker)).
+* Added a new column `_block_number` for MergeTree tables. [#44532](https://github.com/ClickHouse/ClickHouse/issues/44532). [#47532](https://github.com/ClickHouse/ClickHouse/pull/47532) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Add `IF EMPTY` clause for `DROP TABLE` queries. [#48915](https://github.com/ClickHouse/ClickHouse/pull/48915) ([Pavel Novitskiy](https://github.com/pnovitskiy)).
+* SQL functions `toString(datetime, timezone)` and `formatDateTime(datetime, format, timezone)` now support non-constant timezone arguments. [#53680](https://github.com/ClickHouse/ClickHouse/pull/53680) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add support for `ALTER TABLE MODIFY COMMENT`. Note: something similar was added by an external contributor a long time ago, but the feature did not work at all and only confused users. This closes [#36377](https://github.com/ClickHouse/ClickHouse/issues/36377). [#51304](https://github.com/ClickHouse/ClickHouse/pull/51304) ([Alexey Milovidov](https://github.com/alexey-milovidov)). Note: this command does not propagate between replicas, so the replicas of a table could have different comments.
+* Added `GCD` a.k.a. "greatest common denominator" as a new data compression codec. The codec computes the GCD of all column values, and then divides each value by the GCD. The GCD codec is a data preparation codec (similar to Delta and DoubleDelta) and cannot be used stand-alone. It works with data integer, decimal and date/time type. A viable use case for the GCD codec are column values that change (increase/decrease) in multiples of the GCD, e.g. 24 - 28 - 16 - 24 - 8 - 24 (assuming GCD = 4). [#53149](https://github.com/ClickHouse/ClickHouse/pull/53149) ([Alexander Nam](https://github.com/seshWCS)).
+* Two new type aliases `DECIMAL(P)` (as shortcut for `DECIMAL(P, 0)` and `DECIMAL` (as shortcut for `DECIMAL(10, 0)`) were added. This makes ClickHouse more compatible with MySQL's SQL dialect. [#53328](https://github.com/ClickHouse/ClickHouse/pull/53328) ([Val Doroshchuk](https://github.com/valbok)).
+* Added a new system log table `backup_log` to track all `BACKUP` and `RESTORE` operations. [#53638](https://github.com/ClickHouse/ClickHouse/pull/53638) ([Victor Krasnov](https://github.com/sirvickr)).
+* Added a format setting `output_format_markdown_escape_special_characters` (default: false). The setting controls whether special characters like `!`, `#`, `$` etc. are escaped (i.e. prefixed by a backslash) in the `Markdown` output format. [#53860](https://github.com/ClickHouse/ClickHouse/pull/53860) ([irenjj](https://github.com/irenjj)).
+* Add function `decodeHTMLComponent`. [#54097](https://github.com/ClickHouse/ClickHouse/pull/54097) ([Bharat Nallan](https://github.com/bharatnc)).
+* Added `peak_threads_usage` to query_log table. [#54335](https://github.com/ClickHouse/ClickHouse/pull/54335) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Add `SHOW FUNCTIONS` support to clickhouse-client. [#54337](https://github.com/ClickHouse/ClickHouse/pull/54337) ([Julia Kartseva](https://github.com/wat-ze-hex)).
+* Added function `toDaysSinceYearZero` with alias `TO_DAYS` (for compatibility with MySQL) which returns the number of days passed since `0001-01-01` (in Proleptic Gregorian Calendar). [#54479](https://github.com/ClickHouse/ClickHouse/pull/54479) ([Robert Schulze](https://github.com/rschu1ze)). Function `toDaysSinceYearZero()` now supports arguments of type `DateTime` and `DateTime64`. [#54856](https://github.com/ClickHouse/ClickHouse/pull/54856) ([Serge Klochkov](https://github.com/slvrtrn)).
+* Added functions `YYYYMMDDtoDate`, `YYYYMMDDtoDate32`, `YYYYMMDDhhmmssToDateTime` and `YYYYMMDDhhmmssToDateTime64`. They convert a date or date with time encoded as integer (e.g. 20230911) into a native date or date with time. As such, they provide the opposite functionality of existing functions `YYYYMMDDToDate`, `YYYYMMDDToDateTime`, `YYYYMMDDhhmmddToDateTime`, `YYYYMMDDhhmmddToDateTime64`. [#54509](https://github.com/ClickHouse/ClickHouse/pull/54509) ([Quanfa Fu](https://github.com/dentiscalprum)) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add several string distance functions, including `byteHammingDistance`, `editDistance`. [#54935](https://github.com/ClickHouse/ClickHouse/pull/54935) ([flynn](https://github.com/ucasfl)).
+* Allow specifying the expiration date and, optionally, the time for user credentials with `VALID UNTIL datetime` clause. [#51261](https://github.com/ClickHouse/ClickHouse/pull/51261) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Allow S3-style URLs for table functions `s3`, `gcs`, `oss`. URL is automatically converted to HTTP. Example: `'s3://clickhouse-public-datasets/hits.csv'` is converted to `'https://clickhouse-public-datasets.s3.amazonaws.com/hits.csv'`. [#54931](https://github.com/ClickHouse/ClickHouse/pull/54931) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add new setting `print_pretty_type_names` to print pretty deep nested types like Tuple/Maps/Arrays. [#55095](https://github.com/ClickHouse/ClickHouse/pull/55095) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### Performance Improvement
+* Speed up reading from S3 by enabling prefetches by default. [#53709](https://github.com/ClickHouse/ClickHouse/pull/53709) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Do not implicitly read PK and version columns in lonely parts if unnecessary for queries with FINAL. [#53919](https://github.com/ClickHouse/ClickHouse/pull/53919) ([Duc Canh Le](https://github.com/canhld94)).
+* Optimize group by constant keys. Will optimize queries with group by `_file/_path` after https://github.com/ClickHouse/ClickHouse/pull/53529. [#53549](https://github.com/ClickHouse/ClickHouse/pull/53549) ([Kruglov Pavel](https://github.com/Avogar)).
+* Improve performance of sorting for `Decimal` columns. Improve performance of insertion into `MergeTree` if ORDER BY contains a `Decimal` column. Improve performance of sorting when data is already sorted or almost sorted. [#35961](https://github.com/ClickHouse/ClickHouse/pull/35961) ([Maksim Kita](https://github.com/kitaisreal)).
+* Improve performance for huge query analysis. Fixes [#51224](https://github.com/ClickHouse/ClickHouse/issues/51224). [#51469](https://github.com/ClickHouse/ClickHouse/pull/51469) ([frinkr](https://github.com/frinkr)).
+* An optimization to rewrite `COUNT(DISTINCT ...)` and various `uniq` variants to `count` if it is selected from a subquery with GROUP BY. [#52082](https://github.com/ClickHouse/ClickHouse/pull/52082) [#52645](https://github.com/ClickHouse/ClickHouse/pull/52645) ([JackyWoo](https://github.com/JackyWoo)).
+* Remove manual calls to `mmap/mremap/munmap` and delegate all this work to `jemalloc` - and it slightly improves performance. [#52792](https://github.com/ClickHouse/ClickHouse/pull/52792) ([Nikita Taranov](https://github.com/nickitat)).
+* Fixed high in CPU consumption when working with NATS. [#54399](https://github.com/ClickHouse/ClickHouse/pull/54399) ([Vasilev Pyotr](https://github.com/vahpetr)).
+* Since we use separate instructions for executing `toString()` with datetime argument, it is possible to improve performance a bit for non-datetime arguments and have some parts of the code cleaner. Follows up [#53680](https://github.com/ClickHouse/ClickHouse/issues/53680). [#54443](https://github.com/ClickHouse/ClickHouse/pull/54443) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Instead of serializing json elements into a `std::stringstream`, this PR try to put the serialization result into `ColumnString` direclty. [#54613](https://github.com/ClickHouse/ClickHouse/pull/54613) ([lgbo](https://github.com/lgbo-ustc)).
+* Enable ORDER BY optimization for reading data in corresponding order from a MergeTree table in case that the table is behind a view. [#54628](https://github.com/ClickHouse/ClickHouse/pull/54628) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Improve JSON SQL functions by reusing `GeneratorJSONPath` and removing several shared pointers. [#54735](https://github.com/ClickHouse/ClickHouse/pull/54735) ([lgbo](https://github.com/lgbo-ustc)).
+* Keeper tries to batch flush requests for better performance. [#53049](https://github.com/ClickHouse/ClickHouse/pull/53049) ([Antonio Andelic](https://github.com/antonio2368)).
+* Now `clickhouse-client` processes files in parallel in case of `INFILE 'glob_expression'`. Closes [#54218](https://github.com/ClickHouse/ClickHouse/issues/54218). [#54533](https://github.com/ClickHouse/ClickHouse/pull/54533) ([Max K.](https://github.com/mkaynov)).
+* Allow to use primary key for IN function where primary key column types are different from `IN` function right side column types. Example: `SELECT id FROM test_table WHERE id IN (SELECT '5')`. Closes [#48936](https://github.com/ClickHouse/ClickHouse/issues/48936). [#54544](https://github.com/ClickHouse/ClickHouse/pull/54544) ([Maksim Kita](https://github.com/kitaisreal)).
+* Hash JOIN tries to shrink internal buffers consuming half of maximal available memory (set by `max_bytes_in_join`). [#54584](https://github.com/ClickHouse/ClickHouse/pull/54584) ([vdimir](https://github.com/vdimir)).
+* Respect `max_block_size` for array join to avoid possible OOM. Close [#54290](https://github.com/ClickHouse/ClickHouse/issues/54290). [#54664](https://github.com/ClickHouse/ClickHouse/pull/54664) ([李扬](https://github.com/taiyang-li)).
+* Reuse HTTP connections in the `s3` table function. [#54812](https://github.com/ClickHouse/ClickHouse/pull/54812) ([Michael Kolupaev](https://github.com/al13n321)).
+* Replace the linear search in `MergeTreeRangeReader::Stream::ceilRowsToCompleteGranules` with a binary search. [#54869](https://github.com/ClickHouse/ClickHouse/pull/54869) ([usurai](https://github.com/usurai)).
+
+#### Experimental Feature
+* The creation of `Annoy` indexes can now be parallelized using setting `max_threads_for_annoy_index_creation`. [#54047](https://github.com/ClickHouse/ClickHouse/pull/54047) ([Robert Schulze](https://github.com/rschu1ze)).
+* Parallel replicas over distributed don't read from all replicas [#54199](https://github.com/ClickHouse/ClickHouse/pull/54199) ([Igor Nikonov](https://github.com/devcrafter)).
+
+#### Improvement
+* Allow to replace long names of files of columns in `MergeTree` data parts to hashes of names. It helps to avoid `File name too long` error in some cases. [#50612](https://github.com/ClickHouse/ClickHouse/pull/50612) ([Anton Popov](https://github.com/CurtizJ)).
+* Parse data in `JSON` format as `JSONEachRow` if failed to parse metadata. It will allow to read files with `.json` extension even if real format is JSONEachRow. Closes [#45740](https://github.com/ClickHouse/ClickHouse/issues/45740). [#54405](https://github.com/ClickHouse/ClickHouse/pull/54405) ([Kruglov Pavel](https://github.com/Avogar)).
+* Output valid JSON/XML on excetpion during HTTP query execution. Add setting `http_write_exception_in_output_format` to enable/disable this behaviour (enabled by default). [#52853](https://github.com/ClickHouse/ClickHouse/pull/52853) ([Kruglov Pavel](https://github.com/Avogar)).
+* View `information_schema.tables` now has a new field `data_length` which shows the approximate size of the data on disk. Required to run queries generated by Amazon QuickSight. [#55037](https://github.com/ClickHouse/ClickHouse/pull/55037) ([Robert Schulze](https://github.com/rschu1ze)).
+* The MySQL interface gained a minimal implementation of prepared statements, just enough to allow a connection from Tableau Online to ClickHouse via the MySQL connector. [#54115](https://github.com/ClickHouse/ClickHouse/pull/54115) ([Serge Klochkov](https://github.com/slvrtrn)). Please note: the prepared statements implementation is pretty minimal, we do not support arguments binding yet, it is not required in this particular Tableau online use case. It will be implemented as a follow-up if necessary after extensive testing of Tableau Online in case we discover issues.
+* Support case-insensitive and dot-all matching modes in `regexp_tree` dictionaries. [#50906](https://github.com/ClickHouse/ClickHouse/pull/50906) ([Johann Gan](https://github.com/johanngan)).
+* Keeper improvement: Add a `createIfNotExists` Keeper command. [#48855](https://github.com/ClickHouse/ClickHouse/pull/48855) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* More precise integer type inference, fix [#51236](https://github.com/ClickHouse/ClickHouse/issues/51236). [#53003](https://github.com/ClickHouse/ClickHouse/pull/53003) ([Chen768959](https://github.com/Chen768959)).
+* Introduced resolving of charsets in the string literals for MaterializedMySQL. [#53220](https://github.com/ClickHouse/ClickHouse/pull/53220) ([Val Doroshchuk](https://github.com/valbok)).
+* Fix a subtle issue with a rarely used `EmbeddedRocksDB` table engine in an extremely rare scenario: sometimes the `EmbeddedRocksDB` table engine does not close files correctly in NFS after running `DROP TABLE`. [#53502](https://github.com/ClickHouse/ClickHouse/pull/53502) ([Mingliang Pan](https://github.com/liangliangpan)).
+* `RESTORE TABLE ON CLUSTER` must create replicated tables with a matching UUID on hosts. Otherwise the macro `{uuid}` in ZooKeeper path can't work correctly after RESTORE. This PR implements that. [#53765](https://github.com/ClickHouse/ClickHouse/pull/53765) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Added restore setting `restore_broken_parts_as_detached`: if it's true the RESTORE process won't stop on broken parts while restoring, instead all the broken parts will be copied to the `detached` folder with the prefix `broken-from-backup'. If it's false the RESTORE process will stop on the first broken part (if any). The default value is false. [#53877](https://github.com/ClickHouse/ClickHouse/pull/53877) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Add `elapsed_ns` field to HTTP headers X-ClickHouse-Progress and X-ClickHouse-Summary. [#54179](https://github.com/ClickHouse/ClickHouse/pull/54179) ([joelynch](https://github.com/joelynch)).
+* Implementation of `reconfig` (https://github.com/ClickHouse/ClickHouse/pull/49450), `sync`, and `exists` commands for keeper-client. [#54201](https://github.com/ClickHouse/ClickHouse/pull/54201) ([pufit](https://github.com/pufit)).
+* `clickhouse-local` and `clickhouse-client` now allow to specify the `--query` parameter multiple times, e.g. `./clickhouse-client --query "SELECT 1" --query "SELECT 2"`. This syntax is slightly more intuitive than `./clickhouse-client --multiquery "SELECT 1;S ELECT 2"`, a bit easier to script (e.g. `queries.push_back('--query "$q"')`) and more consistent with the behavior of existing parameter `--queries-file` (e.g. `./clickhouse client --queries-file queries1.sql --queries-file queries2.sql`). [#54249](https://github.com/ClickHouse/ClickHouse/pull/54249) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add sub-second precision to `formatReadableTimeDelta`. [#54250](https://github.com/ClickHouse/ClickHouse/pull/54250) ([Andrey Zvonov](https://github.com/zvonand)).
+* Enable `allow_remove_stale_moving_parts` by default. [#54260](https://github.com/ClickHouse/ClickHouse/pull/54260) ([vdimir](https://github.com/vdimir)).
+* Fix using count from cache and improve progress bar for reading from archives. [#54271](https://github.com/ClickHouse/ClickHouse/pull/54271) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add support for S3 credentials using SSO. To define a profile to be used with SSO, set `AWS_PROFILE` environment variable. [#54347](https://github.com/ClickHouse/ClickHouse/pull/54347) ([Antonio Andelic](https://github.com/antonio2368)).
+* Support NULL as default for nested types Array/Tuple/Map for input formats. Closes [#51100](https://github.com/ClickHouse/ClickHouse/issues/51100). [#54351](https://github.com/ClickHouse/ClickHouse/pull/54351) ([Kruglov Pavel](https://github.com/Avogar)).
+* Allow reading some unusual configuration of chunks from Arrow/Parquet formats. [#54370](https://github.com/ClickHouse/ClickHouse/pull/54370) ([Arthur Passos](https://github.com/arthurpassos)).
+* Add `STD` alias to `stddevPop` function for MySQL compatibility. Closes [#54274](https://github.com/ClickHouse/ClickHouse/issues/54274). [#54382](https://github.com/ClickHouse/ClickHouse/pull/54382) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add `addDate` function for compatibility with MySQL and `subDate` for consistency. Reference [#54275](https://github.com/ClickHouse/ClickHouse/issues/54275). [#54400](https://github.com/ClickHouse/ClickHouse/pull/54400) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add `modification_time` into `system.detached_parts`. [#54506](https://github.com/ClickHouse/ClickHouse/pull/54506) ([Azat Khuzhin](https://github.com/azat)).
+* Added a setting `splitby_max_substrings_includes_remaining_string` which controls if functions "splitBy*()" with argument "max_substring" > 0 include the remaining string (if any) in the result array (Python/Spark semantics) or not. The default behavior does not change. [#54518](https://github.com/ClickHouse/ClickHouse/pull/54518) ([Robert Schulze](https://github.com/rschu1ze)).
+* Better integer types inference for `Int64`/`UInt64` fields. Continuation of [#53003](https://github.com/ClickHouse/ClickHouse/pull/53003). Now it works also for nested types like Arrays of Arrays and for functions like `map/tuple`. Issue: [#51236](https://github.com/ClickHouse/ClickHouse/issues/51236). [#54553](https://github.com/ClickHouse/ClickHouse/pull/54553) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added array operations for multiplying, dividing and modulo on scalar. Works in each way, for example `5 * [5, 5]` and `[5, 5] * 5` - both cases are possible. [#54608](https://github.com/ClickHouse/ClickHouse/pull/54608) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add optional `version` argument to `rm` command in `keeper-client` to support safer deletes. [#54708](https://github.com/ClickHouse/ClickHouse/pull/54708) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Disable killing the server by systemd (that may lead to data loss when using Buffer tables). [#54744](https://github.com/ClickHouse/ClickHouse/pull/54744) ([Azat Khuzhin](https://github.com/azat)).
+* Added field `is_deterministic` to system table `system.functions` which indicates whether the result of a function is stable between two invocations (given exactly the same inputs) or not. [#54766](https://github.com/ClickHouse/ClickHouse/pull/54766) [#55035](https://github.com/ClickHouse/ClickHouse/pull/55035) ([Robert Schulze](https://github.com/rschu1ze)).
+* Made the views in schema `information_schema` more compatible with the equivalent views in MySQL (i.e. modified and extended them) up to a point where Tableau Online is able to connect to ClickHouse. More specifically: 1. The type of field `information_schema.tables.table_type` changed from Enum8 to String. 2. Added fields `table_comment` and `table_collation` to view `information_schema.table`. 3. Added views `information_schema.key_column_usage` and `referential_constraints`. 4. Replaced uppercase aliases in `information_schema` views with concrete uppercase columns. [#54773](https://github.com/ClickHouse/ClickHouse/pull/54773) ([Serge Klochkov](https://github.com/slvrtrn)).
+* The query cache now returns an error if the user tries to cache the result of a query with a non-deterministic function such as `now`, `randomString` and `dictGet`. Compared to the previous behavior (silently don't cache the result), this reduces confusion and surprise for users. [#54801](https://github.com/ClickHouse/ClickHouse/pull/54801) ([Robert Schulze](https://github.com/rschu1ze)).
+* Forbid special columns like materialized/ephemeral/alias for `file`/`s3`/`url`/... storages, fix insert into ephemeral columns from files. Closes [#53477](https://github.com/ClickHouse/ClickHouse/issues/53477). [#54803](https://github.com/ClickHouse/ClickHouse/pull/54803) ([Kruglov Pavel](https://github.com/Avogar)).
+* More configurable collecting metadata for backup. [#54804](https://github.com/ClickHouse/ClickHouse/pull/54804) ([Vitaly Baranov](https://github.com/vitlibar)).
+* `clickhouse-local`'s log file (if enabled with --server_logs_file flag) will now prefix each line with timestamp, thread id, etc, just like `clickhouse-server`. [#54807](https://github.com/ClickHouse/ClickHouse/pull/54807) ([Michael Kolupaev](https://github.com/al13n321)).
+* Field `is_obsolete` in the `system.merge_tree_settings` table - it is now 1 for obsolete merge tree settings. Previously, only the description indicated that the setting is obsolete. [#54837](https://github.com/ClickHouse/ClickHouse/pull/54837) ([Robert Schulze](https://github.com/rschu1ze)).
+* Make it possible to use plural when using interval literals. `INTERVAL 2 HOURS` should be equivalent to `INTERVAL 2 HOUR`. [#54860](https://github.com/ClickHouse/ClickHouse/pull/54860) ([Jordi Villar](https://github.com/jrdi)).
+* Always allow the creation of a projection with `Nullable` PK. This fixes [#54814](https://github.com/ClickHouse/ClickHouse/issues/54814). [#54895](https://github.com/ClickHouse/ClickHouse/pull/54895) ([Amos Bird](https://github.com/amosbird)).
+* Retry backup's S3 operations after connection reset failure. [#54900](https://github.com/ClickHouse/ClickHouse/pull/54900) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Make the exception message exact in case of the maximum value of a settings is less than the minimum value. [#54925](https://github.com/ClickHouse/ClickHouse/pull/54925) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* `LIKE`, `match`, and other regular expressions matching functions now allow matching with patterns containing non-UTF-8 substrings by falling back to binary matching. Example: you can use `string LIKE '\xFE\xFF%'` to detect BOM. This closes [#54486](https://github.com/ClickHouse/ClickHouse/issues/54486). [#54942](https://github.com/ClickHouse/ClickHouse/pull/54942) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Added `ContextLockWaitMicroseconds` profile event. [#55029](https://github.com/ClickHouse/ClickHouse/pull/55029) ([Maksim Kita](https://github.com/kitaisreal)).
+* The Keeper dynamically adjusts log levels. [#50372](https://github.com/ClickHouse/ClickHouse/pull/50372) ([helifu](https://github.com/helifu)).
+* Added function `timestamp` for compatibility with MySQL. Closes [#54275](https://github.com/ClickHouse/ClickHouse/issues/54275). [#54639](https://github.com/ClickHouse/ClickHouse/pull/54639) ([Nikolay Degterinsky](https://github.com/evillique)).
+
+#### Build/Testing/Packaging Improvement
+* Bumped the compiler of official and continuous integration builds of ClickHouse from Clang 16 to 17. [#53831](https://github.com/ClickHouse/ClickHouse/pull/53831) ([Robert Schulze](https://github.com/rschu1ze)).
+* Regenerated tld data for lookups (`tldLookup.generated.cpp`). [#54269](https://github.com/ClickHouse/ClickHouse/pull/54269) ([Bharat Nallan](https://github.com/bharatnc)).
+* Remove the redundant `clickhouse-keeper-client` symlink. [#54587](https://github.com/ClickHouse/ClickHouse/pull/54587) ([Tomas Barton](https://github.com/deric)).
+* Use `/usr/bin/env` to resolve bash - now it supports Nix OS. [#54603](https://github.com/ClickHouse/ClickHouse/pull/54603) ([Fionera](https://github.com/fionera)).
+* CMake added `PROFILE_CPU` option needed to perform `perf record` without using a DWARF call graph. [#54917](https://github.com/ClickHouse/ClickHouse/pull/54917) ([Maksim Kita](https://github.com/kitaisreal)).
+* If the linker is different than LLD, stop with a fatal error. [#55036](https://github.com/ClickHouse/ClickHouse/pull/55036) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Replaced the library to handle (encode/decode) base64 values from Turbo-Base64 to aklomp-base64. Both are SIMD-accelerated on x86 and ARM but 1. the license of the latter (BSD-2) is more favorable for ClickHouse, Turbo64 switched in the meantime to GPL-3, 2. with more GitHub stars, aklomp-base64 seems more future-proof, 3. aklomp-base64 has a slightly nicer API (which is arguably subjective), and 4. aklomp-base64 does not require us to hack around bugs (like non-threadsafe initialization). Note: aklomp-base64 rejects unpadded base64 values whereas Turbo-Base64 decodes them on a best-effort basis. RFC-4648 leaves it open whether padding is mandatory or not, but depending on the context this may be a behavioral change to be aware of. [#54119](https://github.com/ClickHouse/ClickHouse/pull/54119) ([Mikhail Koviazin](https://github.com/mkmkme)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Fix REPLACE/MOVE PARTITION with zero-copy replication (note: "zero-copy replication" is an experimental feature) [#54193](https://github.com/ClickHouse/ClickHouse/pull/54193) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix zero copy locks with hardlinks (note: "zero-copy replication" is an experimental feature) [#54859](https://github.com/ClickHouse/ClickHouse/pull/54859) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix zero copy garbage (note: "zero-copy replication" is an experimental feature) [#54550](https://github.com/ClickHouse/ClickHouse/pull/54550) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Pass HTTP retry timeout as milliseconds (it was incorrect before). [#54438](https://github.com/ClickHouse/ClickHouse/pull/54438) ([Duc Canh Le](https://github.com/canhld94)).
+* Fix misleading error message in OUTFILE with `CapnProto`/`Protobuf` [#52870](https://github.com/ClickHouse/ClickHouse/pull/52870) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix summary reporting with parallel replicas with LIMIT [#53050](https://github.com/ClickHouse/ClickHouse/pull/53050) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix throttling of BACKUPs from/to S3 (in case native copy was not used) and in some other places as well [#53336](https://github.com/ClickHouse/ClickHouse/pull/53336) ([Azat Khuzhin](https://github.com/azat)).
+* Fix IO throttling during copying whole directories [#53338](https://github.com/ClickHouse/ClickHouse/pull/53338) ([Azat Khuzhin](https://github.com/azat)).
+* Fix: moved to prewhere condition actions can lose column [#53492](https://github.com/ClickHouse/ClickHouse/pull/53492) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fixed internal error when replacing with byte-equal parts [#53735](https://github.com/ClickHouse/ClickHouse/pull/53735) ([Pedro Riera](https://github.com/priera)).
+* Fix: require columns participating in interpolate expression [#53754](https://github.com/ClickHouse/ClickHouse/pull/53754) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix cluster discovery initialization + setting up fail points in config [#54113](https://github.com/ClickHouse/ClickHouse/pull/54113) ([vdimir](https://github.com/vdimir)).
+* Fix issues in `accurateCastOrNull` [#54136](https://github.com/ClickHouse/ClickHouse/pull/54136) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Fix nullable primary key with the FINAL modifier [#54164](https://github.com/ClickHouse/ClickHouse/pull/54164) ([Amos Bird](https://github.com/amosbird)).
+* Fixed error that prevented insertion in replicated materialized view of new data in presence of duplicated data. [#54184](https://github.com/ClickHouse/ClickHouse/pull/54184) ([Pedro Riera](https://github.com/priera)).
+* Fix: allow `IPv6` for bloom filter [#54200](https://github.com/ClickHouse/ClickHouse/pull/54200) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* fix possible type mismatch with `IPv4` [#54212](https://github.com/ClickHouse/ClickHouse/pull/54212) ([Bharat Nallan](https://github.com/bharatnc)).
+* Fix `system.data_skipping_indices` for recreated indices [#54225](https://github.com/ClickHouse/ClickHouse/pull/54225) ([Artur Malchanau](https://github.com/Hexta)).
+* fix name clash for multiple join rewriter v2 [#54240](https://github.com/ClickHouse/ClickHouse/pull/54240) ([Tao Wang](https://github.com/wangtZJU)).
+* Fix unexpected errors in `system.errors` after join [#54306](https://github.com/ClickHouse/ClickHouse/pull/54306) ([vdimir](https://github.com/vdimir)).
+* Fix `isZeroOrNull(NULL)` [#54316](https://github.com/ClickHouse/ClickHouse/pull/54316) ([flynn](https://github.com/ucasfl)).
+* Fix: parallel replicas over distributed with `prefer_localhost_replica` = 1 [#54334](https://github.com/ClickHouse/ClickHouse/pull/54334) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix logical error in vertical merge + replacing merge tree + optimize cleanup [#54368](https://github.com/ClickHouse/ClickHouse/pull/54368) ([alesapin](https://github.com/alesapin)).
+* Fix possible error `URI contains invalid characters` in the `s3` table function [#54373](https://github.com/ClickHouse/ClickHouse/pull/54373) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix segfault in AST optimization of `arrayExists` function [#54379](https://github.com/ClickHouse/ClickHouse/pull/54379) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Check for overflow before addition in `analysisOfVariance` function [#54385](https://github.com/ClickHouse/ClickHouse/pull/54385) ([Antonio Andelic](https://github.com/antonio2368)).
+* Reproduce and fix the bug in removeSharedRecursive [#54430](https://github.com/ClickHouse/ClickHouse/pull/54430) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix possible incorrect result with SimpleAggregateFunction in PREWHERE and FINAL [#54436](https://github.com/ClickHouse/ClickHouse/pull/54436) ([Azat Khuzhin](https://github.com/azat)).
+* Fix filtering parts with indexHint for non analyzer [#54449](https://github.com/ClickHouse/ClickHouse/pull/54449) ([Azat Khuzhin](https://github.com/azat)).
+* Fix aggregate projections with normalized states [#54480](https://github.com/ClickHouse/ClickHouse/pull/54480) ([Amos Bird](https://github.com/amosbird)).
+* `clickhouse-local`: something for multiquery parameter [#54498](https://github.com/ClickHouse/ClickHouse/pull/54498) ([CuiShuoGuo](https://github.com/bakam412)).
+* `clickhouse-local` supports `--database` command line argument [#54503](https://github.com/ClickHouse/ClickHouse/pull/54503) ([vdimir](https://github.com/vdimir)).
+* Fix possible parsing error in `-WithNames` formats with disabled `input_format_with_names_use_header` [#54513](https://github.com/ClickHouse/ClickHouse/pull/54513) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix rare case of CHECKSUM_DOESNT_MATCH error [#54549](https://github.com/ClickHouse/ClickHouse/pull/54549) ([alesapin](https://github.com/alesapin)).
+* Fix sorting of UNION ALL of already sorted results [#54564](https://github.com/ClickHouse/ClickHouse/pull/54564) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix snapshot install in Keeper [#54572](https://github.com/ClickHouse/ClickHouse/pull/54572) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix race in `ColumnUnique` [#54575](https://github.com/ClickHouse/ClickHouse/pull/54575) ([Nikita Taranov](https://github.com/nickitat)).
+* Annoy/Usearch index: Fix LOGICAL_ERROR during build-up with default values [#54600](https://github.com/ClickHouse/ClickHouse/pull/54600) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix serialization of `ColumnDecimal` [#54601](https://github.com/ClickHouse/ClickHouse/pull/54601) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix schema inference for *Cluster functions for column names with spaces [#54635](https://github.com/ClickHouse/ClickHouse/pull/54635) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix using structure from insertion tables in case of defaults and explicit insert columns [#54655](https://github.com/ClickHouse/ClickHouse/pull/54655) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix: avoid using regex match, possibly containing alternation, as a key condition. [#54696](https://github.com/ClickHouse/ClickHouse/pull/54696) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix ReplacingMergeTree with vertical merge and cleanup [#54706](https://github.com/ClickHouse/ClickHouse/pull/54706) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix virtual columns having incorrect values after ORDER BY [#54811](https://github.com/ClickHouse/ClickHouse/pull/54811) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix filtering parts with indexHint for non analyzer [#54825](https://github.com/ClickHouse/ClickHouse/pull/54825) [#54449](https://github.com/ClickHouse/ClickHouse/pull/54449) ([Azat Khuzhin](https://github.com/azat)).
+* Fix Keeper segfault during shutdown [#54841](https://github.com/ClickHouse/ClickHouse/pull/54841) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix `Invalid number of rows in Chunk` in MaterializedPostgreSQL [#54844](https://github.com/ClickHouse/ClickHouse/pull/54844) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Move obsolete format settings to separate section [#54855](https://github.com/ClickHouse/ClickHouse/pull/54855) ([Kruglov Pavel](https://github.com/Avogar)).
+* Rebuild `minmax_count_projection` when partition key gets modified [#54943](https://github.com/ClickHouse/ClickHouse/pull/54943) ([Amos Bird](https://github.com/amosbird)).
+* Fix bad cast to `ColumnVector` in function `if` [#55019](https://github.com/ClickHouse/ClickHouse/pull/55019) ([Kruglov Pavel](https://github.com/Avogar)).
+* Prevent attaching parts from tables with different projections or indices [#55062](https://github.com/ClickHouse/ClickHouse/pull/55062) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Store NULL in scalar result map for empty subquery result [#52240](https://github.com/ClickHouse/ClickHouse/pull/52240) ([vdimir](https://github.com/vdimir)).
+* Fix `FINAL` produces invalid read ranges in a rare case [#54934](https://github.com/ClickHouse/ClickHouse/pull/54934) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix: insert quorum w/o keeper retries [#55026](https://github.com/ClickHouse/ClickHouse/pull/55026) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix simple state with nullable [#55030](https://github.com/ClickHouse/ClickHouse/pull/55030) ([Pedro Riera](https://github.com/priera)).
+
+
### ClickHouse release 23.8 LTS, 2023-08-31
#### Backward Incompatible Change
@@ -273,7 +441,7 @@
* Add new setting `disable_url_encoding` that allows to disable decoding/encoding path in uri in URL engine. [#52337](https://github.com/ClickHouse/ClickHouse/pull/52337) ([Kruglov Pavel](https://github.com/Avogar)).
#### Performance Improvement
-* Enable automatic selection of the sparse serialization format by default. It improves performance. The format is supported since version 22.1. After this change, downgrading to versions older than 22.1 might not be possible. You can turn off the usage of the sparse serialization format by providing the `ratio_of_defaults_for_sparse_serialization = 1` setting for your MergeTree tables. [#49631](https://github.com/ClickHouse/ClickHouse/pull/49631) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Enable automatic selection of the sparse serialization format by default. It improves performance. The format is supported since version 22.1. After this change, downgrading to versions older than 22.1 might not be possible. A downgrade may require to set `ratio_of_defaults_for_sparse_serialization=0.9375` [55153](https://github.com/ClickHouse/ClickHouse/issues/55153). You can turn off the usage of the sparse serialization format by providing the `ratio_of_defaults_for_sparse_serialization = 1` setting for your MergeTree tables. [#49631](https://github.com/ClickHouse/ClickHouse/pull/49631) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Enable `move_all_conditions_to_prewhere` and `enable_multiple_prewhere_read_steps` settings by default. [#46365](https://github.com/ClickHouse/ClickHouse/pull/46365) ([Alexander Gololobov](https://github.com/davenger)).
* Improves performance of some queries by tuning allocator. [#46416](https://github.com/ClickHouse/ClickHouse/pull/46416) ([Azat Khuzhin](https://github.com/azat)).
* Now we use fixed-size tasks in `MergeTreePrefetchedReadPool` as in `MergeTreeReadPool`. Also from now we use connection pool for S3 requests. [#49732](https://github.com/ClickHouse/ClickHouse/pull/49732) ([Nikita Taranov](https://github.com/nickitat)).
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c0ef323e06bf..4be0c63e3b87 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -318,7 +318,16 @@ set (COMPILER_FLAGS "${COMPILER_FLAGS}")
# Our built-in unwinder only supports DWARF version up to 4.
set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
+# Disable omit frame pointer compiler optimization using -fno-omit-frame-pointer
+option(DISABLE_OMIT_FRAME_POINTER "Disable omit frame pointer compiler optimization" OFF)
+
+if (DISABLE_OMIT_FRAME_POINTER)
+ set (CMAKE_CXX_FLAGS_ADD "${CMAKE_CXX_FLAGS_ADD} -fno-omit-frame-pointer")
+ set (CMAKE_C_FLAGS_ADD "${CMAKE_C_FLAGS_ADD} -fno-omit-frame-pointer")
+ set (CMAKE_ASM_FLAGS_ADD "${CMAKE_ASM_FLAGS_ADD} -fno-omit-frame-pointer")
+endif()
+
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
diff --git a/README.md b/README.md
index 6736f5c67f34..67d4f46988f7 100644
--- a/README.md
+++ b/README.md
@@ -23,10 +23,13 @@ curl https://clickhouse.com/ | sh
## Upcoming Events
* [**v23.9 Community Call**]([https://clickhouse.com/company/events/v23-8-community-release-call](https://clickhouse.com/company/events/v23-9-community-release-call)?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-08) - Sep 28 - 23.9 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
-* [**ClickHouse Meetup in Amsterdam**](https://www.meetup.com/clickhouse-netherlands-user-group/events/296334590/)) - Oct 31
+* [**ClickHouse Meetup in Amsterdam**](https://www.meetup.com/clickhouse-netherlands-user-group/events/296334590/) - Oct 31
* [**ClickHouse Meetup in Beijing**](https://www.meetup.com/clickhouse-beijing-user-group/events/296334856/) - Nov 4
-* [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/296334923/) - Nov 14
+* [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/296334923/) - Nov 8
* [**ClickHouse Meetup in Singapore**](https://www.meetup.com/clickhouse-singapore-meetup-group/events/296334976/) - Nov 15
+* [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/296488501/) - Nov 30
+* [**ClickHouse Meetup in NYC**](https://www.meetup.com/clickhouse-new-york-user-group/events/296488779/) - Dec 11
+* [**ClickHouse Meetup in Boston**](https://www.meetup.com/clickhouse-boston-user-group/events/296488840/) - Dec 12
Also, keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler clickhouse com.
diff --git a/SECURITY.md b/SECURITY.md
index c3e36b90367b..82b7254f8c16 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -13,9 +13,10 @@ The following versions of ClickHouse server are currently being supported with s
| Version | Supported |
|:-|:-|
+| 23.9 | ✔️ |
| 23.8 | ✔️ |
| 23.7 | ✔️ |
-| 23.6 | ✔️ |
+| 23.6 | ❌ |
| 23.5 | ❌ |
| 23.4 | ❌ |
| 23.3 | ✔️ |
diff --git a/base/base/hex.h b/base/base/hex.h
index 937218fec5a4..931f220aa081 100644
--- a/base/base/hex.h
+++ b/base/base/hex.h
@@ -289,3 +289,13 @@ inline void writeBinByte(UInt8 byte, void * out)
{
memcpy(out, &impl::bin_byte_to_char_table[static_cast(byte) * 8], 8);
}
+
+/// Converts byte array to a hex string. Useful for debug logging.
+inline std::string hexString(const void * data, size_t size)
+{
+ const char * p = reinterpret_cast(data);
+ std::string s(size * 2, '\0');
+ for (size_t i = 0; i < size; ++i)
+ writeHexByteLowercase(p[i], s.data() + i * 2);
+ return s;
+}
diff --git a/base/base/sort.h b/base/base/sort.h
index 912545979dc4..1a8145877632 100644
--- a/base/base/sort.h
+++ b/base/base/sort.h
@@ -131,3 +131,29 @@ void sort(RandomIt first, RandomIt last)
using comparator = std::less;
::sort(first, last, comparator());
}
+
+/** Try to fast sort elements for common sorting patterns:
+ * 1. If elements are already sorted.
+ * 2. If elements are already almost sorted.
+ * 3. If elements are already sorted in reverse order.
+ *
+ * Returns true if fast sort was performed or elements were already sorted, false otherwise.
+ */
+template
+bool trySort(RandomIt first, RandomIt last, Compare compare)
+{
+#ifndef NDEBUG
+ ::shuffle(first, last);
+#endif
+
+ ComparatorWrapper compare_wrapper = compare;
+ return ::pdqsort_try_sort(first, last, compare_wrapper);
+}
+
+template
+bool trySort(RandomIt first, RandomIt last)
+{
+ using value_type = typename std::iterator_traits::value_type;
+ using comparator = std::less;
+ return ::trySort(first, last, comparator());
+}
diff --git a/base/poco/MongoDB/include/Poco/MongoDB/Database.h b/base/poco/MongoDB/include/Poco/MongoDB/Database.h
index 3334a673df64..1fa91f4ca1af 100644
--- a/base/poco/MongoDB/include/Poco/MongoDB/Database.h
+++ b/base/poco/MongoDB/include/Poco/MongoDB/Database.h
@@ -70,7 +70,7 @@ namespace MongoDB
Document::Ptr queryBuildInfo(Connection & connection) const;
/// Queries server build info (all wire protocols)
- Document::Ptr queryServerHello(Connection & connection) const;
+ Document::Ptr queryServerHello(Connection & connection, bool old = false) const;
/// Queries hello response from server (all wire protocols)
Int64 count(Connection & connection, const std::string & collectionName) const;
diff --git a/base/poco/MongoDB/src/Database.cpp b/base/poco/MongoDB/src/Database.cpp
index 1a0d3cfe5599..15c46b172511 100644
--- a/base/poco/MongoDB/src/Database.cpp
+++ b/base/poco/MongoDB/src/Database.cpp
@@ -356,11 +356,19 @@ Document::Ptr Database::queryBuildInfo(Connection& connection) const
}
-Document::Ptr Database::queryServerHello(Connection& connection) const
+Document::Ptr Database::queryServerHello(Connection& connection, bool old) const
{
// hello can be issued on "config" system database
Poco::SharedPtr request = createCommand();
- request->selector().add("hello", 1);
+
+ // 'hello' command was previously called 'isMaster'
+ std::string command_name;
+ if (old)
+ command_name = "isMaster";
+ else
+ command_name = "hello";
+
+ request->selector().add(command_name, 1);
Poco::MongoDB::ResponseMessage response;
connection.sendRequest(*request, response);
diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 5c04bf5a77ab..2f6d43d6cd0c 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -2,11 +2,11 @@
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54478)
+SET(VERSION_REVISION 54479)
SET(VERSION_MAJOR 23)
-SET(VERSION_MINOR 9)
+SET(VERSION_MINOR 10)
SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH ebc7d9a9f3b40be89e0b3e738b35d394aabeea3e)
-SET(VERSION_DESCRIBE v23.9.1.1-testing)
-SET(VERSION_STRING 23.9.1.1)
+SET(VERSION_GITHASH 8f9a227de1f530cdbda52c145d41a6b0f1d29961)
+SET(VERSION_DESCRIBE v23.10.1.1-testing)
+SET(VERSION_STRING 23.10.1.1)
# end of autochange
diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake
index 9a70e4aee320..e8bf856332a5 100644
--- a/cmake/ccache.cmake
+++ b/cmake/ccache.cmake
@@ -12,7 +12,7 @@ endif()
set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'")
if(COMPILER_CACHE STREQUAL "auto")
- find_program (CCACHE_EXECUTABLE ccache sccache)
+ find_program (CCACHE_EXECUTABLE NAMES ccache sccache)
elseif (COMPILER_CACHE STREQUAL "ccache")
find_program (CCACHE_EXECUTABLE ccache)
elseif(COMPILER_CACHE STREQUAL "sccache")
diff --git a/cmake/tools.cmake b/cmake/tools.cmake
index 7fb45bc15d80..1ba3007b0f3b 100644
--- a/cmake/tools.cmake
+++ b/cmake/tools.cmake
@@ -49,14 +49,14 @@ endif ()
if (NOT LINKER_NAME)
if (COMPILER_CLANG)
- if (OS_LINUX)
- if (NOT ARCH_S390X) # s390x doesnt support lld
- find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
- endif ()
+ if (OS_LINUX AND NOT ARCH_S390X)
+ find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
+ elseif (OS_DARWIN)
+ find_program (LLD_PATH NAMES "ld")
endif ()
endif ()
- if (OS_LINUX)
- if (LLD_PATH)
+ if (LLD_PATH)
+ if (OS_LINUX OR OS_DARWIN)
if (COMPILER_CLANG)
# Clang driver simply allows full linker path.
set (LINKER_NAME ${LLD_PATH})
@@ -75,9 +75,11 @@ endif ()
if (LINKER_NAME)
message(STATUS "Using linker: ${LINKER_NAME}")
-else()
+elseif (NOT ARCH_S390X AND NOT OS_FREEBSD)
+ message (FATAL_ERROR "The only supported linker is LLVM's LLD, but we cannot find it.")
+else ()
message(STATUS "Using linker: ")
-endif()
+endif ()
# Archiver
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index f9c3fddff40c..390b0241e7d9 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -212,6 +212,8 @@ add_contrib (libbcrypt-cmake libbcrypt)
add_contrib (google-benchmark-cmake google-benchmark)
add_contrib (ulid-c-cmake ulid-c)
+add_contrib (libssh-cmake libssh)
+
# Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs.
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,
diff --git a/contrib/croaring b/contrib/croaring
index f40ed52bcdd6..9b7cc0ff1c41 160000
--- a/contrib/croaring
+++ b/contrib/croaring
@@ -1 +1 @@
-Subproject commit f40ed52bcdd635840a79877cef4857315dba817c
+Subproject commit 9b7cc0ff1c41e9457efb6228cfd2c538d0155303
diff --git a/contrib/croaring-cmake/CMakeLists.txt b/contrib/croaring-cmake/CMakeLists.txt
index 794c0426b969..5f3b90e0f7c0 100644
--- a/contrib/croaring-cmake/CMakeLists.txt
+++ b/contrib/croaring-cmake/CMakeLists.txt
@@ -2,23 +2,25 @@ set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/croaring")
set(SRCS
"${LIBRARY_DIR}/src/array_util.c"
+ "${LIBRARY_DIR}/src/bitset.c"
"${LIBRARY_DIR}/src/bitset_util.c"
+ "${LIBRARY_DIR}/src/isadetection.c"
+ "${LIBRARY_DIR}/src/memory.c"
+ "${LIBRARY_DIR}/src/roaring.c"
+ "${LIBRARY_DIR}/src/roaring_array.c"
+ "${LIBRARY_DIR}/src/roaring_priority_queue.c"
"${LIBRARY_DIR}/src/containers/array.c"
"${LIBRARY_DIR}/src/containers/bitset.c"
"${LIBRARY_DIR}/src/containers/containers.c"
"${LIBRARY_DIR}/src/containers/convert.c"
- "${LIBRARY_DIR}/src/containers/mixed_intersection.c"
- "${LIBRARY_DIR}/src/containers/mixed_union.c"
+ "${LIBRARY_DIR}/src/containers/mixed_andnot.c"
"${LIBRARY_DIR}/src/containers/mixed_equal.c"
- "${LIBRARY_DIR}/src/containers/mixed_subset.c"
+ "${LIBRARY_DIR}/src/containers/mixed_intersection.c"
"${LIBRARY_DIR}/src/containers/mixed_negation.c"
+ "${LIBRARY_DIR}/src/containers/mixed_subset.c"
+ "${LIBRARY_DIR}/src/containers/mixed_union.c"
"${LIBRARY_DIR}/src/containers/mixed_xor.c"
- "${LIBRARY_DIR}/src/containers/mixed_andnot.c"
- "${LIBRARY_DIR}/src/containers/run.c"
- "${LIBRARY_DIR}/src/roaring.c"
- "${LIBRARY_DIR}/src/roaring_priority_queue.c"
- "${LIBRARY_DIR}/src/roaring_array.c"
- "${LIBRARY_DIR}/src/memory.c")
+ "${LIBRARY_DIR}/src/containers/run.c")
add_library(_roaring ${SRCS})
diff --git a/contrib/curl b/contrib/curl
index eb3b049df526..d755a5f7c009 160000
--- a/contrib/curl
+++ b/contrib/curl
@@ -1 +1 @@
-Subproject commit eb3b049df526bf125eda23218e680ce7fa9ec46c
+Subproject commit d755a5f7c009dd63a61b2c745180d8ba937cbfeb
diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt
index 733865d5101f..7e86352befc5 100644
--- a/contrib/curl-cmake/CMakeLists.txt
+++ b/contrib/curl-cmake/CMakeLists.txt
@@ -64,6 +64,7 @@ set (SRCS
"${LIBRARY_DIR}/lib/hostsyn.c"
"${LIBRARY_DIR}/lib/hsts.c"
"${LIBRARY_DIR}/lib/http.c"
+ "${LIBRARY_DIR}/lib/http1.c"
"${LIBRARY_DIR}/lib/http2.c"
"${LIBRARY_DIR}/lib/http_aws_sigv4.c"
"${LIBRARY_DIR}/lib/http_chunks.c"
diff --git a/contrib/googletest b/contrib/googletest
index 71140c3ca7a8..e47544ad31cb 160000
--- a/contrib/googletest
+++ b/contrib/googletest
@@ -1 +1 @@
-Subproject commit 71140c3ca7a87bb1b5b9c9f1500fea8858cce344
+Subproject commit e47544ad31cb3ceecd04cc13e8fe556f8df9fe0b
diff --git a/contrib/googletest-cmake/CMakeLists.txt b/contrib/googletest-cmake/CMakeLists.txt
index 3905df031557..c917a2f31a77 100644
--- a/contrib/googletest-cmake/CMakeLists.txt
+++ b/contrib/googletest-cmake/CMakeLists.txt
@@ -6,20 +6,14 @@ target_compile_definitions (_gtest PUBLIC GTEST_HAS_POSIX_RE=0)
target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/googletest/include")
target_include_directories(_gtest PRIVATE "${SRC_DIR}/googletest")
-add_library(_gtest_main "${SRC_DIR}/googletest/src/gtest_main.cc")
-set_target_properties(_gtest_main PROPERTIES VERSION "1.0.0")
-target_link_libraries(_gtest_main PUBLIC _gtest)
-
-add_library(_gtest_all INTERFACE)
-target_link_libraries(_gtest_all INTERFACE _gtest _gtest_main)
-add_library(ch_contrib::gtest_all ALIAS _gtest_all)
-
+add_library(ch_contrib::gtest ALIAS _gtest)
add_library(_gmock "${SRC_DIR}/googlemock/src/gmock-all.cc")
set_target_properties(_gmock PROPERTIES VERSION "1.0.0")
target_compile_definitions (_gmock PUBLIC GTEST_HAS_POSIX_RE=0)
target_include_directories(_gmock SYSTEM PUBLIC "${SRC_DIR}/googlemock/include" "${SRC_DIR}/googletest/include")
target_include_directories(_gmock PRIVATE "${SRC_DIR}/googlemock")
+target_link_libraries(_gmock PUBLIC _gtest)
add_library(_gmock_main "${SRC_DIR}/googlemock/src/gmock_main.cc")
set_target_properties(_gmock_main PROPERTIES VERSION "1.0.0")
diff --git a/contrib/libssh b/contrib/libssh
new file mode 160000
index 000000000000..2c76332ef56d
--- /dev/null
+++ b/contrib/libssh
@@ -0,0 +1 @@
+Subproject commit 2c76332ef56d90f55965ab24da6b6dbcbef29c4c
diff --git a/contrib/libssh-cmake/CMakeLists.txt b/contrib/libssh-cmake/CMakeLists.txt
new file mode 100644
index 000000000000..58db81cf3526
--- /dev/null
+++ b/contrib/libssh-cmake/CMakeLists.txt
@@ -0,0 +1,74 @@
+set(LIB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libssh")
+set(LIB_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libssh")
+# Specify search path for CMake modules to be loaded by include()
+# and find_package()
+list(APPEND CMAKE_MODULE_PATH "${LIB_SOURCE_DIR}/cmake/Modules")
+
+include(DefineCMakeDefaults)
+include(DefineCompilerFlags)
+
+project(libssh VERSION 0.9.7 LANGUAGES C)
+
+# global needed variable
+set(APPLICATION_NAME ${PROJECT_NAME})
+
+# SOVERSION scheme: CURRENT.AGE.REVISION
+# If there was an incompatible interface change:
+# Increment CURRENT. Set AGE and REVISION to 0
+# If there was a compatible interface change:
+# Increment AGE. Set REVISION to 0
+# If the source code was changed, but there were no interface changes:
+# Increment REVISION.
+set(LIBRARY_VERSION "4.8.7")
+set(LIBRARY_SOVERSION "4")
+
+# where to look first for cmake modules, before ${CMAKE_ROOT}/Modules/ is checked
+
+# add definitions
+
+include(DefinePlatformDefaults)
+
+# Copy library files to a lib sub-directory
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${LIB_BINARY_DIR}/lib")
+
+set(CMAKE_THREAD_PREFER_PTHREADS ON)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+
+set(WITH_ZLIB OFF)
+set(WITH_SYMBOL_VERSIONING OFF)
+set(WITH_SERVER ON)
+
+include(IncludeSources.cmake)
+if (OS_LINUX)
+ if (ARCH_AMD64)
+ if (USE_MUSL)
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/linux/x86-64-musl")
+ else()
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/linux/x86-64")
+ endif ()
+ elseif (ARCH_AARCH64)
+ if (USE_MUSL)
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/linux/aarch64-musl")
+ else()
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/linux/aarch64")
+ endif ()
+ elseif (ARCH_PPC64LE)
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/linux/ppc64le")
+ elseif (ARCH_S390X)
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/linux/s390x")
+ elseif (ARCH_RISCV64)
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/linux/riscv64")
+ else ()
+ message(FATAL_ERROR "Platform is not supported")
+ endif ()
+elseif (OS_DARWIN)
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/darwin")
+elseif (OS_FREEBSD)
+ target_include_directories(_ssh PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libssh-cmake/freebsd")
+else ()
+ message(FATAL_ERROR "Platform is not supported")
+endif()
+
+configure_file(${LIB_SOURCE_DIR}/include/libssh/libssh_version.h.cmake
+ ${LIB_BINARY_DIR}/include/libssh/libssh_version.h
+ @ONLY)
diff --git a/contrib/libssh-cmake/IncludeSources.cmake b/contrib/libssh-cmake/IncludeSources.cmake
new file mode 100644
index 000000000000..d72cf11da1f3
--- /dev/null
+++ b/contrib/libssh-cmake/IncludeSources.cmake
@@ -0,0 +1,140 @@
+set(LIBSSH_LINK_LIBRARIES
+ ${LIBSSH_REQUIRED_LIBRARIES}
+)
+
+
+set(LIBSSH_LINK_LIBRARIES
+ ${LIBSSH_LINK_LIBRARIES}
+ OpenSSL::Crypto
+)
+
+if (MINGW AND Threads_FOUND)
+ set(LIBSSH_LINK_LIBRARIES
+ ${LIBSSH_LINK_LIBRARIES}
+ Threads::Threads
+ )
+endif()
+
+set(libssh_SRCS
+ ${LIB_SOURCE_DIR}/src/agent.c
+ ${LIB_SOURCE_DIR}/src/auth.c
+ ${LIB_SOURCE_DIR}/src/base64.c
+ ${LIB_SOURCE_DIR}/src/bignum.c
+ ${LIB_SOURCE_DIR}/src/buffer.c
+ ${LIB_SOURCE_DIR}/src/callbacks.c
+ ${LIB_SOURCE_DIR}/src/channels.c
+ ${LIB_SOURCE_DIR}/src/client.c
+ ${LIB_SOURCE_DIR}/src/config.c
+ ${LIB_SOURCE_DIR}/src/connect.c
+ ${LIB_SOURCE_DIR}/src/connector.c
+ ${LIB_SOURCE_DIR}/src/curve25519.c
+ ${LIB_SOURCE_DIR}/src/dh.c
+ ${LIB_SOURCE_DIR}/src/ecdh.c
+ ${LIB_SOURCE_DIR}/src/error.c
+ ${LIB_SOURCE_DIR}/src/getpass.c
+ ${LIB_SOURCE_DIR}/src/init.c
+ ${LIB_SOURCE_DIR}/src/kdf.c
+ ${LIB_SOURCE_DIR}/src/kex.c
+ ${LIB_SOURCE_DIR}/src/known_hosts.c
+ ${LIB_SOURCE_DIR}/src/knownhosts.c
+ ${LIB_SOURCE_DIR}/src/legacy.c
+ ${LIB_SOURCE_DIR}/src/log.c
+ ${LIB_SOURCE_DIR}/src/match.c
+ ${LIB_SOURCE_DIR}/src/messages.c
+ ${LIB_SOURCE_DIR}/src/misc.c
+ ${LIB_SOURCE_DIR}/src/options.c
+ ${LIB_SOURCE_DIR}/src/packet.c
+ ${LIB_SOURCE_DIR}/src/packet_cb.c
+ ${LIB_SOURCE_DIR}/src/packet_crypt.c
+ ${LIB_SOURCE_DIR}/src/pcap.c
+ ${LIB_SOURCE_DIR}/src/pki.c
+ ${LIB_SOURCE_DIR}/src/pki_container_openssh.c
+ ${LIB_SOURCE_DIR}/src/poll.c
+ ${LIB_SOURCE_DIR}/src/session.c
+ ${LIB_SOURCE_DIR}/src/scp.c
+ ${LIB_SOURCE_DIR}/src/socket.c
+ ${LIB_SOURCE_DIR}/src/string.c
+ ${LIB_SOURCE_DIR}/src/threads.c
+ ${LIB_SOURCE_DIR}/src/wrapper.c
+ ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c
+ ${LIB_SOURCE_DIR}/src/external/blowfish.c
+ ${LIB_SOURCE_DIR}/src/external/chacha.c
+ ${LIB_SOURCE_DIR}/src/external/poly1305.c
+ ${LIB_SOURCE_DIR}/src/chachapoly.c
+ ${LIB_SOURCE_DIR}/src/config_parser.c
+ ${LIB_SOURCE_DIR}/src/token.c
+ ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c
+)
+
+if (DEFAULT_C_NO_DEPRECATION_FLAGS)
+ set_source_files_properties(known_hosts.c
+ PROPERTIES
+ COMPILE_FLAGS ${DEFAULT_C_NO_DEPRECATION_FLAGS})
+endif()
+
+if (CMAKE_USE_PTHREADS_INIT)
+ set(libssh_SRCS
+ ${libssh_SRCS}
+ ${LIB_SOURCE_DIR}/src/threads/noop.c
+ ${LIB_SOURCE_DIR}/src/threads/pthread.c
+ )
+elseif (CMAKE_USE_WIN32_THREADS_INIT)
+ set(libssh_SRCS
+ ${libssh_SRCS}
+ ${LIB_SOURCE_DIR}/src/threads/noop.c
+ ${LIB_SOURCE_DIR}/src/threads/winlocks.c
+ )
+else()
+ set(libssh_SRCS
+ ${libssh_SRCS}
+ ${LIB_SOURCE_DIR}/src/threads/noop.c
+ )
+endif()
+
+# LIBCRYPT specific
+set(libssh_SRCS
+ ${libssh_SRCS}
+ ${LIB_SOURCE_DIR}/src/threads/libcrypto.c
+ ${LIB_SOURCE_DIR}/src/pki_crypto.c
+ ${LIB_SOURCE_DIR}/src/ecdh_crypto.c
+ ${LIB_SOURCE_DIR}/src/libcrypto.c
+ ${LIB_SOURCE_DIR}/src/dh_crypto.c
+)
+
+if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC))
+ add_compile_definitions(USE_BORINGSSL=1)
+endif()
+
+set(libssh_SRCS
+${libssh_SRCS}
+${LIB_SOURCE_DIR}/src/options.c
+${LIB_SOURCE_DIR}/src/server.c
+${LIB_SOURCE_DIR}/src/bind.c
+${LIB_SOURCE_DIR}/src/bind_config.c
+)
+
+
+add_library(_ssh STATIC ${libssh_SRCS})
+
+target_include_directories(_ssh PRIVATE ${LIB_BINARY_DIR})
+target_include_directories(_ssh PUBLIC "${LIB_SOURCE_DIR}/include" "${LIB_BINARY_DIR}/include")
+target_link_libraries(_ssh
+ PRIVATE ${LIBSSH_LINK_LIBRARIES})
+
+add_library(ch_contrib::ssh ALIAS _ssh)
+
+target_compile_options(_ssh
+ PRIVATE
+ ${DEFAULT_C_COMPILE_FLAGS}
+ -D_GNU_SOURCE)
+
+
+set_target_properties(_ssh
+ PROPERTIES
+ VERSION
+ ${LIBRARY_VERSION}
+ SOVERSION
+ ${LIBRARY_SOVERSION}
+ DEFINE_SYMBOL
+ LIBSSH_EXPORTS
+)
diff --git a/contrib/libssh-cmake/darwin/config.h b/contrib/libssh-cmake/darwin/config.h
new file mode 100644
index 000000000000..12378a64ceaa
--- /dev/null
+++ b/contrib/libssh-cmake/darwin/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/darwin"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UTMP_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UTIL_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+#define HAVE_GLOB_GL_FLAGS_MEMBER 1
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+#define HAVE_NTOHLL 1
+
+/* Define to 1 if you have the `htonll' function. */
+#define HAVE_HTONLL 1
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+/* #undef HAVE_EXPLICIT_BZERO */
+
+/* Define to 1 if you have the `memset_s' function. */
+#define HAVE_MEMSET_S 1
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/libssh-cmake/freebsd/config.h b/contrib/libssh-cmake/freebsd/config.h
new file mode 100644
index 000000000000..8a70acb473c0
--- /dev/null
+++ b/contrib/libssh-cmake/freebsd/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/freebsd"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+#define HAVE_GLOB_GL_FLAGS_MEMBER 1
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+#define HAVE_EXPLICIT_BZERO 1
+
+/* Define to 1 if you have the `memset_s' function. */
+/* #undef HAVE_MEMSET_S */
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/libssh-cmake/linux/aarch64-musl/config.h b/contrib/libssh-cmake/linux/aarch64-musl/config.h
new file mode 100644
index 000000000000..15236527fdf8
--- /dev/null
+++ b/contrib/libssh-cmake/linux/aarch64-musl/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/aarch64-musl"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+/* #undef HAVE_GLOB_GL_FLAGS_MEMBER
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+#define HAVE_EXPLICIT_BZERO 1
+
+/* Define to 1 if you have the `memset_s' function. */
+/* #undef HAVE_MEMSET_S */
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/libssh-cmake/linux/aarch64/config.h b/contrib/libssh-cmake/linux/aarch64/config.h
new file mode 100644
index 000000000000..e65ccb8ba3e6
--- /dev/null
+++ b/contrib/libssh-cmake/linux/aarch64/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/aarch64"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+#define HAVE_GLOB_GL_FLAGS_MEMBER 1
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+#define HAVE_EXPLICIT_BZERO 1
+
+/* Define to 1 if you have the `memset_s' function. */
+#define HAVE_MEMSET_S 1
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/libssh-cmake/linux/ppc64le/config.h b/contrib/libssh-cmake/linux/ppc64le/config.h
new file mode 100644
index 000000000000..c56b1ad03344
--- /dev/null
+++ b/contrib/libssh-cmake/linux/ppc64le/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/ppc64le"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+#define HAVE_GLOB_GL_FLAGS_MEMBER 1
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+/* #undef HAVE_EXPLICIT_BZERO 1 */
+
+/* Define to 1 if you have the `memset_s' function. */
+/* #undef HAVE_MEMSET_S */
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/libssh-cmake/linux/riscv64/config.h b/contrib/libssh-cmake/linux/riscv64/config.h
new file mode 100644
index 000000000000..33c91bf542b9
--- /dev/null
+++ b/contrib/libssh-cmake/linux/riscv64/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/riscv64"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+#define HAVE_GLOB_GL_FLAGS_MEMBER 1
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+/* #undef HAVE_EXPLICIT_BZERO 1 */
+
+/* Define to 1 if you have the `memset_s' function. */
+/* #undef HAVE_MEMSET_S */
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/libssh-cmake/linux/s390x/config.h b/contrib/libssh-cmake/linux/s390x/config.h
new file mode 100644
index 000000000000..289a8cabc8e4
--- /dev/null
+++ b/contrib/libssh-cmake/linux/s390x/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/s390x"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+#define HAVE_GLOB_GL_FLAGS_MEMBER 1
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+/* #undef HAVE_EXPLICIT_BZERO 1 */
+
+/* Define to 1 if you have the `memset_s' function. */
+/* #undef HAVE_MEMSET_S */
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+#define WORDS_BIGENDIAN 1
diff --git a/contrib/libssh-cmake/linux/x86-64-musl/config.h b/contrib/libssh-cmake/linux/x86-64-musl/config.h
new file mode 100644
index 000000000000..fd7c2e2b0c12
--- /dev/null
+++ b/contrib/libssh-cmake/linux/x86-64-musl/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/musl"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+/* #undef HAVE_GLOB_GL_FLAGS_MEMBER
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+#define HAVE_EXPLICIT_BZERO 1
+
+/* Define to 1 if you have the `memset_s' function. */
+/* #undef HAVE_MEMSET_S */
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/libssh-cmake/linux/x86-64/config.h b/contrib/libssh-cmake/linux/x86-64/config.h
new file mode 100644
index 000000000000..4090c5a45ad1
--- /dev/null
+++ b/contrib/libssh-cmake/linux/x86-64/config.h
@@ -0,0 +1,287 @@
+/* Name of package */
+#define PACKAGE "libssh"
+
+/* Version number of package */
+#define VERSION "0.9.7"
+
+#define SYSCONFDIR "etc"
+#define BINARYDIR "/home/ubuntu/workdir/ClickHouse/build/Debug"
+#define SOURCEDIR "/home/ubuntu/workdir/ClickHouse"
+
+/* Global bind configuration file path */
+#define GLOBAL_BIND_CONFIG "/etc/ssh/libssh_server_config"
+
+/* Global client configuration file path */
+#define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config"
+
+/************************** HEADER FILES *************************/
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_ARGP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VALGRIND_VALGRIND_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTY_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTMP_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UTIL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UTIME_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_AES_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WSPIAPI_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_OPENSSL_BLOWFISH_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_DES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDH_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_EC_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_ECDSA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have eliptic curve cryptography in openssl */
+#define HAVE_OPENSSL_ECC 1
+
+/* Define to 1 if you have eliptic curve cryptography in gcrypt */
+/* #undef HAVE_GCRYPT_ECC */
+
+/* Define to 1 if you have eliptic curve cryptography */
+#define HAVE_ECC 1
+
+/* Define to 1 if you have DSA */
+/* #undef HAVE_DSA */
+
+/* Define to 1 if you have gl_flags as a glob_t sturct member */
+#define HAVE_GLOB_GL_FLAGS_MEMBER 1
+
+/* Define to 1 if you have OpenSSL with Ed25519 support */
+#define HAVE_OPENSSL_ED25519 1
+
+/* Define to 1 if you have OpenSSL with X25519 support */
+#define HAVE_OPENSSL_X25519 1
+
+/*************************** FUNCTIONS ***************************/
+
+/* Define to 1 if you have the `EVP_aes128_ctr' function. */
+#define HAVE_OPENSSL_EVP_AES_CTR 1
+
+/* Define to 1 if you have the `EVP_aes128_cbc' function. */
+#define HAVE_OPENSSL_EVP_AES_CBC 1
+
+/* Define to 1 if you have the `EVP_aes128_gcm' function. */
+/* #undef HAVE_OPENSSL_EVP_AES_GCM */
+
+/* Define to 1 if you have the `CRYPTO_THREADID_set_callback' function. */
+#define HAVE_OPENSSL_CRYPTO_THREADID_SET_CALLBACK 1
+
+/* Define to 1 if you have the `CRYPTO_ctr128_encrypt' function. */
+#define HAVE_OPENSSL_CRYPTO_CTR128_ENCRYPT 1
+
+/* Define to 1 if you have the `EVP_CIPHER_CTX_new' function. */
+#define HAVE_OPENSSL_EVP_CIPHER_CTX_NEW 1
+
+/* Define to 1 if you have the `EVP_KDF_CTX_new_id' function. */
+/* #undef HAVE_OPENSSL_EVP_KDF_CTX_NEW_ID */
+
+/* Define to 1 if you have the `FIPS_mode' function. */
+#if USE_BORINGSSL
+#define HAVE_OPENSSL_FIPS_MODE 1
+#endif
+
+/* Define to 1 if you have the `EVP_DigestSign' function. */
+#define HAVE_OPENSSL_EVP_DIGESTSIGN 1
+
+/* Define to 1 if you have the `EVP_DigestVerify' function. */
+#define HAVE_OPENSSL_EVP_DIGESTVERIFY 1
+
+/* Define to 1 if you have the `OPENSSL_ia32cap_loc' function. */
+/* #undef HAVE_OPENSSL_IA32CAP_LOC */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the `_snprintf' function. */
+/* #undef HAVE__SNPRINTF */
+
+/* Define to 1 if you have the `_snprintf_s' function. */
+/* #undef HAVE__SNPRINTF_S */
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* Define to 1 if you have the `_vsnprintf_s' function. */
+/* #undef HAVE__VSNPRINTF_S */
+
+/* Define to 1 if you have the `isblank' function. */
+#define HAVE_ISBLANK 1
+
+/* Define to 1 if you have the `strncpy' function. */
+#define HAVE_STRNCPY 1
+
+/* Define to 1 if you have the `strndup' function. */
+#define HAVE_STRNDUP 1
+
+/* Define to 1 if you have the `cfmakeraw' function. */
+/* #undef HAVE_CFMAKERAW */
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+/* #undef HAVE_CLOCK_GETTIME */
+
+/* Define to 1 if you have the `ntohll' function. */
+/* #undef HAVE_NTOHLL */
+
+/* Define to 1 if you have the `htonll' function. */
+/* #undef HAVE_HTONLL */
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `__strtoull' function. */
+/* #undef HAVE___STRTOULL */
+
+/* Define to 1 if you have the `_strtoui64' function. */
+/* #undef HAVE__STRTOUI64 */
+
+/* Define to 1 if you have the `glob' function. */
+#define HAVE_GLOB 1
+
+/* Define to 1 if you have the `explicit_bzero' function. */
+#define HAVE_EXPLICIT_BZERO 1
+
+/* Define to 1 if you have the `memset_s' function. */
+#define HAVE_MEMSET_S 1
+
+/* Define to 1 if you have the `SecureZeroMemory' function. */
+/* #undef HAVE_SECURE_ZERO_MEMORY */
+
+/* Define to 1 if you have the `cmocka_set_test_filter' function. */
+/* #undef HAVE_CMOCKA_SET_TEST_FILTER */
+
+/*************************** LIBRARIES ***************************/
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#define HAVE_LIBCRYPTO 1
+
+/* Define to 1 if you have the `gcrypt' library (-lgcrypt). */
+/* #undef HAVE_LIBGCRYPT */
+
+/* Define to 1 if you have the 'mbedTLS' library (-lmbedtls). */
+/* #undef HAVE_LIBMBEDCRYPTO */
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_PTHREAD 1
+
+/* Define to 1 if you have the `cmocka' library (-lcmocka). */
+/* #undef HAVE_CMOCKA */
+
+/**************************** OPTIONS ****************************/
+
+#define HAVE_GCC_THREAD_LOCAL_STORAGE 1
+/* #undef HAVE_MSC_THREAD_LOCAL_STORAGE */
+
+#define HAVE_FALLTHROUGH_ATTRIBUTE 1
+#define HAVE_UNUSED_ATTRIBUTE 1
+
+#define HAVE_CONSTRUCTOR_ATTRIBUTE 1
+#define HAVE_DESTRUCTOR_ATTRIBUTE 1
+
+#define HAVE_GCC_VOLATILE_MEMORY_PROTECTION 1
+
+#define HAVE_COMPILER__FUNC__ 1
+#define HAVE_COMPILER__FUNCTION__ 1
+
+/* #undef HAVE_GCC_BOUNDED_ATTRIBUTE */
+
+/* Define to 1 if you want to enable GSSAPI */
+/* #undef WITH_GSSAPI */
+
+/* Define to 1 if you want to enable ZLIB */
+/* #undef WITH_ZLIB */
+
+/* Define to 1 if you want to enable SFTP */
+/* #undef WITH_SFTP */
+
+/* Define to 1 if you want to enable server support */
+#define WITH_SERVER 1
+
+/* Define to 1 if you want to enable DH group exchange algorithms */
+/* #undef WITH_GEX */
+
+/* Define to 1 if you want to enable blowfish cipher support */
+/* #undef WITH_BLOWFISH_CIPHER */
+
+/* Define to 1 if you want to enable debug output for crypto functions */
+/* #undef DEBUG_CRYPTO */
+
+/* Define to 1 if you want to enable debug output for packet functions */
+/* #undef DEBUG_PACKET */
+
+/* Define to 1 if you want to enable pcap output support (experimental) */
+/* #undef WITH_PCAP */
+
+/* Define to 1 if you want to enable calltrace debug output */
+/* #undef DEBUG_CALLTRACE */
+
+/* Define to 1 if you want to enable NaCl support */
+/* #undef WITH_NACL */
+
+/*************************** ENDIAN *****************************/
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+/* #undef WORDS_BIGENDIAN */
diff --git a/contrib/pdqsort/pdqsort.h b/contrib/pdqsort/pdqsort.h
index 01e82b710ee0..cbfc82a4f41e 100644
--- a/contrib/pdqsort/pdqsort.h
+++ b/contrib/pdqsort/pdqsort.h
@@ -54,8 +54,10 @@ namespace pdqsort_detail {
block_size = 64,
// Cacheline size, assumes power of two.
- cacheline_size = 64
+ cacheline_size = 64,
+ /// Try sort allowed iterations
+ try_sort_iterations = 3,
};
#if __cplusplus >= 201103L
@@ -501,6 +503,167 @@ namespace pdqsort_detail {
leftmost = false;
}
}
+
+ template
+ inline bool pdqsort_try_sort_loop(Iter begin,
+ Iter end,
+ Compare comp,
+ size_t bad_allowed,
+ size_t iterations_allowed,
+ bool force_sort = false,
+ bool leftmost = true) {
+ typedef typename std::iterator_traits::difference_type diff_t;
+
+ // Use a while loop for tail recursion elimination.
+ while (true) {
+ if (!force_sort && iterations_allowed == 0) {
+ return false;
+ }
+
+ diff_t size = end - begin;
+
+ // Insertion sort is faster for small arrays.
+ if (size < insertion_sort_threshold) {
+ if (leftmost) insertion_sort(begin, end, comp);
+ else unguarded_insertion_sort(begin, end, comp);
+
+ return true;
+ }
+
+ // Choose pivot as median of 3 or pseudomedian of 9.
+ diff_t s2 = size / 2;
+ if (size > ninther_threshold) {
+ sort3(begin, begin + s2, end - 1, comp);
+ sort3(begin + 1, begin + (s2 - 1), end - 2, comp);
+ sort3(begin + 2, begin + (s2 + 1), end - 3, comp);
+ sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp);
+ std::iter_swap(begin, begin + s2);
+ } else sort3(begin + s2, begin, end - 1, comp);
+
+ // If *(begin - 1) is the end of the right partition of a previous partition operation
+ // there is no element in [begin, end) that is smaller than *(begin - 1). Then if our
+ // pivot compares equal to *(begin - 1) we change strategy, putting equal elements in
+ // the left partition, greater elements in the right partition. We do not have to
+ // recurse on the left partition, since it's sorted (all equal).
+ if (!leftmost && !comp(*(begin - 1), *begin)) {
+ begin = partition_left(begin, end, comp) + 1;
+ continue;
+ }
+
+ // Partition and get results.
+ std::pair part_result =
+ Branchless ? partition_right_branchless(begin, end, comp)
+ : partition_right(begin, end, comp);
+ Iter pivot_pos = part_result.first;
+ bool already_partitioned = part_result.second;
+
+ // Check for a highly unbalanced partition.
+ diff_t l_size = pivot_pos - begin;
+ diff_t r_size = end - (pivot_pos + 1);
+ bool highly_unbalanced = l_size < size / 8 || r_size < size / 8;
+
+ // If we got a highly unbalanced partition we shuffle elements to break many patterns.
+ if (highly_unbalanced) {
+ if (!force_sort) {
+ return false;
+ }
+
+ // If we had too many bad partitions, switch to heapsort to guarantee O(n log n).
+ if (--bad_allowed == 0) {
+ std::make_heap(begin, end, comp);
+ std::sort_heap(begin, end, comp);
+ return true;
+ }
+
+ if (l_size >= insertion_sort_threshold) {
+ std::iter_swap(begin, begin + l_size / 4);
+ std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4);
+
+ if (l_size > ninther_threshold) {
+ std::iter_swap(begin + 1, begin + (l_size / 4 + 1));
+ std::iter_swap(begin + 2, begin + (l_size / 4 + 2));
+ std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1));
+ std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2));
+ }
+ }
+
+ if (r_size >= insertion_sort_threshold) {
+ std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4));
+ std::iter_swap(end - 1, end - r_size / 4);
+
+ if (r_size > ninther_threshold) {
+ std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4));
+ std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4));
+ std::iter_swap(end - 2, end - (1 + r_size / 4));
+ std::iter_swap(end - 3, end - (2 + r_size / 4));
+ }
+ }
+ } else {
+ // If we were decently balanced and we tried to sort an already partitioned
+ // sequence try to use insertion sort.
+ if (already_partitioned && partial_insertion_sort(begin, pivot_pos, comp)
+ && partial_insertion_sort(pivot_pos + 1, end, comp)) {
+ return true;
+ }
+ }
+
+ // Sort the left partition first using recursion and do tail recursion elimination for
+ // the right-hand partition.
+ if (pdqsort_try_sort_loop(begin,
+ pivot_pos,
+ comp,
+ bad_allowed,
+ iterations_allowed - 1,
+ force_sort,
+ leftmost)) {
+ force_sort = true;
+ } else {
+ return false;
+ }
+
+ --iterations_allowed;
+ begin = pivot_pos + 1;
+ leftmost = false;
+ }
+
+ return false;
+ }
+
+ template
+ inline bool pdqsort_try_sort_impl(Iter begin, Iter end, Compare comp, size_t bad_allowed)
+ {
+ typedef typename std::iterator_traits::difference_type diff_t;
+
+ static constexpr size_t iterations_allowed = pdqsort_detail::try_sort_iterations;
+ static constexpr size_t num_to_try = 16;
+
+ diff_t size = end - begin;
+
+ if (size > num_to_try * 10)
+ {
+ size_t out_of_order_elements = 0;
+
+ for (size_t i = 1; i < num_to_try; ++i)
+ {
+ diff_t offset = size / num_to_try;
+
+ diff_t prev_position = offset * (i - 1);
+ diff_t curr_position = offset * i;
+ diff_t next_position = offset * (i + 1) - 1;
+
+ bool prev_less_than_curr = comp(*(begin + prev_position), *(begin + curr_position));
+ bool curr_less_than_next = comp(*(begin + curr_position), *(begin + next_position));
+ if ((prev_less_than_curr && curr_less_than_next) || (!prev_less_than_curr && !curr_less_than_next))
+ continue;
+
+ ++out_of_order_elements;
+ if (out_of_order_elements > iterations_allowed)
+ return false;
+ }
+ }
+
+ return pdqsort_try_sort_loop(begin, end, comp, bad_allowed, iterations_allowed);
+ }
}
@@ -538,6 +701,41 @@ inline void pdqsort_branchless(Iter begin, Iter end) {
pdqsort_branchless(begin, end, std::less());
}
+template
+inline bool pdqsort_try_sort(Iter begin, Iter end, Compare comp) {
+ if (begin == end) return true;
+
+#if __cplusplus >= 201103L
+ return pdqsort_detail::pdqsort_try_sort_impl::type>::value &&
+ std::is_arithmetic::value_type>::value>(
+ begin, end, comp, pdqsort_detail::log2(end - begin));
+#else
+ return pdqsort_detail::pdqsort_try_sort_impl(
+ begin, end, comp, pdqsort_detail::log2(end - begin));
+#endif
+}
+
+template
+inline bool pdqsort_try_sort(Iter begin, Iter end) {
+ typedef typename std::iterator_traits::value_type T;
+ return pdqsort_try_sort(begin, end, std::less());
+}
+
+template
+inline bool pdqsort_try_sort_branchless(Iter begin, Iter end, Compare comp) {
+ if (begin == end) return true;
+
+ return pdqsort_detail::pdqsort_try_sort_impl(
+ begin, end, comp, pdqsort_detail::log2(end - begin));
+}
+
+template
+inline bool pdqsort_try_sort_branchless(Iter begin, Iter end) {
+ typedef typename std::iterator_traits::value_type T;
+ return pdqsort_try_sort_branchless(begin, end, std::less());
+}
+
#undef PDQSORT_PREFER_MOVE
diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile
index 63dbac6a995a..0e10068d79a2 100644
--- a/docker/keeper/Dockerfile
+++ b/docker/keeper/Dockerfile
@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
esac
ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release"
-ARG VERSION="23.8.2.7"
+ARG VERSION="23.9.1.1854"
ARG PACKAGES="clickhouse-keeper"
# user/group precreated explicitly with fixed uid/gid on purpose.
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index 11efffd592c3..6a27e7adcebd 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -15,6 +15,11 @@ if [ "$EXTRACT_TOOLCHAIN_DARWIN" = "1" ]; then
mkdir -p /build/cmake/toolchain/darwin-x86_64
tar xJf /MacOSX11.0.sdk.tar.xz -C /build/cmake/toolchain/darwin-x86_64 --strip-components=1
ln -sf darwin-x86_64 /build/cmake/toolchain/darwin-aarch64
+
+ if [ "$EXPORT_SOURCES_WITH_SUBMODULES" = "1" ]; then
+ cd /build
+ tar --exclude-vcs-ignores --exclude-vcs --exclude build --exclude build_docker --exclude debian --exclude .git --exclude .github --exclude .cache --exclude docs --exclude tests/integration -c . | pigz -9 > /output/source_sub.tar.gz
+ fi
fi
# Uncomment to debug ccache. Don't put ccache log in /output right away, or it
@@ -26,9 +31,6 @@ fi
mkdir -p /build/build_docker
cd /build/build_docker
rm -f CMakeCache.txt
-# Read cmake arguments into array (possibly empty)
-read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
-env
if [ -n "$MAKE_DEB" ]; then
rm -rf /build/packages/root
@@ -55,11 +57,36 @@ ccache_status
# clear cache stats
ccache --zero-stats ||:
+function check_prebuild_exists() {
+ local path="$1"
+ [ -d "$path" ] && [ "$(ls -A "$path")" ]
+}
+
+# Check whether the directory with pre-build scripts exists and not empty.
+if check_prebuild_exists /build/packages/pre-build
+then
+ # Execute all commands
+ for file in /build/packages/pre-build/*.sh ;
+ do
+ # The script may want to modify environment variables. Why not to allow it to do so?
+ # shellcheck disable=SC1090
+ source "$file"
+ done
+else
+ echo "There are no subcommands to execute :)"
+fi
+
+# Read cmake arguments into array (possibly empty)
+# The name of local variable has to be different from the name of environment variable
+# not to override it. And make it usable for other processes.
+read -ra CMAKE_FLAGS_ARRAY <<< "${CMAKE_FLAGS:-}"
+env
+
if [ "$BUILD_MUSL_KEEPER" == "1" ]
then
# build keeper with musl separately
# and without rust bindings
- cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+ cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS_ARRAY[@]}" ..
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
ninja $NINJA_FLAGS clickhouse-keeper
@@ -73,13 +100,13 @@ then
fi
rm -f CMakeCache.txt
- # Build the rest of binaries
- cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
-else
- # Build everything
- cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+ # Modify CMake flags, so we won't overwrite standalone keeper with symlinks
+ CMAKE_FLAGS_ARRAY+=(-DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0)
fi
+# Build everything
+cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS_ARRAY[@]}" ..
+
# No quotes because I want it to expand to nothing if empty.
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
ninja $NINJA_FLAGS $BUILD_TARGET
diff --git a/docker/packager/packager b/docker/packager/packager
index a0e460abacf8..e63a4912e7cc 100755
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -105,7 +105,7 @@ def run_docker_image_with_env(
ccache_mount = ""
cmd = (
- f"docker run --network=host --user={user} --rm {ccache_mount}"
+ f"docker run --network=host --user={user} --rm {ccache_mount} "
f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}"
)
@@ -179,6 +179,7 @@ def parse_env_variables(
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake"
)
result.append("EXTRACT_TOOLCHAIN_DARWIN=1")
+ result.append("EXPORT_SOURCES_WITH_SUBMODULES=1")
elif is_cross_darwin_arm:
cc = compiler[: -len(DARWIN_ARM_SUFFIX)]
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar")
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index 36dacd781bc1..afe1be75bedf 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="23.8.2.7"
+ARG VERSION="23.9.1.1854"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# user/group precreated explicitly with fixed uid/gid on purpose.
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
index 752adf672296..402168c3e157 100644
--- a/docker/server/Dockerfile.ubuntu
+++ b/docker/server/Dockerfile.ubuntu
@@ -23,7 +23,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="23.8.2.7"
+ARG VERSION="23.9.1.1854"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# set non-empty deb_location_url url to create a docker image
diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh
index 0d6c0e368db0..0ff79e24bf85 100755
--- a/docker/test/base/setup_export_logs.sh
+++ b/docker/test/base/setup_export_logs.sh
@@ -15,8 +15,8 @@ CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
# Pre-configured destination cluster, where to export the data
CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
-EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, "}
-EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type, '' AS instance_id"}
+EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime('UTC'), check_name String, instance_type String, instance_id String, "}
+EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"CAST(0 AS UInt32) AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type, '' AS instance_id"}
EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name, "}
function __set_connection_args
@@ -125,9 +125,9 @@ function setup_logs_replication
echo 'Create %_log tables'
clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
do
- # Calculate hash of its structure. Note: 1 is the version of extra columns - increment it if extra columns are changed:
+ # Calculate hash of its structure. Note: 4 is the version of extra columns - increment it if extra columns are changed:
hash=$(clickhouse-client --query "
- SELECT sipHash64(1, groupArray((name, type)))
+ SELECT sipHash64(4, groupArray((name, type)))
FROM (SELECT name, type FROM system.columns
WHERE database = 'system' AND table = '$table'
ORDER BY position)
diff --git a/docker/test/integration/mysql_java_client/Dockerfile b/docker/test/integration/mysql_java_client/Dockerfile
index 83c175514814..38fefac070e7 100644
--- a/docker/test/integration/mysql_java_client/Dockerfile
+++ b/docker/test/integration/mysql_java_client/Dockerfile
@@ -10,6 +10,5 @@ RUN curl -L -o /mysql-connector-j-${ver}.jar https://repo1.maven.org/maven2/com/
ENV CLASSPATH=$CLASSPATH:/mysql-connector-j-${ver}.jar
WORKDIR /jdbc
-COPY Test.java Test.java
-COPY PreparedStatementsTest.java PreparedStatementsTest.java
-RUN javac Test.java PreparedStatementsTest.java
+COPY MySQLJavaClientTest.java MySQLJavaClientTest.java
+RUN javac MySQLJavaClientTest.java
diff --git a/docker/test/integration/mysql_java_client/PreparedStatementsTest.java b/docker/test/integration/mysql_java_client/MySQLJavaClientTest.java
similarity index 81%
rename from docker/test/integration/mysql_java_client/PreparedStatementsTest.java
rename to docker/test/integration/mysql_java_client/MySQLJavaClientTest.java
index 1ffa945718d7..1ac21ffe4b4e 100644
--- a/docker/test/integration/mysql_java_client/PreparedStatementsTest.java
+++ b/docker/test/integration/mysql_java_client/MySQLJavaClientTest.java
@@ -2,7 +2,7 @@
import java.sql.*;
-public class PreparedStatementsTest {
+public class MySQLJavaClientTest {
public static void main(String[] args) {
int i = 0;
String host = "127.0.0.1";
@@ -10,6 +10,7 @@ public static void main(String[] args) {
String user = "default";
String password = "";
String database = "default";
+ String binary = "false";
while (i < args.length) {
switch (args[i]) {
case "--host":
@@ -27,16 +28,19 @@ public static void main(String[] args) {
case "--database":
database = args[++i];
break;
+ case "--binary":
+ binary = args[++i];
+ break;
default:
i++;
break;
}
}
- // useServerPrepStmts uses COM_STMT_PREPARE and COM_STMT_EXECUTE
- // instead of COM_QUERY which allows us to test the binary protocol
- String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=true",
- host, port, database);
+ // useServerPrepStmts=true -> COM_STMT_PREPARE + COM_STMT_EXECUTE -> binary
+ // useServerPrepStmts=false -> COM_QUERY -> text
+ String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=%s",
+ host, port, database, binary);
try {
Class.forName("com.mysql.cj.jdbc.Driver");
@@ -49,6 +53,7 @@ public static void main(String[] args) {
testDateTypes(conn);
testUnusualDateTime64Scales(conn);
testDateTimeTimezones(conn);
+ testSuspiciousNullableLowCardinalityTypes(conn);
conn.close();
} catch (Exception e) {
e.printStackTrace();
@@ -58,7 +63,7 @@ public static void main(String[] args) {
private static void testSimpleDataTypes(Connection conn) throws SQLException {
System.out.println("### testSimpleDataTypes");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_simple_data_types").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM simple_data_types").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -83,7 +88,7 @@ private static void testSimpleDataTypes(Connection conn) throws SQLException {
private static void testStringTypes(Connection conn) throws SQLException {
System.out.println("### testStringTypes");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_string_types").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM string_types").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -97,7 +102,7 @@ private static void testStringTypes(Connection conn) throws SQLException {
private static void testLowCardinalityAndNullableTypes(Connection conn) throws SQLException {
System.out.println("### testLowCardinalityAndNullableTypes");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_low_cardinality_and_nullable_types").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM low_cardinality_and_nullable_types").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -111,7 +116,7 @@ private static void testLowCardinalityAndNullableTypes(Connection conn) throws S
private static void testDecimalTypes(Connection conn) throws SQLException {
System.out.println("### testDecimalTypes");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_decimal_types").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM decimal_types").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -127,7 +132,7 @@ private static void testDecimalTypes(Connection conn) throws SQLException {
private static void testDateTypes(Connection conn) throws SQLException {
System.out.println("### testDateTypes");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_date_types").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM date_types").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -143,7 +148,7 @@ private static void testDateTypes(Connection conn) throws SQLException {
private static void testUnusualDateTime64Scales(Connection conn) throws SQLException {
System.out.println("### testUnusualDateTime64Scales");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_unusual_datetime64_scales").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM unusual_datetime64_scales").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -160,7 +165,7 @@ private static void testUnusualDateTime64Scales(Connection conn) throws SQLExcep
private static void testDateTimeTimezones(Connection conn) throws SQLException {
System.out.println("### testDateTimeTimezones");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_datetime_timezones").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM datetime_timezones").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -172,7 +177,7 @@ private static void testDateTimeTimezones(Connection conn) throws SQLException {
private static void testMiscTypes(Connection conn) throws SQLException {
System.out.println("### testMiscTypes");
- ResultSet rs = conn.prepareStatement("SELECT * FROM ps_misc_types").executeQuery();
+ ResultSet rs = conn.prepareStatement("SELECT * FROM misc_types").executeQuery();
int rowNum = 1;
while (rs.next()) {
System.out.printf("Row #%d\n", rowNum++);
@@ -184,6 +189,20 @@ private static void testMiscTypes(Connection conn) throws SQLException {
System.out.println();
}
+ private static void testSuspiciousNullableLowCardinalityTypes(Connection conn) throws SQLException {
+ System.out.println("### testSuspiciousNullableLowCardinalityTypes");
+ String query = "SELECT * FROM suspicious_nullable_low_cardinality_types";
+ ResultSet rs = conn.prepareStatement(query).executeQuery();
+ int rowNum = 1;
+ while (rs.next()) {
+ System.out.printf("Row #%d\n", rowNum++);
+ System.out.printf("%s, value: %s\n", getMysqlType(rs, "f"), rs.getFloat("f"));
+ System.out.printf("%s, value: %s\n", getMysqlType(rs, "d"), rs.getDate("d"));
+ System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"));
+ }
+ System.out.println();
+ }
+
private static String getMysqlType(ResultSet rs, String columnLabel) throws SQLException {
ResultSetMetaData meta = rs.getMetaData();
return String.format("%s type is %s", columnLabel,
diff --git a/docker/test/integration/mysql_java_client/Test.java b/docker/test/integration/mysql_java_client/Test.java
deleted file mode 100644
index 752441718a98..000000000000
--- a/docker/test/integration/mysql_java_client/Test.java
+++ /dev/null
@@ -1,78 +0,0 @@
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-class JavaConnectorTest {
- private static final String CREATE_TABLE_SQL = "CREATE TABLE IF NOT EXISTS default.test1 (`age` Int32, `name` String, `int_nullable` Nullable(Int32)) Engine = Memory";
- private static final String INSERT_SQL = "INSERT INTO default.test1(`age`, `name`) VALUES(33, 'jdbc'),(44, 'ck')";
- private static final String SELECT_SQL = "SELECT * FROM default.test1";
- private static final String SELECT_NUMBER_SQL = "SELECT * FROM system.numbers LIMIT 13";
- private static final String DROP_TABLE_SQL = "DROP TABLE default.test1";
-
- public static void main(String[] args) {
- int i = 0;
- String host = "127.0.0.1";
- String port = "9004";
- String user = "default";
- String password = "";
- String database = "default";
- while (i < args.length) {
- switch (args[i]) {
- case "--host":
- host = args[++i];
- break;
- case "--port":
- port = args[++i];
- break;
- case "--user":
- user = args[++i];
- break;
- case "--password":
- password = args[++i];
- break;
- case "--database":
- database = args[++i];
- break;
- default:
- i++;
- break;
- }
- }
-
- String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false", host, port, database);
-
- Connection conn = null;
- Statement stmt = null;
- try {
- Class.forName("com.mysql.cj.jdbc.Driver");
- conn = DriverManager.getConnection(jdbcUrl, user, password);
- stmt = conn.createStatement();
- stmt.executeUpdate(CREATE_TABLE_SQL);
- stmt.executeUpdate(INSERT_SQL);
-
- ResultSet rs = stmt.executeQuery(SELECT_SQL);
- while (rs.next()) {
- System.out.print(rs.getString("age"));
- System.out.print(rs.getString("name"));
- System.out.print(rs.getString("int_nullable"));
- System.out.println();
- }
-
- stmt.executeUpdate(DROP_TABLE_SQL);
-
- rs = stmt.executeQuery(SELECT_NUMBER_SQL);
- while (rs.next()) {
- System.out.print(rs.getString(1));
- System.out.println();
- }
-
- stmt.close();
- conn.close();
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(1);
- }
- }
-}
diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile
index d42fcb9baf67..8345e3d57912 100644
--- a/docker/test/integration/runner/Dockerfile
+++ b/docker/test/integration/runner/Dockerfile
@@ -80,7 +80,6 @@ RUN python3 -m pip install --no-cache-dir \
kafka-python \
kazoo \
lz4 \
- meilisearch==0.18.3 \
minio \
nats-py \
protobuf \
diff --git a/docker/test/integration/runner/compose/docker_compose_ldap.yml b/docker/test/integration/runner/compose/docker_compose_ldap.yml
new file mode 100644
index 000000000000..857f8fdce624
--- /dev/null
+++ b/docker/test/integration/runner/compose/docker_compose_ldap.yml
@@ -0,0 +1,16 @@
+version: '2.3'
+services:
+ openldap:
+ image: bitnami/openldap:2.6.6
+ restart: always
+ environment:
+ LDAP_ROOT: dc=example,dc=org
+ LDAP_ADMIN_DN: cn=admin,dc=example,dc=org
+ LDAP_ADMIN_USERNAME: admin
+ LDAP_ADMIN_PASSWORD: clickhouse
+ LDAP_USER_DC: users
+ LDAP_USERS: janedoe,johndoe
+ LDAP_PASSWORDS: qwerty,qwertz
+ LDAP_PORT_NUMBER: ${LDAP_INTERNAL_PORT:-1389}
+ ports:
+ - ${LDAP_EXTERNAL_PORT:-1389}:${LDAP_INTERNAL_PORT:-1389}
diff --git a/docker/test/integration/runner/compose/docker_compose_meili.yml b/docker/test/integration/runner/compose/docker_compose_meili.yml
deleted file mode 100644
index c1fad4aca874..000000000000
--- a/docker/test/integration/runner/compose/docker_compose_meili.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-version: '2.3'
-services:
- meili1:
- image: getmeili/meilisearch:v0.27.0
- restart: always
- ports:
- - ${MEILI_EXTERNAL_PORT:-7700}:${MEILI_INTERNAL_PORT:-7700}
-
- meili_secure:
- image: getmeili/meilisearch:v0.27.0
- restart: always
- ports:
- - ${MEILI_SECURE_EXTERNAL_PORT:-7700}:${MEILI_SECURE_INTERNAL_PORT:-7700}
- environment:
- MEILI_MASTER_KEY: "password"
diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml
index f2979566296f..45e55e7a79cf 100644
--- a/docker/test/integration/runner/compose/docker_compose_minio.yml
+++ b/docker/test/integration/runner/compose/docker_compose_minio.yml
@@ -2,9 +2,7 @@ version: '2.3'
services:
minio1:
- # Newer version of minio results in such errors:
- # "AWSErrorMarshaller: Encountered AWSError 'InternalError': We encountered an internal error, please try again"
- image: minio/minio:RELEASE.2021-09-23T04-46-24Z
+ image: minio/minio:RELEASE.2023-09-30T07-02-29Z
volumes:
- data1-1:/data1
- ${MINIO_CERTS_DIR:-}:/certs
diff --git a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
index 0190516728cf..2db9fb589d2a 100644
--- a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
+++ b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
@@ -2,15 +2,13 @@ version: '2.3'
services:
rabbitmq1:
- image: rabbitmq:3.8-management-alpine
+ image: rabbitmq:3.12.6-management-alpine
hostname: rabbitmq1
expose:
- ${RABBITMQ_PORT:-5672}
- environment:
- RABBITMQ_DEFAULT_USER: "root"
- RABBITMQ_DEFAULT_PASS: "clickhouse"
- RABBITMQ_LOG_BASE: /rabbitmq_logs/
volumes:
- type: ${RABBITMQ_LOGS_FS:-tmpfs}
source: ${RABBITMQ_LOGS:-}
target: /rabbitmq_logs/
+ - "${RABBITMQ_COOKIE_FILE}:/var/lib/rabbitmq/.erlang.cookie"
+ - /misc/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
\ No newline at end of file
diff --git a/docker/test/integration/runner/misc/rabbitmq.conf b/docker/test/integration/runner/misc/rabbitmq.conf
new file mode 100644
index 000000000000..3527c83880b8
--- /dev/null
+++ b/docker/test/integration/runner/misc/rabbitmq.conf
@@ -0,0 +1,8 @@
+loopback_users.guest = false
+listeners.tcp.default = 5672
+default_pass = clickhouse
+default_user = root
+management.tcp.port = 15672
+
+log.file = /rabbitmq_logs/rabbit.log
+log.file.level = debug
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 74ff536b98e7..34fc12d1a72c 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -69,6 +69,16 @@ else
fi
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+ sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
+ | sed "s|/var/lib/clickhouse/filesystem_caches/|/var/lib/clickhouse/filesystem_caches_1/|" \
+ > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
+ mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
+
+ sudo cat /etc/clickhouse-server2/config.d/filesystem_caches_path.xml \
+ | sed "s|/var/lib/clickhouse/filesystem_caches/|/var/lib/clickhouse/filesystem_caches_2/|" \
+ > /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp
+ mv /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
+
mkdir -p /var/run/clickhouse-server1
sudo chown clickhouse:clickhouse /var/run/clickhouse-server1
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib
index e56369ce1613..11945b68f700 100644
--- a/docker/test/stateless/stress_tests.lib
+++ b/docker/test/stateless/stress_tests.lib
@@ -52,6 +52,21 @@ function configure()
| sed "s|100000|10000|" \
> /etc/clickhouse-server/config.d/keeper_port.xml.tmp
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+
+ function randomize_config_boolean_value {
+ value=$(($RANDOM % 2))
+ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
+ | sed "s|<$1>[01]$1>|<$1>$value$1>|" \
+ > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
+ sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+ }
+
+ # Randomize all Keeper feature flags
+ randomize_config_boolean_value filtered_list
+ randomize_config_boolean_value multi_read
+ randomize_config_boolean_value check_not_exists
+ randomize_config_boolean_value create_if_not_exists
+
sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
sudo chgrp clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile
index b75bfb6661cc..cf5ba1eec7fa 100644
--- a/docker/test/unit/Dockerfile
+++ b/docker/test/unit/Dockerfile
@@ -6,5 +6,4 @@ FROM clickhouse/stateless-test:$FROM_TAG
RUN apt-get install gdb
COPY run.sh /
-COPY process_unit_tests_result.py /
CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/unit/process_unit_tests_result.py b/docker/test/unit/process_unit_tests_result.py
deleted file mode 100755
index 0550edc7c251..000000000000
--- a/docker/test/unit/process_unit_tests_result.py
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import logging
-import argparse
-import csv
-
-OK_SIGN = "OK ]"
-FAILED_SIGN = "FAILED ]"
-SEGFAULT = "Segmentation fault"
-SIGNAL = "received signal SIG"
-PASSED = "PASSED"
-
-
-def get_test_name(line):
- elements = reversed(line.split(" "))
- for element in elements:
- if "(" not in element and ")" not in element:
- return element
- raise Exception("No test name in line '{}'".format(line))
-
-
-def process_result(result_folder):
- summary = []
- total_counter = 0
- failed_counter = 0
- result_log_path = "{}/test_result.txt".format(result_folder)
- if not os.path.exists(result_log_path):
- logging.info("No output log on path %s", result_log_path)
- return "exception", "No output log", []
-
- status = "success"
- description = ""
- passed = False
- with open(result_log_path, "r") as test_result:
- for line in test_result:
- if OK_SIGN in line:
- logging.info("Found ok line: '%s'", line)
- test_name = get_test_name(line.strip())
- logging.info("Test name: '%s'", test_name)
- summary.append((test_name, "OK"))
- total_counter += 1
- elif FAILED_SIGN in line and "listed below" not in line and "ms)" in line:
- logging.info("Found fail line: '%s'", line)
- test_name = get_test_name(line.strip())
- logging.info("Test name: '%s'", test_name)
- summary.append((test_name, "FAIL"))
- total_counter += 1
- failed_counter += 1
- elif SEGFAULT in line:
- logging.info("Found segfault line: '%s'", line)
- status = "failure"
- description += "Segmentation fault. "
- break
- elif SIGNAL in line:
- logging.info("Received signal line: '%s'", line)
- status = "failure"
- description += "Exit on signal. "
- break
- elif PASSED in line:
- logging.info("PASSED record found: '%s'", line)
- passed = True
-
- if not passed:
- status = "failure"
- description += "PASSED record not found. "
-
- if failed_counter != 0:
- status = "failure"
-
- if not description:
- description += "fail: {}, passed: {}".format(
- failed_counter, total_counter - failed_counter
- )
-
- return status, description, summary
-
-
-def write_results(results_file, status_file, results, status):
- with open(results_file, "w") as f:
- out = csv.writer(f, delimiter="\t")
- out.writerows(results)
- with open(status_file, "w") as f:
- out = csv.writer(f, delimiter="\t")
- out.writerow(status)
-
-
-if __name__ == "__main__":
- logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
- parser = argparse.ArgumentParser(
- description="ClickHouse script for parsing results of unit tests"
- )
- parser.add_argument("--in-results-dir", default="/test_output/")
- parser.add_argument("--out-results-file", default="/test_output/test_results.tsv")
- parser.add_argument("--out-status-file", default="/test_output/check_status.tsv")
- args = parser.parse_args()
-
- state, description, test_results = process_result(args.in_results_dir)
- logging.info("Result parsed")
- status = (state, description)
- write_results(args.out_results_file, args.out_status_file, test_results, status)
- logging.info("Result written")
diff --git a/docker/test/unit/run.sh b/docker/test/unit/run.sh
index a4784466e27b..e87432214d84 100644
--- a/docker/test/unit/run.sh
+++ b/docker/test/unit/run.sh
@@ -3,5 +3,4 @@
set -x
service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test '';
-timeout 40m gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt
-./process_unit_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
+timeout 40m gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms --gtest_output='json:test_output/test_result.json' | tee test_output/test_result.txt
diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh
index 96c11e75f503..c69d90b9af07 100644
--- a/docker/test/upgrade/run.sh
+++ b/docker/test/upgrade/run.sh
@@ -60,11 +60,19 @@ install_packages previous_release_package_folder
# available for dump via clickhouse-local
configure
+function remove_keeper_config()
+{
+ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
+ | sed "/<$1>$2<\/$1>/d" \
+ > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
+ sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+}
+
# async_replication setting doesn't exist on some older versions
-sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
- | sed "/1<\/async_replication>/d" \
- > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+remove_keeper_config "async_replication" "1"
+
+# create_if_not_exists feature flag doesn't exist on some older versions
+remove_keeper_config "create_if_not_exists" "[01]"
# it contains some new settings, but we can safely remove it
rm /etc/clickhouse-server/config.d/merge_tree.xml
@@ -89,10 +97,10 @@ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
# async_replication setting doesn't exist on some older versions
-sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
- | sed "/1<\/async_replication>/d" \
- > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
-sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
+remove_keeper_config "async_replication" "1"
+
+# create_if_not_exists feature flag doesn't exist on some older versions
+remove_keeper_config "create_if_not_exists" "[01]"
# But we still need default disk because some tables loaded only into it
sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
diff --git a/docs/changelogs/v23.8.3.48-lts.md b/docs/changelogs/v23.8.3.48-lts.md
new file mode 100644
index 000000000000..af669c5adc88
--- /dev/null
+++ b/docs/changelogs/v23.8.3.48-lts.md
@@ -0,0 +1,43 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.8.3.48-lts (ebe4eb3d23e) FIXME as compared to v23.8.2.7-lts (f73c8f37874)
+
+#### Improvement
+* Backported in [#54287](https://github.com/ClickHouse/ClickHouse/issues/54287): Enable allow_remove_stale_moving_parts by default. [#54260](https://github.com/ClickHouse/ClickHouse/pull/54260) ([vdimir](https://github.com/vdimir)).
+* Backported in [#55057](https://github.com/ClickHouse/ClickHouse/issues/55057): ProfileEvents added ContextLockWaitMicroseconds event. [#55029](https://github.com/ClickHouse/ClickHouse/pull/55029) ([Maksim Kita](https://github.com/kitaisreal)).
+
+#### Build/Testing/Packaging Improvement
+* Backported in [#54702](https://github.com/ClickHouse/ClickHouse/issues/54702): Enrich `changed_images.json` with the latest tag from master for images that are not changed in the pull request. [#54369](https://github.com/ClickHouse/ClickHouse/pull/54369) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#54743](https://github.com/ClickHouse/ClickHouse/issues/54743): Remove redundant `clickhouse-keeper-client` symlink. [#54587](https://github.com/ClickHouse/ClickHouse/pull/54587) ([Tomas Barton](https://github.com/deric)).
+* Backported in [#54685](https://github.com/ClickHouse/ClickHouse/issues/54685): We build and upload them for every push, which isn't worth it. [#54675](https://github.com/ClickHouse/ClickHouse/pull/54675) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Fix: moved to prewhere condition actions can lose column [#53492](https://github.com/ClickHouse/ClickHouse/pull/53492) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix: parallel replicas over distributed with prefer_localhost_replica=1 [#54334](https://github.com/ClickHouse/ClickHouse/pull/54334) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix possible error 'URI contains invalid characters' in s3 table function [#54373](https://github.com/ClickHouse/ClickHouse/pull/54373) ([Kruglov Pavel](https://github.com/Avogar)).
+* Check for overflow before addition in `analysisOfVariance` function [#54385](https://github.com/ClickHouse/ClickHouse/pull/54385) ([Antonio Andelic](https://github.com/antonio2368)).
+* reproduce and fix the bug in removeSharedRecursive [#54430](https://github.com/ClickHouse/ClickHouse/pull/54430) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix aggregate projections with normalized states [#54480](https://github.com/ClickHouse/ClickHouse/pull/54480) ([Amos Bird](https://github.com/amosbird)).
+* Fix possible parsing error in WithNames formats with disabled input_format_with_names_use_header [#54513](https://github.com/ClickHouse/ClickHouse/pull/54513) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix zero copy garbage [#54550](https://github.com/ClickHouse/ClickHouse/pull/54550) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix race in `ColumnUnique` [#54575](https://github.com/ClickHouse/ClickHouse/pull/54575) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix serialization of `ColumnDecimal` [#54601](https://github.com/ClickHouse/ClickHouse/pull/54601) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix virtual columns having incorrect values after ORDER BY [#54811](https://github.com/ClickHouse/ClickHouse/pull/54811) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix Keeper segfault during shutdown [#54841](https://github.com/ClickHouse/ClickHouse/pull/54841) ([Antonio Andelic](https://github.com/antonio2368)).
+* Rebuild minmax_count_projection when partition key gets modified [#54943](https://github.com/ClickHouse/ClickHouse/pull/54943) ([Amos Bird](https://github.com/amosbird)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Test libunwind changes. [#51436](https://github.com/ClickHouse/ClickHouse/pull/51436) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Use pathlib.Path in S3Helper, rewrite build reports, improve small things [#54010](https://github.com/ClickHouse/ClickHouse/pull/54010) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Properly re-initialize ZooKeeper fault injection [#54251](https://github.com/ClickHouse/ClickHouse/pull/54251) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix segfault in system.zookeeper [#54326](https://github.com/ClickHouse/ClickHouse/pull/54326) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Update automated commit status comment [#54441](https://github.com/ClickHouse/ClickHouse/pull/54441) ([vdimir](https://github.com/vdimir)).
+* Adjusting `num_streams` by expected work in StorageS3 [#54815](https://github.com/ClickHouse/ClickHouse/pull/54815) ([pufit](https://github.com/pufit)).
+
diff --git a/docs/changelogs/v23.9.1.1854-stable.md b/docs/changelogs/v23.9.1.1854-stable.md
new file mode 100644
index 000000000000..655dd54d81bb
--- /dev/null
+++ b/docs/changelogs/v23.9.1.1854-stable.md
@@ -0,0 +1,381 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.9.1.1854-stable (8f9a227de1f) FIXME as compared to v23.8.1.2992-lts (ebc7d9a9f3b)
+
+#### Backward Incompatible Change
+* Remove the `status_info` configuration option and dictionaries status from the default Prometheus handler. [#54090](https://github.com/ClickHouse/ClickHouse/pull/54090) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* The experimental parts metadata cache is removed from the codebase. [#54215](https://github.com/ClickHouse/ClickHouse/pull/54215) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Disable setting `input_format_json_try_infer_numbers_from_strings` by default, so we don't try to infer numbers from strings in JSON formats by default to avoid possible parsing errors when sample data contains strings that looks like a number. [#55099](https://github.com/ClickHouse/ClickHouse/pull/55099) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### New Feature
+* Added new type of authentication based on SSH keys. It works only for Native TCP protocol. [#41109](https://github.com/ClickHouse/ClickHouse/pull/41109) ([George Gamezardashvili](https://github.com/InfJoker)).
+* Added IO Scheduling support for remote disks. Storage configuration for disk types `s3`, `s3_plain`, `hdfs` and `azure_blob_storage` can now contain `read_resource` and `write_resource` elements holding resource names. Scheduling policies for these resources can be configured in a separate server configuration section `resources`. Queries can be marked using setting `workload` and classified using server configuration section `workload_classifiers` to achieve diverse resource scheduling goals. More details in docs/en/operations/workload-scheduling.md. [#47009](https://github.com/ClickHouse/ClickHouse/pull/47009) ([Sergei Trifonov](https://github.com/serxa)).
+* Added a new column _block_number resolves [#44532](https://github.com/ClickHouse/ClickHouse/issues/44532). [#47532](https://github.com/ClickHouse/ClickHouse/pull/47532) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Add options `partial_result_update_duration_ms` and `max_rows_in_partial_result` to show updates of a partial result of output table in real-time during query execution. [#48607](https://github.com/ClickHouse/ClickHouse/pull/48607) ([Alexey Perevyshin](https://github.com/alexX512)).
+* Support case-insensitive and dot-all matching modes in RegExpTree dictionaries. [#50906](https://github.com/ClickHouse/ClickHouse/pull/50906) ([Johann Gan](https://github.com/johanngan)).
+* Add support for `ALTER TABLE MODIFY COMMENT`. Note: something similar was added by an external contributor a long time ago, but the feature did not work at all and only confused users. This closes [#36377](https://github.com/ClickHouse/ClickHouse/issues/36377). [#51304](https://github.com/ClickHouse/ClickHouse/pull/51304) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Added "GCD" aka. "greatest common denominator" as a new data compression codec. The codec computes the GCD of all column values, and then divides each value by the GCD. The GCD codec is a data preparation codec (similar to Delta and DoubleDelta) and cannot be used stand-alone. It works with data integer, decimal and date/time type. A viable use case for the GCD codec are column values that change (increase/decrease) in multiples of the GCD, e.g. 24 - 28 - 16 - 24 - 8 - 24 (assuming GCD = 4). [#53149](https://github.com/ClickHouse/ClickHouse/pull/53149) ([Alexander Nam](https://github.com/seshWCS)).
+* Two new type aliases "DECIMAL(P)" (as shortcut for "DECIMAL(P, 0") and "DECIMAL" (as shortcut for "DECIMAL(10, 0)") were added. This makes ClickHouse more compatible with MySQL's SQL dialect. [#53328](https://github.com/ClickHouse/ClickHouse/pull/53328) ([Val Doroshchuk](https://github.com/valbok)).
+* Added a new system log table `backup_log` to track all `BACKUP` and `RESTORE` operations. [#53638](https://github.com/ClickHouse/ClickHouse/pull/53638) ([Victor Krasnov](https://github.com/sirvickr)).
+* Added a format setting "output_format_markdown_escape_special_characters" (default: false). The setting controls whether special characters like "!", "#", "$" etc. are escaped (i.e. prefixed by a backslash) in the "Markdown" output format. [#53860](https://github.com/ClickHouse/ClickHouse/pull/53860) ([irenjj](https://github.com/irenjj)).
+* Add function `decodeHTMLComponent`. [#54097](https://github.com/ClickHouse/ClickHouse/pull/54097) ([Bharat Nallan](https://github.com/bharatnc)).
+* Added peak_threads_usage to query_log table. [#54335](https://github.com/ClickHouse/ClickHouse/pull/54335) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Add SHOW FUNCTIONS support to clickhouse-client. [#54337](https://github.com/ClickHouse/ClickHouse/pull/54337) ([Julia Kartseva](https://github.com/wat-ze-hex)).
+* This PRs improves schema inference from JSON formats: 1) Now it's possible to infer named Tuples from JSON objects without experimantal JSON type under a setting `input_format_json_try_infer_named_tuples_from_objects` in JSON formats. Previously without experimantal type JSON we could only infer JSON objects as Strings or Maps, now we can infer named Tuple. Resulting Tuple type will conain all keys of objects that were read in data sample during schema inference. It can be useful for reading structured JSON data without sparse objects. The setting is enabled by default. 2) Allow parsing JSON array into a column with type String under setting `input_format_json_read_arrays_as_strings`. It can help reading arrays with values with different types. 3) Allow to use type String for JSON keys with unkown types (`null`/`[]`/`{}`) in sample data under setting `input_format_json_infer_incomplete_types_as_strings`. Now in JSON formats we can read any value into String column and we can avoid getting error `Cannot determine type for column 'column_name' by first 25000 rows of data, most likely this column contains only Nulls or empty Arrays/Maps` during schema inference by using type String for unknown types, so the data will be read successfully. [#54427](https://github.com/ClickHouse/ClickHouse/pull/54427) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added function "toDaysSinceYearZero" with alias "TO_DAYS()" (for compatibility with MySQL) which returns the number of days passed since 0001-01-01. [#54479](https://github.com/ClickHouse/ClickHouse/pull/54479) ([Robert Schulze](https://github.com/rschu1ze)).
+* Added functions YYYYMMDDtoDate(), YYYYMMDDtoDate32(), YYYYMMDDhhmmssToDateTime() and YYYYMMDDhhmmssToDateTime64(). They convert a date or date with time encoded as integer (e.g. 20230911) into a native date or date with time. As such, they provide the opposite functionality of existing functions YYYYMMDDToDate(), YYYYMMDDToDateTime(), YYYYMMDDhhmmddToDateTime(), YYYYMMDDhhmmddToDateTime64(). [#54509](https://github.com/ClickHouse/ClickHouse/pull/54509) ([Robert Schulze](https://github.com/rschu1ze)).
+* Added "bandwidth_limit" IO scheduling node type. It allows you to specify `max_speed` and `max_burst` constraints on traffic passing though this node. More details in docs/en/operations/workload-scheduling.md. [#54618](https://github.com/ClickHouse/ClickHouse/pull/54618) ([Sergei Trifonov](https://github.com/serxa)).
+* Function `toDaysSinceYearZero()` now supports arguments of type `DateTime` and `DateTime64`. [#54856](https://github.com/ClickHouse/ClickHouse/pull/54856) ([Serge Klochkov](https://github.com/slvrtrn)).
+* Allow S3-style URLs for table functions `s3`, `gcs`, `oss`. URL is automatically converted to HTTP. Example: `'s3://clickhouse-public-datasets/hits.csv'` is converted to `'https://clickhouse-public-datasets.s3.amazonaws.com/hits.csv'`. [#54931](https://github.com/ClickHouse/ClickHouse/pull/54931) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add several string distance functions, include `byteHammingDistance`, `byteJaccardIndex`, `byteEditDistance`. ### Documentation entry for user-facing changes. [#54935](https://github.com/ClickHouse/ClickHouse/pull/54935) ([flynn](https://github.com/ucasfl)).
+* Add new setting `print_pretty_type_names` to print pretty deep nested types like Tuple/Maps/Arrays. [#55095](https://github.com/ClickHouse/ClickHouse/pull/55095) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### Performance Improvement
+* Improve performance of sorting for decimal columns. Improve performance of insertion into MergeTree if ORDER BY contains Decimal column. Improve performance of sorting when data is already sorted or almost sorted. [#35961](https://github.com/ClickHouse/ClickHouse/pull/35961) ([Maksim Kita](https://github.com/kitaisreal)).
+* Improve performance for huge query analysis. Fixes [#51224](https://github.com/ClickHouse/ClickHouse/issues/51224). [#51469](https://github.com/ClickHouse/ClickHouse/pull/51469) ([frinkr](https://github.com/frinkr)).
+* 1. Add rewriter for new analyzer. [#52082](https://github.com/ClickHouse/ClickHouse/pull/52082) ([JackyWoo](https://github.com/JackyWoo)).
+* 1. Add rewriter for both old and new analyzer. 2. Add settings `optimize_uniq_to_count`. [#52645](https://github.com/ClickHouse/ClickHouse/pull/52645) ([JackyWoo](https://github.com/JackyWoo)).
+* Remove manual calls to `mmap/mremap/munmap` and delegate all this work to `jemalloc`. [#52792](https://github.com/ClickHouse/ClickHouse/pull/52792) ([Nikita Taranov](https://github.com/nickitat)).
+* Now roaringBitmaps being optimized before serialization. [#52842](https://github.com/ClickHouse/ClickHouse/pull/52842) ([UnamedRus](https://github.com/UnamedRus)).
+* Optimize group by constant keys. Will optimize queries with group by `_file/_path` after https://github.com/ClickHouse/ClickHouse/pull/53529. [#53549](https://github.com/ClickHouse/ClickHouse/pull/53549) ([Kruglov Pavel](https://github.com/Avogar)).
+* Speed up reading from S3 by enabling prefetches by default. [#53709](https://github.com/ClickHouse/ClickHouse/pull/53709) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Do not implicitly read pk and version columns in lonely parts if unnecessary. [#53919](https://github.com/ClickHouse/ClickHouse/pull/53919) ([Duc Canh Le](https://github.com/canhld94)).
+* Fixed high in CPU consumption when working with NATS. [#54399](https://github.com/ClickHouse/ClickHouse/pull/54399) ([Vasilev Pyotr](https://github.com/vahpetr)).
+* Since we use separate instructions for executing `toString()` with datetime argument, it is possible to improve performance a bit for non-datetime arguments and have some parts of the code cleaner. Follows up [#53680](https://github.com/ClickHouse/ClickHouse/issues/53680). [#54443](https://github.com/ClickHouse/ClickHouse/pull/54443) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Instead of serializing json elements into a `std::stringstream`, this PR try to put the serialization result into `ColumnString` direclty. [#54613](https://github.com/ClickHouse/ClickHouse/pull/54613) ([lgbo](https://github.com/lgbo-ustc)).
+* Enable ORDER BY optimization for reading data in corresponding order from a MergeTree table in case that the table is behind a view. [#54628](https://github.com/ClickHouse/ClickHouse/pull/54628) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Improve JSON SQL functions by reusing `GeneratorJSONPath`. Since there are several `make_shared` in `GenerateorJSONPath`'s constructor, it has bad performance. [#54735](https://github.com/ClickHouse/ClickHouse/pull/54735) ([lgbo](https://github.com/lgbo-ustc)).
+
+#### Improvement
+* Keeper improvement: Add a `createIfNotExists` Keeper command. [#48855](https://github.com/ClickHouse/ClickHouse/pull/48855) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Add IF EMPTY clause for DROP TABLE queries. [#48915](https://github.com/ClickHouse/ClickHouse/pull/48915) ([Pavel Novitskiy](https://github.com/pnovitskiy)).
+* The Keeper dynamically adjusts log levels. [#50372](https://github.com/ClickHouse/ClickHouse/pull/50372) ([helifu](https://github.com/helifu)).
+* Allow to replace long names of files of columns in `MergeTree` data parts to hashes of names. It helps to avoid `File name too long` error in some cases. [#50612](https://github.com/ClickHouse/ClickHouse/pull/50612) ([Anton Popov](https://github.com/CurtizJ)).
+* Allow specifying the expiration date and, optionally, the time for user credentials with `VALID UNTIL datetime` clause. [#51261](https://github.com/ClickHouse/ClickHouse/pull/51261) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add setting `ignore_access_denied_multidirectory_globs`. [#52839](https://github.com/ClickHouse/ClickHouse/pull/52839) ([Andrey Zvonov](https://github.com/zvonand)).
+* Output valid JSON/XML on excetpion during HTTP query execution. Add setting `http_write_exception_in_output_format` to enable/disable this behaviour (enabled by default). [#52853](https://github.com/ClickHouse/ClickHouse/pull/52853) ([Kruglov Pavel](https://github.com/Avogar)).
+* More precise Integer type inference, fix [#51236](https://github.com/ClickHouse/ClickHouse/issues/51236). [#53003](https://github.com/ClickHouse/ClickHouse/pull/53003) ([Chen768959](https://github.com/Chen768959)).
+* Keeper tries to batch flush requests for better performance. [#53049](https://github.com/ClickHouse/ClickHouse/pull/53049) ([Antonio Andelic](https://github.com/antonio2368)).
+* Introduced resolving of charsets in the string literals for MaterializedMySQL. [#53220](https://github.com/ClickHouse/ClickHouse/pull/53220) ([Val Doroshchuk](https://github.com/valbok)).
+* Fix a subtle issue with a rarely used `EmbeddedRocksDB` table engine in an extremely rare scenario: sometimes the `EmbeddedRocksDB` table engine does not close files correctly in NFS after running `DROP TABLE`. [#53502](https://github.com/ClickHouse/ClickHouse/pull/53502) ([Mingliang Pan](https://github.com/liangliangpan)).
+* SQL functions "toString(datetime)" and "formatDateTime()" now support non-constant timezone arguments. [#53680](https://github.com/ClickHouse/ClickHouse/pull/53680) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* `RESTORE TABLE ON CLUSTER` must create replicated tables with a matching UUID on hosts. Otherwise the macro `{uuid}` in ZooKeeper path can't work correctly after RESTORE. This PR implements that. [#53765](https://github.com/ClickHouse/ClickHouse/pull/53765) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Added restore setting `restore_broken_parts_as_detached`: if it's true the RESTORE process won't stop on broken parts while restoring, instead all the broken parts will be copied to the `detached` folder with the prefix `broken-from-backup'. If it's false the RESTORE process will stop on the first broken part (if any). The default value is false. [#53877](https://github.com/ClickHouse/ClickHouse/pull/53877) ([Vitaly Baranov](https://github.com/vitlibar)).
+* The creation of Annoy indexes can now be parallelized using setting `max_threads_for_annoy_index_creation`. [#54047](https://github.com/ClickHouse/ClickHouse/pull/54047) ([Robert Schulze](https://github.com/rschu1ze)).
+* The MySQL interface gained a minimal implementation of prepared statements, just enough to allow a connection from Tableau Online to ClickHouse via the MySQL connector. [#54115](https://github.com/ClickHouse/ClickHouse/pull/54115) ([Serge Klochkov](https://github.com/slvrtrn)).
+* Replaced the library to handle (encode/decode) base64 values from Turbo-Base64 to aklomp-base64. Both are SIMD-accelerated on x86 and ARM but 1. the license of the latter (BSD-2) is more favorable for ClickHouse, Turbo64 switched in the meantime to GPL-3, 2. with more GitHub stars, aklomp-base64 seems more future-proof, 3. aklomp-base64 has a slightly nicer API (which is arguably subjective), and 4. aklomp-base64 does not require us to hack around bugs (like non-threadsafe initialization). Note: aklomp-base64 rejects unpadded base64 values whereas Turbo-Base64 decodes them on a best-effort basis. RFC-4648 leaves it open whether padding is mandatory or not, but depending on the context this may be a behavioral change to be aware of. [#54119](https://github.com/ClickHouse/ClickHouse/pull/54119) ([Mikhail Koviazin](https://github.com/mkmkme)).
+* Add elapsed_ns to HTTP headers X-ClickHouse-Progress and X-ClickHouse-Summary. [#54179](https://github.com/ClickHouse/ClickHouse/pull/54179) ([joelynch](https://github.com/joelynch)).
+* Implementation of `reconfig` (https://github.com/ClickHouse/ClickHouse/pull/49450), `sync`, and `exists` commands for keeper-client. [#54201](https://github.com/ClickHouse/ClickHouse/pull/54201) ([pufit](https://github.com/pufit)).
+* "clickhouse-local" and "clickhouse-client" now allow to specify the "--query" parameter multiple times, e.g. './clickhouse-client --query "SELECT 1" --query "SELECT 2"'. This syntax is slightly more intuitive than `./clickhouse-client --multiquery "SELECT 1;SELECT2", a bit easier to script (e.g. "queries.push_back('--query "$q"')") and more consistent with the behavior of existing parameter "--queries-file" (e.g. "./clickhouse client --queries-file queries1.sql --queries-file queries2.sql"). [#54249](https://github.com/ClickHouse/ClickHouse/pull/54249) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add sub-second precision to `formatReadableTimeDelta`. [#54250](https://github.com/ClickHouse/ClickHouse/pull/54250) ([Andrey Zvonov](https://github.com/zvonand)).
+* Fix wrong reallocation in HashedArrayDictionary:. [#54254](https://github.com/ClickHouse/ClickHouse/pull/54254) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Enable allow_remove_stale_moving_parts by default. [#54260](https://github.com/ClickHouse/ClickHouse/pull/54260) ([vdimir](https://github.com/vdimir)).
+* Fix using count from cache and improve progress bar for reading from archives. [#54271](https://github.com/ClickHouse/ClickHouse/pull/54271) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add support for S3 credentials using SSO. To define a profile to be used with SSO, set `AWS_PROFILE` environment variable. [#54347](https://github.com/ClickHouse/ClickHouse/pull/54347) ([Antonio Andelic](https://github.com/antonio2368)).
+* Support NULL as default for nested types Array/Tuple/Map for input formats. Closes [#51100](https://github.com/ClickHouse/ClickHouse/issues/51100). [#54351](https://github.com/ClickHouse/ClickHouse/pull/54351) ([Kruglov Pavel](https://github.com/Avogar)).
+* This is actually a bug fix, but not sure I'll be able to add a test to support the case, so I have put it as an improvement. This issue was introduced in https://github.com/ClickHouse/ClickHouse/pull/45878, which is when CH started reading arrow in batches. [#54370](https://github.com/ClickHouse/ClickHouse/pull/54370) ([Arthur Passos](https://github.com/arthurpassos)).
+* Add STD alias to stddevPop function for MySQL compatibility. Closes [#54274](https://github.com/ClickHouse/ClickHouse/issues/54274). [#54382](https://github.com/ClickHouse/ClickHouse/pull/54382) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add `addDate` function for compatibility with MySQL and `subDate` for consistency. Reference [#54275](https://github.com/ClickHouse/ClickHouse/issues/54275). [#54400](https://github.com/ClickHouse/ClickHouse/pull/54400) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Parse data in JSON format as JSONEachRow if failed to parse metadata. It will allow to read files with `.json` extension even if real format is JSONEachRow. Closes [#45740](https://github.com/ClickHouse/ClickHouse/issues/45740). [#54405](https://github.com/ClickHouse/ClickHouse/pull/54405) ([Kruglov Pavel](https://github.com/Avogar)).
+* Pass http retry timeout as milliseconds. [#54438](https://github.com/ClickHouse/ClickHouse/pull/54438) ([Duc Canh Le](https://github.com/canhld94)).
+* Support SAMPLE BY for VIEW. [#54477](https://github.com/ClickHouse/ClickHouse/pull/54477) ([Azat Khuzhin](https://github.com/azat)).
+* Add modification_time into system.detached_parts. [#54506](https://github.com/ClickHouse/ClickHouse/pull/54506) ([Azat Khuzhin](https://github.com/azat)).
+* Added a setting "splitby_max_substrings_includes_remaining_string" which controls if functions "splitBy*()" with argument "max_substring" > 0 include the remaining string (if any) in the result array (Python/Spark semantics) or not. The default behavior does not change. [#54518](https://github.com/ClickHouse/ClickHouse/pull/54518) ([Robert Schulze](https://github.com/rschu1ze)).
+* Now clickhouse-client process files in parallel in case of `INFILE 'glob_expression'`. Closes [#54218](https://github.com/ClickHouse/ClickHouse/issues/54218). [#54533](https://github.com/ClickHouse/ClickHouse/pull/54533) ([Max K.](https://github.com/mkaynov)).
+* Allow to use primary key for IN function where primary key column types are different from `IN` function right side column types. Example: `SELECT id FROM test_table WHERE id IN (SELECT '5')`. Closes [#48936](https://github.com/ClickHouse/ClickHouse/issues/48936). [#54544](https://github.com/ClickHouse/ClickHouse/pull/54544) ([Maksim Kita](https://github.com/kitaisreal)).
+* Better integer types inference for Int64/UInt64 fields. Continuation of https://github.com/ClickHouse/ClickHouse/pull/53003. Now it works also for nested types like Arrays of Arrays anf for functions like `map/tuple`. Issue: [#51236](https://github.com/ClickHouse/ClickHouse/issues/51236). [#54553](https://github.com/ClickHouse/ClickHouse/pull/54553) ([Kruglov Pavel](https://github.com/Avogar)).
+* HashJoin tries to shrink internal buffers consuming half of maximal available memory (set by `max_bytes_in_join`). [#54584](https://github.com/ClickHouse/ClickHouse/pull/54584) ([vdimir](https://github.com/vdimir)).
+* Added array operations for multiplying, dividing and modulo on scalar. Works in each way, for example `5 * [5, 5]` and `[5, 5] * 5` - both cases are possible. [#54608](https://github.com/ClickHouse/ClickHouse/pull/54608) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Added function `timestamp` for compatibility with MySQL. Closes [#54275](https://github.com/ClickHouse/ClickHouse/issues/54275). [#54639](https://github.com/ClickHouse/ClickHouse/pull/54639) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Respect max_block_size for array join to avoid possible OOM. Close [#54290](https://github.com/ClickHouse/ClickHouse/issues/54290). [#54664](https://github.com/ClickHouse/ClickHouse/pull/54664) ([李扬](https://github.com/taiyang-li)).
+* Add optional `version` argument to `rm` command in `keeper-client` to support safer deletes. [#54708](https://github.com/ClickHouse/ClickHouse/pull/54708) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Disable killing the server by systemd (that may lead to data loss when using Buffer tables). [#54744](https://github.com/ClickHouse/ClickHouse/pull/54744) ([Azat Khuzhin](https://github.com/azat)).
+* Added field "is_deterministic" to system table "system.functions" which indicates whether the result of a function is stable between two invocations (given exactly the same inputs) or not. [#54766](https://github.com/ClickHouse/ClickHouse/pull/54766) ([Robert Schulze](https://github.com/rschu1ze)).
+* Made the views in schema "information_schema" more compatible with the equivalent views in MySQL (i.e. modified and extended them) up to a point where Tableau Online is able to connect to ClickHouse. More specifically: 1. The type of field "information_schema.tables.table_type" changed from Enum8 to String. 2. Added fields "table_comment" and "table_collation" to view "information_schema.table". 3. Added views "information_schema.key_column_usage" and "referential_constraints". 4. Replaced uppercase aliases in "information_schema" views with concrete uppercase columns. [#54773](https://github.com/ClickHouse/ClickHouse/pull/54773) ([Serge Klochkov](https://github.com/slvrtrn)).
+* The query cache now returns an error if the user tries to cache the result of a query with a non-deterministic function such as "now()", "randomString()" and "dictGet()". Compared to the previous behavior (silently don't cache the result), this reduces confusion and surprise for users. [#54801](https://github.com/ClickHouse/ClickHouse/pull/54801) ([Robert Schulze](https://github.com/rschu1ze)).
+* Forbid special columns for file/s3/url/... storages, fix insert into ephemeral columns from files. Closes [#53477](https://github.com/ClickHouse/ClickHouse/issues/53477). [#54803](https://github.com/ClickHouse/ClickHouse/pull/54803) ([Kruglov Pavel](https://github.com/Avogar)).
+* More configurable collecting metadata for backup. [#54804](https://github.com/ClickHouse/ClickHouse/pull/54804) ([Vitaly Baranov](https://github.com/vitlibar)).
+* `clickhouse-local`'s log file (if enabled with --server_logs_file flag) will now prefix each line with timestamp, thread id, etc, just like `clickhouse-server`. [#54807](https://github.com/ClickHouse/ClickHouse/pull/54807) ([Michael Kolupaev](https://github.com/al13n321)).
+* Reuse HTTP connections in s3 table function. [#54812](https://github.com/ClickHouse/ClickHouse/pull/54812) ([Michael Kolupaev](https://github.com/al13n321)).
+* Avoid excessive calls to getifaddrs in isLocalAddress. [#54819](https://github.com/ClickHouse/ClickHouse/pull/54819) ([Duc Canh Le](https://github.com/canhld94)).
+* Field "is_obsolete" in system.merge_tree_settings is now 1 for obsolete merge tree settings. Previously, only the description indicated that the setting is obsolete. [#54837](https://github.com/ClickHouse/ClickHouse/pull/54837) ([Robert Schulze](https://github.com/rschu1ze)).
+* Make it possible to use plural when using interval literals. `INTERVAL 2 HOURS` should be equivalent to `INTERVAL 2 HOUR`. [#54860](https://github.com/ClickHouse/ClickHouse/pull/54860) ([Jordi Villar](https://github.com/jrdi)).
+* Replace the linear method in `MergeTreeRangeReader::Stream::ceilRowsToCompleteGranules` with a binary search. [#54869](https://github.com/ClickHouse/ClickHouse/pull/54869) ([usurai](https://github.com/usurai)).
+* Always allow the creation of a projection with `Nullable` PK. This fixes [#54814](https://github.com/ClickHouse/ClickHouse/issues/54814). [#54895](https://github.com/ClickHouse/ClickHouse/pull/54895) ([Amos Bird](https://github.com/amosbird)).
+* Retry backup S3 operations after connection reset failure. [#54900](https://github.com/ClickHouse/ClickHouse/pull/54900) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Make the exception message exact in case of the maximum value of a settings is less than the minimum value. [#54925](https://github.com/ClickHouse/ClickHouse/pull/54925) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* LIKE, match, and other regular expressions matching functions now allow matching with patterns containing non-UTF-8 substrings by falling back to binary matching. Example: you can use `string LIKE '\xFE\xFF%'` to detect BOM. This closes [#54486](https://github.com/ClickHouse/ClickHouse/issues/54486). [#54942](https://github.com/ClickHouse/ClickHouse/pull/54942) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* ProfileEvents added ContextLockWaitMicroseconds event. [#55029](https://github.com/ClickHouse/ClickHouse/pull/55029) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added field "is_deterministic" to system table "system.functions" which indicates whether the result of a function is stable between two invocations (given exactly the same inputs) or not. [#55035](https://github.com/ClickHouse/ClickHouse/pull/55035) ([Robert Schulze](https://github.com/rschu1ze)).
+* View information_schema.tables now has a new field `data_length` which shows the approximate size of the data on disk. Required to run queries generated by Amazon QuickSight. [#55037](https://github.com/ClickHouse/ClickHouse/pull/55037) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### Build/Testing/Packaging Improvement
+* ClickHouse is built with Musl instead of GLibc by default. [#52550](https://github.com/ClickHouse/ClickHouse/pull/52550) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* ClickHouse is built with Musl instead of GLibc. [#52721](https://github.com/ClickHouse/ClickHouse/pull/52721) ([Azat Khuzhin](https://github.com/azat)).
+* Bumped the compiler of official and continuous integration builds of ClickHouse from Clang 16 to 17. [#53831](https://github.com/ClickHouse/ClickHouse/pull/53831) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix flaky test. `wait_resolver` function was asserting the response to be == proxy1, but it might actually return proxy2. Account for that as well. [#54191](https://github.com/ClickHouse/ClickHouse/pull/54191) ([Arthur Passos](https://github.com/arthurpassos)).
+* Regenerated tld data for lookups (`tldLookup.generated.cpp`). [#54269](https://github.com/ClickHouse/ClickHouse/pull/54269) ([Bharat Nallan](https://github.com/bharatnc)).
+* Report properly timeout for check itself in `fast_test_check`/`stress_check`. [#54278](https://github.com/ClickHouse/ClickHouse/pull/54278) ([Igor Nikonov](https://github.com/devcrafter)).
+* Suddenly, `test_host_regexp_multiple_ptr_records_concurrent` became flaky. [#54307](https://github.com/ClickHouse/ClickHouse/pull/54307) ([Arthur Passos](https://github.com/arthurpassos)).
+* Fixed precise float parsing issue on s390x. [#54330](https://github.com/ClickHouse/ClickHouse/pull/54330) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Enrich `changed_images.json` with the latest tag from master for images that are not changed in the pull request. [#54369](https://github.com/ClickHouse/ClickHouse/pull/54369) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fixed endian issue in jemalloc_bins system table for s390x. [#54517](https://github.com/ClickHouse/ClickHouse/pull/54517) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Fixed random generation issue for UInt256 and IPv4 on s390x. [#54576](https://github.com/ClickHouse/ClickHouse/pull/54576) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Remove redundant `clickhouse-keeper-client` symlink. [#54587](https://github.com/ClickHouse/ClickHouse/pull/54587) ([Tomas Barton](https://github.com/deric)).
+* Use `/usr/bin/env` to resolve bash. [#54603](https://github.com/ClickHouse/ClickHouse/pull/54603) ([Fionera](https://github.com/fionera)).
+* Move all `tests/ci/*.lib files` to `stateless-tests` image. Closes [#54540](https://github.com/ClickHouse/ClickHouse/issues/54540). [#54668](https://github.com/ClickHouse/ClickHouse/pull/54668) ([Kruglov Pavel](https://github.com/Avogar)).
+* We build and upload them for every push, which isn't worth it. [#54675](https://github.com/ClickHouse/ClickHouse/pull/54675) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fixed SimHash function endian issue for s390x. [#54793](https://github.com/ClickHouse/ClickHouse/pull/54793) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Do not clone the fast tests repo twice; parallelize submodules checkout; use the current user in the fast-tests container. [#54849](https://github.com/ClickHouse/ClickHouse/pull/54849) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Avoid running pull request ci workflow for fixes touching .md files only. [#54914](https://github.com/ClickHouse/ClickHouse/pull/54914) ([Max K.](https://github.com/mkaynov)).
+* CMake added `PROFILE_CPU` option needed to perform `perf record` without using DWARF call graph. [#54917](https://github.com/ClickHouse/ClickHouse/pull/54917) ([Maksim Kita](https://github.com/kitaisreal)).
+* Use `--gtest_output='json:'` to parse unit test results. [#54922](https://github.com/ClickHouse/ClickHouse/pull/54922) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Added support for additional scripts (you need to mound a volume) to extend build process. [#55000](https://github.com/ClickHouse/ClickHouse/pull/55000) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* If the linker is different than LLD, stop with a fatal error. [#55036](https://github.com/ClickHouse/ClickHouse/pull/55036) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+
+* Store NULL in scalar result map for empty subquery result [#52240](https://github.com/ClickHouse/ClickHouse/pull/52240) ([vdimir](https://github.com/vdimir)).
+* Fix misleading error message in OUTFILE with CapnProto/Protobuf [#52870](https://github.com/ClickHouse/ClickHouse/pull/52870) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix summary reporting with parallel replicas with LIMIT [#53050](https://github.com/ClickHouse/ClickHouse/pull/53050) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix throttling of BACKUPs from/to S3 (in case native copy was not used) and in some other places as well [#53336](https://github.com/ClickHouse/ClickHouse/pull/53336) ([Azat Khuzhin](https://github.com/azat)).
+* Fix IO throttling during copying whole directories [#53338](https://github.com/ClickHouse/ClickHouse/pull/53338) ([Azat Khuzhin](https://github.com/azat)).
+* Fix: moved to prewhere condition actions can lose column [#53492](https://github.com/ClickHouse/ClickHouse/pull/53492) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* 37737 fixed internal error when replacing with byte-equal parts [#53735](https://github.com/ClickHouse/ClickHouse/pull/53735) ([Pedro Riera](https://github.com/priera)).
+* Fix: require columns participating in interpolate expression [#53754](https://github.com/ClickHouse/ClickHouse/pull/53754) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix cluster discovery initialization + setting up fail points in config [#54113](https://github.com/ClickHouse/ClickHouse/pull/54113) ([vdimir](https://github.com/vdimir)).
+* Fix issues in accurateCastOrNull [#54136](https://github.com/ClickHouse/ClickHouse/pull/54136) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Fix nullable primary key in final [#54164](https://github.com/ClickHouse/ClickHouse/pull/54164) ([Amos Bird](https://github.com/amosbird)).
+* Inserting only non-duplicate chunks in MV [#54184](https://github.com/ClickHouse/ClickHouse/pull/54184) ([Pedro Riera](https://github.com/priera)).
+* Fix REPLACE/MOVE PARTITION with zero-copy replication [#54193](https://github.com/ClickHouse/ClickHouse/pull/54193) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix: parallel replicas over distributed don't read from all replicas [#54199](https://github.com/ClickHouse/ClickHouse/pull/54199) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix: allow IPv6 for bloom filter [#54200](https://github.com/ClickHouse/ClickHouse/pull/54200) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* fix possible type mismatch with IPv4 [#54212](https://github.com/ClickHouse/ClickHouse/pull/54212) ([Bharat Nallan](https://github.com/bharatnc)).
+* Fix system.data_skipping_indices for recreated indices [#54225](https://github.com/ClickHouse/ClickHouse/pull/54225) ([Artur Malchanau](https://github.com/Hexta)).
+* fix name clash for multiple join rewriter v2 [#54240](https://github.com/ClickHouse/ClickHouse/pull/54240) ([Tao Wang](https://github.com/wangtZJU)).
+* Fix unexpected errors in system.errors after join [#54306](https://github.com/ClickHouse/ClickHouse/pull/54306) ([vdimir](https://github.com/vdimir)).
+* Fix isZeroOrNull(NULL) [#54316](https://github.com/ClickHouse/ClickHouse/pull/54316) ([flynn](https://github.com/ucasfl)).
+* Fix: parallel replicas over distributed with prefer_localhost_replica=1 [#54334](https://github.com/ClickHouse/ClickHouse/pull/54334) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix logical error in vertical merge + replacing merge tree + optimize cleanup [#54368](https://github.com/ClickHouse/ClickHouse/pull/54368) ([alesapin](https://github.com/alesapin)).
+* Fix possible error 'URI contains invalid characters' in s3 table function [#54373](https://github.com/ClickHouse/ClickHouse/pull/54373) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix segfault in AST optimization of `arrayExists` function [#54379](https://github.com/ClickHouse/ClickHouse/pull/54379) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Check for overflow before addition in `analysisOfVariance` function [#54385](https://github.com/ClickHouse/ClickHouse/pull/54385) ([Antonio Andelic](https://github.com/antonio2368)).
+* reproduce and fix the bug in removeSharedRecursive [#54430](https://github.com/ClickHouse/ClickHouse/pull/54430) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix possible incorrect result with SimpleAggregateFunction in PREWHERE and FINAL [#54436](https://github.com/ClickHouse/ClickHouse/pull/54436) ([Azat Khuzhin](https://github.com/azat)).
+* Fix filtering parts with indexHint for non analyzer [#54449](https://github.com/ClickHouse/ClickHouse/pull/54449) ([Azat Khuzhin](https://github.com/azat)).
+* Fix aggregate projections with normalized states [#54480](https://github.com/ClickHouse/ClickHouse/pull/54480) ([Amos Bird](https://github.com/amosbird)).
+* Bugfix/local multiquery parameter [#54498](https://github.com/ClickHouse/ClickHouse/pull/54498) ([CuiShuoGuo](https://github.com/bakam412)).
+* clickhouse-local support --database command line argument [#54503](https://github.com/ClickHouse/ClickHouse/pull/54503) ([vdimir](https://github.com/vdimir)).
+* Fix possible parsing error in WithNames formats with disabled input_format_with_names_use_header [#54513](https://github.com/ClickHouse/ClickHouse/pull/54513) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix rare case of CHECKSUM_DOESNT_MATCH error [#54549](https://github.com/ClickHouse/ClickHouse/pull/54549) ([alesapin](https://github.com/alesapin)).
+* Fix zero copy garbage [#54550](https://github.com/ClickHouse/ClickHouse/pull/54550) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix sorting of UNION ALL of already sorted results [#54564](https://github.com/ClickHouse/ClickHouse/pull/54564) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix snapshot install in Keeper [#54572](https://github.com/ClickHouse/ClickHouse/pull/54572) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix race in `ColumnUnique` [#54575](https://github.com/ClickHouse/ClickHouse/pull/54575) ([Nikita Taranov](https://github.com/nickitat)).
+* Annoy/Usearch index: Fix LOGICAL_ERROR during build-up with default values [#54600](https://github.com/ClickHouse/ClickHouse/pull/54600) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix serialization of `ColumnDecimal` [#54601](https://github.com/ClickHouse/ClickHouse/pull/54601) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix schema inference for *Cluster functions for column names with spaces [#54635](https://github.com/ClickHouse/ClickHouse/pull/54635) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix using structure from insertion tables in case of defaults and explicit insert columns [#54655](https://github.com/ClickHouse/ClickHouse/pull/54655) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix: avoid using regex match, possibly containing alternation, as a key condition. [#54696](https://github.com/ClickHouse/ClickHouse/pull/54696) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix ReplacingMergeTree with vertical merge and cleanup [#54706](https://github.com/ClickHouse/ClickHouse/pull/54706) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix virtual columns having incorrect values after ORDER BY [#54811](https://github.com/ClickHouse/ClickHouse/pull/54811) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix filtering parts with indexHint for non analyzer (resubmit) [#54825](https://github.com/ClickHouse/ClickHouse/pull/54825) ([Azat Khuzhin](https://github.com/azat)).
+* Fix Keeper segfault during shutdown [#54841](https://github.com/ClickHouse/ClickHouse/pull/54841) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix "Invalid number of rows in Chunk" in MaterializedPostgreSQL [#54844](https://github.com/ClickHouse/ClickHouse/pull/54844) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Move obsolete format settings to separate section [#54855](https://github.com/ClickHouse/ClickHouse/pull/54855) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix zero copy locks with hardlinks [#54859](https://github.com/ClickHouse/ClickHouse/pull/54859) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix `FINAL` produces invalid read ranges in a rare case [#54934](https://github.com/ClickHouse/ClickHouse/pull/54934) ([Nikita Taranov](https://github.com/nickitat)).
+* Rebuild minmax_count_projection when partition key gets modified [#54943](https://github.com/ClickHouse/ClickHouse/pull/54943) ([Amos Bird](https://github.com/amosbird)).
+* Fix bad cast to ColumnVector in function if [#55019](https://github.com/ClickHouse/ClickHouse/pull/55019) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix: insert quorum w/o keeper retries [#55026](https://github.com/ClickHouse/ClickHouse/pull/55026) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix simple state with nullable [#55030](https://github.com/ClickHouse/ClickHouse/pull/55030) ([Pedro Riera](https://github.com/priera)).
+* Prevent attaching parts from tables with different projections or indices [#55062](https://github.com/ClickHouse/ClickHouse/pull/55062) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY: 'Revert "Revert "Fixed wrong python test name pattern""'. [#54043](https://github.com/ClickHouse/ClickHouse/pull/54043) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* NO CL ENTRY: 'Revert "Fix: respect skip_unavailable_shards with parallel replicas"'. [#54189](https://github.com/ClickHouse/ClickHouse/pull/54189) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY: 'Revert "Add settings for real-time updates during query execution"'. [#54470](https://github.com/ClickHouse/ClickHouse/pull/54470) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Fix issues in accurateCastOrNull"'. [#54472](https://github.com/ClickHouse/ClickHouse/pull/54472) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Revert "Add settings for real-time updates during query execution""'. [#54476](https://github.com/ClickHouse/ClickHouse/pull/54476) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* NO CL ENTRY: 'Revert "add runOptimize call in bitmap write method"'. [#54528](https://github.com/ClickHouse/ClickHouse/pull/54528) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Optimize uniq to count"'. [#54566](https://github.com/ClickHouse/ClickHouse/pull/54566) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Add stateless test for clickhouse keeper-client --no-confirmation"'. [#54616](https://github.com/ClickHouse/ClickHouse/pull/54616) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* NO CL ENTRY: 'Revert "Remove flaky tests for the experimental `UNDROP` feature"'. [#54671](https://github.com/ClickHouse/ClickHouse/pull/54671) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY: 'Revert "Fix filtering parts with indexHint for non analyzer"'. [#54806](https://github.com/ClickHouse/ClickHouse/pull/54806) ([Azat Khuzhin](https://github.com/azat)).
+* NO CL ENTRY: 'Revert "refine error code of duplicated index in create query"'. [#54840](https://github.com/ClickHouse/ClickHouse/pull/54840) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* NO CL ENTRY: 'Revert "Avoid excessive calls to getifaddrs in isLocalAddress"'. [#54893](https://github.com/ClickHouse/ClickHouse/pull/54893) ([Igor Nikonov](https://github.com/devcrafter)).
+* NO CL ENTRY: 'Revert "Fix NATS high cpu usage"'. [#55005](https://github.com/ClickHouse/ClickHouse/pull/55005) ([Nikolay Degterinsky](https://github.com/evillique)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* libFuzzer: add CI fuzzers build, add tcp protocol fuzzer, fix other fuzzers. [#42599](https://github.com/ClickHouse/ClickHouse/pull/42599) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Add new exceptions to 4xx error [#50722](https://github.com/ClickHouse/ClickHouse/pull/50722) ([Boris Kuschel](https://github.com/bkuschel)).
+* Test libunwind changes. [#51436](https://github.com/ClickHouse/ClickHouse/pull/51436) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix data race in copyFromIStreamWithProgressCallback [#51449](https://github.com/ClickHouse/ClickHouse/pull/51449) ([Michael Kolupaev](https://github.com/al13n321)).
+* Abort on `std::logic_error` in CI [#51907](https://github.com/ClickHouse/ClickHouse/pull/51907) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Unify setting http keep-alive timeout, increase default to 30s [#53068](https://github.com/ClickHouse/ClickHouse/pull/53068) ([Nikita Taranov](https://github.com/nickitat)).
+* Add a regression test for broken Vertical merge after ADD+DROP COLUMN [#53214](https://github.com/ClickHouse/ClickHouse/pull/53214) ([Azat Khuzhin](https://github.com/azat)).
+* Revert "Revert "dateDiff: add support for plural units."" [#53803](https://github.com/ClickHouse/ClickHouse/pull/53803) ([Han Fei](https://github.com/hanfei1991)).
+* Fix some tests [#53892](https://github.com/ClickHouse/ClickHouse/pull/53892) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Refactoring of reading from `MergeTree` tables [#53931](https://github.com/ClickHouse/ClickHouse/pull/53931) ([Anton Popov](https://github.com/CurtizJ)).
+* Use pathlib.Path in S3Helper, rewrite build reports, improve small things [#54010](https://github.com/ClickHouse/ClickHouse/pull/54010) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Correct UniquesHashSet to be endianness-independent. [#54045](https://github.com/ClickHouse/ClickHouse/pull/54045) ([Austin Kothig](https://github.com/kothiga)).
+* Increase retries for test_merge_tree_azure_blob_storage [#54069](https://github.com/ClickHouse/ClickHouse/pull/54069) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix SipHash128 reference for big-endian platforms [#54095](https://github.com/ClickHouse/ClickHouse/pull/54095) ([ltrk2](https://github.com/ltrk2)).
+* Small usearch index improvements: metrics and configurable internal data type [#54103](https://github.com/ClickHouse/ClickHouse/pull/54103) ([Michael Kolupaev](https://github.com/al13n321)).
+* Small refactoring for read from object storage [#54134](https://github.com/ClickHouse/ClickHouse/pull/54134) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Minor changes [#54171](https://github.com/ClickHouse/ClickHouse/pull/54171) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix hostname and co result constness in new analyzer [#54174](https://github.com/ClickHouse/ClickHouse/pull/54174) ([vdimir](https://github.com/vdimir)).
+* Amend a confusing line of code in Loggers.cpp [#54183](https://github.com/ClickHouse/ClickHouse/pull/54183) ([Victor Krasnov](https://github.com/sirvickr)).
+* Fix partition id pruning for analyzer. [#54185](https://github.com/ClickHouse/ClickHouse/pull/54185) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Update version after release [#54186](https://github.com/ClickHouse/ClickHouse/pull/54186) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update version_date.tsv and changelogs after v23.8.1.2992-lts [#54188](https://github.com/ClickHouse/ClickHouse/pull/54188) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix pager in client/local interactive mode when not all data had been read [#54190](https://github.com/ClickHouse/ClickHouse/pull/54190) ([Azat Khuzhin](https://github.com/azat)).
+* Fix flaky test `01099_operators_date_and_timestamp` [#54195](https://github.com/ClickHouse/ClickHouse/pull/54195) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Save system tables from s3_disk in the report [#54198](https://github.com/ClickHouse/ClickHouse/pull/54198) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix timezones in the CI Logs database [#54210](https://github.com/ClickHouse/ClickHouse/pull/54210) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* 2R: Fix: respect skip_unavailable_shards with parallel replicas [#54213](https://github.com/ClickHouse/ClickHouse/pull/54213) ([Igor Nikonov](https://github.com/devcrafter)).
+* S3Queue is experimental [#54214](https://github.com/ClickHouse/ClickHouse/pull/54214) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Improve vars with refernce in Settings cpps [#54220](https://github.com/ClickHouse/ClickHouse/pull/54220) ([xuzifu666](https://github.com/xuzifu666)).
+* Add ProfileEvents::Timer class [#54221](https://github.com/ClickHouse/ClickHouse/pull/54221) ([Stig Bakken](https://github.com/stigsb)).
+* Test: extend cluster_all_replicas integration test with skip_unavailable_shards [#54223](https://github.com/ClickHouse/ClickHouse/pull/54223) ([Igor Nikonov](https://github.com/devcrafter)).
+* remove semicolon [#54236](https://github.com/ClickHouse/ClickHouse/pull/54236) ([YinZheng-Sun](https://github.com/YinZheng-Sun)).
+* Fix bad code in the `system.filesystem_cache`: catching exceptions [#54237](https://github.com/ClickHouse/ClickHouse/pull/54237) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Merge [#54236](https://github.com/ClickHouse/ClickHouse/issues/54236) [#54238](https://github.com/ClickHouse/ClickHouse/pull/54238) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Minor improvement, load from config [#54244](https://github.com/ClickHouse/ClickHouse/pull/54244) ([zhanglistar](https://github.com/zhanglistar)).
+* Follow-up to [#54198](https://github.com/ClickHouse/ClickHouse/issues/54198) [#54246](https://github.com/ClickHouse/ClickHouse/pull/54246) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Properly re-initialize ZooKeeper fault injection [#54251](https://github.com/ClickHouse/ClickHouse/pull/54251) ([Alexander Gololobov](https://github.com/davenger)).
+* Update ci-slack-bot.py [#54253](https://github.com/ClickHouse/ClickHouse/pull/54253) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix clickhouse-test --no-drop-if-fail on reference mismatch [#54256](https://github.com/ClickHouse/ClickHouse/pull/54256) ([Azat Khuzhin](https://github.com/azat)).
+* Improve slack-bot-ci lambda [#54258](https://github.com/ClickHouse/ClickHouse/pull/54258) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Update version_date.tsv and changelogs after v23.3.12.11-lts [#54259](https://github.com/ClickHouse/ClickHouse/pull/54259) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Minor change [#54261](https://github.com/ClickHouse/ClickHouse/pull/54261) ([flynn](https://github.com/ucasfl)).
+* Add a note of where the lambda is deployed [#54268](https://github.com/ClickHouse/ClickHouse/pull/54268) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Query cache: Log caching of entries [#54270](https://github.com/ClickHouse/ClickHouse/pull/54270) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update version_date.tsv and changelogs after v23.8.2.7-lts [#54273](https://github.com/ClickHouse/ClickHouse/pull/54273) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Fix test `02783_parsedatetimebesteffort_syslog` [#54279](https://github.com/ClickHouse/ClickHouse/pull/54279) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix `test_keeper_disks` [#54291](https://github.com/ClickHouse/ClickHouse/pull/54291) ([Antonio Andelic](https://github.com/antonio2368)).
+* Code improvement for reading from archives [#54293](https://github.com/ClickHouse/ClickHouse/pull/54293) ([Antonio Andelic](https://github.com/antonio2368)).
+* Rollback testing part from [#42599](https://github.com/ClickHouse/ClickHouse/issues/42599) [#54301](https://github.com/ClickHouse/ClickHouse/pull/54301) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* CI: libFuzzer integration [#54310](https://github.com/ClickHouse/ClickHouse/pull/54310) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Update version_date.tsv and changelogs after v23.3.13.6-lts [#54313](https://github.com/ClickHouse/ClickHouse/pull/54313) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Add logs for parallel replica over distributed [#54315](https://github.com/ClickHouse/ClickHouse/pull/54315) ([Igor Nikonov](https://github.com/devcrafter)).
+* Increase timeout for system.stack_trace in 01051_system_stack_trace [#54321](https://github.com/ClickHouse/ClickHouse/pull/54321) ([Azat Khuzhin](https://github.com/azat)).
+* Fix replace_partition test [#54322](https://github.com/ClickHouse/ClickHouse/pull/54322) ([Pedro Riera](https://github.com/priera)).
+* Fix segfault in system.zookeeper [#54326](https://github.com/ClickHouse/ClickHouse/pull/54326) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fixed flaky test `02841_parallel_replicas_summary` [#54331](https://github.com/ClickHouse/ClickHouse/pull/54331) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Consolidate GCD codec tests (Follow up to [#53149](https://github.com/ClickHouse/ClickHouse/issues/53149)) [#54332](https://github.com/ClickHouse/ClickHouse/pull/54332) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fixed wrong dereference problem in Context::setTemporaryStorageInCache [#54333](https://github.com/ClickHouse/ClickHouse/pull/54333) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Used assert_cast instead of dynamic_cast in ExternalDataSourceCache [#54336](https://github.com/ClickHouse/ClickHouse/pull/54336) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Fix bad punctuation in Keeper's logs [#54338](https://github.com/ClickHouse/ClickHouse/pull/54338) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Improved protection from dereferencing of nullptr [#54339](https://github.com/ClickHouse/ClickHouse/pull/54339) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Fix filesystem cache test [#54343](https://github.com/ClickHouse/ClickHouse/pull/54343) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
+* Parallel replicas: remove unused code [#54354](https://github.com/ClickHouse/ClickHouse/pull/54354) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix flaky test test_storage_azure_blob_storage/test.py::test_schema_iference_cache [#54367](https://github.com/ClickHouse/ClickHouse/pull/54367) ([Kruglov Pavel](https://github.com/Avogar)).
+* Enable hedged requests integration tests with tsan, use max_distributed_connections=1 to fix possible flakiness [#54371](https://github.com/ClickHouse/ClickHouse/pull/54371) ([Kruglov Pavel](https://github.com/Avogar)).
+* Use abiv2 when generating OpenSSL .s files for powerpc64le [#54375](https://github.com/ClickHouse/ClickHouse/pull/54375) ([Boris Kuschel](https://github.com/bkuschel)).
+* Disable prefer_localhost_replica in test for parallel replicas [#54377](https://github.com/ClickHouse/ClickHouse/pull/54377) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix incorrect formatting of CREATE query with PRIMARY KEY [#54403](https://github.com/ClickHouse/ClickHouse/pull/54403) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix failed assert in attach thread during startup retries [#54408](https://github.com/ClickHouse/ClickHouse/pull/54408) ([Antonio Andelic](https://github.com/antonio2368)).
+* Hashtable order fix on big endian platform [#54409](https://github.com/ClickHouse/ClickHouse/pull/54409) ([Suzy Wang](https://github.com/SuzyWangIBMer)).
+* Small fine-tune for using ColumnNullable pointer [#54435](https://github.com/ClickHouse/ClickHouse/pull/54435) ([Alex Cheng](https://github.com/Alex-Cheng)).
+* Update automated commit status comment [#54441](https://github.com/ClickHouse/ClickHouse/pull/54441) ([vdimir](https://github.com/vdimir)).
+* Remove useless line [#54466](https://github.com/ClickHouse/ClickHouse/pull/54466) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add a log message on replicated table drop [#54467](https://github.com/ClickHouse/ClickHouse/pull/54467) ([Dmitry Novik](https://github.com/novikd)).
+* Cleanup: unnecessary SelectQueryInfo usage around distributed [#54468](https://github.com/ClickHouse/ClickHouse/pull/54468) ([Igor Nikonov](https://github.com/devcrafter)).
+* Add `instance_type` column to CI Logs and the `checks` table [#54469](https://github.com/ClickHouse/ClickHouse/pull/54469) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Refactor IHints [#54481](https://github.com/ClickHouse/ClickHouse/pull/54481) ([flynn](https://github.com/ucasfl)).
+* Fix strange message [#54489](https://github.com/ClickHouse/ClickHouse/pull/54489) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Bump re2 to latest main [#54492](https://github.com/ClickHouse/ClickHouse/pull/54492) ([Robert Schulze](https://github.com/rschu1ze)).
+* S3 artifacts [#54504](https://github.com/ClickHouse/ClickHouse/pull/54504) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Flush logs for system.backup_log test. [#54507](https://github.com/ClickHouse/ClickHouse/pull/54507) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix use-after-free in `MergeTreePrefetchedReadPool` [#54512](https://github.com/ClickHouse/ClickHouse/pull/54512) ([Anton Popov](https://github.com/CurtizJ)).
+* Disable parallel replicas on shards with not enough nodes [#54519](https://github.com/ClickHouse/ClickHouse/pull/54519) ([Igor Nikonov](https://github.com/devcrafter)).
+* Parallel replicas: cleanup unused params [#54520](https://github.com/ClickHouse/ClickHouse/pull/54520) ([Igor Nikonov](https://github.com/devcrafter)).
+* FunctionHelpers remove areTypesEqual function [#54546](https://github.com/ClickHouse/ClickHouse/pull/54546) ([Maksim Kita](https://github.com/kitaisreal)).
+* Add stateless test for clickhouse keeper-client --no-confirmation [#54547](https://github.com/ClickHouse/ClickHouse/pull/54547) ([Azat Khuzhin](https://github.com/azat)).
+* Increase default timeout in tests for keeper-client [#54551](https://github.com/ClickHouse/ClickHouse/pull/54551) ([pufit](https://github.com/pufit)).
+* clang-format: Disable namespace indentation and omit {} in if/for/while [#54554](https://github.com/ClickHouse/ClickHouse/pull/54554) ([Robert Schulze](https://github.com/rschu1ze)).
+* ngramDistance* queries fix for big endian platform [#54555](https://github.com/ClickHouse/ClickHouse/pull/54555) ([Suzy Wang](https://github.com/SuzyWangIBMer)).
+* Fix AST fuzzer crash in MergeTreeIndex{FullText|Inverted} [#54563](https://github.com/ClickHouse/ClickHouse/pull/54563) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove output_format_markdown_escape_special_characters from settings changes history [#54585](https://github.com/ClickHouse/ClickHouse/pull/54585) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add basic logic to find releasable commits [#54604](https://github.com/ClickHouse/ClickHouse/pull/54604) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix reading of virtual columns in reverse order [#54610](https://github.com/ClickHouse/ClickHouse/pull/54610) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix possible CANNOT_READ_ALL_DATA during ZooKeeper client finalization and add some tests [#54632](https://github.com/ClickHouse/ClickHouse/pull/54632) ([Azat Khuzhin](https://github.com/azat)).
+* Fix a bug in addData and subData functions [#54636](https://github.com/ClickHouse/ClickHouse/pull/54636) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Follow-up to [#54550](https://github.com/ClickHouse/ClickHouse/issues/54550) [#54641](https://github.com/ClickHouse/ClickHouse/pull/54641) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Remove broken lockless variant of re2 [#54642](https://github.com/ClickHouse/ClickHouse/pull/54642) ([Robert Schulze](https://github.com/rschu1ze)).
+* Bump abseil [#54646](https://github.com/ClickHouse/ClickHouse/pull/54646) ([Robert Schulze](https://github.com/rschu1ze)).
+* limit the delay before next try in S3 [#54651](https://github.com/ClickHouse/ClickHouse/pull/54651) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix parser unit tests [#54670](https://github.com/ClickHouse/ClickHouse/pull/54670) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix: Log engine Mark file to read and write in little Endian for s390x [#54677](https://github.com/ClickHouse/ClickHouse/pull/54677) ([bhavnajindal](https://github.com/bhavnajindal)).
+* Update WebObjectStorage.cpp [#54695](https://github.com/ClickHouse/ClickHouse/pull/54695) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* add cancelation point to s3 retries [#54697](https://github.com/ClickHouse/ClickHouse/pull/54697) ([Sema Checherinda](https://github.com/CheSema)).
+* Revert default batch size for Keeper [#54745](https://github.com/ClickHouse/ClickHouse/pull/54745) ([Antonio Andelic](https://github.com/antonio2368)).
+* Enable `allow_experimental_undrop_table_query` [#54754](https://github.com/ClickHouse/ClickHouse/pull/54754) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix 02882_clickhouse_keeper_client_no_confirmation test [#54761](https://github.com/ClickHouse/ClickHouse/pull/54761) ([Azat Khuzhin](https://github.com/azat)).
+* Better exception message in checkDataPart [#54768](https://github.com/ClickHouse/ClickHouse/pull/54768) ([alesapin](https://github.com/alesapin)).
+* Don't use default move assignment in TimerDescriptor [#54769](https://github.com/ClickHouse/ClickHouse/pull/54769) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add retries to rests test_async_query_sending/test_async_connect [#54772](https://github.com/ClickHouse/ClickHouse/pull/54772) ([Kruglov Pavel](https://github.com/Avogar)).
+* update comment [#54780](https://github.com/ClickHouse/ClickHouse/pull/54780) ([flynn](https://github.com/ucasfl)).
+* Fix broken tests for clickhouse-diagnostics [#54790](https://github.com/ClickHouse/ClickHouse/pull/54790) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* refine error code of duplicated index in create query [#54791](https://github.com/ClickHouse/ClickHouse/pull/54791) ([Han Fei](https://github.com/hanfei1991)).
+* Do not set PR status label [#54799](https://github.com/ClickHouse/ClickHouse/pull/54799) ([vdimir](https://github.com/vdimir)).
+* Prevent parquet schema inference reading the first 1 MB of the file unnecessarily [#54808](https://github.com/ClickHouse/ClickHouse/pull/54808) ([Michael Kolupaev](https://github.com/al13n321)).
+* Prevent ParquetMetadata reading 40 MB from each file unnecessarily [#54809](https://github.com/ClickHouse/ClickHouse/pull/54809) ([Michael Kolupaev](https://github.com/al13n321)).
+* Use appropriate error code instead of LOGICAL_ERROR [#54810](https://github.com/ClickHouse/ClickHouse/pull/54810) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Adjusting `num_streams` by expected work in StorageS3 [#54815](https://github.com/ClickHouse/ClickHouse/pull/54815) ([pufit](https://github.com/pufit)).
+* Fix test_backup_restore_on_cluster/test.py::test_stop_other_host_during_backup flakiness [#54816](https://github.com/ClickHouse/ClickHouse/pull/54816) ([Azat Khuzhin](https://github.com/azat)).
+* Remove config files sizes check [#54824](https://github.com/ClickHouse/ClickHouse/pull/54824) ([Igor Nikonov](https://github.com/devcrafter)).
+* Set correct size for signal pipe buffer [#54836](https://github.com/ClickHouse/ClickHouse/pull/54836) ([Antonio Andelic](https://github.com/antonio2368)).
+* Refactor and split up vector search tests [#54839](https://github.com/ClickHouse/ClickHouse/pull/54839) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add some logging to StorageRabbitMQ [#54842](https://github.com/ClickHouse/ClickHouse/pull/54842) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Update CHANGELOG.md [#54843](https://github.com/ClickHouse/ClickHouse/pull/54843) ([Ilya Yatsishin](https://github.com/qoega)).
+* Refactor and simplify multi-directory globs [#54863](https://github.com/ClickHouse/ClickHouse/pull/54863) ([Andrey Zvonov](https://github.com/zvonand)).
+* KeeperTCPHandler.cpp: Fix clang-17 build [#54874](https://github.com/ClickHouse/ClickHouse/pull/54874) ([Robert Schulze](https://github.com/rschu1ze)).
+* Decrease timeout for fast tests with a commit [#54878](https://github.com/ClickHouse/ClickHouse/pull/54878) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* More stable `02703_keeper_map_concurrent_create_drop` [#54879](https://github.com/ClickHouse/ClickHouse/pull/54879) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix division by zero in StorageS3 [#54904](https://github.com/ClickHouse/ClickHouse/pull/54904) ([pufit](https://github.com/pufit)).
+* Set exception for promise in `CreatingSetsTransform` [#54920](https://github.com/ClickHouse/ClickHouse/pull/54920) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix an exception message in Pipe::addTransform [#54926](https://github.com/ClickHouse/ClickHouse/pull/54926) ([Alex Cheng](https://github.com/Alex-Cheng)).
+* Fix data race during BackupsWorker::backup_log initialization [#54928](https://github.com/ClickHouse/ClickHouse/pull/54928) ([Victor Krasnov](https://github.com/sirvickr)).
+* Provide support for BSON on BE [#54933](https://github.com/ClickHouse/ClickHouse/pull/54933) ([Austin Kothig](https://github.com/kothiga)).
+* Set a minimum limit of `num_streams` in StorageS3 [#54936](https://github.com/ClickHouse/ClickHouse/pull/54936) ([pufit](https://github.com/pufit)).
+* Ipv4 read big endian [#54938](https://github.com/ClickHouse/ClickHouse/pull/54938) ([Suzy Wang](https://github.com/SuzyWangIBMer)).
+* Fix data race in SYSTEM STOP LISTEN [#54939](https://github.com/ClickHouse/ClickHouse/pull/54939) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Add desperate instrumentation for debugging deadlock in MultiplexedConnections [#54940](https://github.com/ClickHouse/ClickHouse/pull/54940) ([Michael Kolupaev](https://github.com/al13n321)).
+* Respect max_block_size while generating rows for system.stack_trace (will fix flakiness of the test) [#54946](https://github.com/ClickHouse/ClickHouse/pull/54946) ([Azat Khuzhin](https://github.com/azat)).
+* Remove test `01051_system_stack_trace` [#54951](https://github.com/ClickHouse/ClickHouse/pull/54951) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add a test for compatibility [#54960](https://github.com/ClickHouse/ClickHouse/pull/54960) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove test `02151_hash_table_sizes_stats` [#54961](https://github.com/ClickHouse/ClickHouse/pull/54961) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove 02151_hash_table_sizes_stats_distributed (fixes broken CI) [#54969](https://github.com/ClickHouse/ClickHouse/pull/54969) ([Azat Khuzhin](https://github.com/azat)).
+* Use pregenerated gRPC protocol pb2 files to fix test flakyness. [#54976](https://github.com/ClickHouse/ClickHouse/pull/54976) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Delete a test [#54984](https://github.com/ClickHouse/ClickHouse/pull/54984) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add assertion [#54985](https://github.com/ClickHouse/ClickHouse/pull/54985) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix test parallel replicas over distributed [#54987](https://github.com/ClickHouse/ClickHouse/pull/54987) ([Igor Nikonov](https://github.com/devcrafter)).
+* Update README.md [#54990](https://github.com/ClickHouse/ClickHouse/pull/54990) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Re-enable clang-tidy checks disabled in the Clang 17 update [#54999](https://github.com/ClickHouse/ClickHouse/pull/54999) ([Robert Schulze](https://github.com/rschu1ze)).
+* Print more information about one logical error in MergeTreeDataWriter [#55001](https://github.com/ClickHouse/ClickHouse/pull/55001) ([Michael Kolupaev](https://github.com/al13n321)).
+* Add a test [#55003](https://github.com/ClickHouse/ClickHouse/pull/55003) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Lower log levels for `SSOCredentialsProvider` [#55012](https://github.com/ClickHouse/ClickHouse/pull/55012) ([Antonio Andelic](https://github.com/antonio2368)).
+* Set exception for promise in `CreatingSetsTransform` in more cases [#55013](https://github.com/ClickHouse/ClickHouse/pull/55013) ([Antonio Andelic](https://github.com/antonio2368)).
+* Setting compile_aggregate_expressions comment fix [#55020](https://github.com/ClickHouse/ClickHouse/pull/55020) ([Maksim Kita](https://github.com/kitaisreal)).
+* Revert "Added field "is_deterministic" to system.functions" [#55022](https://github.com/ClickHouse/ClickHouse/pull/55022) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Get rid of the most of `os.path` stuff [#55028](https://github.com/ClickHouse/ClickHouse/pull/55028) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix pre-build scripts for old branches [#55032](https://github.com/ClickHouse/ClickHouse/pull/55032) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Review fix for [#54935](https://github.com/ClickHouse/ClickHouse/issues/54935) [#55042](https://github.com/ClickHouse/ClickHouse/pull/55042) ([flynn](https://github.com/ucasfl)).
+* Update gtest_lru_file_cache.cpp [#55053](https://github.com/ClickHouse/ClickHouse/pull/55053) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix prebuild scripts one more time [#55059](https://github.com/ClickHouse/ClickHouse/pull/55059) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Use different names for variables inside build.sh [#55067](https://github.com/ClickHouse/ClickHouse/pull/55067) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Remove String Jaccard Index [#55080](https://github.com/ClickHouse/ClickHouse/pull/55080) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* I don't understand why backup log is not enabled by default [#55081](https://github.com/ClickHouse/ClickHouse/pull/55081) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix typo in packager when ccache is used [#55104](https://github.com/ClickHouse/ClickHouse/pull/55104) ([Ilya Yatsishin](https://github.com/qoega)).
+* Reduce flakiness of 01455_opentelemetry_distributed [#55111](https://github.com/ClickHouse/ClickHouse/pull/55111) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix build [#55113](https://github.com/ClickHouse/ClickHouse/pull/55113) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
diff --git a/docs/en/engines/table-engines/integrations/hive.md b/docs/en/engines/table-engines/integrations/hive.md
index 48867394418f..ea289b27ad72 100644
--- a/docs/en/engines/table-engines/integrations/hive.md
+++ b/docs/en/engines/table-engines/integrations/hive.md
@@ -60,7 +60,7 @@ Before using cache, add it to `config.xml`
- limit_size: Required. The maximum size(in bytes) of local cache files.
- bytes_read_before_flush: Control bytes before flush to local filesystem when downloading file from remote filesystem. The default value is 1MB.
-When ClickHouse is started up with local cache for remote filesystem enabled, users can still choose not to use cache with `settings use_local_cache_for_remote_fs = 0` in their query. `use_local_cache_for_remote_fs` is `false` in default.
+When ClickHouse is started up with local cache for remote filesystem enabled, users can still choose not to use cache with `settings use_local_cache_for_remote_storage = 0` in their query. `use_local_cache_for_remote_storage` is `1` by default.
### Query Hive Table with ORC Input Format
diff --git a/docs/en/engines/table-engines/integrations/s3queue.md b/docs/en/engines/table-engines/integrations/s3queue.md
index 50330962b86c..b8f8e51b3711 100644
--- a/docs/en/engines/table-engines/integrations/s3queue.md
+++ b/docs/en/engines/table-engines/integrations/s3queue.md
@@ -1,12 +1,19 @@
---
slug: /en/engines/table-engines/integrations/s3queue
-sidebar_position: 7
+sidebar_position: 181
sidebar_label: S3Queue
---
-# S3Queue Table Engine
+# [experimental] S3Queue Table Engine
This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ecosystem and allows streaming import. This engine is similar to the [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) engines, but provides S3-specific features.
+:::note
+This table engine is experimental. To use it, set `allow_experimental_s3queue` to 1 by using the `SET` command:
+```sql
+SET allow_experimental_s3queue=1
+```
+:::
+
## Create Table {#creating-a-table}
``` sql
diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md
index 23ab2699cc16..e37fd3b72884 100644
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@@ -37,8 +37,8 @@ The [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine does
``` sql
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
(
- name1 [type1] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr1] [TTL expr1] [CODEC(codec1)] [[NOT] NULL|PRIMARY KEY],
- name2 [type2] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr2] [TTL expr2] [CODEC(codec2)] [[NOT] NULL|PRIMARY KEY],
+ name1 [type1] [[NOT] NULL] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr1] [COMMENT ...] [CODEC(codec1)] [TTL expr1] [PRIMARY KEY],
+ name2 [type2] [[NOT] NULL] [DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL expr2] [COMMENT ...] [CODEC(codec2)] [TTL expr2] [PRIMARY KEY],
...
INDEX index_name1 expr1 TYPE type1(...) [GRANULARITY value1],
INDEX index_name2 expr2 TYPE type2(...) [GRANULARITY value2],
diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
index 7db2f3b465a2..6de818c130f0 100644
--- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -100,8 +100,41 @@ SELECT * FROM mySecondReplacingMT FINAL;
The row is deleted when `OPTIMIZE ... FINAL CLEANUP` or `OPTIMIZE ... FINAL` is used, or if the engine setting `clean_deleted_rows` has been set to `Always`.
No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
+
:::
+Example:
+```sql
+-- with ver and is_deleted
+CREATE OR REPLACE TABLE myThirdReplacingMT
+(
+ `key` Int64,
+ `someCol` String,
+ `eventTime` DateTime,
+ `is_deleted` UInt8
+)
+ENGINE = ReplacingMergeTree(eventTime, is_deleted)
+ORDER BY key;
+
+INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 0);
+INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 1);
+
+select * from myThirdReplacingMT final;
+
+0 rows in set. Elapsed: 0.003 sec.
+
+-- delete rows with is_deleted
+OPTIMIZE TABLE myThirdReplacingMT FINAL CLEANUP;
+
+INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 00:00:00', 0);
+
+select * from myThirdReplacingMT final;
+
+┌─key─┬─someCol─┬───────────eventTime─┬─is_deleted─┐
+│ 1 │ first │ 2020-01-01 00:00:00 │ 0 │
+└─────┴─────────┴─────────────────────┴────────────┘
+```
+
## Query clauses
When creating a `ReplacingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table.
diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index d059d158d549..58c6280da35c 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -1259,9 +1259,13 @@ SELECT * FROM json_each_row_nested
- [input_format_import_nested_json](/docs/en/operations/settings/settings-formats.md/#input_format_import_nested_json) - map nested JSON data to nested tables (it works for JSONEachRow format). Default value - `false`.
- [input_format_json_read_bools_as_numbers](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_bools_as_numbers) - allow to parse bools as numbers in JSON input formats. Default value - `true`.
-- [input_format_json_read_numbers_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `false`.
-- [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`.
+- [input_format_json_read_numbers_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `true`.
+- [input_format_json_read_arrays_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_arrays_as_strings) - allow to parse JSON arrays as strings in JSON input formats. Default value - `true`.
+- [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `true`.
- [input_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings-formats.md/#input_format_json_named_tuples_as_objects) - parse named tuple columns as JSON objects. Default value - `true`.
+- [input_format_json_try_infer_numbers_from_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_numbers_from_strings) - Try to infer numbers from string fields while schema inference. Default value - `false`.
+- [input_format_json_try_infer_named_tuples_from_objects](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_named_tuples_from_objects) - try to infer named tuple from JSON objects during schema inference. Default value - `true`.
+- [input_format_json_infer_incomplete_types_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_infer_incomplete_types_as_strings) - use type String for keys that contains only Nulls or empty objects/arrays during schema inference in JSON input formats. Default value - `true`.
- [input_format_json_defaults_for_missing_elements_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_defaults_for_missing_elements_in_named_tuple) - insert default values for missing elements in JSON object while parsing named tuple. Default value - `true`.
- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - Ignore unknown keys in json object for named tuples. Default value - `false`.
- [input_format_json_compact_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_json_compact_allow_variable_number_of_columns) - allow variable number of columns in JSONCompact/JSONCompactEachRow format, ignore extra columns and use default values on missing columns. Default value - `false`.
@@ -1920,6 +1924,14 @@ SELECT * FROM test.hits format Protobuf SETTINGS format_protobuf_use_autogenerat
In this case autogenerated Protobuf schema will be saved in file `path/to/schema/schema.capnp`.
+### Drop Protobuf cache
+
+To reload Protobuf schema loaded from [format_schema_path](../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-format_schema_path) use [SYSTEM DROP ... FORMAT CACHE](../sql-reference/statements/system.md/#system-drop-schema-format) statement.
+
+```sql
+SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf
+```
+
## ProtobufSingle {#protobufsingle}
Same as [Protobuf](#protobuf) but for storing/parsing single Protobuf message without length delimiters.
diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md
index 8789fe836cb7..0e2c0c00e4c1 100644
--- a/docs/en/interfaces/http.md
+++ b/docs/en/interfaces/http.md
@@ -58,7 +58,7 @@ Connection: Close
Content-Type: text/tab-separated-values; charset=UTF-8
X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal
X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f
-X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334,"peak_memory_usage":"0"}
+X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
1
```
@@ -288,9 +288,9 @@ Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you
You can receive information about the progress of a query in `X-ClickHouse-Progress` response headers. To do this, enable [send_progress_in_http_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Example of the header sequence:
``` text
-X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","elapsed_ns":"662334","peak_memory_usage":"4371480"}
-X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","elapsed_ns":"992334","peak_memory_usage":"13621616"}
-X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","elapsed_ns":"1232334","peak_memory_usage":"23155600"}
+X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","elapsed_ns":"662334"}
+X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","elapsed_ns":"992334"}
+X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","elapsed_ns":"1232334"}
```
Possible header fields:
@@ -439,7 +439,7 @@ $ curl -v 'http://localhost:8123/predefined_query'
< X-ClickHouse-Format: Template
< X-ClickHouse-Timezone: Asia/Shanghai
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
# HELP "Query" "Number of executing queries"
# TYPE "Query" counter
@@ -604,7 +604,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
* Connection #0 to host localhost left intact
Say Hi!%
@@ -644,7 +644,7 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
< Content-Type: text/plain; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
* Connection #0 to host localhost left intact
%
@@ -696,7 +696,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
Absolute Path File
* Connection #0 to host localhost left intact
@@ -715,8 +715,88 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
Relative Path File
* Connection #0 to host localhost left intact
```
+
+## Valid JSON/XML response on exception during HTTP streaming {valid-output-on-exception-http-streaming}
+
+While query execution over HTTP an exception can happen when part of the data has already been sent. Usually an exception is sent to the client in plain text
+even if some specific data format was used to output data and the output may become invalid in terms of specified data format.
+To prevent it, you can use setting `http_write_exception_in_output_format` (enabled by default) that will tell ClickHouse to write an exception in specified format (currently supported for XML and JSON* formats).
+
+Examples:
+
+```bash
+$ curl 'http://localhost:8123/?query=SELECT+number,+throwIf(number>3)+from+system.numbers+format+JSON+settings+max_block_size=1&http_write_exception_in_output_format=1'
+{
+ "meta":
+ [
+ {
+ "name": "number",
+ "type": "UInt64"
+ },
+ {
+ "name": "throwIf(greater(number, 2))",
+ "type": "UInt8"
+ }
+ ],
+
+ "data":
+ [
+ {
+ "number": "0",
+ "throwIf(greater(number, 2))": 0
+ },
+ {
+ "number": "1",
+ "throwIf(greater(number, 2))": 0
+ },
+ {
+ "number": "2",
+ "throwIf(greater(number, 2))": 0
+ }
+ ],
+
+ "rows": 3,
+
+ "exception": "Code: 395. DB::Exception: Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 2) :: 2) -> throwIf(greater(number, 2)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) (version 23.8.1.1)"
+}
+```
+
+```bash
+$ curl 'http://localhost:8123/?query=SELECT+number,+throwIf(number>2)+from+system.numbers+format+XML+settings+max_block_size=1&http_write_exception_in_output_format=1'
+
+
+
+
+
+ number
+ UInt64
+
+
+ throwIf(greater(number, 2))
+ UInt8
+
+
+
+
+
+ 0
+ 0
+
+
+ 1
+ 0
+
+
+ 2
+ 0
+
+
+ 3
+ Code: 395. DB::Exception: Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 2) :: 2) -> throwIf(greater(number, 2)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) (version 23.8.1.1)
+
+```
diff --git a/docs/en/interfaces/mysql.md b/docs/en/interfaces/mysql.md
index 32c612dfa5f7..ce5ab24ecb08 100644
--- a/docs/en/interfaces/mysql.md
+++ b/docs/en/interfaces/mysql.md
@@ -10,10 +10,6 @@ ClickHouse supports the MySQL wire protocol. This allow tools that are MySQL-com
## Enabling the MySQL Interface On ClickHouse Cloud
-:::note
-The MySQL interface for ClickHouse Cloud is currently in private preview. Please contact support@clickhouse.com to enable this feature for your ClickHouse Cloud service.
-:::
-
1. After creating your ClickHouse Cloud Service, on the credentials screen, select the MySQL tab
![Credentials screen - Prompt](./images/mysql1.png)
diff --git a/docs/en/interfaces/schema-inference.md b/docs/en/interfaces/schema-inference.md
index 3022d5e05e9c..0aadb09730a6 100644
--- a/docs/en/interfaces/schema-inference.md
+++ b/docs/en/interfaces/schema-inference.md
@@ -389,9 +389,25 @@ DESC format(JSONEachRow, '{"arr" : [null, 42, null]}')
└──────┴────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
-Tuples:
+Named tuples:
+
+When setting `input_format_json_try_infer_named_tuples_from_objects` is enabled, during schema inference ClickHouse will try to infer named Tuple from JSON objects.
+The resulting named Tuple will contain all elements from all corresponding JSON objects from sample data.
-In JSON formats we treat Arrays with elements of different types as Tuples.
+```sql
+SET input_format_json_try_infer_named_tuples_from_objects = 1;
+DESC format(JSONEachRow, '{"obj" : {"a" : 42, "b" : "Hello"}}, {"obj" : {"a" : 43, "c" : [1, 2, 3]}}, {"obj" : {"d" : {"e" : 42}}}')
+```
+
+```response
+┌─name─┬─type───────────────────────────────────────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ obj │ Tuple(a Nullable(Int64), b Nullable(String), c Array(Nullable(Int64)), d Tuple(e Nullable(Int64))) │ │ │ │ │ │
+└──────┴────────────────────────────────────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+```
+
+Unnamed Tuples:
+
+In JSON formats we treat Arrays with elements of different types as Unnamed Tuples.
```sql
DESC format(JSONEachRow, '{"tuple" : [1, "Hello, World!", [1, 2, 3]]}')
```
@@ -418,7 +434,10 @@ DESC format(JSONEachRow, $$
Maps:
In JSON we can read objects with values of the same type as Map type.
+Note: it will work only when settings `input_format_json_read_objects_as_strings` and `input_format_json_try_infer_named_tuples_from_objects` are disabled.
+
```sql
+SET input_format_json_read_objects_as_strings = 0, input_format_json_try_infer_named_tuples_from_objects = 0;
DESC format(JSONEachRow, '{"map" : {"key1" : 42, "key2" : 24, "key3" : 4}}')
```
```response
@@ -448,14 +467,22 @@ Nested complex types:
DESC format(JSONEachRow, '{"value" : [[[42, 24], []], {"key1" : 42, "key2" : 24}]}')
```
```response
-┌─name──┬─type───────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
-│ value │ Tuple(Array(Array(Nullable(Int64))), Map(String, Nullable(Int64))) │ │ │ │ │ │
-└───────┴────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+┌─name──┬─type─────────────────────────────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ value │ Tuple(Array(Array(Nullable(String))), Tuple(key1 Nullable(Int64), key2 Nullable(Int64))) │ │ │ │ │ │
+└───────┴──────────────────────────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
-If ClickHouse cannot determine the type, because the data contains only nulls, an exception will be thrown:
+If ClickHouse cannot determine the type for some key, because the data contains only nulls/empty objects/empty arrays, type `String` will be used if setting `input_format_json_infer_incomplete_types_as_strings` is enabled or an exception will be thrown otherwise:
+```sql
+DESC format(JSONEachRow, '{"arr" : [null, null]}') SETTINGS input_format_json_infer_incomplete_types_as_strings = 1;
+```
+```response
+┌─name─┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ arr │ Array(Nullable(String)) │ │ │ │ │ │
+└──────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+```
```sql
-DESC format(JSONEachRow, '{"arr" : [null, null]}')
+DESC format(JSONEachRow, '{"arr" : [null, null]}') SETTINGS input_format_json_infer_incomplete_types_as_strings = 0;
```
```response
Code: 652. DB::Exception: Received from localhost:9000. DB::Exception:
@@ -466,31 +493,11 @@ most likely this column contains only Nulls or empty Arrays/Maps.
#### JSON settings {#json-settings}
-##### input_format_json_read_objects_as_strings
-
-Enabling this setting allows reading nested JSON objects as strings.
-This setting can be used to read nested JSON objects without using JSON object type.
-
-This setting is enabled by default.
-
-```sql
-SET input_format_json_read_objects_as_strings = 1;
-DESC format(JSONEachRow, $$
- {"obj" : {"key1" : 42, "key2" : [1,2,3,4]}}
- {"obj" : {"key3" : {"nested_key" : 1}}}
- $$)
-```
-```response
-┌─name─┬─type─────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
-│ obj │ Nullable(String) │ │ │ │ │ │
-└──────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
-```
-
##### input_format_json_try_infer_numbers_from_strings
Enabling this setting allows inferring numbers from string values.
-This setting is enabled by default.
+This setting is disabled by default.
**Example:**
@@ -507,11 +514,69 @@ DESC format(JSONEachRow, $$
└───────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
+##### input_format_json_try_infer_named_tuples_from_objects
+
+Enabling this setting allows inferring named Tuples from JSON objects. The resulting named Tuple will contain all elements from all corresponding JSON objects from sample data.
+It can be useful when JSON data is not sparse so the sample of data will contain all possible object keys.
+
+This setting is enabled by default.
+
+**Example**
+
+```sql
+SET input_format_json_try_infer_named_tuples_from_objects = 1;
+DESC format(JSONEachRow, '{"obj" : {"a" : 42, "b" : "Hello"}}, {"obj" : {"a" : 43, "c" : [1, 2, 3]}}, {"obj" : {"d" : {"e" : 42}}}')
+```
+
+Result:
+
+```
+┌─name─┬─type───────────────────────────────────────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ obj │ Tuple(a Nullable(Int64), b Nullable(String), c Array(Nullable(Int64)), d Tuple(e Nullable(Int64))) │ │ │ │ │ │
+└──────┴────────────────────────────────────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+```
+
+```sql
+SET input_format_json_try_infer_named_tuples_from_objects = 1;
+DESC format(JSONEachRow, '{"array" : [{"a" : 42, "b" : "Hello"}, {}, {"c" : [1,2,3]}, {"d" : "2020-01-01"}]}')
+```
+
+Result:
+
+```
+┌─name──┬─type────────────────────────────────────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ array │ Array(Tuple(a Nullable(Int64), b Nullable(String), c Array(Nullable(Int64)), d Nullable(Date))) │ │ │ │ │ │
+└───────┴─────────────────────────────────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+```
+
+##### input_format_json_read_objects_as_strings
+
+Enabling this setting allows reading nested JSON objects as strings.
+This setting can be used to read nested JSON objects without using JSON object type.
+
+This setting is enabled by default.
+
+Note: enabling this setting will take effect only if setting `input_format_json_try_infer_named_tuples_from_objects` is disabled.
+
+```sql
+SET input_format_json_read_objects_as_strings = 1, input_format_json_try_infer_named_tuples_from_objects = 0;
+DESC format(JSONEachRow, $$
+ {"obj" : {"key1" : 42, "key2" : [1,2,3,4]}}
+ {"obj" : {"key3" : {"nested_key" : 1}}}
+ $$)
+```
+```response
+┌─name─┬─type─────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ obj │ Nullable(String) │ │ │ │ │ │
+└──────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+```
+
+
##### input_format_json_read_numbers_as_strings
Enabling this setting allows reading numeric values as strings.
-This setting is disabled by default.
+This setting is enabled by default.
**Example**
@@ -549,6 +614,49 @@ DESC format(JSONEachRow, $$
└───────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
+##### input_format_json_read_arrays_as_strings
+
+Enabling this setting allows reading JSON array values as strings.
+
+This setting is enabled by default.
+
+**Example**
+
+```sql
+SET input_format_json_read_arrays_as_strings = 1;
+SELECT arr, toTypeName(arr), JSONExtractArrayRaw(arr)[3] from format(JSONEachRow, 'arr String', '{"arr" : [1, "Hello", [1,2,3]]}');
+```
+```response
+┌─arr───────────────────┬─toTypeName(arr)─┬─arrayElement(JSONExtractArrayRaw(arr), 3)─┐
+│ [1, "Hello", [1,2,3]] │ String │ [1,2,3] │
+└───────────────────────┴─────────────────┴───────────────────────────────────────────┘
+```
+
+##### input_format_json_infer_incomplete_types_as_strings
+
+Enabling this setting allows to use String type for JSON keys that contain only `Null`/`{}`/`[]` in data sample during schema inference.
+In JSON formats any value can be read as String if all corresponding settings are enabled (they are all enabled by default), and we can avoid errors like `Cannot determine type for column 'column_name' by first 25000 rows of data, most likely this column contains only Nulls or empty Arrays/Maps` during schema inference
+by using String type for keys with unknown types.
+
+Example:
+
+```sql
+SET input_format_json_infer_incomplete_types_as_strings = 1, input_format_json_try_infer_named_tuples_from_objects = 1;
+DESCRIBE format(JSONEachRow, '{"obj" : {"a" : [1,2,3], "b" : "hello", "c" : null, "d" : {}, "e" : []}}');
+SELECT * FROM format(JSONEachRow, '{"obj" : {"a" : [1,2,3], "b" : "hello", "c" : null, "d" : {}, "e" : []}}');
+```
+
+Result:
+```
+┌─name─┬─type───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ obj │ Tuple(a Array(Nullable(Int64)), b Nullable(String), c Nullable(String), d Nullable(String), e Array(Nullable(String))) │ │ │ │ │ │
+└──────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+
+┌─obj────────────────────────────┐
+│ ([1,2,3],'hello',NULL,'{}',[]) │
+└────────────────────────────────┘
+```
+
### CSV {#csv}
In CSV format ClickHouse extracts column values from the row according to delimiters. ClickHouse expects all types except numbers and strings to be enclosed in double quotes. If the value is in double quotes, ClickHouse tries to parse
diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md
index e229198bdfda..7b5c4f27a2a9 100644
--- a/docs/en/interfaces/third-party/client-libraries.md
+++ b/docs/en/interfaces/third-party/client-libraries.md
@@ -44,6 +44,8 @@ ClickHouse Inc does **not** maintain the libraries listed below and hasn’t don
- [nestjs-clickhouse](https://github.com/depyronick/nestjs-clickhouse)
- [clickhouse-client](https://github.com/depyronick/clickhouse-client)
- [node-clickhouse-orm](https://github.com/zimv/node-clickhouse-orm)
+ - [clickhouse-ts](https://github.com/bytadaniel/clickhouse-ts)
+ - [clickcache](https://github.com/bytadaniel/clickcache)
### Perl
- [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
- [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md
index 687bd66056b2..6068b185ede2 100644
--- a/docs/en/operations/backup.md
+++ b/docs/en/operations/backup.md
@@ -406,7 +406,7 @@ RESTORE TABLE data AS data_restored FROM Disk('s3_plain', 'cloud_backup');
:::note
But keep in mind that:
- This disk should not be used for `MergeTree` itself, only for `BACKUP`/`RESTORE`
-- It has excessive API calls
+- If your tables are backed by S3 storage, it doesn't use `CopyObject` calls to copy parts to the destination bucket, instead, it downloads and uploads them, which is very inefficient. Prefer to use `BACKUP ... TO S3()` syntax for this use-case.
:::
## Alternatives
diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md
index 6e21b0b3658b..665ae6cdfdca 100644
--- a/docs/en/operations/query-cache.md
+++ b/docs/en/operations/query-cache.md
@@ -61,17 +61,17 @@ FROM table
SETTINGS use_query_cache = true, enable_writes_to_query_cache = false;
```
-For maximum control, it is generally recommended to provide settings "use_query_cache", "enable_writes_to_query_cache" and
-"enable_reads_from_query_cache" only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET
+For maximum control, it is generally recommended to provide settings `use_query_cache`, `enable_writes_to_query_cache` and
+`enable_reads_from_query_cache` only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET
use_query_cache = true`) but one should keep in mind that all `SELECT` queries including monitoring or debugging queries to system tables
may return cached results then.
The query cache can be cleared using statement `SYSTEM DROP QUERY CACHE`. The content of the query cache is displayed in system table
-`system.query_cache`. The number of query cache hits and misses since database start are shown as events "QueryCacheHits" and
-"QueryCacheMisses" in system table [system.events](system-tables/events.md). Both counters are only updated for `SELECT` queries which run
-with setting `use_query_cache = true`, other queries do not affect "QueryCacheMisses". Field `query_log_usage` in system table
-[system.query_log](system-tables/query_log.md) shows for each executed query whether the query result was written into or read from the
-query cache. Asynchronous metrics "QueryCacheEntries" and "QueryCacheBytes" in system table
+[system.query_cache](system-tables/query_cache.md). The number of query cache hits and misses since database start are shown as events
+"QueryCacheHits" and "QueryCacheMisses" in system table [system.events](system-tables/events.md). Both counters are only updated for
+`SELECT` queries which run with setting `use_query_cache = true`, other queries do not affect "QueryCacheMisses". Field `query_cache_usage`
+in system table [system.query_log](system-tables/query_log.md) shows for each executed query whether the query result was written into or
+read from the query cache. Asynchronous metrics "QueryCacheEntries" and "QueryCacheBytes" in system table
[system.asynchronous_metrics](system-tables/asynchronous_metrics.md) show how many entries / bytes the query cache currently contains.
The query cache exists once per ClickHouse server process. However, cache results are by default not shared between users. This can be
@@ -86,9 +86,18 @@ If the query was aborted due to an exception or user cancellation, no entry is w
The size of the query cache in bytes, the maximum number of cache entries and the maximum size of individual cache entries (in bytes and in
records) can be configured using different [server configuration options](server-configuration-parameters/settings.md#server_configuration_parameters_query-cache).
+```xml
+
+ 1073741824
+ 1024
+ 1048576
+ 30000000
+
+```
+
It is also possible to limit the cache usage of individual users using [settings profiles](settings/settings-profiles.md) and [settings
constraints](settings/constraints-on-settings.md). More specifically, you can restrict the maximum amount of memory (in bytes) a user may
-allocate in the query cache and the the maximum number of stored query results. For that, first provide configurations
+allocate in the query cache and the maximum number of stored query results. For that, first provide configurations
[query_cache_max_size_in_bytes](settings/settings.md#query-cache-max-size-in-bytes) and
[query_cache_max_entries](settings/settings.md#query-cache-size-max-entries) in a user profile in `users.xml`, then make both settings
readonly:
@@ -158,6 +167,7 @@ Also, results of queries with non-deterministic functions are not cached by defa
- functions which depend on the environment: [`currentUser()`](../sql-reference/functions/other-functions.md#currentUser),
[`queryID()`](../sql-reference/functions/other-functions.md#queryID),
[`getMacro()`](../sql-reference/functions/other-functions.md#getMacro) etc.
+
To force caching of results of queries with non-deterministic functions regardless, use setting
[query_cache_store_results_of_queries_with_nondeterministic_functions](settings/settings.md#query-cache-store-results-of-queries-with-nondeterministic-functions).
diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 5586dbafd88d..bd8e1da2f1ed 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -2403,7 +2403,8 @@ This section contains the following parameters:
- zookeeper_load_balancing - Specifies the algorithm of ZooKeeper node selection.
* random - randomly selects one of ZooKeeper nodes.
* in_order - selects the first ZooKeeper node, if it's not available then the second, and so on.
- * nearest_hostname - selects a ZooKeeper node with a hostname that is most similar to the server’s hostname.
+ * nearest_hostname - selects a ZooKeeper node with a hostname that is most similar to the server’s hostname, hostname is compared with name prefix.
+ * hostname_levenshtein_distance - just like nearest_hostname, but it compares hostname in a levenshtein distance manner.
* first_or_random - selects the first ZooKeeper node, if it's not available then randomly selects one of remaining ZooKeeper nodes.
* round_robin - selects the first ZooKeeper node, if reconnection happens selects the next.
@@ -2425,7 +2426,7 @@ This section contains the following parameters:
/path/to/zookeeper/node
user:password
-
+
random
```
diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md
index 483dcf2e61cf..a0428bd09777 100644
--- a/docs/en/operations/settings/merge-tree-settings.md
+++ b/docs/en/operations/settings/merge-tree-settings.md
@@ -502,7 +502,7 @@ Possible values:
Default value: 480.
After merging several parts into a new part, ClickHouse marks the original parts as inactive and deletes them only after `old_parts_lifetime` seconds.
-Inactive parts are removed if they are not used by current queries, i.e. if the `refcount` of the part is zero.
+Inactive parts are removed if they are not used by current queries, i.e. if the `refcount` of the part is 1.
`fsync` is not called for new parts, so for some time new parts exist only in the server's RAM (OS cache). If the server is rebooted spontaneously, new parts can be lost or damaged.
To protect data inactive parts are not deleted immediately.
@@ -555,7 +555,7 @@ Merge reads rows from parts in blocks of `merge_max_block_size` rows, then merge
## number_of_free_entries_in_pool_to_lower_max_size_of_merge {#number-of-free-entries-in-pool-to-lower-max-size-of-merge}
-When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue).
+When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue).
This is to allow small merges to process - not filling the pool with long running merges.
Possible values:
@@ -566,7 +566,7 @@ Default value: 8
## number_of_free_entries_in_pool_to_execute_mutation {#number-of-free-entries-in-pool-to-execute-mutation}
-When there is less than specified number of free entries in pool, do not execute part mutations.
+When there is less than specified number of free entries in pool, do not execute part mutations.
This is to leave free threads for regular merges and avoid "Too many parts".
Possible values:
@@ -746,14 +746,14 @@ Default value: `0` (limit never applied).
Minimal ratio of the number of _default_ values to the number of _all_ values in a column. Setting this value causes the column to be stored using sparse serializations.
-If a column is sparse (contains mostly zeros), ClickHouse can encode it in a sparse format and automatically optimize calculations - the data does not require full decompression during queries. To enable this sparse serialization, define the `ratio_of_defaults_for_sparse_serialization` setting to be less than 1.0. If the value is greater than or equal to 1.0 (the default), then the columns will be always written using the normal full serialization.
+If a column is sparse (contains mostly zeros), ClickHouse can encode it in a sparse format and automatically optimize calculations - the data does not require full decompression during queries. To enable this sparse serialization, define the `ratio_of_defaults_for_sparse_serialization` setting to be less than 1.0. If the value is greater than or equal to 1.0, then the columns will be always written using the normal full serialization.
Possible values:
- Float between 0 and 1 to enable sparse serialization
- 1.0 (or greater) if you do not want to use sparse serialization
-Default value: `1.0` (sparse serialization is disabled)
+Default value: `0.9375`
**Example**
@@ -845,6 +845,13 @@ You can see which parts of `s` were stored using the sparse serialization:
└────────┴────────────────────┘
```
+## replace_long_file_name_to_hash {#replace_long_file_name_to_hash}
+If the file name for column is too long (more than `max_file_name_length` bytes) replace it to SipHash128. Default value: `false`.
+
+## max_file_name_length {#max_file_name_length}
+
+The maximal length of the file name to keep it as is without hashing. Takes effect only if setting `replace_long_file_name_to_hash` is enabled. The value of this setting does not include the length of file extension. So, it is recommended to set it below the maximum filename length (usually 255 bytes) with some gap to avoid filesystem errors. Default value: 127.
+
## clean_deleted_rows
Enable/disable automatic deletion of rows flagged as `is_deleted` when perform `OPTIMIZE ... FINAL` on a table using the ReplacingMergeTree engine. When disabled, the `CLEANUP` keyword has to be added to the `OPTIMIZE ... FINAL` to have the same behaviour.
diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md
index 507f4dd23b91..bb59402079e5 100644
--- a/docs/en/operations/settings/settings-formats.md
+++ b/docs/en/operations/settings/settings-formats.md
@@ -381,6 +381,13 @@ Enabled by default.
Allow parsing numbers as strings in JSON input formats.
+Enabled by default.
+
+## input_format_json_try_infer_numbers_from_strings {#input_format_json_try_infer_numbers_from_strings}
+
+If enabled, during schema inference ClickHouse will try to infer numbers from string fields.
+It can be useful if JSON data contains quoted UInt64 numbers.
+
Disabled by default.
## input_format_json_read_objects_as_strings {#input_format_json_read_objects_as_strings}
@@ -404,7 +411,76 @@ Result:
└────┴──────────────────────────┴────────────┘
```
-Disabled by default.
+Enabled by default.
+
+## input_format_json_try_infer_named_tuples_from_objects {#input_format_json_try_infer_named_tuples_from_objects}
+
+If enabled, during schema inference ClickHouse will try to infer named Tuple from JSON objects.
+The resulting named Tuple will contain all elements from all corresponding JSON objects from sample data.
+
+Example:
+
+```sql
+SET input_format_json_try_infer_named_tuples_from_objects = 1;
+DESC format(JSONEachRow, '{"obj" : {"a" : 42, "b" : "Hello"}}, {"obj" : {"a" : 43, "c" : [1, 2, 3]}}, {"obj" : {"d" : {"e" : 42}}}')
+```
+
+Result:
+
+```
+┌─name─┬─type───────────────────────────────────────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ obj │ Tuple(a Nullable(Int64), b Nullable(String), c Array(Nullable(Int64)), d Tuple(e Nullable(Int64))) │ │ │ │ │ │
+└──────┴────────────────────────────────────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+```
+
+Enabled by default.
+
+## input_format_json_read_arrays_as_strings {#input_format_json_read_arrays_as_strings}
+
+Allow parsing JSON arrays as strings in JSON input formats.
+
+Example:
+
+```sql
+SET input_format_json_read_arrays_as_strings = 1;
+SELECT arr, toTypeName(arr), JSONExtractArrayRaw(arr)[3] from format(JSONEachRow, 'arr String', '{"arr" : [1, "Hello", [1,2,3]]}');
+```
+
+Result:
+```
+┌─arr───────────────────┬─toTypeName(arr)─┬─arrayElement(JSONExtractArrayRaw(arr), 3)─┐
+│ [1, "Hello", [1,2,3]] │ String │ [1,2,3] │
+└───────────────────────┴─────────────────┴───────────────────────────────────────────┘
+```
+
+Enabled by default.
+
+## input_format_json_infer_incomplete_types_as_strings {#input_format_json_infer_incomplete_types_as_strings}
+
+Allow to use String type for JSON keys that contain only `Null`/`{}`/`[]` in data sample during schema inference.
+In JSON formats any value can be read as String, and we can avoid errors like `Cannot determine type for column 'column_name' by first 25000 rows of data, most likely this column contains only Nulls or empty Arrays/Maps` during schema inference
+by using String type for keys with unknown types.
+
+Example:
+
+```sql
+SET input_format_json_infer_incomplete_types_as_strings = 1, input_format_json_try_infer_named_tuples_from_objects = 1;
+DESCRIBE format(JSONEachRow, '{"obj" : {"a" : [1,2,3], "b" : "hello", "c" : null, "d" : {}, "e" : []}}');
+SELECT * FROM format(JSONEachRow, '{"obj" : {"a" : [1,2,3], "b" : "hello", "c" : null, "d" : {}, "e" : []}}');
+```
+
+Result:
+```
+┌─name─┬─type───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
+│ obj │ Tuple(a Array(Nullable(Int64)), b Nullable(String), c Nullable(String), d Nullable(String), e Array(Nullable(String))) │ │ │ │ │ │
+└──────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
+
+┌─obj────────────────────────────┐
+│ ([1,2,3],'hello',NULL,'{}',[]) │
+└────────────────────────────────┘
+```
+
+Enabled by default.
## input_format_json_validate_types_from_metadata {#input_format_json_validate_types_from_metadata}
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index bf705c6e2639..fc04906451af 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1413,6 +1413,7 @@ ClickHouse supports the following algorithms of choosing replicas:
- [Random](#load_balancing-random) (by default)
- [Nearest hostname](#load_balancing-nearest_hostname)
+- [Hostname levenshtein distance](#load_balancing-hostname_levenshtein_distance)
- [In order](#load_balancing-in_order)
- [First or random](#load_balancing-first_or_random)
- [Round robin](#load_balancing-round_robin)
@@ -1444,6 +1445,25 @@ This method might seem primitive, but it does not require external data about ne
Thus, if there are equivalent replicas, the closest one by name is preferred.
We can also assume that when sending a query to the same server, in the absence of failures, a distributed query will also go to the same servers. So even if different data is placed on the replicas, the query will return mostly the same results.
+### Hostname levenshtein distance {#load_balancing-hostname_levenshtein_distance}
+
+``` sql
+load_balancing = hostname_levenshtein_distance
+```
+
+Just like `nearest_hostname`, but it compares hostname in a [levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) manner. For example:
+
+``` text
+example-clickhouse-0-0 ample-clickhouse-0-0
+1
+
+example-clickhouse-0-0 example-clickhouse-1-10
+2
+
+example-clickhouse-0-0 example-clickhouse-12-0
+3
+```
+
### In Order {#load_balancing-in_order}
``` sql
@@ -3279,6 +3299,17 @@ Possible values:
Default value: `0`.
+## use_mysql_types_in_show_columns {#use_mysql_types_in_show_columns}
+
+Show the names of MySQL data types corresponding to ClickHouse data types in [SHOW COLUMNS](../../sql-reference/statements/show.md#show_columns).
+
+Possible values:
+
+- 0 - Show names of native ClickHouse data types.
+- 1 - Show names of MySQL data types corresponding to ClickHouse data types.
+
+Default value: `0`.
+
## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold}
Enables special logic to perform merges on replicas.
@@ -4659,6 +4690,10 @@ SELECT toFloat64('1.7091'), toFloat64('1.5008753E7') SETTINGS precise_float_pars
Interval (in milliseconds) for sending updates with partial data about the result table to the client (in interactive mode) during query execution. Setting to 0 disables partial results. Only supported for single-threaded GROUP BY without key, ORDER BY, LIMIT and OFFSET.
+:::note
+It's an experimental feature. Enable `allow_experimental_partial_result` setting first to use it.
+:::
+
## max_rows_in_partial_result
Maximum rows to show in the partial result after every real-time update while the query runs (use partial result limit + OFFSET as a value in case of OFFSET in the query).
@@ -4678,3 +4713,36 @@ The default value is `false`.
``` xml
true
```
+
+## print_pretty_type_names {#print_pretty_type_names}
+
+Allows to print deep-nested type names in a pretty way with indents in `DESCRIBE` query and in `toTypeName()` function.
+
+Example:
+
+```sql
+CREATE TABLE test (a Tuple(b String, c Tuple(d Nullable(UInt64), e Array(UInt32), f Array(Tuple(g String, h Map(String, Array(Tuple(i String, j UInt64))))), k Date), l Nullable(String))) ENGINE=Memory;
+DESCRIBE TABLE test FORMAT TSVRaw SETTINGS print_pretty_type_names=1;
+```
+
+```
+a Tuple(
+ b String,
+ c Tuple(
+ d Nullable(UInt64),
+ e Array(UInt32),
+ f Array(Tuple(
+ g String,
+ h Map(
+ String,
+ Array(Tuple(
+ i String,
+ j UInt64
+ ))
+ )
+ )),
+ k Date
+ ),
+ l Nullable(String)
+)
+```
diff --git a/docs/en/operations/system-tables/functions.md b/docs/en/operations/system-tables/functions.md
index 60bfa08975b7..d52bf24f2891 100644
--- a/docs/en/operations/system-tables/functions.md
+++ b/docs/en/operations/system-tables/functions.md
@@ -7,28 +7,34 @@ Contains information about normal and aggregate functions.
Columns:
-- `name`(`String`) – The name of the function.
-- `is_aggregate`(`UInt8`) — Whether the function is aggregate.
+- `name` ([String](../../sql-reference/data-types/string.md)) – The name of the function.
+- `is_aggregate` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Whether the function is an aggregate function.
+- `is_deterministic` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md))) - Whether the function is deterministic.
+- `case_insensitive`, ([UInt8](../../sql-reference/data-types/int-uint.md)) - Whether the function name can be used case-insensitively.
+- `alias_to`, ([String](../../sql-reference/data-types/string.md)) - The original function name, if the function name is an alias.
+- `create_query`, ([String](../../sql-reference/data-types/enum.md)) - Unused.
+- `origin`, ([Enum8](../../sql-reference/data-types/string.md)) - Unused.
+- `description`, ([String](../../sql-reference/data-types/string.md)) - A high-level description what the function does.
+- `syntax`, ([String](../../sql-reference/data-types/string.md)) - Signature of the function.
+- `arguments`, ([String](../../sql-reference/data-types/string.md)) - What arguments does the function take.
+- `returned_value`, ([String](../../sql-reference/data-types/string.md)) - What does the function return.
+- `examples`, ([String](../../sql-reference/data-types/string.md)) - Example usage of the function.
+- `categories`, ([String](../../sql-reference/data-types/string.md)) - The category of the function.
**Example**
```sql
- SELECT * FROM system.functions LIMIT 10;
+ SELECT name, is_aggregate, is_deterministic, case_insensitive, alias_to FROM system.functions LIMIT 5;
```
```text
-┌─name──────────────────┬─is_aggregate─┬─case_insensitive─┬─alias_to─┬─create_query─┬─origin─┐
-│ logTrace │ 0 │ 0 │ │ │ System │
-│ aes_decrypt_mysql │ 0 │ 0 │ │ │ System │
-│ aes_encrypt_mysql │ 0 │ 0 │ │ │ System │
-│ decrypt │ 0 │ 0 │ │ │ System │
-│ encrypt │ 0 │ 0 │ │ │ System │
-│ toBool │ 0 │ 0 │ │ │ System │
-│ windowID │ 0 │ 0 │ │ │ System │
-│ hopStart │ 0 │ 0 │ │ │ System │
-│ hop │ 0 │ 0 │ │ │ System │
-│ snowflakeToDateTime64 │ 0 │ 0 │ │ │ System │
-└───────────────────────┴──────────────┴──────────────────┴──────────┴──────────────┴────────┘
+┌─name─────────────────────┬─is_aggregate─┬─is_deterministic─┬─case_insensitive─┬─alias_to─┐
+│ BLAKE3 │ 0 │ 1 │ 0 │ │
+│ sipHash128Reference │ 0 │ 1 │ 0 │ │
+│ mapExtractKeyLike │ 0 │ 1 │ 0 │ │
+│ sipHash128ReferenceKeyed │ 0 │ 1 │ 0 │ │
+│ mapPartialSort │ 0 │ 1 │ 0 │ │
+└──────────────────────────┴──────────────┴──────────────────┴──────────────────┴──────────┘
-10 rows in set. Elapsed: 0.002 sec.
+5 rows in set. Elapsed: 0.002 sec.
```
diff --git a/docs/en/operations/system-tables/information_schema.md b/docs/en/operations/system-tables/information_schema.md
index bca37b473129..8470ac838a49 100644
--- a/docs/en/operations/system-tables/information_schema.md
+++ b/docs/en/operations/system-tables/information_schema.md
@@ -69,6 +69,7 @@ Columns:
- `domain_catalog` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — `NULL`, not supported.
- `domain_schema` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — `NULL`, not supported.
- `domain_name` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — `NULL`, not supported.
+- `extra` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — `STORED GENERATED` for `MATERIALIZED`-type columns, `VIRTUAL GENERATED` for `ALIAS`-type columns, `DEFAULT_GENERATED` for `DEFAULT`-type columns, or `NULL`.
**Example**
@@ -198,8 +199,12 @@ Columns:
- `FOREIGN TABLE`
- `LOCAL TEMPORARY`
- `SYSTEM VIEW`
+- `table_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The total
+ number of rows. NULL if it could not be determined.
+- `data_length` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — The size of
+ the data on-disk. NULL if it could not be determined.
+- `table_collation` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — The table default collation. Always `utf8mb4_0900_ai_ci`.
- `table_comment` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — The comment used when creating the table.
-- `table_collation` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — The table default collation. Always `utf8mb4`.
**Example**
@@ -291,7 +296,7 @@ is_trigger_deletable: NO
is_trigger_insertable_into: NO
```
-## KEY_COLUMN_USAGE (#key_column_usage)
+## KEY_COLUMN_USAGE {#key_column_usage}
Contains columns from the [system.tables](../../operations/system-tables/tables.md) system table which are restricted by constraints.
@@ -350,7 +355,7 @@ referenced_table_name: ᴺᵁᴸᴸ
referenced_column_name: ᴺᵁᴸᴸ
```
-## REFERENTIAL_CONSTRAINTS (#referential_constraints)
+## REFERENTIAL_CONSTRAINTS {#referential_constraints}
Contains information about foreign keys. Currently returns an empty result (no rows) which is just enough to provide compatibility with 3rd party tools like Tableau Online.
diff --git a/docs/en/operations/system-tables/merge_tree_settings.md b/docs/en/operations/system-tables/merge_tree_settings.md
index 557835ce3b6b..48217d63f9db 100644
--- a/docs/en/operations/system-tables/merge_tree_settings.md
+++ b/docs/en/operations/system-tables/merge_tree_settings.md
@@ -17,7 +17,7 @@ Columns:
- `0` — Current user can change the setting.
- `1` — Current user can’t change the setting.
- `type` ([String](../../sql-reference/data-types/string.md)) — Setting type (implementation specific string value).
-- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete.
+- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete.
**Example**
```sql
diff --git a/docs/en/operations/system-tables/part_log.md b/docs/en/operations/system-tables/part_log.md
index c9e34962c793..b9185434e016 100644
--- a/docs/en/operations/system-tables/part_log.md
+++ b/docs/en/operations/system-tables/part_log.md
@@ -11,21 +11,21 @@ The `system.part_log` table contains the following columns:
- `query_id` ([String](../../sql-reference/data-types/string.md)) — Identifier of the `INSERT` query that created this data part.
- `event_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the event that occurred with the data part. Can have one of the following values:
- - `NEW_PART` — Inserting of a new data part.
- - `MERGE_PARTS` — Merging of data parts.
- - `DOWNLOAD_PART` — Downloading a data part.
- - `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
- - `MUTATE_PART` — Mutating of a data part.
- - `MOVE_PART` — Moving the data part from the one disk to another one.
+ - `NewPart` — Inserting of a new data part.
+ - `MergeParts` — Merging of data parts.
+ - `DownloadParts` — Downloading a data part.
+ - `RemovePart` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
+ - `MutatePart` — Mutating of a data part.
+ - `MovePart` — Moving the data part from the one disk to another one.
- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — The reason for the event with type `MERGE_PARTS`. Can have one of the following values:
- - `NOT_A_MERGE` — The current event has the type other than `MERGE_PARTS`.
- - `REGULAR_MERGE` — Some regular merge.
- - `TTL_DELETE_MERGE` — Cleaning up expired data.
- - `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
+ - `NotAMerge` — The current event has the type other than `MERGE_PARTS`.
+ - `RegularMerge` — Some regular merge.
+ - `TTLDeleteMerge` — Cleaning up expired data.
+ - `TTLRecompressMerge` — Recompressing data part with the.
- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values:
- - `UNDECIDED`
- - `HORIZONTAL`
- - `VERTICAL`
+ - `Undecided`
+ - `Horizontal`
+ - `Vertical`
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
diff --git a/docs/en/operations/system-tables/query_cache.md b/docs/en/operations/system-tables/query_cache.md
new file mode 100644
index 000000000000..a9f86f5fc2b3
--- /dev/null
+++ b/docs/en/operations/system-tables/query_cache.md
@@ -0,0 +1,36 @@
+---
+slug: /en/operations/system-tables/query_cache
+---
+# query_cache
+
+Shows the content of the [query cache](../query-cache.md).
+
+Columns:
+
+- `query` ([String](../../sql-reference/data-types/string.md)) — Query string.
+- `result_size` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Size of the query cache entry.
+- `stale` ([UInt8](../../sql-reference/data-types/int-uint.md)) — If the query cache entry is stale.
+- `shared` ([UInt8](../../sql-reference/data-types/int-uint.md)) — If the query cache entry is shared between multiple users.
+- `compressed` ([UInt8](../../sql-reference/data-types/int-uint.md)) — If the query cache entry is compressed.
+- `expires_at` ([DateTime](../../sql-reference/data-types/datetime.md)) — When the query cache entry becomes stale.
+- `key_hash` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — A hash of the query string, used as a key to find query cache entries.
+
+**Example**
+
+``` sql
+SELECT * FROM system.query_cache FORMAT Vertical;
+```
+
+``` text
+Row 1:
+──────
+query: SELECT 1 SETTINGS use_query_cache = 1
+result_size: 128
+stale: 0
+shared: 0
+compressed: 1
+expires_at: 2023-10-13 13:35:45
+key_hash: 12188185624808016954
+
+1 row in set. Elapsed: 0.004 sec.
+```
diff --git a/docs/en/operations/system-tables/server_settings.md b/docs/en/operations/system-tables/server_settings.md
index df482261ae82..7efe605ccef6 100644
--- a/docs/en/operations/system-tables/server_settings.md
+++ b/docs/en/operations/system-tables/server_settings.md
@@ -14,7 +14,7 @@ Columns:
- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting was specified in `config.xml`
- `description` ([String](../../sql-reference/data-types/string.md)) — Short server setting description.
- `type` ([String](../../sql-reference/data-types/string.md)) — Server setting value type.
-- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete.
+- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete.
**Example**
diff --git a/docs/en/operations/system-tables/settings.md b/docs/en/operations/system-tables/settings.md
index 7dd2345a2d0f..a04e095e990a 100644
--- a/docs/en/operations/system-tables/settings.md
+++ b/docs/en/operations/system-tables/settings.md
@@ -17,7 +17,7 @@ Columns:
- `0` — Current user can change the setting.
- `1` — Current user can’t change the setting.
- `default` ([String](../../sql-reference/data-types/string.md)) — Setting default value.
-- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) _ Shows whether a setting is obsolete.
+- `is_obsolete` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) - Shows whether a setting is obsolete.
**Example**
diff --git a/docs/en/sql-reference/aggregate-functions/reference/argmin.md b/docs/en/sql-reference/aggregate-functions/reference/argmin.md
index fdfce0833e07..616fa3d33b83 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/argmin.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/argmin.md
@@ -104,7 +104,7 @@ SELECT argMin((a, b), (b, a)), min(tuple(b, a)) FROM test;
└──────────────────────────────────┴──────────────────┘
SELECT argMin(a, tuple(b)) FROM test;
-┌─argMax(a, tuple(b))─┐
+┌─argMin(a, tuple(b))─┐
│ d │ -- `Tuple` can be used in `min` to not skip rows with `NULL` values as b.
└─────────────────────┘
```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md
index 29b43851f446..fa320b4e3367 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md
@@ -33,8 +33,8 @@ The null hypothesis is that means of populations are equal. Normal distribution
- calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md).
- calculated p-value. [Float64](../../../sql-reference/data-types/float.md).
-- [calculated confidence-interval-low.] [Float64](../../../sql-reference/data-types/float.md).
-- [calculated confidence-interval-high.] [Float64](../../../sql-reference/data-types/float.md).
+- [calculated confidence-interval-low. [Float64](../../../sql-reference/data-types/float.md).]
+- [calculated confidence-interval-high. [Float64](../../../sql-reference/data-types/float.md).]
**Example**
diff --git a/docs/en/sql-reference/data-types/date.md b/docs/en/sql-reference/data-types/date.md
index 048466f7ae44..26e4610aec76 100644
--- a/docs/en/sql-reference/data-types/date.md
+++ b/docs/en/sql-reference/data-types/date.md
@@ -26,7 +26,12 @@ ENGINE = TinyLog;
```
``` sql
-INSERT INTO dt VALUES (1546300800, 1), ('2019-01-01', 2);
+-- Parse Date
+-- - from string,
+-- - from 'small' integer interpreted as number of days since 1970-01-01, and
+-- - from 'big' integer interpreted as number of seconds since 1970-01-01.
+INSERT INTO dt VALUES ('2019-01-01', 1), (17897, 2), (1546300800, 3);
+
SELECT * FROM dt;
```
@@ -34,6 +39,7 @@ SELECT * FROM dt;
┌──timestamp─┬─event_id─┐
│ 2019-01-01 │ 1 │
│ 2019-01-01 │ 2 │
+│ 2019-01-01 │ 3 │
└────────────┴──────────┘
```
diff --git a/docs/en/sql-reference/data-types/date32.md b/docs/en/sql-reference/data-types/date32.md
index 7cf8b1b95fe5..38a07cd817da 100644
--- a/docs/en/sql-reference/data-types/date32.md
+++ b/docs/en/sql-reference/data-types/date32.md
@@ -13,7 +13,7 @@ A date. Supports the date range same with [DateTime64](../../sql-reference/data-
Creating a table with a `Date32`-type column and inserting data into it:
``` sql
-CREATE TABLE new
+CREATE TABLE dt32
(
`timestamp` Date32,
`event_id` UInt8
@@ -22,8 +22,13 @@ ENGINE = TinyLog;
```
``` sql
-INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2);
-SELECT * FROM new;
+-- Parse Date
+-- - from string,
+-- - from 'small' integer interpreted as number of days since 1970-01-01, and
+-- - from 'big' integer interpreted as number of seconds since 1970-01-01.
+INSERT INTO dt32 VALUES ('2100-01-01', 1), (47482, 2), (4102444800, 3);
+
+SELECT * FROM dt32;
```
``` text
diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md
index c99c87915423..42a1ca5aaaca 100644
--- a/docs/en/sql-reference/data-types/datetime.md
+++ b/docs/en/sql-reference/data-types/datetime.md
@@ -48,17 +48,18 @@ ENGINE = TinyLog;
```
``` sql
-INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2);
-```
+-- Parse DateTime
+-- - from string,
+-- - from integer interpreted as number of seconds since 1970-01-01.
+INSERT INTO dt VALUES ('2019-01-01 00:00:00', 1), (1546300800, 3);
-``` sql
SELECT * FROM dt;
```
``` text
┌───────────timestamp─┬─event_id─┐
-│ 2019-01-01 03:00:00 │ 1 │
│ 2019-01-01 00:00:00 │ 2 │
+│ 2019-01-01 03:00:00 │ 1 │
└─────────────────────┴──────────┘
```
@@ -73,7 +74,7 @@ SELECT * FROM dt WHERE timestamp = toDateTime('2019-01-01 00:00:00', 'Asia/Istan
``` text
┌───────────timestamp─┬─event_id─┐
-│ 2019-01-01 00:00:00 │ 2 │
+│ 2019-01-01 00:00:00 │ 1 │
└─────────────────────┴──────────┘
```
@@ -85,7 +86,7 @@ SELECT * FROM dt WHERE timestamp = '2019-01-01 00:00:00'
``` text
┌───────────timestamp─┬─event_id─┐
-│ 2019-01-01 03:00:00 │ 1 │
+│ 2019-01-01 00:00:00 │ 1 │
└─────────────────────┴──────────┘
```
diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md
index 3b80e8b1a8bf..8c7fa17ae92b 100644
--- a/docs/en/sql-reference/data-types/datetime64.md
+++ b/docs/en/sql-reference/data-types/datetime64.md
@@ -28,7 +28,7 @@ Note: The precision of the maximum value is 8. If the maximum precision of 9 dig
1. Creating a table with `DateTime64`-type column and inserting data into it:
``` sql
-CREATE TABLE dt
+CREATE TABLE dt64
(
`timestamp` DateTime64(3, 'Asia/Istanbul'),
`event_id` UInt8
@@ -37,11 +37,12 @@ ENGINE = TinyLog;
```
``` sql
-INSERT INTO dt Values (1546300800123, 1), (1546300800.123, 2), ('2019-01-01 00:00:00', 3);
-```
+-- Parse DateTime
+-- - from integer interpreted as number of seconds since 1970-01-01.
+-- - from string,
+INSERT INTO dt64 VALUES (1546300800123, 1), (1546300800.123, 2), ('2019-01-01 00:00:00', 3);
-``` sql
-SELECT * FROM dt;
+SELECT * FROM dt64;
```
``` text
@@ -58,7 +59,7 @@ SELECT * FROM dt;
2. Filtering on `DateTime64` values
``` sql
-SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul');
+SELECT * FROM dt64 WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul');
```
``` text
@@ -70,7 +71,7 @@ SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Asia/
Unlike `DateTime`, `DateTime64` values are not converted from `String` automatically.
``` sql
-SELECT * FROM dt WHERE timestamp = toDateTime64(1546300800.123, 3);
+SELECT * FROM dt64 WHERE timestamp = toDateTime64(1546300800.123, 3);
```
``` text
@@ -101,7 +102,7 @@ SELECT toDateTime64(now(), 3, 'Asia/Istanbul') AS column, toTypeName(column) AS
SELECT
toDateTime64(timestamp, 3, 'Europe/London') as lon_time,
toDateTime64(timestamp, 3, 'Asia/Istanbul') as istanbul_time
-FROM dt;
+FROM dt64;
```
``` text
diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md
index 3add754e6b65..8f87eeca075f 100644
--- a/docs/en/sql-reference/data-types/tuple.md
+++ b/docs/en/sql-reference/data-types/tuple.md
@@ -12,7 +12,7 @@ Tuples are used for temporary column grouping. Columns can be grouped when an IN
Tuples can be the result of a query. In this case, for text formats other than JSON, values are comma-separated in brackets. In JSON formats, tuples are output as arrays (in square brackets).
-## Creating a Tuple
+## Creating Tuples
You can use a function to create a tuple:
@@ -23,7 +23,7 @@ tuple(T1, T2, ...)
Example of creating a tuple:
``` sql
-SELECT tuple(1,'a') AS x, toTypeName(x)
+SELECT tuple(1, 'a') AS x, toTypeName(x)
```
``` text
@@ -32,7 +32,7 @@ SELECT tuple(1,'a') AS x, toTypeName(x)
└─────────┴───────────────────────────┘
```
-Tuple can contain a single element
+A Tuple can contain a single element
Example:
@@ -46,12 +46,12 @@ SELECT tuple('a') AS x;
└───────┘
```
-There is a syntax sugar using parentheses `( tuple_element1, tuple_element2 )` to create a tuple of several elements without tuple function.
+Syntax `(tuple_element1, tuple_element2)` may be used to create a tuple of several elements without calling the `tuple()` function.
Example:
``` sql
-SELECT (1, 'a') AS x, (today(), rand(), 'someString') y, ('a') not_a_tuple;
+SELECT (1, 'a') AS x, (today(), rand(), 'someString') AS y, ('a') AS not_a_tuple;
```
``` text
@@ -60,9 +60,9 @@ SELECT (1, 'a') AS x, (today(), rand(), 'someString') y, ('a') not_a_tuple;
└─────────┴────────────────────────────────────────┴─────────────┘
```
-## Working with Data Types
+## Data Type Detection
-When creating a tuple on the fly, ClickHouse automatically detects the type of each argument as the minimum of the types which can store the argument value. If the argument is [NULL](../../sql-reference/syntax.md#null-literal), the type of the tuple element is [Nullable](../../sql-reference/data-types/nullable.md).
+When creating tuples on the fly, ClickHouse interferes the type of the tuples arguments as the smallest types which can hold the provided argument value. If the value is [NULL](../../sql-reference/syntax.md#null-literal), the interfered type is [Nullable](../../sql-reference/data-types/nullable.md).
Example of automatic data type detection:
@@ -71,23 +71,21 @@ SELECT tuple(1, NULL) AS x, toTypeName(x)
```
``` text
-┌─x────────┬─toTypeName(tuple(1, NULL))──────┐
-│ (1,NULL) │ Tuple(UInt8, Nullable(Nothing)) │
-└──────────┴─────────────────────────────────┘
+┌─x─────────┬─toTypeName(tuple(1, NULL))──────┐
+│ (1, NULL) │ Tuple(UInt8, Nullable(Nothing)) │
+└───────────┴─────────────────────────────────┘
```
-## Addressing Tuple Elements
+## Referring to Tuple Elements
-It is possible to read elements of named tuples using indexes and names:
+Tuple elements can be referred to by name or by index:
``` sql
CREATE TABLE named_tuples (`a` Tuple(s String, i Int64)) ENGINE = Memory;
-
INSERT INTO named_tuples VALUES (('y', 10)), (('x',-10));
-SELECT a.s FROM named_tuples;
-
-SELECT a.2 FROM named_tuples;
+SELECT a.s FROM named_tuples; -- by name
+SELECT a.2 FROM named_tuples; -- by index
```
Result:
@@ -106,7 +104,7 @@ Result:
## Comparison operations with Tuple
-The operation of comparing two tuples is performed sequentially element by element from left to right. If the element of the first tuple is greater than the corresponding element of the second tuple, then the first tuple is greater than the second, if the elements are equal, the next element is compared.
+Two tuples are compared by sequentially comparing their elements from the left to the right. If first tuples element is greater (smaller) than the second tuples corresponding element, then the first tuple is greater (smaller) than the second, otherwise (both elements are equal), the next element is compared.
Example:
diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md
index 69f1816b7dfa..6d95f3dc3583 100644
--- a/docs/en/sql-reference/functions/arithmetic-functions.md
+++ b/docs/en/sql-reference/functions/arithmetic-functions.md
@@ -73,7 +73,7 @@ Calculates the product of two values `a` and `b`.
multiply(a, b)
```
-Alias: `a \* b` (operator)
+Alias: `a * b` (operator)
## divide
@@ -345,9 +345,9 @@ Result:
┌─multiply(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐
│ -26.8609633 │
└───────────────────────────────────────────────────────────┘
-┌─multiplyDecimal(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐
-│ -26.8609 │
-└──────────────────────────────────────────────────────────────────┘
+┌───────a─┬──────b─┬─multiplyDecimal(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐
+│ -12.647 │ 2.1239 │ -26.8609 │
+└─────────┴────────┴──────────────────────────────────────────────────────────────────┘
```
```sql
@@ -441,3 +441,40 @@ DB::Exception: Decimal result's scale is less than argument's one: While process
│ -12 │ 2.1 │ -5.7 │ -5.71428 │
└─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘
```
+
+## byteSwap
+
+Reverses the bytes of an integer, i.e. changes its [endianness](https://en.wikipedia.org/wiki/Endianness).
+
+**Syntax**
+
+```sql
+byteSwap(a)
+```
+
+**Example**
+
+```sql
+byteSwap(3351772109)
+```
+
+Result:
+
+```result
+┌─byteSwap(3351772109)─┐
+│ 3455829959 │
+└──────────────────────┘
+```
+
+The above example can be worked out in the following manner:
+1. Convert the base-10 integer to its equivalent hexadecimal format in big-endian format, i.e. 3351772109 -> C7 C7 FB CD (4 bytes)
+2. Reverse the bytes, i.e. C7 C7 FB CD -> CD FB C7 C7
+3. Convert the result back to an integer assuming big-endian, i.e. CD FB C7 C7 -> 3455829959
+
+One use case of this function is reversing IPv4s:
+
+```result
+┌─toIPv4(byteSwap(toUInt32(toIPv4('205.251.199.199'))))─┐
+│ 199.199.251.205 │
+└───────────────────────────────────────────────────────┘
+```
diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md
index 59af1b909caa..73c51a41dfba 100644
--- a/docs/en/sql-reference/functions/array-functions.md
+++ b/docs/en/sql-reference/functions/array-functions.md
@@ -1123,6 +1123,32 @@ Result:
└─────────────────────────────┘
```
+## arrayFold
+
+Applies a lambda function to one or more equally-sized arrays and collects the result in an accumulator.
+
+**Syntax**
+
+``` sql
+arrayFold(lambda_function, arr1, arr2, ..., accumulator)
+```
+
+**Example**
+
+Query:
+
+``` sql
+SELECT arrayFold( x,acc -> acc + x*2, [1, 2, 3, 4], toInt64(3)) AS res;
+```
+
+Result:
+
+``` text
+┌─arrayFold(lambda(tuple(x, acc), plus(acc, multiply(x, 2))), [1, 2, 3, 4], toInt64(3))─┐
+│ 3 │
+└───────────────────────────────────────────────────────────────────────────────────────┘
+```
+
## arrayReverse(arr)
Returns an array of the same size as the original array containing the elements in reverse order.
@@ -2118,6 +2144,80 @@ Result:
└─────────────────────┘
```
+
+## arrayRandomSample
+
+Function `arrayRandomSample` returns a subset with `samples`-many random elements of an input array. If `samples` exceeds the size of the input array, the sample size is limited to the size of the array. In this case, all elements of the input array are returned, but the order is not guaranteed. The function can handle both flat arrays and nested arrays.
+
+**Syntax**
+
+```sql
+arrayRandomSample(arr, samples)
+```
+
+**Arguments**
+
+- `arr` — The input array from which to sample elements. This may be flat or nested arrays.
+- `samples` — An unsigned integer specifying the number of elements to include in the random sample.
+
+**Returned Value**
+
+- An array containing a random sample of elements from the input array.
+
+**Examples**
+
+Query:
+
+```sql
+SELECT arrayRandomSample(['apple', 'banana', 'cherry', 'date'], 2) as res;
+```
+
+Result:
+```
+┌─res────────────────┐
+│ ['banana','apple'] │
+└────────────────────┘
+```
+
+Query:
+
+```sql
+SELECT arrayRandomSample([[1, 2], [3, 4], [5, 6]], 2) as res;
+```
+
+Result:
+```
+┌─res───────────┐
+│ [[3,4],[5,6]] │
+└───────────────┘
+```
+
+Query:
+
+```sql
+SELECT arrayRandomSample([1, 2, 3, 4, 5], 0) as res;
+```
+
+Result:
+```
+┌─res─┐
+│ [] │
+└─────┘
+```
+
+Query:
+
+```sql
+SELECT arrayRandomSample([1, 2, 3], 5) as res;
+```
+
+Result:
+```
+┌─res─────┐
+│ [3,1,2] │
+└─────────┘
+```
+
## Distance functions
All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).
diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 0364a6104041..0602cf31c58e 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -134,9 +134,57 @@ Like [makeDateTime](#makedatetime) but produces a [DateTime64](../../sql-referen
**Syntax**
``` sql
-makeDateTime32(year, month, day, hour, minute, second[, fraction[, precision[, timezone]]])
+makeDateTime64(year, month, day, hour, minute, second[, fraction[, precision[, timezone]]])
```
+## timestamp
+
+Converts the first argument 'expr' to type [DateTime64(6)](../../sql-reference/data-types/datetime64.md).
+If a second argument 'expr_time' is provided, it adds the specified time to the converted value.
+
+**Syntax**
+
+``` sql
+timestamp(expr[, expr_time])
+```
+
+Alias: `TIMESTAMP`
+
+**Arguments**
+
+- `expr` - Date or date with time. Type: [String](../../sql-reference/data-types/string.md).
+- `expr_time` - Optional parameter. Time to add. [String](../../sql-reference/data-types/string.md).
+
+**Examples**
+
+``` sql
+SELECT timestamp('2023-12-31') as ts;
+```
+
+Result:
+
+``` text
+┌─────────────────────────ts─┐
+│ 2023-12-31 00:00:00.000000 │
+└────────────────────────────┘
+```
+
+``` sql
+SELECT timestamp('2023-12-31 12:00:00', '12:00:00.11') as ts;
+```
+
+Result:
+
+``` text
+┌─────────────────────────ts─┐
+│ 2024-01-01 00:00:00.110000 │
+└────────────────────────────┘
+```
+
+**Returned value**
+
+- [DateTime64](../../sql-reference/data-types/datetime64.md)(6)
+
## timeZone
Returns the timezone of the current session, i.e. the value of setting [session_timezone](../../operations/settings/settings.md#session_timezone).
@@ -156,6 +204,20 @@ Alias: `timezone`.
Type: [String](../../sql-reference/data-types/string.md).
+**Example**
+
+```sql
+SELECT timezone()
+```
+
+Result:
+
+```response
+┌─timezone()─────┐
+│ America/Denver │
+└────────────────┘
+```
+
**See also**
- [serverTimeZone](#serverTimeZone)
@@ -179,6 +241,20 @@ Alias: `serverTimezone`.
Type: [String](../../sql-reference/data-types/string.md).
+**Example**
+
+```sql
+SELECT serverTimeZone()
+```
+
+Result:
+
+```response
+┌─serverTimeZone()─┐
+│ UTC │
+└──────────────────┘
+```
+
**See also**
- [timeZone](#timeZone)
@@ -318,37 +394,189 @@ Result:
## toYear
-Converts a date or date with time to the year number (AD) as UInt16 value.
+Converts a date or date with time to the year number (AD) as `UInt16` value.
+
+
+**Syntax**
+
+```sql
+toYear(value)
+```
+
+Alias: `YEAR`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The year of the given date/time
+
+Type: `UInt16`
+
+**Example**
+
+```sql
+SELECT toYear(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
-Alias: `YEAR`.
+```response
+┌─toYear(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2023 │
+└───────────────────────────────────────────┘
+```
## toQuarter
-Converts a date or date with time to the quarter number as UInt8 value.
+Converts a date or date with time to the quarter number (1-4) as `UInt8` value.
+
+**Syntax**
+
+```sql
+toQuarter(value)
+```
+
+Alias: `QUARTER`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The quarter of the year (1, 2, 3 or 4) of the given date/time
+
+Type: `UInt8`
+
+**Example**
+
+```sql
+SELECT toQuarter(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toQuarter(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2 │
+└──────────────────────────────────────────────┘
+```
-Alias: `QUARTER`.
## toMonth
-Converts a date or date with time to the month number (1-12) as UInt8 value.
+Converts a date or date with time to the month number (1-12) as `UInt8` value.
+
+**Syntax**
+
+```sql
+toMonth(value)
+```
+
+Alias: `MONTH`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The month of the year (1 - 12) of the given date/time
+
+Type: `UInt8`
+
+**Example**
+
+```sql
+SELECT toMonth(toDateTime('2023-04-21 10:20:30'))
+```
-Alias: `MONTH`.
+Result:
+
+```response
+┌─toMonth(toDateTime('2023-04-21 10:20:30'))─┐
+│ 4 │
+└────────────────────────────────────────────┘
+```
## toDayOfYear
-Converts a date or date with time to the number of the day of the year (1-366) as UInt16 value.
+Converts a date or date with time to the number of the day of the year (1-366) as `UInt16` value.
+
+**Syntax**
+
+```sql
+toDayOfYear(value)
+```
+
+Alias: `DAYOFYEAR`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The day of the year (1 - 366) of the given date/time
+
+Type: `UInt16`
+
+**Example**
-Alias: `DAYOFYEAR`.
+```sql
+SELECT toDayOfYear(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toDayOfYear(toDateTime('2023-04-21 10:20:30'))─┐
+│ 111 │
+└────────────────────────────────────────────────┘
+```
## toDayOfMonth
-Converts a date or date with time to the number of the day in the month (1-31) as UInt8 value.
+Converts a date or date with time to the number of the day in the month (1-31) as `UInt8` value.
+
+**Syntax**
+
+```sql
+toDayOfMonth(value)
+```
+
+Aliases: `DAYOFMONTH`, `DAY`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The day of the month (1 - 31) of the given date/time
+
+Type: `UInt8`
+
+**Example**
+
+```sql
+SELECT toDayOfMonth(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
-Aliases: `DAYOFMONTH`, `DAY`.
+```response
+┌─toDayOfMonth(toDateTime('2023-04-21 10:20:30'))─┐
+│ 21 │
+└─────────────────────────────────────────────────┘
+```
## toDayOfWeek
-Converts a date or date with time to the number of the day in the week as UInt8 value.
+Converts a date or date with time to the number of the day in the week as `UInt8` value.
The two-argument form of `toDayOfWeek()` enables you to specify whether the week starts on Monday or Sunday, and whether the return value should be in the range from 0 to 6 or 1 to 7. If the mode argument is omitted, the default mode is 0. The time zone of the date can be specified as the third argument.
@@ -367,25 +595,143 @@ Alias: `DAYOFWEEK`.
toDayOfWeek(t[, mode[, timezone]])
```
+**Arguments**
+
+- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+- `mode` - determines what the first day of the week is. Possible values are 0, 1, 2 or 3. See the table above for the differences.
+- `timezone` - optional parameter, it behaves like any other conversion function
+
+**Returned value**
+
+- The day of the month (1 - 31) of the given date/time
+
+**Example**
+
+The following date is April 21, 2023, which was a Friday:
+
+```sql
+SELECT
+ toDayOfWeek(toDateTime('2023-04-21')),
+ toDayOfWeek(toDateTime('2023-04-21'), 1)
+```
+
+Result:
+
+```response
+┌─toDayOfWeek(toDateTime('2023-04-21'))─┬─toDayOfWeek(toDateTime('2023-04-21'), 1)─┐
+│ 5 │ 4 │
+└───────────────────────────────────────┴──────────────────────────────────────────┘
+```
+
## toHour
-Converts a date with time the number of the hour in 24-hour time (0-23) as UInt8 value.
+Converts a date with time to the number of the hour in 24-hour time (0-23) as `UInt8` value.
+
+Assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always exactly when it occurs - it depends on the timezone).
+
+**Syntax**
+
+```sql
+toHour(value)
+```
+
+Alias: `HOUR`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The hour of the day (0 - 23) of the given date/time
+
+Type: `UInt8`
+
+**Example**
-Assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always true – even in Moscow the clocks were twice changed at a different time).
+```sql
+SELECT toHour(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
-Alias: `HOUR`.
+```response
+┌─toHour(toDateTime('2023-04-21 10:20:30'))─┐
+│ 10 │
+└───────────────────────────────────────────┘
+```
## toMinute
-Converts a date with time to the number of the minute of the hour (0-59) as UInt8 value.
+Converts a date with time to the number of the minute of the hour (0-59) as `UInt8` value.
+
+**Syntax**
+
+```sql
+toMinute(value)
+```
+
+Alias: `MINUTE`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The minute of the hour (0 - 59) of the given date/time
+
+Type: `UInt8`
+
+**Example**
+
+```sql
+SELECT toMinute(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
-Alias: `MINUTE`.
+```response
+┌─toMinute(toDateTime('2023-04-21 10:20:30'))─┐
+│ 20 │
+└─────────────────────────────────────────────┘
+```
## toSecond
-Converts a date with time to the second in the minute (0-59) as UInt8 value. Leap seconds are not considered.
+Converts a date with time to the second in the minute (0-59) as `UInt8` value. Leap seconds are not considered.
+
+**Syntax**
+
+```sql
+toSecond(value)
+```
+
+Alias: `SECOND`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The second in the minute (0 - 59) of the given date/time
-Alias: `SECOND`.
+Type: `UInt8`
+
+**Example**
+
+```sql
+SELECT toSecond(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toSecond(toDateTime('2023-04-21 10:20:30'))─┐
+│ 30 │
+└─────────────────────────────────────────────┘
+```
## toUnixTimestamp
@@ -448,48 +794,220 @@ Behavior for
## toStartOfYear
-Rounds down a date or date with time to the first day of the year.
-Returns the date.
+Rounds down a date or date with time to the first day of the year. Returns the date as a `Date` object.
-## toStartOfISOYear
+**Syntax**
-Rounds down a date or date with time to the first day of ISO year.
-Returns the date.
+```sql
+toStartOfYear(value)
+```
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The first day of the year of the input date/time
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT toStartOfYear(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toStartOfYear(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2023-01-01 │
+└──────────────────────────────────────────────────┘
+```
+
+## toStartOfISOYear
+
+Rounds down a date or date with time to the first day of the ISO year, which can be different than a "regular" year. (See [https://en.wikipedia.org/wiki/ISO_week_date](https://en.wikipedia.org/wiki/ISO_week_date).)
+
+**Syntax**
+
+```sql
+toStartOfISOYear(value)
+```
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The first day of the year of the input date/time
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT toStartOfISOYear(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toStartOfISOYear(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2023-01-02 │
+└─────────────────────────────────────────────────────┘
+```
## toStartOfQuarter
-Rounds down a date or date with time to the first day of the quarter.
-The first day of the quarter is either 1 January, 1 April, 1 July, or 1 October.
+Rounds down a date or date with time to the first day of the quarter. The first day of the quarter is either 1 January, 1 April, 1 July, or 1 October.
Returns the date.
+**Syntax**
+
+```sql
+toStartOfQuarter(value)
+```
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The first day of the quarter of the given date/time
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT toStartOfQuarter(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toStartOfQuarter(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2023-04-01 │
+└─────────────────────────────────────────────────────┘
+```
+
## toStartOfMonth
-Rounds down a date or date with time to the first day of the month.
-Returns the date.
+Rounds down a date or date with time to the first day of the month. Returns the date.
+
+**Syntax**
+
+```sql
+toStartOfMonth(value)
+```
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The first day of the month of the given date/time
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT toStartOfMonth(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toStartOfMonth(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2023-04-01 │
+└───────────────────────────────────────────────────┘
+```
:::note
-The behavior of parsing incorrect dates is implementation specific. ClickHouse may return zero date, throw an exception or do “natural” overflow.
+The behavior of parsing incorrect dates is implementation specific. ClickHouse may return zero date, throw an exception, or do “natural” overflow.
:::
## toLastDayOfMonth
-Rounds a date, or date with time, to the last day of the month.
-Returns the date.
+Rounds a date or date with time to the last day of the month. Returns the date.
+
+**Syntax**
-Alias: `LAST_DAY`.
+```sql
+toLastDayOfMonth(value)
+```
-If `toLastDayOfMonth` is called with an argument of type `Date` greater then 2149-05-31, the result will be calculated from the argument 2149-05-31 instead.
+Alias: `LAST_DAY`
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The last day of the month of the given date/time
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT toLastDayOfMonth(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toLastDayOfMonth(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2023-04-30 │
+└─────────────────────────────────────────────────────┘
+```
## toMonday
-Rounds down a date, or date with time, to the nearest Monday.
-Returns the date.
+Rounds down a date or date with time to the nearest Monday. Returns the date.
+
+**Syntax**
+
+```sql
+toMonday(value)
+```
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The date of the nearest Monday on or prior to the given date
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT
+ toMonday(toDateTime('2023-04-21 10:20:30')), /* a Friday */
+ toMonday(toDate('2023-04-24')), /* already a Monday */
+```
+
+Result:
+
+```response
+┌─toMonday(toDateTime('2023-04-21 10:20:30'))─┬─toMonday(toDate('2023-04-24'))─┐
+│ 2023-04-17 │ 2023-04-24 │
+└─────────────────────────────────────────────┴────────────────────────────────┘
+```
## toStartOfWeek
-Rounds a date or date with time down to the nearest Sunday or Monday.
-Returns the date.
-The mode argument works exactly like the mode argument in function `toWeek()`. If no mode is specified, mode is assumed as 0.
+Rounds a date or date with time down to the nearest Sunday or Monday. Returns the date. The mode argument works exactly like the mode argument in function `toWeek()`. If no mode is specified, it defaults to 0.
**Syntax**
@@ -497,10 +1015,43 @@ The mode argument works exactly like the mode argument in function `toWeek()`. I
toStartOfWeek(t[, mode[, timezone]])
```
+**Arguments**
+
+- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+- `mode` - determines the first day of the week as described in the [toWeek()](date-time-functions#toweek) function
+- `timezone` - Optional parameter, it behaves like any other conversion function
+
+**Returned value**
+
+- The date of the nearest Sunday or Monday on or prior to the given date, depending on the mode
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT
+ toStartOfWeek(toDateTime('2023-04-21 10:20:30')), /* a Friday */
+ toStartOfWeek(toDateTime('2023-04-21 10:20:30'), 1), /* a Friday */
+ toStartOfWeek(toDate('2023-04-24')), /* a Monday */
+ toStartOfWeek(toDate('2023-04-24'), 1) /* a Monday */
+FORMAT Vertical
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toStartOfWeek(toDateTime('2023-04-21 10:20:30')): 2023-04-16
+toStartOfWeek(toDateTime('2023-04-21 10:20:30'), 1): 2023-04-17
+toStartOfWeek(toDate('2023-04-24')): 2023-04-23
+toStartOfWeek(toDate('2023-04-24'), 1): 2023-04-24
+```
+
## toLastDayOfWeek
-Rounds a date or date with time up to the nearest Saturday or Sunday.
-Returns the date.
+Rounds a date or date with time up to the nearest Saturday or Sunday. Returns the date.
The mode argument works exactly like the mode argument in function `toWeek()`. If no mode is specified, mode is assumed as 0.
**Syntax**
@@ -509,18 +1060,148 @@ The mode argument works exactly like the mode argument in function `toWeek()`. I
toLastDayOfWeek(t[, mode[, timezone]])
```
+**Arguments**
+
+- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+- `mode` - determines the last day of the week as described in the [toWeek()](date-time-functions#toweek) function
+- `timezone` - Optional parameter, it behaves like any other conversion function
+
+**Returned value**
+
+- The date of the nearest Sunday or Monday on or after the given date, depending on the mode
+
+Type: `Date`
+
+**Example**
+
+```sql
+SELECT
+ toLastDayOfWeek(toDateTime('2023-04-21 10:20:30')), /* a Friday */
+ toLastDayOfWeek(toDateTime('2023-04-21 10:20:30'), 1), /* a Friday */
+ toLastDayOfWeek(toDate('2023-04-22')), /* a Saturday */
+ toLastDayOfWeek(toDate('2023-04-22'), 1) /* a Saturday */
+FORMAT Vertical
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toLastDayOfWeek(toDateTime('2023-04-21 10:20:30')): 2023-04-22
+toLastDayOfWeek(toDateTime('2023-04-21 10:20:30'), 1): 2023-04-23
+toLastDayOfWeek(toDate('2023-04-22')): 2023-04-22
+toLastDayOfWeek(toDate('2023-04-22'), 1): 2023-04-23
+```
+
## toStartOfDay
Rounds down a date with time to the start of the day.
+**Syntax**
+
+```sql
+toStartOfDay(value)
+```
+
+**Arguments**
+
+- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The start of the day of the given date/time
+
+Type: `DateTime`
+
+**Example**
+
+```sql
+SELECT toStartOfDay(toDateTime('2023-04-21 10:20:30'))
+```
+
+Result:
+
+```response
+┌─toStartOfDay(toDateTime('2023-04-21 10:20:30'))─┐
+│ 2023-04-21 00:00:00 │
+└─────────────────────────────────────────────────┘
+```
+
## toStartOfHour
Rounds down a date with time to the start of the hour.
+**Syntax**
+
+```sql
+toStartOfHour(value)
+```
+
+**Arguments**
+
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The start of the hour of the given date/time
+
+Type: `DateTime`
+
+**Example**
+
+```sql
+SELECT
+ toStartOfHour(toDateTime('2023-04-21 10:20:30')),
+ toStartOfHour(toDateTime64('2023-04-21', 6))
+```
+
+Result:
+
+```response
+┌─toStartOfHour(toDateTime('2023-04-21 10:20:30'))─┬─toStartOfHour(toDateTime64('2023-04-21', 6))─┐
+│ 2023-04-21 10:00:00 │ 2023-04-21 00:00:00 │
+└──────────────────────────────────────────────────┴──────────────────────────────────────────────┘
+```
+
## toStartOfMinute
Rounds down a date with time to the start of the minute.
+**Syntax**
+
+```sql
+toStartOfMinute(value)
+```
+
+**Arguments**
+
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The start of the minute of the given date/time
+
+Type: `DateTime`
+
+**Example**
+
+```sql
+SELECT
+ toStartOfMinute(toDateTime('2023-04-21 10:20:30')),
+ toStartOfMinute(toDateTime64('2023-04-21 10:20:30.5300', 8))
+FORMAT Vertical
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toStartOfMinute(toDateTime('2023-04-21 10:20:30')): 2023-04-21 10:20:00
+toStartOfMinute(toDateTime64('2023-04-21 10:20:30.5300', 8)): 2023-04-21 10:20:00
+```
+
## toStartOfSecond
Truncates sub-seconds.
@@ -582,21 +1263,152 @@ Result:
Rounds down a date with time to the start of the five-minute interval.
+**Syntax**
+
+```sql
+toStartOfFiveMinutes(value)
+```
+
+**Arguments**
+
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The start of the five-minute interval of the given date/time
+
+Type: `DateTime`
+
+**Example**
+
+```sql
+SELECT
+ toStartOfFiveMinutes(toDateTime('2023-04-21 10:17:00')),
+ toStartOfFiveMinutes(toDateTime('2023-04-21 10:20:00')),
+ toStartOfFiveMinutes(toDateTime('2023-04-21 10:23:00'))
+FORMAT Vertical
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toStartOfFiveMinutes(toDateTime('2023-04-21 10:17:00')): 2023-04-21 10:15:00
+toStartOfFiveMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:20:00
+toStartOfFiveMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:20:00
+```
+
## toStartOfTenMinutes
Rounds down a date with time to the start of the ten-minute interval.
+**Syntax**
+
+```sql
+toStartOfTenMinutes(value)
+```
+
+**Arguments**
+
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The start of the ten-minute interval of the given date/time
+
+Type: `DateTime`
+
+**Example**
+
+```sql
+SELECT
+ toStartOfTenMinutes(toDateTime('2023-04-21 10:17:00')),
+ toStartOfTenMinutes(toDateTime('2023-04-21 10:20:00')),
+ toStartOfTenMinutes(toDateTime('2023-04-21 10:23:00'))
+FORMAT Vertical
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toStartOfTenMinutes(toDateTime('2023-04-21 10:17:00')): 2023-04-21 10:10:00
+toStartOfTenMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:20:00
+toStartOfTenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:20:00
+```
+
## toStartOfFifteenMinutes
Rounds down the date with time to the start of the fifteen-minute interval.
+**Syntax**
+
+```sql
+toStartOfFifteenMinutes(value)
+```
+
+**Arguments**
+
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+**Returned value**
+
+- The start of the fifteen-minute interval of the given date/time
+
+Type: `DateTime`
+
+**Example**
+
+```sql
+SELECT
+ toStartOfFifteenMinutes(toDateTime('2023-04-21 10:17:00')),
+ toStartOfFifteenMinutes(toDateTime('2023-04-21 10:20:00')),
+ toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00'))
+FORMAT Vertical
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toStartOfFifteenMinutes(toDateTime('2023-04-21 10:17:00')): 2023-04-21 10:15:00
+toStartOfFifteenMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:15:00
+toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:15:00
+```
+
## toStartOfInterval(time_or_data, INTERVAL x unit \[, time_zone\])
-This is a generalization of other functions named `toStartOf*`. For example,
-`toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`,
-`toStartOfInterval(t, INTERVAL 1 month)` returns the same as `toStartOfMonth(t)`,
-`toStartOfInterval(t, INTERVAL 1 day)` returns the same as `toStartOfDay(t)`,
-`toStartOfInterval(t, INTERVAL 15 minute)` returns the same as `toStartOfFifteenMinutes(t)` etc.
+This function generalizes other `toStartOf*()` functions. For example,
+- `toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`,
+- `toStartOfInterval(t, INTERVAL 1 month)` returns the same as `toStartOfMonth(t)`,
+- `toStartOfInterval(t, INTERVAL 1 day)` returns the same as `toStartOfDay(t)`,
+- `toStartOfInterval(t, INTERVAL 15 minute)` returns the same as `toStartOfFifteenMinutes(t)`.
+
+The calculation is performed relative to specific points in time:
+
+| Interval | Start |
+|-------------|------------------------|
+| year | year 0 |
+| quarter | 1900 Q1 |
+| month | 1900 January |
+| week | 1970, 1st week (01-05) |
+| day | 1970-01-01 |
+| hour | (*) |
+| minute | 1970-01-01 00:00:00 |
+| second | 1970-01-01 00:00:00 |
+| millisecond | 1970-01-01 00:00:00 |
+| microsecond | 1970-01-01 00:00:00 |
+| nanosecond | 1970-01-01 00:00:00 |
+
+(*) hour intervals are special: the calculation is always performed relative to 00:00:00 (midnight) of the current day. As a result, only
+ hour values between 1 and 23 are useful.
+
+**See Also**
+
+- [date_trunc](#date_trunc)
## toTime
diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index 378fc821df9a..22a362cd39c1 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -68,6 +68,45 @@ WHERE macro = 'test';
└───────┴──────────────┘
```
+## getHttpHeader
+Returns the value of specified http header.If there is no such header or the request method is not http, it will return empty string.
+
+**Syntax**
+
+```sql
+getHttpHeader(name);
+```
+
+**Arguments**
+
+- `name` — Http header name .[String](../../sql-reference/data-types/string.md#string)
+
+**Returned value**
+
+Value of the specified header.
+Type:[String](../../sql-reference/data-types/string.md#string).
+
+
+When we use `clickhouse-client` to execute this function, we'll always get empty string, because client doesn't use http protocol.
+```sql
+SELECT getHttpHeader('test')
+```
+result:
+
+```text
+┌─getHttpHeader('test')─┐
+│ │
+└───────────────────────┘
+```
+Try to use http request:
+```shell
+echo "select getHttpHeader('X-Clickhouse-User')" | curl -H 'X-ClickHouse-User: default' -H 'X-ClickHouse-Key: ' 'http://localhost:8123/' -d @-
+
+#result
+default
+```
+
+
## FQDN
Returns the fully qualified domain name of the ClickHouse server.
diff --git a/docs/en/sql-reference/functions/splitting-merging-functions.md b/docs/en/sql-reference/functions/splitting-merging-functions.md
index 614bf556c8e4..8e50637cf300 100644
--- a/docs/en/sql-reference/functions/splitting-merging-functions.md
+++ b/docs/en/sql-reference/functions/splitting-merging-functions.md
@@ -34,10 +34,10 @@ Returns an array of selected substrings. Empty substrings may be selected when:
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
:::note
-The behavior of parameter `max_substrings` changed starting with ClickHouse v22.11. In versions older than that, `max_substrings` > 0 meant that `max_substring`-many splits were performed and that the remainder of the string was returned as the final element of the list.
+The behavior of parameter `max_substrings` changed starting with ClickHouse v22.11. In versions older than that, `max_substrings > 0` meant that `max_substring`-many splits were performed and that the remainder of the string was returned as the final element of the list.
For example,
-- in v22.10: `SELECT splitByChar('=', 'a=b=c=d', 2); -- ['a','b','c=d']`
-- in v22.11: `SELECT splitByChar('=', 'a=b=c=d', 2); -- ['a','b']`
+- in v22.10: `SELECT splitByChar('=', 'a=b=c=d', 2);` returned `['a','b','c=d']`
+- in v22.11: `SELECT splitByChar('=', 'a=b=c=d', 2);` returned `['a','b']`
A behavior similar to ClickHouse pre-v22.11 can be achieved by setting
[splitby_max_substrings_includes_remaining_string](../../operations/settings/settings.md#splitby_max_substrings_includes_remaining_string)
diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md
index 8a2e257ed346..264708513fae 100644
--- a/docs/en/sql-reference/functions/string-search-functions.md
+++ b/docs/en/sql-reference/functions/string-search-functions.md
@@ -684,12 +684,12 @@ Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively.
## byteHammingDistance
-Calculates the hamming distance between two byte strings.
+Calculates the [hamming distance](https://en.wikipedia.org/wiki/Hamming_distance) between two byte strings.
**Syntax**
```sql
-byteHammingDistance(haystack, needle)
+byteHammingDistance(string2, string2)
```
**Examples**
@@ -708,53 +708,52 @@ Result:
- Alias: mismatches
-## byteJaccardIndex
+## jaccardIndex
-Calculates the jaccard similarity index between two byte strings.
+Calculates the [Jaccard similarity index](https://en.wikipedia.org/wiki/Jaccard_index) between two byte strings.
**Syntax**
```sql
-byteJaccardIndex(haystack, needle)
+byteJaccardIndex(string1, string2)
```
**Examples**
``` sql
-SELECT byteJaccardIndex('clickhouse', 'mouse');
+SELECT jaccardIndex('clickhouse', 'mouse');
```
Result:
``` text
-┌─byteJaccardIndex('clickhouse', 'mouse')─┐
+┌─jaccardIndex('clickhouse', 'mouse')─┐
│ 0.4 │
└─────────────────────────────────────────┘
```
-## byteEditDistance
+## editDistance
-Calculates the edit distance between two byte strings.
+Calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two byte strings.
**Syntax**
```sql
-byteEidtDistance(haystack, needle)
+editDistance(string1, string2)
```
**Examples**
``` sql
-SELECT byteEditDistance('clickhouse', 'mouse');
+SELECT editDistance('clickhouse', 'mouse');
```
Result:
``` text
-┌─byteEditDistance('clickhouse', 'mouse')─┐
+┌─editDistance('clickhouse', 'mouse')─┐
│ 6 │
└─────────────────────────────────────────┘
```
-- Alias: byteLevenshteinDistance
-
+- Alias: levenshteinDistance
diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md
index 88e4ac03fdb6..2f7c6377ee14 100644
--- a/docs/en/sql-reference/functions/tuple-functions.md
+++ b/docs/en/sql-reference/functions/tuple-functions.md
@@ -37,6 +37,8 @@ tupleElement(tuple, name, [, default_value])
Performs syntactic substitution of [tuple](../../sql-reference/data-types/tuple.md#tuplet1-t2) elements in the call location.
+The names of the result columns are implementation-specific and subject to change. Do not assume specific column names after `untuple`.
+
**Syntax**
``` sql
@@ -87,8 +89,6 @@ Result:
└───────┴───────┘
```
-Note: the names are implementation specific and are subject to change. You should not assume specific names of the columns after application of the `untuple`.
-
Example of using an `EXCEPT` expression:
Query:
diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md
index a0aa74e6d252..114b8d5ffe32 100644
--- a/docs/en/sql-reference/statements/alter/partition.md
+++ b/docs/en/sql-reference/statements/alter/partition.md
@@ -113,6 +113,7 @@ For the query to run successfully, the following conditions must be met:
- Both tables must have the same structure.
- Both tables must have the same partition key, the same order by key and the same primary key.
+- Both tables must have the same indices and projections.
- Both tables must have the same storage policy.
## REPLACE PARTITION
@@ -132,6 +133,7 @@ For the query to run successfully, the following conditions must be met:
- Both tables must have the same structure.
- Both tables must have the same partition key, the same order by key and the same primary key.
+- Both tables must have the same indices and projections.
- Both tables must have the same storage policy.
## MOVE PARTITION TO TABLE
@@ -146,6 +148,7 @@ For the query to run successfully, the following conditions must be met:
- Both tables must have the same structure.
- Both tables must have the same partition key, the same order by key and the same primary key.
+- Both tables must have the same indices and projections.
- Both tables must have the same storage policy.
- Both tables must be the same engine family (replicated or non-replicated).
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index 5e94a5fdc6f0..c6fad5dc2da6 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -487,7 +487,7 @@ ClickHouse supports temporary tables which have the following characteristics:
- The DB can’t be specified for a temporary table. It is created outside of databases.
- Impossible to create a temporary table with distributed DDL query on all cluster servers (by using `ON CLUSTER`): this table exists only in the current session.
- If a temporary table has the same name as another one and a query specifies the table name without specifying the DB, the temporary table will be used.
-- For distributed query processing, temporary tables used in a query are passed to remote servers.
+- For distributed query processing, temporary tables with Memory engine used in a query are passed to remote servers.
To create a temporary table, use the following syntax:
diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md
index f6d1960a4f32..20cd9a47e982 100644
--- a/docs/en/sql-reference/statements/create/user.md
+++ b/docs/en/sql-reference/statements/create/user.md
@@ -12,7 +12,7 @@ Syntax:
``` sql
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
- [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}]
+ [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'}]
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[VALID UNTIL datetime]
[IN access_storage_type]
@@ -39,6 +39,7 @@ There are multiple ways of user identification:
- `IDENTIFIED WITH ldap SERVER 'server_name'`
- `IDENTIFIED WITH kerberos` or `IDENTIFIED WITH kerberos REALM 'realm'`
- `IDENTIFIED WITH ssl_certificate CN 'mysite.com:user'`
+- `IDENTIFIED WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa', KEY 'another_public_key' TYPE 'ssh-ed25519'`
- `IDENTIFIED BY 'qwerty'`
Password complexity requirements can be edited in [config.xml](/docs/en/operations/configuration-files). Below is an example configuration that requires passwords to be at least 12 characters long and contain 1 number. Each password complexity rule requires a regex to match against passwords and a description of the rule.
diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md
index 82af20fd18c6..21bd674bd248 100644
--- a/docs/en/sql-reference/statements/show.md
+++ b/docs/en/sql-reference/statements/show.md
@@ -189,7 +189,7 @@ Result:
- [Create Tables](https://clickhouse.com/docs/en/getting-started/tutorial/#create-tables)
- [SHOW CREATE TABLE](https://clickhouse.com/docs/en/sql-reference/statements/show/#show-create-table)
-## SHOW COLUMNS
+## SHOW COLUMNS {#show_columns}
Displays a list of columns
@@ -206,15 +206,15 @@ The optional keyword `EXTENDED` currently has no effect, it only exists for MySQ
The optional keyword `FULL` causes the output to include the collation, comment and privilege columns.
The statement produces a result table with the following structure:
-- field - The name of the column (String)
-- type - The column data type (String)
-- null - `YES` if the column data type is Nullable, `NO` otherwise (String)
-- key - `PRI` if the column is part of the primary key, `SOR` if the column is part of the sorting key, empty otherwise (String)
-- default - Default expression of the column if it is of type `ALIAS`, `DEFAULT`, or `MATERIALIZED`, otherwise `NULL`. (Nullable(String))
-- extra - Additional information, currently unused (String)
-- collation - (only if `FULL` keyword was specified) Collation of the column, always `NULL` because ClickHouse has no per-column collations (Nullable(String))
-- comment - (only if `FULL` keyword was specified) Comment on the column (String)
-- privilege - (only if `FULL` keyword was specified) The privilege you have on this column, currently not available (String)
+- `field` - The name of the column (String)
+- `type` - The column data type. If setting `[use_mysql_types_in_show_columns](../../operations/settings/settings.md#use_mysql_types_in_show_columns) = 1` (default: 0), then the equivalent type name in MySQL is shown. (String)
+- `null` - `YES` if the column data type is Nullable, `NO` otherwise (String)
+- `key` - `PRI` if the column is part of the primary key, `SOR` if the column is part of the sorting key, empty otherwise (String)
+- `default` - Default expression of the column if it is of type `ALIAS`, `DEFAULT`, or `MATERIALIZED`, otherwise `NULL`. (Nullable(String))
+- `extra` - Additional information, currently unused (String)
+- `collation` - (only if `FULL` keyword was specified) Collation of the column, always `NULL` because ClickHouse has no per-column collations (Nullable(String))
+- `comment` - (only if `FULL` keyword was specified) Comment on the column (String)
+- `privilege` - (only if `FULL` keyword was specified) The privilege you have on this column, currently not available (String)
**Examples**
@@ -286,21 +286,21 @@ equivalent. If no database is specified, the query assumes the current database
The optional keyword `EXTENDED` currently has no effect, it only exists for MySQL compatibility.
The statement produces a result table with the following structure:
-- table - The name of the table. (String)
-- non_unique - Always `1` as ClickHouse does not support uniqueness constraints. (UInt8)
-- key_name - The name of the index, `PRIMARY` if the index is a primary key index. (String)
-- seq_in_index - For a primary key index, the position of the column starting from `1`. For a data skipping index: always `1`. (UInt8)
-- column_name - For a primary key index, the name of the column. For a data skipping index: `''` (empty string), see field "expression". (String)
-- collation - The sorting of the column in the index: `A` if ascending, `D` if descending, `NULL` if unsorted. (Nullable(String))
-- cardinality - An estimation of the index cardinality (number of unique values in the index). Currently always 0. (UInt64)
-- sub_part - Always `NULL` because ClickHouse does not support index prefixes like MySQL. (Nullable(String))
-- packed - Always `NULL` because ClickHouse does not support packed indexes (like MySQL). (Nullable(String))
-- null - Currently unused
-- index_type - The index type, e.g. `PRIMARY`, `MINMAX`, `BLOOM_FILTER` etc. (String)
-- comment - Additional information about the index, currently always `''` (empty string). (String)
-- index_comment - `''` (empty string) because indexes in ClickHouse cannot have a `COMMENT` field (like in MySQL). (String)
-- visible - If the index is visible to the optimizer, always `YES`. (String)
-- expression - For a data skipping index, the index expression. For a primary key index: `''` (empty string). (String)
+- `table` - The name of the table. (String)
+- `non_unique` - Always `1` as ClickHouse does not support uniqueness constraints. (UInt8)
+- `key_name` - The name of the index, `PRIMARY` if the index is a primary key index. (String)
+- `seq_in_index` - For a primary key index, the position of the column starting from `1`. For a data skipping index: always `1`. (UInt8)
+- `column_name` - For a primary key index, the name of the column. For a data skipping index: `''` (empty string), see field "expression". (String)
+- `collation` - The sorting of the column in the index: `A` if ascending, `D` if descending, `NULL` if unsorted. (Nullable(String))
+- `cardinality` - An estimation of the index cardinality (number of unique values in the index). Currently always 0. (UInt64)
+- `sub_part` - Always `NULL` because ClickHouse does not support index prefixes like MySQL. (Nullable(String))
+- `packed` - Always `NULL` because ClickHouse does not support packed indexes (like MySQL). (Nullable(String))
+- `null` - Currently unused
+- `index_type` - The index type, e.g. `PRIMARY`, `MINMAX`, `BLOOM_FILTER` etc. (String)
+- `comment` - Additional information about the index, currently always `''` (empty string). (String)
+- `index_comment` - `''` (empty string) because indexes in ClickHouse cannot have a `COMMENT` field (like in MySQL). (String)
+- `visible` - If the index is visible to the optimizer, always `YES`. (String)
+- `expression` - For a data skipping index, the index expression. For a primary key index: `''` (empty string). (String)
**Examples**
diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index 4e1e7aa6a491..1558e64f99b2 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -119,6 +119,18 @@ The compiled expression cache is enabled/disabled with the query/user/profile-le
Clears the [query cache](../../operations/query-cache.md).
+## DROP FORMAT SCHEMA CACHE {#system-drop-schema-format}
+
+Clears cache for schemas loaded from [format_schema_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path).
+
+Supported formats:
+
+- Protobuf
+
+```sql
+SYSTEM DROP FORMAT SCHEMA CACHE [FOR Protobuf]
+```
+
## FLUSH LOGS
Flushes buffered log messages to system tables, e.g. system.query_log. Mainly useful for debugging since most system tables have a default flush interval of 7.5 seconds.
diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md
index 4b46210aa09d..029815a43922 100644
--- a/docs/en/sql-reference/statements/truncate.md
+++ b/docs/en/sql-reference/statements/truncate.md
@@ -25,7 +25,7 @@ If the `alter_sync` is set to `2` and some replicas are not active for more than
## TRUNCATE DATABASE
``` sql
-TRUNCATE DATBASE [IF EXISTS] [db.]name [ON CLUSTER cluster]
+TRUNCATE DATABASE [IF EXISTS] [db.]name [ON CLUSTER cluster]
```
Removes all tables from a database but keeps the database itself. When the clause `IF EXISTS` is omitted, the query returns an error if the database does not exist.
diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md
index 07addafcf58e..8649295e8150 100644
--- a/docs/en/sql-reference/table-functions/s3.md
+++ b/docs/en/sql-reference/table-functions/s3.md
@@ -162,6 +162,28 @@ The below get data from all `test-data.csv.gz` files from any folder inside `my-
SELECT * FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip');
```
+Note. It is possible to specify custom URL mappers in the server configuration file. Example:
+``` sql
+SELECT * FROM s3('s3://clickhouse-public-datasets/my-test-bucket-768/**/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip');
+```
+The URL `'s3://clickhouse-public-datasets/my-test-bucket-768/**/test-data.csv.gz'` would be replaced to `'http://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/**/test-data.csv.gz'`
+
+
+Custom mapper can be added into `config.xml`:
+``` xml
+
+
+ https://{bucket}.s3.amazonaws.com
+
+
+ https://{bucket}.storage.googleapis.com
+
+
+ https://{bucket}.oss.aliyuncs.com
+
+
+```
+
## Partitioned Write
If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency.
diff --git a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md
index cd8b55b0259e..e8089b2c42b7 100644
--- a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md
+++ b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md
@@ -20,7 +20,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1],
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2],
...
-) ENGINE = ReplacingMergeTree([ver])
+) ENGINE = ReplacingMergeTree([ver [, is_deleted]])
[PARTITION BY expr]
[ORDER BY expr]
[SAMPLE BY expr]
@@ -86,6 +86,59 @@ SELECT * FROM mySecondReplacingMT FINAL;
│ 1 │ first │ 2020-01-01 01:01:01 │
└─────┴─────────┴─────────────────────┘
```
+### is_deleted
+
+`is_deleted` — Имя столбца, который используется во время слияния для обозначения того, нужно ли отображать строку или она подлежит удалению; `1` - для удаления строки, `0` - для отображения строки.
+
+ Тип данных столбца — `UInt8`.
+
+:::note
+`is_deleted` может быть использован, если `ver` используется.
+
+Строка удаляется в следующих случаях:
+
+ - при использовании инструкции `OPTIMIZE ... FINAL CLEANUP`
+ - при использовании инструкции `OPTIMIZE ... FINAL`
+ - параметр движка `clean_deleted_rows` установлен в значение `Always` (по умолчанию - `Never`)
+ - есть новые версии строки
+
+Не рекомендуется выполнять `FINAL CLEANUP` или использовать параметр движка `clean_deleted_rows` со значением `Always`, это может привести к неожиданным результатам, например удаленные строки могут вновь появиться.
+
+Вне зависимости от производимых изменений над данными, версия должна увеличиваться. Если у двух строк одна и та же версия, то остается только последняя вставленная строка.
+:::
+
+Пример:
+
+```sql
+-- with ver and is_deleted
+CREATE OR REPLACE TABLE myThirdReplacingMT
+(
+ `key` Int64,
+ `someCol` String,
+ `eventTime` DateTime,
+ `is_deleted` UInt8
+)
+ENGINE = ReplacingMergeTree(eventTime, is_deleted)
+ORDER BY key;
+
+INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 0);
+INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 1);
+
+select * from myThirdReplacingMT final;
+
+0 rows in set. Elapsed: 0.003 sec.
+
+-- delete rows with is_deleted
+OPTIMIZE TABLE myThirdReplacingMT FINAL CLEANUP;
+
+INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 00:00:00', 0);
+
+select * from myThirdReplacingMT final;
+
+┌─key─┬─someCol─┬───────────eventTime─┬─is_deleted─┐
+│ 1 │ first │ 2020-01-01 00:00:00 │ 0 │
+└─────┴─────────┴─────────────────────┴────────────┘
+```
## Секции запроса
diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md
index ae3f100a6e89..16927408bc43 100644
--- a/docs/ru/interfaces/http.md
+++ b/docs/ru/interfaces/http.md
@@ -50,7 +50,7 @@ Connection: Close
Content-Type: text/tab-separated-values; charset=UTF-8
X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal
X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f
-X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
1
```
@@ -267,9 +267,9 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
Прогресс выполнения запроса можно отслеживать с помощью заголовков ответа `X-ClickHouse-Progress`. Для этого включите [send_progress_in_http_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Пример последовательности заголовков:
``` text
-X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","elapsed_ns":"662334","peak_memory_usage":"4371480"}
-X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","elapsed_ns":"992334","peak_memory_usage":"13621616"}
-X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","elapsed_ns":"1232334","peak_memory_usage":"23155600"}
+X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","elapsed_ns":"662334"}
+X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","elapsed_ns":"992334"}
+X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","elapsed_ns":"1232334"}
```
Возможные поля заголовка:
@@ -530,7 +530,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
* Connection #0 to host localhost left intact
Say Hi!%
@@ -570,7 +570,7 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
< Content-Type: text/plain; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
* Connection #0 to host localhost left intact
%
@@ -622,7 +622,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
Absolute Path File
* Connection #0 to host localhost left intact
@@ -641,7 +641,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
Relative Path File
* Connection #0 to host localhost left intact
diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md
index 08961e8dd2e4..a9210964f3ef 100644
--- a/docs/ru/interfaces/third-party/client-libraries.md
+++ b/docs/ru/interfaces/third-party/client-libraries.md
@@ -37,6 +37,8 @@ ClickHouse Inc. не поддерживает перечисленные ниж
- [nestjs-clickhouse](https://github.com/depyronick/nestjs-clickhouse)
- [clickhouse-client](https://github.com/depyronick/clickhouse-client)
- [node-clickhouse-orm](https://github.com/zimv/node-clickhouse-orm)
+ - [clickhouse-ts](https://github.com/bytadaniel/clickhouse-ts)
+ - [clickcache](https://github.com/bytadaniel/clickcache)
- Perl
- [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
- [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md
index 9c1f25bf6d2c..76cfdb251dc6 100644
--- a/docs/ru/sql-reference/statements/create/user.md
+++ b/docs/ru/sql-reference/statements/create/user.md
@@ -13,7 +13,7 @@ sidebar_label: "Пользователь"
``` sql
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
- [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}]
+ [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'}]
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[DEFAULT ROLE role [,...]]
[DEFAULT DATABASE database | NONE]
@@ -27,14 +27,19 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
Существует несколько способов идентификации пользователя:
-- `IDENTIFIED WITH no_password`
-- `IDENTIFIED WITH plaintext_password BY 'qwerty'`
-- `IDENTIFIED WITH sha256_password BY 'qwerty'` or `IDENTIFIED BY 'password'`
-- `IDENTIFIED WITH sha256_hash BY 'hash'` or `IDENTIFIED WITH sha256_hash BY 'hash' SALT 'salt'`
-- `IDENTIFIED WITH double_sha1_password BY 'qwerty'`
-- `IDENTIFIED WITH double_sha1_hash BY 'hash'`
-- `IDENTIFIED WITH ldap SERVER 'server_name'`
-- `IDENTIFIED WITH kerberos` or `IDENTIFIED WITH kerberos REALM 'realm'`
+- `IDENTIFIED WITH no_password`
+- `IDENTIFIED WITH plaintext_password BY 'qwerty'`
+- `IDENTIFIED WITH sha256_password BY 'qwerty'` or `IDENTIFIED BY 'password'`
+- `IDENTIFIED WITH sha256_hash BY 'hash'` or `IDENTIFIED WITH sha256_hash BY 'hash' SALT 'salt'`
+- `IDENTIFIED WITH double_sha1_password BY 'qwerty'`
+- `IDENTIFIED WITH double_sha1_hash BY 'hash'`
+- `IDENTIFIED WITH bcrypt_password BY 'qwerty'`
+- `IDENTIFIED WITH bcrypt_hash BY 'hash'`
+- `IDENTIFIED WITH ldap SERVER 'server_name'`
+- `IDENTIFIED WITH kerberos` or `IDENTIFIED WITH kerberos REALM 'realm'`
+- `IDENTIFIED WITH ssl_certificate CN 'mysite.com:user'`
+- `IDENTIFIED WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa', KEY 'another_public_key' TYPE 'ssh-ed25519'`
+- `IDENTIFIED BY 'qwerty'`
Для идентификации с sha256_hash используя `SALT` - хэш должен быть вычислен от конкатенации 'password' и 'salt'.
diff --git a/docs/zh/engines/table-engines/integrations/hive.md b/docs/zh/engines/table-engines/integrations/hive.md
index fa1bf959ed64..e7af43863e31 100644
--- a/docs/zh/engines/table-engines/integrations/hive.md
+++ b/docs/zh/engines/table-engines/integrations/hive.md
@@ -61,7 +61,7 @@ PARTITION BY expr
- limit_size: 必需的。本地缓存文件的最大大小(单位为字节)。
- bytes_read_before_flush: 从远程文件系统下载文件时,刷新到本地文件系统前的控制字节数。缺省值为1MB。
-当ClickHouse为远程文件系统启用了本地缓存时,用户仍然可以选择不使用缓存,并在查询中设置`use_local_cache_for_remote_fs = 0 `, `use_local_cache_for_remote_fs` 默认为 `false`。
+当ClickHouse为远程文件系统启用了本地缓存时,用户仍然可以选择不使用缓存,并在查询中设置 `use_local_cache_for_remote_storage = 0`, `use_local_cache_for_remote_storage` 默认为 `1`。
### 查询 ORC 输入格式的Hive 表
diff --git a/docs/zh/interfaces/http.md b/docs/zh/interfaces/http.md
index b1501cbb5d66..dfdcf53bd3ff 100644
--- a/docs/zh/interfaces/http.md
+++ b/docs/zh/interfaces/http.md
@@ -53,7 +53,7 @@ Connection: Close
Content-Type: text/tab-separated-values; charset=UTF-8
X-ClickHouse-Server-Display-Name: clickhouse.ru-central1.internal
X-ClickHouse-Query-Id: 5abe861c-239c-467f-b955-8a201abb8b7f
-X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
1
```
@@ -262,9 +262,9 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812
您可以在`X-ClickHouse-Progress`响应头中收到查询进度的信息。为此,启用[Http Header携带进度](../operations/settings/settings.md#settings-send_progress_in_http_headers)。示例:
``` text
-X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","elapsed_ns":"662334","peak_memory_usage":"4371480"}
-X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","elapsed_ns":"992334","peak_memory_usage":"13621616"}
-X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","elapsed_ns":"1232334","peak_memory_usage":"23155600"}
+X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128","elapsed_ns":"662334"}
+X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128","elapsed_ns":"992334"}
+X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128","elapsed_ns":"1232334"}
```
显示字段信息:
@@ -363,7 +363,7 @@ $ curl -v 'http://localhost:8123/predefined_query'
< X-ClickHouse-Format: Template
< X-ClickHouse-Timezone: Asia/Shanghai
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
# HELP "Query" "Number of executing queries"
# TYPE "Query" counter
@@ -521,7 +521,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/hi'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
* Connection #0 to host localhost left intact
Say Hi!%
@@ -561,7 +561,7 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
< Content-Type: text/plain; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
* Connection #0 to host localhost left intact
%
@@ -613,7 +613,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_absolute_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
Absolute Path File
* Connection #0 to host localhost left intact
@@ -632,7 +632,7 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler'
< Content-Type: text/html; charset=UTF-8
< Transfer-Encoding: chunked
< Keep-Alive: timeout=3
-< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334","peak_memory_usage":"0"}
+< X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","written_bytes":"0","total_rows_to_read":"0","elapsed_ns":"662334"}
<
Relative Path File
* Connection #0 to host localhost left intact
diff --git a/docs/zh/interfaces/third-party/client-libraries.md b/docs/zh/interfaces/third-party/client-libraries.md
index 4ee74f1d2c39..ed7825486be0 100644
--- a/docs/zh/interfaces/third-party/client-libraries.md
+++ b/docs/zh/interfaces/third-party/client-libraries.md
@@ -39,6 +39,8 @@ Yandex**没有**维护下面列出的库,也没有做过任何广泛的测试
- [nestjs-clickhouse](https://github.com/depyronick/nestjs-clickhouse)
- [clickhouse-client](https://github.com/depyronick/clickhouse-client)
- [node-clickhouse-orm](https://github.com/zimv/node-clickhouse-orm)
+ - [clickhouse-ts](https://github.com/bytadaniel/clickhouse-ts)
+ - [clickcache](https://github.com/bytadaniel/clickcache)
- Perl
- [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
- [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
diff --git a/packages/.gitignore b/packages/.gitignore
index 355164c12651..69cf3a785e4f 100644
--- a/packages/.gitignore
+++ b/packages/.gitignore
@@ -1 +1,2 @@
*/
+!pre-build/
diff --git a/packages/build b/packages/build
index 6ec991aad079..cc089dd08ad4 100755
--- a/packages/build
+++ b/packages/build
@@ -84,8 +84,8 @@ function deb2tgz {
FILE=$1
PKG_NAME=${FILE##*/}; PKG_NAME=${PKG_NAME%%_*}
PKG_DIR="$PKG_NAME-$CLICKHOUSE_VERSION_STRING"
- PKG_PATH="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING"
- TARBALL="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING-$DEB_ARCH.tgz"
+ PKG_PATH="$OUTPUT_DIR/$PKG_DIR"
+ TARBALL="$OUTPUT_DIR/$PKG_DIR-$DEB_ARCH.tgz"
rm -rf "$PKG_PATH"
dpkg-deb -R "$FILE" "$PKG_PATH"
mkdir -p "$PKG_PATH/install"
diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml
index d4fd9300208e..4d707b28ad90 100644
--- a/packages/clickhouse-client.yaml
+++ b/packages/clickhouse-client.yaml
@@ -1,6 +1,13 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-client"
+description: |
+ Client binary for ClickHouse
+ ClickHouse is a column-oriented database management system.
+ that allows generating analytical data reports in real time.
+ This package provides clickhouse-client, clickhouse-local and clickhouse-benchmark.
+
+# Common packages config
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
@@ -9,19 +16,17 @@ homepage: "https://clickhouse.com"
license: "Apache"
section: "database"
priority: "optional"
+maintainer: "ClickHouse Dev Team "
+deb:
+ fields:
+ Source: clickhouse
+# Package specific content
replaces:
- clickhouse-compressor
conflicts:
- clickhouse-compressor
-maintainer: "ClickHouse Dev Team "
-description: |
- Client binary for ClickHouse
- ClickHouse is a column-oriented database management system.
- that allows generating analytical data reports in real time.
- This package provides clickhouse-client, clickhouse-local and clickhouse-benchmark.
-
overrides:
deb:
depends:
@@ -30,10 +35,6 @@ overrides:
depends:
- clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING}
-deb:
- fields:
- Source: clickhouse
-
contents:
- src: root/etc/clickhouse-client/config.xml
dst: /etc/clickhouse-client/config.xml
diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml
index b2d2b3aaf262..96de4c17d88f 100644
--- a/packages/clickhouse-common-static-dbg.yaml
+++ b/packages/clickhouse-common-static-dbg.yaml
@@ -1,6 +1,13 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-common-static-dbg"
+description: |
+ debugging symbols for clickhouse-common-static
+ This package contains the debugging symbols for clickhouse-common.
+
+
+#
+# Common packages config
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
@@ -9,21 +16,17 @@ homepage: "https://clickhouse.com"
license: "Apache"
section: "database"
priority: "optional"
+maintainer: "ClickHouse Dev Team "
+deb:
+ fields:
+ Source: clickhouse
+# Package specific content
replaces:
- clickhouse-common-dbg
conflicts:
- clickhouse-common-dbg
-maintainer: "ClickHouse Dev Team "
-description: |
- debugging symbols for clickhouse-common-static
- This package contains the debugging symbols for clickhouse-common.
-
-deb:
- fields:
- Source: clickhouse
-
contents:
- src: root/usr/lib/debug/usr/bin/clickhouse.debug
dst: /usr/lib/debug/usr/bin/clickhouse.debug
diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml
index c77914d0d692..95532726d945 100644
--- a/packages/clickhouse-common-static.yaml
+++ b/packages/clickhouse-common-static.yaml
@@ -1,6 +1,13 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-common-static"
+description: |
+ Common files for ClickHouse
+ ClickHouse is a column-oriented database management system
+ that allows generating analytical data reports in real time.
+ This package provides common files for both clickhouse server and client
+
+# Common packages config
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
@@ -9,7 +16,12 @@ homepage: "https://clickhouse.com"
license: "Apache"
section: "database"
priority: "optional"
+maintainer: "ClickHouse Dev Team "
+deb:
+ fields:
+ Source: clickhouse
+# Package specific content
replaces:
- clickhouse-common
- clickhouse-server-base
@@ -19,17 +31,6 @@ provides:
suggests:
- clickhouse-common-static-dbg
-maintainer: "ClickHouse Dev Team "
-description: |
- Common files for ClickHouse
- ClickHouse is a column-oriented database management system
- that allows generating analytical data reports in real time.
- This package provides common files for both clickhouse server and client
-
-deb:
- fields:
- Source: clickhouse
-
contents:
- src: root/usr/bin/clickhouse
dst: /usr/bin/clickhouse
diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml
index a6be9ec9e971..28d53b39518d 100644
--- a/packages/clickhouse-keeper-dbg.yaml
+++ b/packages/clickhouse-keeper-dbg.yaml
@@ -1,6 +1,13 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-keeper-dbg"
+description: |
+ debugging symbols for clickhouse-keeper
+ This package contains the debugging symbols for clickhouse-keeper.
+
+
+#
+# Common packages config
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
@@ -10,14 +17,11 @@ license: "Apache"
section: "database"
priority: "optional"
maintainer: "ClickHouse Dev Team "
-description: |
- debugging symbols for clickhouse-keeper
- This package contains the debugging symbols for clickhouse-keeper.
-
deb:
fields:
Source: clickhouse
+# Package specific content
contents:
- src: root/usr/lib/debug/usr/bin/clickhouse-keeper.debug
dst: /usr/lib/debug/usr/bin/clickhouse-keeper.debug
diff --git a/packages/clickhouse-keeper.postinstall b/packages/clickhouse-keeper.postinstall
index 3d6cd484146e..2b4f303b6849 100644
--- a/packages/clickhouse-keeper.postinstall
+++ b/packages/clickhouse-keeper.postinstall
@@ -3,12 +3,12 @@ set -e
# set -x
PROGRAM=clickhouse-keeper
-KEEPER_USER=${KEEPER_USER:=clickhouse}
-KEEPER_GROUP=${KEEPER_GROUP:=clickhouse}
+KEEPER_USER=${KEEPER_USER:-clickhouse}
+KEEPER_GROUP=${KEEPER_GROUP:-clickhouse}
# Please note that we don't support paths with whitespaces. This is rather ignorant.
-KEEPER_CONFDIR=${KEEPER_CONFDIR:=/etc/$PROGRAM}
-KEEPER_DATADIR=${KEEPER_DATADIR:=/var/lib/clickhouse}
-KEEPER_LOGDIR=${KEEPER_LOGDIR:=/var/log/$PROGRAM}
+KEEPER_CONFDIR=${KEEPER_CONFDIR:-/etc/$PROGRAM}
+KEEPER_DATADIR=${KEEPER_DATADIR:-/var/lib/clickhouse}
+KEEPER_LOGDIR=${KEEPER_LOGDIR:-/var/log/$PROGRAM}
[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule
[ -f /etc/default/clickhouse-keeper ] && . /etc/default/clickhouse-keeper
diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml
index 48037964cfa3..9dad5382c082 100644
--- a/packages/clickhouse-keeper.yaml
+++ b/packages/clickhouse-keeper.yaml
@@ -1,6 +1,13 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-keeper"
+description: |
+ Static clickhouse-keeper binary
+ A stand-alone clickhouse-keeper package
+
+
+#
+# Common packages config
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
@@ -9,29 +16,25 @@ homepage: "https://clickhouse.com"
license: "Apache"
section: "database"
priority: "optional"
+maintainer: "ClickHouse Dev Team "
+deb:
+ fields:
+ Source: clickhouse
+# Package specific content
conflicts:
- clickhouse-server
suggests:
- clickhouse-keeper-dbg
-maintainer: "ClickHouse Dev Team "
-description: |
- Static clickhouse-keeper binary
- A stand-alone clickhouse-keeper package
-
-deb:
- fields:
- Source: clickhouse
-
contents:
- src: root/etc/clickhouse-keeper/keeper_config.xml
dst: /etc/clickhouse-keeper/keeper_config.xml
type: config|noreplace
-- src: root/usr/bin/clickhouse-keeper
- dst: /usr/bin/clickhouse-keeper
- src: clickhouse-keeper.service
dst: /lib/systemd/system/clickhouse-keeper.service
+- src: root/usr/bin/clickhouse-keeper
+ dst: /usr/bin/clickhouse-keeper
- src: clickhouse-keeper
dst: /usr/bin/clickhouse-keeper-client
type: symlink
diff --git a/packages/clickhouse-server.postinstall b/packages/clickhouse-server.postinstall
index 2b9830faf3b0..d3b49db758f0 100644
--- a/packages/clickhouse-server.postinstall
+++ b/packages/clickhouse-server.postinstall
@@ -3,16 +3,21 @@ set -e
# set -x
PROGRAM=clickhouse-server
-CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse}
-CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}}
+CLICKHOUSE_USER=${CLICKHOUSE_USER:-clickhouse}
+CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:-${CLICKHOUSE_USER}}
# Please note that we don't support paths with whitespaces. This is rather ignorant.
-CLICKHOUSE_CONFDIR=${CLICKHOUSE_CONFDIR:=/etc/clickhouse-server}
-CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:=/var/lib/clickhouse}
-CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:=/var/log/clickhouse-server}
-CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin}
-CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse}
+CLICKHOUSE_CONFDIR=${CLICKHOUSE_CONFDIR:-/etc/clickhouse-server}
+CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:-/var/lib/clickhouse}
+CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:-/var/log/clickhouse-server}
+CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:-/usr/bin}
+CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:-clickhouse}
CLICKHOUSE_PIDDIR=/var/run/$PROGRAM
+# Provide clickhouse-keeper
+KEEPER_CONFDIR=${KEEPER_CONFDIR:-/etc/clickhouse-keeper}
+KEEPER_DATADIR=${KEEPER_DATADIR:-/var/lib/clickhouse}
+KEEPER_LOGDIR=${KEEPER_LOGDIR:-/var/log/clickhouse-keeper}
+
[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule
[ -f /etc/default/clickhouse ] && . /etc/default/clickhouse
@@ -54,4 +59,20 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
fi
done
fi
+
+ # Setup clickhouse-keeper directories
+ chown -R "${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}" "${KEEPER_CONFDIR}"
+ chmod 0755 "${KEEPER_CONFDIR}"
+
+ if ! [ -d "${KEEPER_DATADIR}" ]; then
+ mkdir -p "${KEEPER_DATADIR}"
+ chown -R "${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}" "${KEEPER_DATADIR}"
+ chmod 0700 "${KEEPER_DATADIR}"
+ fi
+
+ if ! [ -d "${KEEPER_LOGDIR}" ]; then
+ mkdir -p "${KEEPER_LOGDIR}"
+ chown -R "${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}" "${KEEPER_LOGDIR}"
+ chmod 0770 "${KEEPER_LOGDIR}"
+ fi
fi
diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml
index 66299fddd4aa..5e2bc7c74125 100644
--- a/packages/clickhouse-server.yaml
+++ b/packages/clickhouse-server.yaml
@@ -1,6 +1,13 @@
# package sources should be placed in ${PWD}/root
# nfpm should run from the same directory with a config
name: "clickhouse-server"
+description: |
+ Server binary for ClickHouse
+ ClickHouse is a column-oriented database management system
+ that allows generating analytical data reports in real time.
+ This package provides clickhouse common configuration files
+
+# Common packages config
arch: "${DEB_ARCH}" # amd64, arm64
platform: "linux"
version: "${CLICKHOUSE_VERSION_STRING}"
@@ -9,24 +16,21 @@ homepage: "https://clickhouse.com"
license: "Apache"
section: "database"
priority: "optional"
+maintainer: "ClickHouse Dev Team "
+deb:
+ fields:
+ Source: clickhouse
-conflicts:
-- clickhouse-keeper
+# Package specific content
replaces:
- clickhouse-server-common
- clickhouse-server-base
provides:
+- clickhouse-keeper
- clickhouse-server-common
recommends:
- libcap2-bin
-maintainer: "ClickHouse Dev Team "
-description: |
- Server binary for ClickHouse
- ClickHouse is a column-oriented database management system
- that allows generating analytical data reports in real time.
- This package provides clickhouse common configuration files
-
overrides:
deb:
depends:
@@ -35,10 +39,6 @@ overrides:
depends:
- clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING}
-deb:
- fields:
- Source: clickhouse
-
contents:
- src: root/etc/clickhouse-server/config.xml
dst: /etc/clickhouse-server/config.xml
@@ -52,16 +52,25 @@ contents:
dst: /lib/systemd/system/clickhouse-server.service
- src: root/usr/bin/clickhouse-copier
dst: /usr/bin/clickhouse-copier
+- src: root/usr/bin/clickhouse-report
+ dst: /usr/bin/clickhouse-report
+- src: root/usr/bin/clickhouse-server
+ dst: /usr/bin/clickhouse-server
+# clickhouse-keeper part
+- src: root/etc/clickhouse-keeper/keeper_config.xml
+ dst: /etc/clickhouse-keeper/keeper_config.xml
+ type: config|noreplace
+- src: clickhouse-keeper.service
+ dst: /lib/systemd/system/clickhouse-keeper.service
- src: clickhouse
dst: /usr/bin/clickhouse-keeper
type: symlink
- src: clickhouse
dst: /usr/bin/clickhouse-keeper-client
type: symlink
-- src: root/usr/bin/clickhouse-report
- dst: /usr/bin/clickhouse-report
-- src: root/usr/bin/clickhouse-server
- dst: /usr/bin/clickhouse-server
+- src: clickhouse
+ dst: /usr/bin/clickhouse-keeper-converter
+ type: symlink
# docs
- src: ../AUTHORS
dst: /usr/share/doc/clickhouse-server/AUTHORS
diff --git a/packages/pre-build/example.sh b/packages/pre-build/example.sh
new file mode 100755
index 000000000000..2e361d6ca44b
--- /dev/null
+++ b/packages/pre-build/example.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "This is an example pre-build script!"
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index 64823f9ec7f1..9f316e54f859 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -125,7 +125,7 @@ void Client::showWarnings()
std::cout << std::endl;
}
}
- catch (...)
+ catch (...) // NOLINT(bugprone-empty-catch)
{
/// Ignore exception
}
@@ -706,6 +706,17 @@ bool Client::processWithFuzzing(const String & full_query)
return true;
}
+ // Kusto is not a subject for fuzzing (yet)
+ if (global_context->getSettingsRef().dialect == DB::Dialect::kusto)
+ {
+ return true;
+ }
+ if (auto *q = orig_ast->as())
+ {
+ if (auto *setDialect = q->changes.tryGet("dialect"); setDialect && setDialect->safeGet() == "kusto")
+ return true;
+ }
+
// Don't repeat:
// - INSERT -- Because the tables may grow too big.
// - CREATE -- Because first we run the unmodified query, it will succeed,
@@ -994,6 +1005,8 @@ void Client::addOptions(OptionsDescription & options_description)
("user,u", po::value()->default_value("default"), "user")
("password", po::value(), "password")
("ask-password", "ask-password")
+ ("ssh-key-file", po::value(), "File containing ssh private key needed for authentication. If not set does password authentication.")
+ ("ssh-key-passphrase", po::value(), "Passphrase for imported ssh key.")
("quota_key", po::value(), "A string to differentiate quotas when the user have keyed quotas configured on server")
("max_client_network_bandwidth", po::value