diff --git a/clickhouse-25.3.yaml b/clickhouse-25.3.yaml new file mode 100644 index 00000000000..984e2616802 --- /dev/null +++ b/clickhouse-25.3.yaml @@ -0,0 +1,456 @@ +package: + name: clickhouse-25.3 + version: "25.3.6.56" + epoch: 0 + description: ClickHouse is the fastest and most resource efficient open-source database for real-time apps and analytics. + copyright: + - license: Apache-2.0 + resources: + cpu: 65 + memory: 128Gi + dependencies: + provides: + - clickhouse=${{package.full-version}} + runtime: + - merged-usrsbin + - wolfi-baselayout + +var-transforms: + - from: ${{package.version}} + match: ^(\d+).* + replace: $1 + to: major-version + +environment: + contents: + packages: + - bash + - build-base + - busybox + - ca-certificates-bundle + - clang-19 + - cmake + - coreutils + - findutils + - git + - grep + - ini-file + - libcxx1 + - lld-19 + - nasm + - ninja + - perl + - python3 + - shadow + - xmlstarlet + - yasm + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/ClickHouse/ClickHouse + tag: v${{package.version}}-lts + expected-commit: 39f0a39e4bba45eea18c12814607719245ba3153 + + # The default build script is defensive and tries to protect against defining cflags. + - uses: patch + with: + patches: allow_cflags.patch + + - runs: | + git submodule update --init + + # Overwrite VERSION_STRING to properly set the version of the binary. + sed -i "s|^SET(VERSION_STRING [^)]*)|SET(VERSION_STRING ${{package.version}})|" cmake/autogenerated_versions.txt + + mkdir build + cd build + + # Reference for cmake options: https://github.com/ClickHouse/ClickHouse/blob/master/ci/jobs/build_clickhouse.py + cmake \ + -DCOMPILER_CACHE=disabled \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=RELEASE \ + -DNO_ARMV81_OR_HIGHER=1 \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_CXX_FLAGS="-Wno-error=unused-result" \ + -DVERSION_STRING=${{package.version}} \ + -DCLICKHOUSE_OFFICIAL_BUILD=1 \ + .. + + - runs: | + cd build + ninja -j $(nproc) + mkdir -p ${{targets.destdir}}/var/log/clickhouse-server + DESTDIR=${{targets.destdir}} ninja install + rm -rf ${{targets.destdir}}/usr/lib/debug + + - uses: strip + +subpackages: + - name: "${{package.name}}-dev" + description: "headers for clickhouse" + pipeline: + - uses: split/dev + dependencies: + runtime: + - merged-usrsbin + - wolfi-baselayout + + - name: "${{package.name}}-bash-completion" + description: "bash completion for clickhouse" + pipeline: + - runs: | + mkdir -p ${{targets.subpkgdir}}/usr/share/bash-completion/completions + mv ${{targets.destdir}}/usr/share/bash-completion/completions/clickhouse ${{targets.subpkgdir}}/usr/share/bash-completion/completions + dependencies: + runtime: + - merged-usrsbin + - wolfi-baselayout + + - name: "${{package.name}}-compat" + description: "docker compat for clickhouse" + pipeline: + - runs: | + cd build + install -Dm755 ../docker/server/entrypoint.sh ${{targets.subpkgdir}}/entrypoint.sh + mkdir -p ${{targets.subpkgdir}}/etc/clickhouse-server/config.d/ + # users.d dir required by entrypoint script + mkdir -p ${{targets.subpkgdir}}/etc/clickhouse-server/users.d/ + cp ../docker/server/docker_related_config.xml ${{targets.subpkgdir}}/etc/clickhouse-server/config.d/docker_related_config.xml + dependencies: + runtime: + - merged-usrsbin + - wolfi-baselayout + + - name: ${{package.name}}-bitnami-compat + dependencies: + provides: + - clickhouse-bitnami-compat=${{package.full-version}} + runtime: + - bash + - busybox + - coreutils + - merged-usrsbin + - wolfi-baselayout + - xmlstarlet + pipeline: + - uses: bitnami/compat + with: + image: clickhouse + version-path: 25/debian-12 + - runs: | + set -x + mkdir -p ${{targets.contextdir}}/bitnami/clickhouse/etc + mkdir -p ${{targets.contextdir}}/bitnami/clickhouse/data + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/etc.default + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/etc/config.d + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/etc/conf.d + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/etc/users.d + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/bin + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/logs + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/tmp + mkdir -p ${{targets.contextdir}}/opt/bitnami/clickhouse/licenses + mkdir -p ${{targets.contextdir}}/var/log/clickhouse-server + + mkdir ${{targets.contextdir}}/docker-entrypoint-initdb.d + mkdir ${{targets.contextdir}}/docker-entrypoint-startdb.d + + install -m755 ${{targets.destdir}}/etc/clickhouse-keeper/keeper_config.xml ${{targets.contextdir}}/opt/bitnami/clickhouse/etc/keeper_config.xml + install -m755 ${{targets.destdir}}/etc/clickhouse-server/users.xml ${{targets.contextdir}}/opt/bitnami/clickhouse/etc/users.xml + install -m755 ${{targets.destdir}}/etc/clickhouse-server/config.xml ${{targets.contextdir}}/opt/bitnami/clickhouse/etc/config.xml + + mkdir -p ${{targets.contextdir}}/var/log/ + + # Disable some commands used in Bitnami scripts. These commands more likely fail in this since this image take non root approach + # sed -i 's/owned_by "$dir" "$owner_user" "$owner_group"/continue/g' ${{targets.contextdir}}/opt/bitnami/scripts/libfs.sh + sed -i 's/ensure_user_exists/# ensure_user_exists/g' ${{targets.contextdir}}/opt/bitnami/scripts/clickhouse/postunpack.sh + # sed -i 's/am_i_root/# am_i_root/g' ${{targets.contextdir}}/opt/bitnami/scripts/clickhouse/setup.sh + + # The `--userspec`` flag belongs to GNU's chroot, whereas we are use BusyBox's. As a workaround, use `su-exec` instead. + sed -i 's|exec chroot --userspec="$userspec" /|exec chroot / su-exec "$userspec"|' ${{targets.contextdir}}/opt/bitnami/scripts/libos.sh + sed -i 's|chroot --userspec="$userspec" /|chroot / su-exec "$userspec"|' ${{targets.contextdir}}/opt/bitnami/scripts/libos.sh + + # Use package path while unpacking + find . -iname "*.sh" -exec sed 's#/opt/bitnami#${{targets.contextdir}}/opt/bitnami#g' -i {} \; + find . -iname "*.sh" -exec sed -i '/chown/c\continue' -i {} \; + find . -iname "*.sh" -exec sed 's#/BITNAMI_VOLUME_DIR="/bitnami"#BITNAMI_VOLUME_DIR="${{targets.contextdir}}/bitnami"#g' -i {} \; + find . -iname "*.sh" -exec sed 's#"/bitnami/clickhouse"#"${{targets.contextdir}}/bitnami/clickhouse"#g' -i {} \; + + ${{targets.contextdir}}/opt/bitnami/scripts/clickhouse/postunpack.sh + # Restore path + find ${{targets.contextdir}}/opt/bitnami -type f -exec sed 's#${{targets.contextdir}}##g' -i {} \; + + # Find all files in /usr/bin that are either named "clickhouse" or symlinks pointing to "clickhouse" + for file in ${{targets.destdir}}/usr/bin/*; do + if [ -f "$file" ] && [ "$(basename "$file")" = "clickhouse" ]; then + # Found a direct match for "clickhouse" + ln -sf /usr/bin/clickhouse ${{targets.contextdir}}/opt/bitnami/clickhouse/bin/clickhouse + elif [ -L "$file" ]; then + # Check if the symlink points to clickhouse + target=$(readlink -f "$file") + if [ "$(basename "$target")" = "clickhouse" ]; then + link_name=$(basename "$file") + ln -sf /usr/bin/clickhouse "${{targets.contextdir}}/opt/bitnami/clickhouse/bin/$link_name" + fi + fi + done + + ln -s /dev/stdout ${{targets.contextdir}}/var/log/clickhouse-server/clickhouse.log + ln -s /dev/stderr ${{targets.contextdir}}/var/log/clickhouse-server/clickhouse_error.log + mkdir -p ${{targets.contextdir}}/var/lib + ln -s /bitnami/clickhouse/data ${{targets.contextdir}}/var/lib/clickhouse + test: + environment: + contents: + packages: + - ${{package.name}} + pipeline: + - uses: bitnami/validate-welcome-message + with: + app-name: clickhouse + + - name: ${{package.name}}-iamguarded-compat + dependencies: + provides: + - clickhouse-iamguarded-compat=${{package.full-version}} + runtime: + - bash + - busybox + - coreutils + - merged-usrsbin + - wolfi-baselayout + - xmlstarlet + pipeline: + - uses: iamguarded/build-compat + with: + package: clickhouse + version: ${{vars.major-version}} + - runs: | + set -x + mkdir -p /iamguarded/clickhouse/etc + mkdir -p /iamguarded/clickhouse/data + mkdir -p /opt/iamguarded/clickhouse/etc.default + mkdir -p /opt/iamguarded/clickhouse/etc/config.d + mkdir -p /opt/iamguarded/clickhouse/etc/conf.d + mkdir -p /opt/iamguarded/clickhouse/etc/users.d + mkdir -p /opt/iamguarded/clickhouse/bin + mkdir -p /opt/iamguarded/clickhouse/logs + mkdir -p /opt/iamguarded/clickhouse/tmp + mkdir -p /opt/iamguarded/clickhouse/licenses + mkdir -p ${{targets.contextdir}}/var/log/clickhouse-server + mkdir ${{targets.contextdir}}/docker-entrypoint-initdb.d + mkdir ${{targets.contextdir}}/docker-entrypoint-startdb.d + install -m755 ${{targets.destdir}}/etc/clickhouse-keeper/keeper_config.xml /opt/iamguarded/clickhouse/etc/keeper_config.xml + install -m755 ${{targets.destdir}}/etc/clickhouse-server/users.xml /opt/iamguarded/clickhouse/etc/users.xml + install -m755 ${{targets.destdir}}/etc/clickhouse-server/config.xml /opt/iamguarded/clickhouse/etc/config.xml + mkdir -p ${{targets.contextdir}}/var/log/ + # Disable some commands used in iamguarded scripts. These commands more likely fail in this since this image take non root approach + # sed -i 's/owned_by "$dir" "$owner_user" "$owner_group"/continue/g' /opt/iamguarded/scripts/libfs.sh + sed -i 's/ensure_user_exists/# ensure_user_exists/g' /opt/iamguarded/scripts/clickhouse/postunpack.sh + # sed -i 's/am_i_root/# am_i_root/g' /opt/iamguarded/scripts/clickhouse/setup.sh + # The `--userspec`` flag belongs to GNU's chroot, whereas we are use BusyBox's. As a workaround, use `su-exec` instead. + sed -i 's|exec chroot --userspec="$userspec" /|exec chroot / su-exec "$userspec"|' /opt/iamguarded/scripts/libos.sh + sed -i 's|chroot --userspec="$userspec" /|chroot / su-exec "$userspec"|' /opt/iamguarded/scripts/libos.sh + # Use package path while unpacking + find /opt/iamguarded/scripts -iname "*.sh" -exec sed -i '/chown/c\continue' -i {} \; + # Remove existing symlinks that might conflict + rm -f /etc/clickhouse-server /var/lib/clickhouse /var/log/clickhouse-server /var/lib/clickhouse/tmp + /opt/iamguarded/scripts/clickhouse/postunpack.sh + # Find all files in /usr/bin that are either named "clickhouse" or symlinks pointing to "clickhouse" + for file in ${{targets.destdir}}/usr/bin/*; do + if [ -f "$file" ] && [ "$(basename "$file")" = "clickhouse" ]; then + # Found a direct match for "clickhouse" + ln -sf /usr/bin/clickhouse /opt/iamguarded/clickhouse/bin/clickhouse + elif [ -L "$file" ]; then + # Check if the symlink points to clickhouse + target=$(readlink -f "$file") + if [ "$(basename "$target")" = "clickhouse" ]; then + link_name=$(basename "$file") + ln -sf /usr/bin/clickhouse "/opt/iamguarded/clickhouse/bin/$link_name" + fi + fi + done + ln -s /dev/stdout ${{targets.contextdir}}/var/log/clickhouse-server/clickhouse.log + ln -s /dev/stderr ${{targets.contextdir}}/var/log/clickhouse-server/clickhouse_error.log + mkdir -p ${{targets.contextdir}}/var/lib + ln -s /iamguarded/clickhouse/data ${{targets.contextdir}}/var/lib/clickhouse + - uses: iamguarded/finalize-compat + with: + package: clickhouse + version: ${{vars.major-version}} + test: + environment: + contents: + packages: + - ${{package.name}} + pipeline: + - uses: iamguarded/test-compat + with: + package: clickhouse + version: ${{vars.major-version}} + + - name: "clickhouse-keeper-25.3" + description: "clickhouse keeper" + dependencies: + provides: + - clickhouse-keeper=${{package.full-version}} + pipeline: + - runs: | + install -Dm755 ${{targets.destdir}}/usr/bin/clickhouse-keeper ${{targets.subpkgdir}}/usr/bin/clickhouse-keeper + + # multicall symlinks + ln -sf /usr/bin/clickhouse-keeper ${{targets.subpkgdir}}/usr/bin/clickhouse-keeper-converter + ln -sf /usr/bin/clickhouse-keeper ${{targets.subpkgdir}}/usr/bin/clickhouse-keeper-client + + - name: clickhouse-keeper-iamguarded-compat-25.3 + dependencies: + provides: + - clickhouse-keeper-iamguarded-compat=${{package.full-version}} + runtime: + - bash + - busybox + - coreutils + - merged-usrsbin + - wolfi-baselayout + - clickhouse-keeper + - xmlstarlet + pipeline: + - uses: iamguarded/build-compat + with: + package: clickhouse-keeper + version: ${{vars.major-version}} + - runs: | + mkdir -p /iamguarded/clickhouse-keeper/etc + mkdir -p /iamguarded/clickhouse-keeper/data + mkdir -p /opt/iamguarded/clickhouse-keeper/etc + mkdir -p /opt/iamguarded/clickhouse-keeper/etc.default + mkdir -p /opt/iamguarded/clickhouse-keeper/bin + mkdir -p /opt/iamguarded/clickhouse-keeper/logs + mkdir -p /opt/iamguarded/clickhouse-keeper/tmp + mkdir -p /opt/iamguarded/clickhouse-keeper/licenses + + mkdir -p ${{targets.contextdir}}/var/log/clickhouse-server + + install -m755 ${{targets.destdir}}/etc/clickhouse-keeper/keeper_config.xml /opt/iamguarded/clickhouse-keeper/etc/keeper_config.xml + + mkdir -p ${{targets.contextdir}}/var/log/ + + sed -i 's/ensure_user_exists/# ensure_user_exists/g' /opt/iamguarded/scripts/clickhouse-keeper/postunpack.sh + + sed -i 's|exec chroot --userspec="$userspec" /|exec chroot / su-exec "$userspec"|' /opt/iamguarded/scripts/libos.sh + sed -i 's|chroot --userspec="$userspec" /|chroot / su-exec "$userspec"|' /opt/iamguarded/scripts/libos.sh + + # Use package path while unpacking + find /opt/iamguarded/scripts -iname "*.sh" -exec sed -i '/chown/c\continue' -i {} \; + # Remove existing symlinks that might conflict + rm -f /etc/clickhouse-keeper /var/lib/clickhouse /var/log/clickhouse-server + /opt/iamguarded/scripts/clickhouse-keeper/postunpack.sh + + # Find all files in /usr/bin that are either named "clickhouse-keeper" or symlinks pointing to "clickhouse-keeper" + for file in ${{targets.destdir}}/usr/bin/*; do + if [ -f "$file" ] && [ "$(basename "$file")" = "clickhouse-keeper" ]; then + # Found a direct match for "clickhouse-keeper" + ln -sf /usr/bin/clickhouse-keeper /opt/iamguarded/clickhouse-keeper/bin/clickhouse-keeper + elif [ -L "$file" ]; then + # Check if the symlink points to clickhouse-keeper + target=$(readlink -f "$file") + if [ "$(basename "$target")" = "clickhouse-keeper" ]; then + link_name=$(basename "$file") + ln -sf /usr/bin/clickhouse-keeper "/opt/iamguarded/clickhouse-keeper/bin/$link_name" + fi + fi + done + - uses: iamguarded/finalize-compat + with: + package: clickhouse-keeper + version: ${{vars.major-version}} + test: + environment: + contents: + packages: + - ${{package.name}} + pipeline: + - uses: iamguarded/test-compat + with: + package: clickhouse-keeper + version: ${{vars.major-version}} + +update: + enabled: true + ignore-regex-patterns: + - '-lts$' + git: + tag-filter-prefix: v25.3. + strip-prefix: v + strip-suffix: -lts + +test: + environment: + accounts: + groups: + - groupname: nonroot + gid: 1001 + users: + - username: nonroot + gid: 1001 + uid: 1001 + run-as: 0 + contents: + packages: + - bash + - coreutils + - findutils + - procps # for checking server process + - curl # for HTTP interface testing + pipeline: + # AUTOGENERATED + - runs: | + ch --version + chc --version + chl --version + clickhouse --version + clickhouse-benchmark --help + clickhouse-client --version + clickhouse-compressor --help + # clickhouse-disks does not support --version + # clickhouse-disks --version + clickhouse-format version + clickhouse-git-import version + clickhouse-keeper --version + clickhouse-keeper-client --help + clickhouse-keeper-converter --help + clickhouse-local --version + clickhouse-obfuscator --help + clickhouse-server --version + clickhouse-static-files-disk-uploader --help + clickhouse-su --version + ch --help + chc --help + chl --help + clickhouse --help + clickhouse-client --help + clickhouse-disks --help + clickhouse-format help + clickhouse-git-import help + clickhouse-keeper --help + clickhouse-local --help + clickhouse-obfuscator version + clickhouse-server --help + clickhouse-static-files-disk-uploader version + clickhouse-su --help + - name: "Test server" + uses: test/daemon-check-output + with: + # NOTE(joshrwolf): This is required because for whatever reason, + # specifying this with melange "paths" isn't working on linux + docker + # runners as of 06/20/2025 + setup: | + chown -R nonroot:nonroot /home/build + start: clickhouse-su nonroot:nonroot clickhouse-server + expected_output: | + Starting ClickHouse + starting up + Scanning + post: | + ./test-daemon.sh diff --git a/clickhouse-25.3/allow_cflags.patch b/clickhouse-25.3/allow_cflags.patch new file mode 100644 index 00000000000..775826aee73 --- /dev/null +++ b/clickhouse-25.3/allow_cflags.patch @@ -0,0 +1,44 @@ +--- a/PreLoad.cmake ++++ b/PreLoad.cmake +@@ -15,41 +15,6 @@ + endif () + endif() + +-# Check if environment is polluted. +-if (NOT "$ENV{CFLAGS}" STREQUAL "" +- OR NOT "$ENV{CXXFLAGS}" STREQUAL "" +- OR NOT "$ENV{LDFLAGS}" STREQUAL "" +- OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS +- OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT) +- +- # if $ENV +- message("CFLAGS: $ENV{CFLAGS}") +- message("CXXFLAGS: $ENV{CXXFLAGS}") +- message("LDFLAGS: $ENV{LDFLAGS}") +- # if *_FLAGS +- message("CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") +- message("CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") +- message("CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}") +- message("CMAKE_SHARED_LINKER_FLAGS: ${CMAKE_SHARED_LINKER_FLAGS}") +- message("CMAKE_MODULE_LINKER_FLAGS: ${CMAKE_MODULE_LINKER_FLAGS}") +- # if *_FLAGS_INIT +- message("CMAKE_C_FLAGS_INIT: ${CMAKE_C_FLAGS_INIT}") +- message("CMAKE_CXX_FLAGS_INIT: ${CMAKE_CXX_FLAGS_INIT}") +- message("CMAKE_EXE_LINKER_FLAGS_INIT: ${CMAKE_EXE_LINKER_FLAGS_INIT}") +- message("CMAKE_MODULE_LINKER_FLAGS_INIT: ${CMAKE_MODULE_LINKER_FLAGS_INIT}") +- +- message(FATAL_ERROR " +- Some of the variables like CFLAGS, CXXFLAGS, LDFLAGS are not empty. +- It is not possible to build ClickHouse with custom flags. +- These variables can be set up by previous invocation of some other build tools. +- You should cleanup these variables and start over again. +- +- Run the `env` command to check the details. +- You will also need to remove the contents of the build directory. +- +- Note: if you don't like this behavior, you can manually edit the cmake files, but please don't complain to developers.") +-endif() +- + # Default toolchain - this is needed to avoid dependency on OS files. + execute_process(COMMAND uname -s + OUTPUT_VARIABLE OS diff --git a/clickhouse-25.3/test-daemon.sh b/clickhouse-25.3/test-daemon.sh new file mode 100755 index 00000000000..357de2178c6 --- /dev/null +++ b/clickhouse-25.3/test-daemon.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -o errexit -o nounset -o errtrace -o pipefail -x + +# Test CREATE DATABASE +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test;" +# Test CREATE TABLE +clickhouse-client --query "CREATE TABLE IF NOT EXISTS test.sample (id UInt32, name String) ENGINE = MergeTree() ORDER BY id;" +# Test INSERT +clickhouse-client --query "INSERT INTO test.sample VALUES (1, 'test');" +# Test SELECT +clickhouse-client --query "SELECT * FROM test.sample" | grep -q "test" + +# Test basic HTTP query +curl -s "http://localhost:8123/?query=SELECT%201" | grep -q "1" +# Test database creation via HTTP +curl -s -X POST "http://localhost:8123/?query=CREATE%20DATABASE%20IF%20NOT%20EXISTS%20http_test" + +# Check access to system tables +clickhouse-client --query "SELECT * FROM system.databases WHERE name = 'system'" | grep -q "system" +clickhouse-client --query "SELECT * FROM system.tables WHERE database = 'system' LIMIT 1" + +# Test data types handling +clickhouse-client --query " + CREATE TABLE IF NOT EXISTS test.types ( + int8_col Int8, + uint64_col UInt64, + float_col Float64, + string_col String, + date_col Date + ) ENGINE = MergeTree() ORDER BY int8_col" +clickhouse-client --query " + INSERT INTO test.types VALUES + (1, 18446744073709551615, 3.14159, 'test string', '2024-01-01')" +clickhouse-client --query "SELECT * FROM test.types" | grep -q "test string" + +# Simple benchmark test with default options +clickhouse-benchmark --query "SELECT 1" --iterations 10 +# Test with concurrency +clickhouse-benchmark --concurrency 2 --query "SELECT number FROM system.numbers LIMIT 10" --iterations 3