Merge pull request #843 from Altinity/v2.5

release v2.5

.github/workflows/build.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         golang-version:
-          - "1.21"
+          - "1.22"
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -49,7 +49,7 @@ jobs:
       - name: Build clickhouse-backup binary
         id: make-race
         env:
-          GOROOT: ${{ env.GOROOT_1_20_X64 }}
+          GOROOT: ${{ env.GOROOT_1_22_X64 }}
         run: |
           make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup
           make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips 
@@ -114,15 +114,24 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
+        golang-version:
+          - "1.22"
         clickhouse:
           - '22.3'
           - '22.8'
           - '23.3'
           - '23.8'
+          - '24.3'
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
 
+      - name: Setup golang
+        id: setup-go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '^${{ matrix.golang-version }}'
+
       - uses: actions/download-artifact@v4
         with:
           name: build-test-artifacts
@@ -181,9 +190,12 @@ jobs:
           tfs --debug --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./test/testflows/raw.log - --confidential --copyright "Altinity LTD" --logo ./test/testflows/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./test/testflows/report.html
           sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances
       - name: Format testflows coverage
+        env:
+          GOROOT: ${{ env.GOROOT_1_22_X64 }}
         run: |
           sudo chmod -Rv a+rw test/testflows/_coverage_/
-          ls -la test/testflows/_coverage_ 
+          ls -la test/testflows/_coverage_
+          go env
           go tool covdata textfmt -i test/testflows/_coverage_/ -o test/testflows/_coverage_/coverage.out
       - name: Report testflows coverage
         uses: coverallsapp/github-action@v2
@@ -192,7 +204,7 @@ jobs:
           parallel: true
           format: golang
           flag-name: testflows-${{ matrix.clickhouse }}
-# todo wait when resolve https://github.com/actions/upload-artifact/issues/270 and uncomment
+      # todo possible failures  https://github.com/actions/upload-artifact/issues/270
       - name: Upload testflows logs
         uses: actions/upload-artifact@v4
         with:
@@ -211,7 +223,7 @@ jobs:
     strategy:
       matrix:
         golang-version:
-          - "1.21"
+          - "1.22"
         clickhouse:
           - '1.1.54394'
           - '19.17'
@@ -223,6 +235,7 @@ jobs:
           - '22.8'
           - '23.3'
           - '23.8'
+          - '24.3'
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -256,10 +269,12 @@ jobs:
 
       - name: Running integration tests
         env:
+          GOROOT: ${{ env.GOROOT_1_22_X64 }}
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
-          # just options for advanced logging
-          # RUN_TESTS: "TestFIPS"
+          # options for advanced debug CI/CD
+          # RUN_TESTS: "TestIntegrationGCS"
           # LOG_LEVEL: "debug"
+          # GCS_DEBUG: "true"
           # SFTP_DEBUG: "true"
           # AZBLOB_DEBUG: "true"
           # FTP_DEBUG: "true"
@@ -303,6 +318,8 @@ jobs:
           docker-compose -f test/integration/${COMPOSE_FILE} ps -a
           go test -timeout 60m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go
       - name: Format integration coverage
+        env:
+          GOROOT: ${{ env.GOROOT_1_22_X64 }}
         run: |
           sudo chmod -Rv a+rw test/integration/_coverage_/
           ls -la test/integration/_coverage_ 

.github/workflows/release.yaml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         golang-version:
-          - "1.21"
+          - "1.22"
 
     steps:
       - name: Checkout project
@@ -55,7 +55,7 @@ jobs:
       - name: Prepare binaries
         id: make
         env:
-          GOROOT: ${{ env.GOROOT_1_20_X64 }}
+          GOROOT: ${{ env.GOROOT_1_22_X64 }}
         run: |
           make build build-fips config test
           #make build-fips-darwin

ChangeLog.md

@@ -1,3 +1,39 @@
+# v2.5.0
+IMPROVEMENTS
+- complete removed support for legacy backups, created with version prior v1.0 
+- removed `disable_progress_bar` config option and related progress bar code
+- added `--delete-source` parameter for `upload` and `create_remote` commands to explicitly delete local backup during upload, fix [777](https://github.com/Altinity/clickhouse-backup/issues/777)
+- added support for `--env ENV_NAME=value` cli parameter for allow dynamically override any config parameter, fix [821](https://github.com/Altinity/clickhouse-backup/issues/821)
+- added support for `use_embedded_backup_restore: true` with empty `embedded_backup_disk` value, tested on S3/GCS over S3/AzureBlobStorage, fix [695](https://github.com/Altinity/clickhouse-backup/issues/695)
+- `--rbac, --rbac-only, --configs, --configs-only` now works with `use_embedded_backup_restore: true`
+-- `--data` for `restore` with `use_embedded_backup_restore: true` will use `allow_non_empty_tables=true` to allow fix [756](https://github.com/Altinity/clickhouse-backup/issues/756)
+- added `--diff-from-remote` parameter for `create` command, will copy only new data parts object disk data, also allows to download properly object disk data from required backup during `restore`, fix [865](https://github.com/Altinity/clickhouse-backup/issues/865)
+- added support of native Clickhouse incremental backup for `use_embedded_backup_restore: true` fix [735](https://github.com/Altinity/clickhouse-backup/issues/735)
+- added `GCS_CHUNK_SIZE` config parameter, try to speedup GCS upload fix [874](https://github.com/Altinity/clickhouse-backup/pull/874), thanks @dermasmid
+- added `--remote-backup` cli parameter to `tables` command and `GET /backup/table`, fix [778](https://github.com/Altinity/clickhouse-backup/issues/778)
+- added `rbac_always_backup: true` option to default config, will create backup for RBAC objects automatically, restore still require `--rbac` to avoid destructive actions, fix [793](https://github.com/Altinity/clickhouse-backup/issues/793)
+- added `rbac_conflict_resolution: recreate` option for RBAC object name conflicts during restore, fix [851](https://github.com/Altinity/clickhouse-backup/issues/851)
+- added `upload_max_bytes_per_seconds` and `download_max_bytes_per_seconds` config options to allow throttling without CAP_SYS_NICE, fix [817](https://github.com/Altinity/clickhouse-backup/issues/817)
+- added `clickhouse_backup_in_progress_commands` metric, fix [836](https://github.com/Altinity/clickhouse-backup/issues/836)
+- switched to golang 1.22
+- updated all third-party SDK to latest versions
+- added `clickhouse/clickhouse-server:24.3` to CI/CD
+
+BUG FIXES
+- continue `S3_MAX_PARTS_COUNT` default value from `2000` to `4000` to continue decrease memory usage for S3 
+- changed minimal part size for multipart upload in CopyObject from `5Mb` to `10Mb`
+- restore SQL UDF functions after restore tables
+- execute `ALTER TABLE ... DROP PARTITION` instead of `DROP TABLE` for `restore` and `restore_remote` with parameters `--data --partitions=...`, fix [756](https://github.com/Altinity/clickhouse-backup/issues/756) 
+- fix wrong behavior for `freeze_by_part` + `freeze_by_part_where`, fix [855](https://github.com/Altinity/clickhouse-backup/issues/855)
+- apply `CLICKHOUSE_SKIP_TABLES_ENGINES` during `create` command
+- fixed behavior for upload / download when .inner. table missing for MATERIALIZED VIEW  by table pattern, fix [765](https://github.com/Altinity/clickhouse-backup/issues/765)
+- fixed `ObjectDisks` + `CLICKHOUSE_USE_EMBEDDED_BACKUP_RESTORE: true` - shall skip upload object disk content, fix [799](https://github.com/Altinity/clickhouse-backup/issues/799)
+- fixed connection to clickhouse-server behavior when long clickhouse-server startup time and `docker-entrypoint.d` processing, will infinite reconnect each 5 seconds, until success, fix [857](https://github.com/Altinity/clickhouse-backup/issues/857)
+- fixed `USE_EMBEDDED_BACKUP_RESTORE=true` behavior to allow use backup disk with type `local`, fix [882](https://github.com/Altinity/clickhouse-backup/issues/882)
+- fixed wrong list command behavior, it shall  scann all system.disks path not only default disk to find pratially created backups, fix [873](https://github.com/Altinity/clickhouse-backup/issues/873)
+- fixed create `--rbac` behavior, don't create access folder if no RBAC objects is present
+- fixed behavior when `system.disks` contains disk which not present in any `storage_policies`, fix [845](https://github.com/Altinity/clickhouse-backup/issues/845)
+
 # v2.4.35
 IMPROVEMENTS
 - set part size for `s3:CopyObject` minimum 128Mb, look details https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data

Dockerfile

@@ -9,22 +9,22 @@ RUN rm -fv /etc/apt/sources.list.d/clickhouse.list && \
     find /etc/apt/ -type f -name *.list -exec sed -i 's/ru.archive.ubuntu.com/archive.ubuntu.com/g' {} + && \
     ( apt-get update || true ) && \
     apt-get install -y --no-install-recommends gnupg ca-certificates wget && update-ca-certificates && \
-    for srv in "keyserver.ubuntu.com" "pool.sks-keyservers.net" "keys.gnupg.net"; do apt-key adv --keyserver $srv --recv-keys 52B59B1571A79DBC054901C0F6BC817356A3D45E; if [[ $? -eq 0 ]]; then break; fi; done && \
+    for srv in "keyserver.ubuntu.com" "pool.sks-keyservers.net" "keys.gnupg.net"; do apt-key adv --keyserver $srv --recv-keys 52B59B1571A79DBC054901C0F6BC817356A3D45E; if [ $? -eq 0 ]; then break; fi; done && \
     DISTRIB_CODENAME=$(cat /etc/lsb-release | grep DISTRIB_CODENAME | cut -d "=" -f 2) && \
     echo ${DISTRIB_CODENAME} && \
     echo "deb https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu ${DISTRIB_CODENAME} main" > /etc/apt/sources.list.d/golang.list && \
     echo "deb-src https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu ${DISTRIB_CODENAME} main" >> /etc/apt/sources.list.d/golang.list && \
     ( apt-get update || true ) && \
-    apt-get install -y --no-install-recommends libc-dev golang-1.21 make git gcc musl-dev musl-tools && \
+    apt-get install -y --no-install-recommends libc-dev golang-1.22 make git gcc musl-dev musl-tools && \
     wget -q -P /root/ https://musl.cc/aarch64-linux-musl-cross.tgz && \
     tar -xvf /root/aarch64-linux-musl-cross.tgz -C /root/ && \
     mkdir -p /root/go/
 
-RUN ln -nsfv /usr/lib/go-1.21/bin/go /usr/bin/go
+RUN ln -nsfv /usr/lib/go-1.22/bin/go /usr/bin/go
 VOLUME /root/.cache/go
 ENV GOCACHE=/root/.cache/go
 ENV GOPATH=/root/go/
-ENV GOROOT=/usr/lib/go-1.21/
+ENV GOROOT=/usr/lib/go-1.22/
 RUN go env
 WORKDIR /src/
 # cache modules when go.mod go.sum changed
@@ -76,16 +76,17 @@ COPY --from=builder-fips /src/build/ /src/build/
 CMD /src/build/${TARGETPLATFORM}/clickhouse-backup-fips --help
 
 
-FROM alpine:3.18 AS image_short
+FROM alpine:3.19 AS image_short
 ARG TARGETPLATFORM
 MAINTAINER Eugene Klimov <eklimov@altinity.com>
 RUN addgroup -S -g 101 clickhouse \
     && adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse server" -u 101 clickhouse
-RUN apk update && apk add --no-cache ca-certificates tzdata bash curl && update-ca-certificates
+RUN apk update && apk add --no-cache ca-certificates tzdata bash curl libcap-setcap && update-ca-certificates
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 COPY build/${TARGETPLATFORM}/clickhouse-backup /bin/clickhouse-backup
 RUN chmod +x /bin/clickhouse-backup
+RUN setcap cap_sys_nice=+ep /bin/clickhouse-backup
 # USER clickhouse
 ENTRYPOINT ["/entrypoint.sh"]
 CMD [ "/bin/clickhouse-backup", "--help" ]
@@ -96,6 +97,7 @@ ARG TARGETPLATFORM
 MAINTAINER Eugene Klimov <eklimov@altinity.com>
 COPY build/${TARGETPLATFORM}/clickhouse-backup-fips /bin/clickhouse-backup
 RUN chmod +x /bin/clickhouse-backup
+RUN setcap cap_sys_nice=+ep /bin/clickhouse-backup
 
 
 FROM ${CLICKHOUSE_IMAGE}:${CLICKHOUSE_VERSION} AS image_full
@@ -106,14 +108,15 @@ RUN apt-get update && apt-get install -y gpg xxd bsdmainutils parallel && wget -
     echo "deb [signed-by=/usr/share/keyrings/kopia-keyring.gpg] https://packages.kopia.io/apt/ stable main" > /etc/apt/sources.list.d/kopia.list && \
     wget -c "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$(dpkg --print-architecture)" -O /usr/bin/yq && chmod +x /usr/bin/yq && \
     apt-get update -y && \
-    apt-get install -y ca-certificates tzdata bash curl restic rsync rclone jq gpg kopia && \
+    apt-get install -y ca-certificates tzdata bash curl restic rsync rclone jq gpg kopia libcap2-bin && \
     update-ca-certificates && \
     rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/*
 
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 COPY build/${TARGETPLATFORM}/clickhouse-backup /bin/clickhouse-backup
 RUN chmod +x /bin/clickhouse-backup
+RUN setcap cap_sys_nice=+ep /bin/clickhouse-backup
 
 # USER clickhouse