fix: gcs endpoint should be independently configured with credential

ReadMe.md

@@ -529,6 +529,7 @@ gcs:
   credentials_file: ""         # GCS_CREDENTIALS_FILE
   credentials_json: ""         # GCS_CREDENTIALS_JSON
   credentials_json_encoded: "" # GCS_CREDENTIALS_JSON_ENCODED
+  endpoint: ""                 # GCS_ENDPOINT, use it for custom GCS endpoint/compatible storage. For example, when using custom endpoint via private service connect
   bucket: ""                   # GCS_BUCKET
   path: ""                     # GCS_PATH, `system.macros` values can be applied as {macro_name}
   object_disk_path: ""         # GCS_OBJECT_DISK_PATH, path for backup of part from `s3` object disk (clickhouse support only gcs over s3 protocol), if disk present, then shall not be zero and shall not be prefixed by `path`

pkg/storage/gcs.go

@@ -90,15 +90,18 @@ func (gcs *GCS) Connect(ctx context.Context) error {
 
 	if gcs.Config.Endpoint != "" {
 		endpoint = gcs.Config.Endpoint
-		clientOptions = append([]option.ClientOption{option.WithoutAuthentication()}, clientOptions...)
 		clientOptions = append(clientOptions, option.WithEndpoint(endpoint))
-	} else if gcs.Config.CredentialsJSON != "" {
+	}
+
+	if gcs.Config.CredentialsJSON != "" {
 		clientOptions = append(clientOptions, option.WithCredentialsJSON([]byte(gcs.Config.CredentialsJSON)))
 	} else if gcs.Config.CredentialsJSONEncoded != "" {
 		d, _ := base64.StdEncoding.DecodeString(gcs.Config.CredentialsJSONEncoded)
 		clientOptions = append(clientOptions, option.WithCredentialsJSON(d))
 	} else if gcs.Config.CredentialsFile != "" {
 		clientOptions = append(clientOptions, option.WithCredentialsFile(gcs.Config.CredentialsFile))
+	} else {
+		clientOptions = append(clientOptions, option.WithoutAuthentication())
 	}
 
 	if gcs.Config.ForceHttp {

test/integration/config-gcs-custom-endpoint.yml

@@ -0,0 +1,29 @@
+general:
+  disable_progress_bar: true
+  remote_storage: gcs
+  upload_concurrency: 4
+  download_concurrency: 4
+  skip_tables:
+    - " system.*"
+    - "INFORMATION_SCHEMA.*"
+    - "information_schema.*"
+    - "_temporary_and_external_tables.*"
+  restore_schema_on_cluster: "{cluster}"
+clickhouse:
+  host: clickhouse
+  port: 9440
+  username: backup
+  password: meow=& 123?*%# МЯУ
+  secure: true
+  skip_verify: true
+  sync_replicated_tables: true
+  timeout: 5s
+  restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; exec:ls -la /var/lib/clickhouse/access; sql:SYSTEM SHUTDOWN"
+  # restart_command: bash -c 'echo "FAKE RESTART"'
+  backup_mutations: true
+gcs:
+  bucket: altinity-qa-test
+  path: backup/{cluster}/{shard}
+  object_disk_path: object_disks/{cluster}/{shard}
+  compression_format: tar
+  endpoint: http://gcs:8080/storage/v1/

test/integration/docker-compose.yml

@@ -43,17 +43,19 @@ services:
       - clickhouse-backup
 
 # todo need to reproduce download after upload
-#  gcs:
-#    image: fsouza/fake-gcs-server:latest
-#    hostname: gcs
-#    entrypoint:
-#      - /bin/sh
-#    command:
-#      - -c
-#      - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080"
-#    networks:
-#      - clickhouse-backup
-
+  gcs:
+    image: fsouza/fake-gcs-server:latest
+    hostname: gcs
+    container_name: gcs
+    entrypoint:
+      - /bin/sh
+    command:
+      - -c
+      - "mkdir -p /data/altinity-qa-test && mkdir -p /data/${QA_GCS_OVER_S3_BUCKET} && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080"
+    networks:
+      - clickhouse-backup
+    environment:
+      QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}"
 
   azure:
     image: mcr.microsoft.com/azure-storage/azurite:latest
@@ -126,9 +128,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
@@ -166,9 +165,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
@@ -196,6 +192,7 @@ services:
       - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml
       - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml
       - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml
+      - ./config-gcs-custom-endpoint.yml:/etc/clickhouse-backup/config-gcs-custom-endpoint.yml
       - ./config-s3.yml:/etc/clickhouse-backup/config-s3.yml
       - ./config-s3-embedded.yml:/etc/clickhouse-backup/config-s3-embedded.yml
       - ./config-s3-fips.yml:/etc/clickhouse-backup/config-s3-fips.yml.template

test/integration/docker-compose_advanced.yml

@@ -57,17 +57,19 @@ services:
       - clickhouse-backup
 
 # todo need to reproduce download after upload
-#  gcs:
-#    image: fsouza/fake-gcs-server:latest
-#    hostname: gcs
-#    entrypoint:
-#      - /bin/sh
-#    command:
-#      - -c
-#      - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080"
-#    networks:
-#      - clickhouse-backup
-
+  gcs:
+    image: fsouza/fake-gcs-server:latest
+    hostname: gcs
+    container_name: gcs
+    entrypoint:
+      - /bin/sh
+    command:
+      - -c
+      - "mkdir -p /data/altinity-qa-test && mkdir -p /data/${QA_GCS_OVER_S3_BUCKET} && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080"
+    networks:
+      - clickhouse-backup
+    environment:
+      QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}"
 
   azure:
     image: mcr.microsoft.com/azure-storage/azurite:latest
@@ -177,9 +179,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
@@ -217,9 +216,6 @@ services:
       AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}"
       CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}"
       GOCOVERDIR: "/tmp/_coverage_/"
-# fake-gcs-server
-#       STORAGE_EMULATOR_HOST: "http://gsc:8080"
-#       GOOGLE_API_USE_CLIENT_CERTIFICATE: "false"
 # FIPS
       QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY}
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
@@ -254,6 +250,7 @@ services:
       - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml
       - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml
       - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml
+      - ./config-gcs-custom-endpoint.yml:/etc/clickhouse-backup/config-gcs-custom-endpoint.yml
       - ./config-s3.yml:/etc/clickhouse-backup/config-s3.yml
       - ./config-s3-embedded.yml:/etc/clickhouse-backup/config-s3-embedded.yml
       - ./config-s3-fips.yml:/etc/clickhouse-backup/config-s3-fips.yml.template

test/integration/integration_test.go

@@ -1720,6 +1720,15 @@ func TestIntegrationGCS(t *testing.T) {
 	runMainIntegrationScenario(t, "GCS", "config-gcs.yml")
 }
 
+func TestIntegrationGCSWithCustomEndpoint(t *testing.T) {
+	if isTestShouldSkip("GCS_TESTS") {
+		t.Skip("Skipping GCS integration tests...")
+		return
+	}
+	//t.Parallel()
+	runMainIntegrationScenario(t, "GCS_EMULATOR", "config-gcs-custom-endpoint.yml")
+}
+
 func TestIntegrationSFTPAuthPassword(t *testing.T) {
 	//t.Parallel()
 	runMainIntegrationScenario(t, "SFTP", "config-sftp-auth-password.yaml")
@@ -2110,7 +2119,6 @@ func checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorag
 	}
 	if remoteStorageType == "SFTP" {
 		checkRemoteDir("total 0", "sshd", "bash", "-c", "ls -lh /root/")
-
 	}
 	if remoteStorageType == "FTP" {
 		if strings.Contains(os.Getenv("COMPOSE_FILE"), "advanced") {
@@ -2119,9 +2127,8 @@ func checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorag
 			checkRemoteDir("total 0", "ftp", "bash", "-c", "ls -lh /home/vsftpd/test_backup/backup/")
 		}
 	}
-	//todo check gcs backup is empty
-	if remoteStorageType == "GCS" {
-
+	if remoteStorageType == "GCS_EMULATOR" {
+		checkRemoteDir("total 0", "gcs", "sh", "-c", "ls -lh /data/altinity-qa-test/")
 	}
 }