branch-3.1: [test](iceberg) add polaris test #55484

docker/thirdparties/docker-compose/polaris/docker-compose.yaml.tpl

@@ -0,0 +1,121 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3.8"
+
+services:
+  # MinIO: S3 compatible object storage for local dev
+  minio:
+    image: minio/minio:RELEASE.2025-01-20T14-49-07Z
+    container_name: ${CONTAINER_UID}polaris-minio
+    ports:
+      - "${MINIO_API_PORT}:9000"
+      - "${MINIO_CONSOLE_PORT}:9001"
+    environment:
+      - MINIO_ROOT_USER=${MINIO_ACCESS_KEY}
+      - MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY}
+      - MINIO_DOMAIN=minio
+    command: ["server", "/data", "--console-address", ":9001"]
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
+      interval: 10s
+      timeout: 60s
+      retries: 30
+    networks:
+      ${CONTAINER_UID}polaris:
+        aliases:
+          - warehouse.minio
+
+  # MinIO client to bootstrap bucket and path
+  minio-client:
+    image: minio/mc:RELEASE.2025-01-17T23-25-50Z
+    container_name: ${CONTAINER_UID}polaris-mc
+    depends_on:
+      minio:
+        condition: service_healthy
+    entrypoint: >
+      /bin/sh -c "
+      until (mc alias set minio http://minio:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}) do echo '...waiting...' && sleep 1; done;
+      mc rm -r --force minio/${MINIO_BUCKET} || echo 'warehouse bucket does not exist yet, continuing...';
+      mc mb minio/${MINIO_BUCKET} || echo 'warehouse bucket already exists, skipping...';
+      mc anonymous set public minio/${MINIO_BUCKET};
+      echo 'MinIO setup completed successfully';
+      "
+    networks:
+      - ${CONTAINER_UID}polaris
+
+  # S3 Catalog with AWS S3
+  polaris:
+    image: apache/polaris:1.0.1-incubating
+    container_name: ${CONTAINER_UID}polaris
+    depends_on:
+      minio:
+        condition: service_healthy
+      minio-client:
+        condition: service_completed_successfully
+    ports:
+      - "${POLARIS_S3_PORT}:8181"
+      - "${POLARIS_S3_ADMIN_PORT}:8182"
+    environment:
+      # Basic configuration
+      POLARIS_BOOTSTRAP_CREDENTIALS: default-realm,root,${POLARIS_BOOTSTRAP_PASSWORD}
+      polaris.features.DROP_WITH_PURGE_ENABLED: "true"
+      polaris.realm-context.realms: default-realm
+
+      # MinIO credentials and endpoints (S3-compatible)
+      AWS_REGION: ${AWSRegion}
+      AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}
+      AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}
+      AWS_ENDPOINT_URL_S3: http://minio:9000
+      AWS_ENDPOINT_URL_STS: http://minio:9000
+
+      # Logging configuration
+      QUARKUS_LOG_LEVEL: INFO
+      QUARKUS_LOG_CATEGORY_"org.apache.polaris".LEVEL: DEBUG
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8182/q/health"]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    networks:
+      - ${CONTAINER_UID}polaris
+
+  # Initialize a Polaris INTERNAL catalog pointing to MinIO
+  polaris-init:
+    image: curlimages/curl:8.11.1
+    container_name: ${CONTAINER_UID}polaris-init
+    depends_on:
+      polaris:
+        condition: service_healthy
+    environment:
+      POLARIS_HOST: polaris
+      POLARIS_PORT: 8181
+      POLARIS_BOOTSTRAP_USER: root
+      POLARIS_BOOTSTRAP_PASSWORD: ${POLARIS_BOOTSTRAP_PASSWORD}
+      POLARIS_CATALOG_NAME: ${POLARIS_CATALOG_NAME}
+      CATALOG_BASE_LOCATION: ${CATALOG_BASE_LOCATION}
+      AWSRegion: ${AWSRegion}
+    volumes:
+      - ./init-catalog.sh:/init-catalog.sh:ro
+    entrypoint: ["/bin/sh","-c","/init-catalog.sh"]
+    networks:
+      - ${CONTAINER_UID}polaris
+
+networks:
+  ${CONTAINER_UID}polaris:
+    name: ${CONTAINER_UID}polaris

docker/thirdparties/docker-compose/polaris/init-catalog.sh

@@ -0,0 +1,145 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+HOST=${POLARIS_HOST:-polaris-s3}
+PORT=${POLARIS_PORT:-8181}
+USER=${POLARIS_BOOTSTRAP_USER:-root}
+PASS=${POLARIS_BOOTSTRAP_PASSWORD:-secret123}
+CATALOG=${POLARIS_CATALOG_NAME:-minio}
+BASE_LOCATION=${CATALOG_BASE_LOCATION:-s3://warehouse/wh/}
+
+echo "[polaris-init] Waiting for Polaris health check at http://$HOST:$PORT/q/health ..."
+for i in $(seq 1 120); do
+  if curl -sSf "http://$HOST:8182/q/health" >/dev/null; then
+    break
+  fi
+  sleep 2
+done
+
+echo "[polaris-init] Fetching OAuth token via client_credentials ..."
+# Try to obtain token using correct OAuth endpoint
+TOKEN_JSON=$(curl -sS \
+  -X POST "http://$HOST:$PORT/api/catalog/v1/oauth/tokens" \
+  -H 'Content-Type: application/x-www-form-urlencoded' \
+  -d "grant_type=client_credentials&client_id=$USER&client_secret=$PASS&scope=PRINCIPAL_ROLE:ALL")
+
+# Extract access_token field
+TOKEN=$(printf "%s" "$TOKEN_JSON" | sed -n 's/.*"access_token"\s*:\s*"\([^"]*\)".*/\1/p')
+
+if [ -z "$TOKEN" ]; then
+  echo "[polaris-init] ERROR: Failed to obtain OAuth token. Response: $TOKEN_JSON" >&2
+  exit 1
+fi
+
+echo "[polaris-init] Creating catalog '$CATALOG' with base '$BASE_LOCATION' ..."
+CREATE_PAYLOAD=$(cat <<JSON
+{
+  "name": "$CATALOG",
+  "type": "INTERNAL",
+  "properties": {
+    "default-base-location": "$BASE_LOCATION",
+    "s3.endpoint": "http://minio:9000",
+    "s3.path-style-access": "true",
+    "s3.access-key-id": "admin",
+    "s3.secret-access-key": "password",
+    "s3.region": "${AWSRegion:-us-east-1}"
+  },
+  "storageConfigInfo": {
+    "roleArn": "arn:aws:iam::000000000000:role/minio-polaris-role",
+    "storageType": "S3",
+    "allowedLocations": ["$BASE_LOCATION"]
+  }
+}
+JSON
+)
+
+# Try create; on 409 Conflict, treat as success
+HTTP_CODE=$(curl -sS -o /tmp/resp.json -w "%{http_code}" \
+  -X POST "http://$HOST:$PORT/api/management/v1/catalogs" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d "$CREATE_PAYLOAD")
+
+if [ "$HTTP_CODE" = "201" ]; then
+  echo "[polaris-init] Catalog created."
+elif [ "$HTTP_CODE" = "409" ]; then
+  echo "[polaris-init] Catalog already exists. Skipping."
+else
+  echo "[polaris-init] Create catalog failed (HTTP $HTTP_CODE):"
+  cat /tmp/resp.json || true
+  exit 1
+fi
+
+echo "[polaris-init] Setting up permissions for catalog '$CATALOG' ..."
+
+# Create a catalog admin role grants
+echo "[polaris-init] Creating catalog admin role grants ..."
+HTTP_CODE=$(curl -sS -o /tmp/resp.json -w "%{http_code}" \
+  -X PUT "http://$HOST:$PORT/api/management/v1/catalogs/$CATALOG/catalog-roles/catalog_admin/grants" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"grant":{"type":"catalog", "privilege":"CATALOG_MANAGE_CONTENT"}}')
+
+if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ]; then
+  echo "[polaris-init] Warning: Failed to create catalog admin grants (HTTP $HTTP_CODE)"
+  cat /tmp/resp.json || true
+fi
+
+# Create a data engineer role
+echo "[polaris-init] Creating data engineer role ..."
+HTTP_CODE=$(curl -sS -o /tmp/resp.json -w "%{http_code}" \
+  -X POST "http://$HOST:$PORT/api/management/v1/principal-roles" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"principalRole":{"name":"data_engineer"}}')
+
+if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ] && [ "$HTTP_CODE" != "409" ]; then
+  echo "[polaris-init] Warning: Failed to create data engineer role (HTTP $HTTP_CODE)"
+  cat /tmp/resp.json || true
+fi
+
+# Connect the roles
+echo "[polaris-init] Connecting roles ..."
+HTTP_CODE=$(curl -sS -o /tmp/resp.json -w "%{http_code}" \
+  -X PUT "http://$HOST:$PORT/api/management/v1/principal-roles/data_engineer/catalog-roles/$CATALOG" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"catalogRole":{"name":"catalog_admin"}}')
+
+if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ]; then
+  echo "[polaris-init] Warning: Failed to connect roles (HTTP $HTTP_CODE)"
+  cat /tmp/resp.json || true
+fi
+
+# Give root the data engineer role
+echo "[polaris-init] Assigning data engineer role to root ..."
+HTTP_CODE=$(curl -sS -o /tmp/resp.json -w "%{http_code}" \
+  -X PUT "http://$HOST:$PORT/api/management/v1/principals/root/principal-roles" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"principalRole": {"name":"data_engineer"}}')
+
+if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ]; then
+  echo "[polaris-init] Warning: Failed to assign data engineer role to root (HTTP $HTTP_CODE)"
+  cat /tmp/resp.json || true
+fi
+
+echo "[polaris-init] Permissions setup completed."
+echo "[polaris-init] Done."
+

docker/thirdparties/docker-compose/polaris/polaris_settings.env

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Polaris uses 20181-20182 for S3
+export POLARIS_S3_PORT=20181
+export POLARIS_S3_ADMIN_PORT=20182
+
+# Polaris authentication
+export POLARIS_BOOTSTRAP_PASSWORD="secret123"
+
+# MinIO (S3-compatible) configuration for local development
+export MINIO_API_PORT=20001
+export MINIO_CONSOLE_PORT=20002
+export MINIO_ACCESS_KEY="admin"
+export MINIO_SECRET_KEY="password"
+export MINIO_BUCKET="warehouse"
+export MINIO_BUCKET_PREFIX="wh"
+
+# Polaris catalog defaults (point to MinIO bucket/prefix)
+export AWSRegion="us-east-1"
+export POLARIS_CATALOG_NAME="doris_test"
+export CATALOG_BASE_LOCATION="s3://${MINIO_BUCKET}/${MINIO_BUCKET_PREFIX}/"

docker/thirdparties/run-thirdparties-docker.sh

@@ -39,12 +39,12 @@ Usage: $0 <options>
      --no-load-data     do not load data into the components
 
   All valid components:
-    mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,trino,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger
+    mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,trino,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger,polaris
   "
     exit 1
 }
 DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2,oceanbase,kerberos,minio"
-ALL_COMPONENTS="${DEFAULT_COMPONENTS},hudi,trino,kafka,spark,lakesoul,ranger"
+ALL_COMPONENTS="${DEFAULT_COMPONENTS},hudi,trino,kafka,spark,lakesoul,ranger,polaris"
 COMPONENTS=$2
 HELP=0
 STOP=0
@@ -160,6 +160,7 @@ RUN_LAKESOUL=0
 RUN_KERBEROS=0
 RUN_MINIO=0
 RUN_RANGER=0
+RUN_POLARIS=0
 
 RESERVED_PORTS="65535"
 
@@ -207,6 +208,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
         RUN_MINIO=1
     elif [[ "${element}"x == "ranger"x ]]; then
         RUN_RANGER=1
+    elif [[ "${element}"x == "polaris"x ]]; then
+        RUN_POLARIS=1
     else
         echo "Invalid component: ${element}"
         usage
@@ -664,6 +667,24 @@ start_minio() {
     fi
 }
 
+start_polaris() {
+    echo "RUN_POLARIS"
+    local POLARIS_DIR="${ROOT}/docker-compose/polaris"
+    # Render compose with envsubst since settings is a bash export file
+    export CONTAINER_UID=${CONTAINER_UID}
+    . "${POLARIS_DIR}/polaris_settings.env"
+    if command -v envsubst >/dev/null 2>&1; then
+        envsubst <"${POLARIS_DIR}/docker-compose.yaml.tpl" >"${POLARIS_DIR}/docker-compose.yaml"
+    else
+        # Fallback: let docker compose handle variable substitution from current shell env
+        cp "${POLARIS_DIR}/docker-compose.yaml.tpl" "${POLARIS_DIR}/docker-compose.yaml"
+    fi
+    sudo docker compose -f "${POLARIS_DIR}/docker-compose.yaml" down
+    if [[ "${STOP}" -ne 1 ]]; then
+        sudo docker compose -f "${POLARIS_DIR}/docker-compose.yaml" up -d --wait --remove-orphans
+    fi
+}
+
 start_ranger() {
     echo "RUN_RANGER"
     export CONTAINER_UID=${CONTAINER_UID}
@@ -794,6 +815,12 @@ if [[ "${RUN_MINIO}" -eq 1 ]]; then
     start_minio > start_minio.log 2>&1 &
     pids["minio"]=$!
 fi
+
+if [[ "${RUN_POLARIS}" -eq 1 ]]; then
+    start_polaris > start_polaris.log 2>&1 &
+    pids["polaris"]=$!
+fi
+
 if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
     start_kerberos > start_kerberos.log 2>&1 &
     pids["kerberos"]=$!

regression-test/conf/regression-conf.groovy

@@ -235,6 +235,10 @@ iceberg_rest_uri_port_s3=19181
 iceberg_rest_uri_port_oss=19182
 iceberg_rest_uri_port_cos=19183
 
+// polaris rest catalog config
+polaris_rest_uri_port=20181
+polaris_minio_port=20001
+
 // If the failure suite num exceeds this config
 // all following suite will be skipped to fast quit the run.
 // <=0 means no limit.

regression-test/pipeline/external/conf/regression-conf.groovy

@@ -134,6 +134,10 @@ iceberg_rest_uri_port_cos=19183
 iceberg_minio_port=19001
 enableIcebergTest=true
 
+// polaris rest catalog config
+polaris_rest_uri_port=20181
+polaris_minio_port=20001
+
 enableEsTest=true
 es_5_port=59200
 es_6_port=19200