Dremio merge 2025 08 12 09 21 (apache#102)

* Helm Chart: remove duplicate line in authenticationOptions template (apache#2297)

* Move python client Makefile into the root level one (apache#2140)

* Move python client Makefile into the root level one

* Update workflow

* add client-lint to pre-commit

* Update README.md to include client

* fix(deps): update dependency boto3 to v1.40.5 (apache#2300)

* fix(deps): update dependency org.assertj:assertj-core to v3.27.4 (apache#2298)

* Fix python CI client-integration-test (apache#2305)

`CONTAINER_TOOL` got renamed to `DOCKER` in
6764a88
but then
eda7074
added more `CONTAINER_TOOL` usage

* fix(docs): update BasePolarisAuthenticator to DefaultAuthenticator (apache#2303)

* Fix Hadoop federation to initialize the configuration prior to catalog initialization (apache#2282)

This PR addresses the gap in Hadoop federation to ensure that we initialize the hadoop configuration prior to initializing the catalog object.

The iceberg library expects the hadoop configuration to be initialized before creating a HadoopCatalog object. This change ensures that Polaris is compatible with the underlying iceberg library. Additionally, since the config initialization is based on the underlying (default) core-site.xml file, the change ensures that the federated catalog was created using `IMPLICIT` authentication mode. 

Testing: 
Due to current limitations in the current test setup, tested the change manually.
[TODO] Add a regtest with hadoop federation once the change is baked into the apache/polaris docker image.

* fix(deps): update dependency boto3 to v1.40.6 (apache#2308)

* fix(deps): update dependency io.opentelemetry:opentelemetry-bom to v1.53.0 (apache#2309)

* Perform force repair when polaris not found (apache#2313)

* fix(deps): update dependency com.github.spotbugs:spotbugs-annotations to v4.9.4 (apache#2312)

* chore(deps): update dependency pre-commit to v4.3.0 (apache#2311)

* fix(deps): update dependency software.amazon.awssdk:bom to v2.32.19 (apache#2314)

* fix(deps): update dependency com.gradleup.shadow:shadow-gradle-plugin to v9.0.1 (apache#2310)

* Separate Cloud Integration Tests (apache#2283)

Currently, cloud integration tests are part of the regular test suite
but require cloud credentials to execute properly. This creates
unnecessary overhead for developers who just want to run local builds
or for CI jobs that don't have cloud access configured.

* chore(deps): update actions/checkout action to v5 (apache#2319)

* CatalogEntity: internal endpoint not considered (apache#2292)

Setting an S3 internal endpoint doesn't work, because the property's not carried over from the OpenAPI model type into `AwsStorageConfigInfo`.

* client/python: loosen boto3 dep (apache#2188)

It's hard to depend on this package when the boto3 dependency is so strict.

* fix(deps): update dependency io.micrometer:micrometer-bom to v1.15.3 (apache#2321)

* NoSQL: test adjustment

* Last merged commit 45e8e02

---------

Co-authored-by: Alexandre Dutra <adutra@apache.org>
Co-authored-by: Yong Zheng <yongzheng0809@gmail.com>
Co-authored-by: Mend Renovate <bot@renovateapp.com>
Co-authored-by: Christopher Lambert <xn137@gmx.de>
Co-authored-by: Yujiang Zhong <42907416+zhongyujiang@users.noreply.github.com>
Co-authored-by: Pooja Nilangekar <poojan@umd.edu>
Co-authored-by: Tamas Mate <50709850+tmater@users.noreply.github.com>
Co-authored-by: Tyler Rockwood <rockwotj@gmail.com>

Author: 9 people

.github/workflows/gradle.yml

@@ -40,7 +40,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - name: Set up JDK 21
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
         with:
@@ -80,7 +80,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - name: Set up JDK 21
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
         with:
@@ -117,7 +117,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - name: Set up JDK 21
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
         with:
@@ -154,7 +154,7 @@ jobs:
       - quarkus-tests
       - integration-tests
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - name: Set up JDK 21
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
         with:

.github/workflows/helm.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
         with:
           fetch-depth: 0
 

.github/workflows/nightly.yml

@@ -46,7 +46,7 @@ jobs:
     if: github.repository == 'apache/polaris'
     steps:
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - name: Set up JDK 21
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
         with:

.github/workflows/python-client.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
       - name: Checkout Polaris project
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
 
       - name: Set up JDK for Gradle
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
@@ -59,14 +59,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Lint
-        working-directory: client/python
         run: |
-          make lint
+          make client-lint
 
       - name: Generated Client Tests
-        working-directory: client/python
         run: |
-          make test-client
+          make client-unit-test
 
       - name: Image build
         run: |
@@ -76,6 +74,5 @@ jobs:
             -Dquarkus.container-image.build=true
 
       - name: Integration Tests
-        working-directory: client/python
         run: |
-          make test-integration
+          make client-integration-test

.github/workflows/regtest.yml

@@ -32,7 +32,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
 
       - name: Set up JDK 21
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4

.github/workflows/site.yml

@@ -29,8 +29,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
         with:
           ref: "versioned-docs"
           path: site/content/releases

.github/workflows/spark_client_regtests.yml

@@ -32,7 +32,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
 
       - name: Set up JDK 21
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4

Makefile

@@ -26,10 +26,14 @@ DOCKER ?= docker
 MINIKUBE_PROFILE ?= minikube
 DEPENDENCIES ?= ct helm helm-docs java21 git
 OPTIONAL_DEPENDENCIES := jq kubectl minikube
+VENV_DIR := .venv
+PYTHON_CLIENT_DIR := client/python
+ACTIVATE_AND_CD = source $(VENV_DIR)/bin/activate && cd $(PYTHON_CLIENT_DIR)
 
 ## Version information
 BUILD_VERSION := $(shell cat version.txt)
 GIT_COMMIT := $(shell git rev-parse HEAD)
+POETRY_VERSION := $(shell cat client/python/pyproject.toml | grep requires-poetry | sed 's/requires-poetry *= *"\(.*\)"/\1/')
 
 ##@ General
 
@@ -41,6 +45,7 @@ help: ## Display this help
 version: ## Display version information
 	@echo "Build version: ${BUILD_VERSION}"
 	@echo "Git commit: ${GIT_COMMIT}"
+	@echo "Poetry version: ${POETRY_VERSION}"
 
 ##@ Polaris Build
 
@@ -99,6 +104,79 @@ spotless-apply: check-dependencies ## Apply code formatting using Spotless Gradl
 	@./gradlew spotlessApply
 	@echo "--- Spotless formatting applied ---"
 
+##@ Polaris Client
+
+# Target to create the virtual environment directory
+$(VENV_DIR):
+	@echo "Setting up Python virtual environment at $(VENV_DIR)..."
+	@python3 -m venv $(VENV_DIR)
+	@echo "Virtual environment created."
+
+.PHONY: client-install-dependencies
+client-install-dependencies: $(VENV_DIR)
+	@echo "Installing Poetry and project dependencies into $(VENV_DIR)..."
+	@$(VENV_DIR)/bin/pip install --upgrade pip
+	@if [ ! -f "$(VENV_DIR)/bin/poetry" ]; then \
+		$(VENV_DIR)/bin/pip install --upgrade "poetry$(POETRY_VERSION)"; \
+	fi
+	@$(ACTIVATE_AND_CD) && poetry install --all-extras
+	@echo "Poetry and dependencies installed."
+
+.PHONY: client-setup-env
+client-setup-env: $(VENV_DIR) client-install-dependencies
+
+.PHONY: client-lint
+client-lint: client-setup-env ## Run linting checks for Polaris client
+	@echo "--- Running client linting checks ---"
+	@$(ACTIVATE_AND_CD) && poetry run pre-commit run --files integration_tests/* python/cli/*
+	@echo "--- Client linting checks complete ---"
+
+.PHONY: client-regenerate
+client-regenerate: client-setup-env ## Regenerate the client code
+	@echo "--- Regenerating client code ---"
+	@client/templates/regenerate.sh
+	@echo "--- Client code regeneration complete ---"
+
+.PHONY: client-unit-test
+client-unit-test: client-setup-env ## Run client unit tests
+	@echo "--- Running client unit tests ---"
+	@$(ACTIVATE_AND_CD) && SCRIPT_DIR="non-existing-mock-directory" poetry run pytest test/
+	@echo "--- Client unit tests complete ---"
+
+.PHONY: client-integration-test
+client-integration-test: client-setup-env ## Run client integration tests
+	@echo "--- Starting client integration tests ---"
+	@echo "Ensuring Docker Compose services are stopped and removed..."
+	@$(DOCKER) compose -f $(PYTHON_CLIENT_DIR)/docker-compose.yml kill || true # `|| true` prevents make from failing if containers don't exist
+	@$(DOCKER) compose -f $(PYTHON_CLIENT_DIR)/docker-compose.yml rm -f || true # `|| true` prevents make from failing if containers don't exist
+	@echo "Bringing up Docker Compose services in detached mode..."
+	@$(DOCKER) compose -f $(PYTHON_CLIENT_DIR)/docker-compose.yml up -d
+	@echo "Waiting for Polaris HTTP health check to pass..."
+	@until curl -s -f http://localhost:8182/q/health > /dev/null; do \
+		echo "Still waiting for HTTP 200 from /q/health (sleeping 2s)..."; \
+		sleep 2; \
+	done
+	@echo "Polaris is healthy. Starting integration tests..."
+	@$(ACTIVATE_AND_CD) && poetry run pytest integration_tests/
+	@echo "--- Client integration tests complete ---"
+	@echo "Tearing down Docker Compose services..."
+	@$(DOCKER) compose -f $(PYTHON_CLIENT_DIR)/docker-compose.yml down || true # Ensure teardown even if tests fail
+
+.PHONY: client-cleanup
+client-cleanup: ## Cleanup virtual environment and Python cache files
+	@echo "--- Cleaning up virtual environment and Python cache files ---"
+	@echo "Attempting to remove virtual environment directory: $(VENV_DIR)..."
+	@if [ -n "$(VENV_DIR)" ] && [ -d "$(VENV_DIR)" ]; then \
+		rm -rf "$(VENV_DIR)"; \
+		echo "Virtual environment removed."; \
+	else \
+		echo "Virtual environment directory '$(VENV_DIR)' not found or VENV_DIR is empty. No action taken."; \
+	fi
+	@echo "Cleaning up Python cache files..."
+	@find $(PYTHON_CLIENT_DIR) -type f -name "*.pyc" -delete
+	@find $(PYTHON_CLIENT_DIR) -type d -name "__pycache__" -delete
+	@echo "--- Virtual environment and Python cache cleanup complete ---"
+
 ##@ Helm
 
 helm-doc-generate: DEPENDENCIES := helm-docs
@@ -178,7 +256,7 @@ minikube-cleanup: check-dependencies ## Cleanup the Minikube cluster
 ##@ Pre-commit
 
 .PHONY: pre-commit
-pre-commit: spotless-apply helm-doc-generate ## Run tasks for pre-commit
+pre-commit: spotless-apply helm-doc-generate client-lint ## Run tasks for pre-commit
 
 ##@ Dependencies
 

README.md

@@ -87,6 +87,7 @@ To streamline the developer experience, especially for common setup and build ta
   - Managing development clusters: e.g., `make minikube-start-cluster, make minikube-cleanup`
   - Automating Helm tasks: e.g., `make helm-doc-generate, make helm-unittest`
   - Handling dependencies: e.g., `make install-dependencies-brew`
+  - Managing client operations: e.g., `make client-lint, make client-regenerate`
 
 To see available commands:
 ```bash

build-logic/src/main/kotlin/polaris-runtime.gradle.kts

@@ -31,8 +31,11 @@ testing {
           // See https://github.com/quarkusio/quarkus/issues/22844
           systemProperty("junit.jupiter.extensions.autodetection.enabled", "true")
         }
-
-        if (testTask.name != "test") {
+      }
+    }
+    fun intTestSuiteConfigure(testSuite: JvmTestSuite) =
+      testSuite.run {
+        targets.all {
           testTask.configure {
             // For Quarkus...
             //
@@ -42,15 +45,22 @@ testing {
             dependsOn(tasks.named("quarkusBuild"))
           }
         }
-      }
-    }
-    register<JvmTestSuite>("intTest") {
-      targets.all {
-        tasks.named("compileIntTestJava").configure {
-          dependsOn(tasks.named("compileQuarkusTestGeneratedSourcesJava"))
+        tasks.named(sources.compileJavaTaskName).configure {
+          dependsOn("compileQuarkusTestGeneratedSourcesJava")
         }
+        configurations.named(sources.runtimeOnlyConfigurationName).configure {
+          extendsFrom(configurations.getByName("testRuntimeOnly"))
+        }
+        configurations.named(sources.implementationConfigurationName).configure {
+          // Let the test's implementation config extend testImplementation, so it also inherits the
+          // project's "main" implementation dependencies (not just the "api" configuration)
+          extendsFrom(configurations.getByName("testImplementation"))
+        }
+        sources { java.srcDirs(tasks.named("quarkusGenerateCodeTests")) }
       }
-      sources { java.srcDirs(tasks.named("quarkusGenerateCodeTests")) }
+
+    listOf("intTest", "cloudTest").forEach {
+      register<JvmTestSuite>(it).configure { intTestSuiteConfigure(this) }
     }
   }
 }
@@ -72,14 +82,6 @@ configurations.all {
   }
 }
 
-// Let the test's implementation config extend testImplementation, so it also inherits the
-// project's "main" implementation dependencies (not just the "api" configuration)
-configurations.named("intTestImplementation").configure {
-  extendsFrom(configurations.getByName("testImplementation"))
-}
-
-dependencies { add("intTestImplementation", java.sourceSets.getByName("test").output.dirs) }
-
 configurations.named("intTestRuntimeOnly").configure {
   extendsFrom(configurations.getByName("testRuntimeOnly"))
 }