Dremio merge 2025 09 09 13 05 (apache#111)

snazy · renovate-bot · dimas-b · web-flow · commit 86bf1dba0f79 · 2025-09-09T14:28:39.000+02:00
* Update dependency io.netty:netty-codec-http2 to v4.2.6.Final (apache#2520) * Update dependency org.testcontainers:localstack to v1.21.3 (apache#2497) * Allow overriding createCatalog calls in integrations tests (apache#2516) This is mostly to add testing flexibility in downstream projects. * Allow `PolarisServerManager` implementations to define custom client headers (apache#2510) `PolarisServerManager` is a plugin point for downstream builds to define runtime env. for tests under `integration-tests`. This change allows more flexibility for test runtime environments by allowing injecting extra headers into test clients. * Fix deprecation warnings around RandomStringUtils (apache#2507) the `RandomStringUtils.random` method is deprecated since commons-lang 3.17.0 apache/commons-lang@69cb996 * Add support for poetry build with wheel (apache#2425) * Core: Clarify the purpose of REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY (apache#2509) * Update gradle/actions digest to ed40850 (apache#2524) * Update dependency com.fasterxml.jackson:jackson-bom to v2.20.0 (apache#2470) * Update hadoop to v3.4.2 (apache#2466) * Last merged commit a1d1176 --------- Co-authored-by: Mend Renovate <bot@renovateapp.com> Co-authored-by: Dmitri Bourlatchkov <dmitri.bourlatchkov@gmail.com> Co-authored-by: Christopher Lambert <xn137@gmx.de> Co-authored-by: Yong Zheng <yongzheng0809@gmail.com> Co-authored-by: Yufei Gu <yufei@apache.org>
diff --git a/.github/actions/ci-incr-build-cache-prepare/action.yml b/.github/actions/ci-incr-build-cache-prepare/action.yml
@@ -45,7 +45,7 @@ runs:
         fi
 
     - name: Gradle / Setup
-      uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+      uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
       with:
         cache-read-only: ${{ inputs.cache-read-only }}
         validate-wrappers: false
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -47,7 +47,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+        uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
         with:
           validate-wrappers: false
       - name: Prepare Gradle build cache
@@ -88,7 +88,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+        uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
         with:
           validate-wrappers: false
       - name: Prepare Gradle build cache
@@ -126,7 +126,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+        uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
         with:
           validate-wrappers: false
       - name: Prepare Gradle build cache
@@ -164,7 +164,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+        uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
         with:
           validate-wrappers: false
       - name: Collect partial Gradle build caches
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -53,7 +53,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+        uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
         with:
           validate-wrappers: false
       - name: Publish SNAPSHOTs to Apache Nexus Repository
diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml
@@ -41,7 +41,7 @@ jobs:
           distribution: 'temurin'
 
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+        uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
         with:
           validate-wrappers: false
 
diff --git a/.github/workflows/spark_client_regtests.yml b/.github/workflows/spark_client_regtests.yml
@@ -41,7 +41,7 @@ jobs:
           distribution: 'temurin'
 
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@017a9effdb900e5b5b2fddfb590a105619dca3c3 # v4
+        uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
         with:
           validate-wrappers: false
 
diff --git a/Makefile b/Makefile
@@ -162,6 +162,12 @@ client-integration-test: client-setup-env ## Run client integration tests
 	@echo "Tearing down Docker Compose services..."
 	@$(DOCKER) compose -f $(PYTHON_CLIENT_DIR)/docker-compose.yml down || true # Ensure teardown even if tests fail
 
+.PHONY: client-build
+client-build: client-setup-env ## Build client distribution
+	@echo "--- Building client distribution ---"
+	@$(ACTIVATE_AND_CD) && poetry build -f wheel
+	@echo "--- Client distribution build complete ---"
+
 .PHONY: client-cleanup
 client-cleanup: ## Cleanup virtual environment and Python cache files
 	@echo "--- Cleaning up virtual environment and Python cache files ---"
diff --git a/client/python/generate_clients.py b/client/python/generate_clients.py
@@ -82,6 +82,7 @@
     Path("README.md"),
     Path("generate_clients.py"),
     Path(".venv"),
+    Path("dist/"),
 ]
 EXCLUDE_EXTENSIONS = [
     "json",
diff --git a/client/python/pyproject.toml b/client/python/pyproject.toml
@@ -46,12 +46,9 @@ polaris = "cli.polaris_cli:main"
 
 [tool.poetry]
 requires-poetry = "==2.1.4"
-packages = [
-    { include = "polaris" },
-    { include = "cli" }
-]
 include = [
-  "polaris/**"
+    { path = "polaris/**", format = "wheel" },
+    { path = "cli/**", format = "wheel" },
 ]
 
 [tool.poetry.group.dev.dependencies]
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -19,7 +19,7 @@
 
 [versions]
 checkstyle = "10.25.0"
-hadoop = "3.4.1"
+hadoop = "3.4.2"
 hive = "3.1.3"
 iceberg = "1.9.2" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
 quarkus = "3.25.4"
@@ -67,7 +67,7 @@ immutables-builder = { module = "org.immutables:builder", version.ref = "immutab
 immutables-value-annotations = { module = "org.immutables:value-annotations", version.ref = "immutables" }
 immutables-value-processor = { module = "org.immutables:value-processor", version.ref = "immutables" }
 iceberg-bom = { module = "org.apache.iceberg:iceberg-bom", version.ref = "iceberg" }
-jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version = "2.19.2" }
+jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version = "2.20.0" }
 jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version = "3.0.0" }
 jakarta-enterprise-cdi-api = { module = "jakarta.enterprise:jakarta.enterprise.cdi-api", version = "4.1.0" }
 jakarta-inject-api = { module = "jakarta.inject:jakarta.inject-api", version = "2.0.1" }
@@ -82,7 +82,7 @@ jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" }
 jmh-generator-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" }
 junit-bom = { module = "org.junit:junit-bom", version = "5.13.4" }
 keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version = "26.0.6" }
-localstack = { module = "org.testcontainers:localstack", version = "1.19.7" }
+localstack = { module = "org.testcontainers:localstack", version = "1.21.3" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" }
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.15.4" }
 microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" }
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java
@@ -39,7 +39,7 @@ public static RESTCatalog restCatalog(
             .put(CatalogProperties.URI, endpoints.catalogApiEndpoint().toString())
             .put(OAuth2Properties.TOKEN, authToken)
             .put("warehouse", catalog)
-            .put("header." + endpoints.realmHeaderName(), endpoints.realmId())
+            .putAll(endpoints.extraHeaders("header."))
             .putAll(extraProperties);
 
     restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast());
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java
@@ -20,6 +20,8 @@
 
 import java.io.Serializable;
 import java.net.URI;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * This class contains the most fundamental information for accessing Polaris APIs, such as the base
@@ -30,12 +32,16 @@ public final class PolarisApiEndpoints implements Serializable {
 
   private final URI baseUri;
   private final String realmId;
-  private final String realmHeaderName;
+  private final Map<String, String> headers;
 
   public PolarisApiEndpoints(URI baseUri, String realmId, String realmHeaderName) {
+    this(baseUri, realmId, Map.of(realmHeaderName, realmId));
+  }
+
+  public PolarisApiEndpoints(URI baseUri, String realmId, Map<String, String> headers) {
     this.baseUri = baseUri;
     this.realmId = realmId;
-    this.realmHeaderName = realmHeaderName;
+    this.headers = headers;
   }
 
   public URI catalogApiEndpoint() {
@@ -50,7 +56,13 @@ public String realmId() {
     return realmId;
   }
 
-  public String realmHeaderName() {
-    return realmHeaderName;
+  public Map<String, String> extraHeaders() {
+    return headers;
+  }
+
+  public Map<String, String> extraHeaders(String keyPrefix) {
+    return headers.entrySet().stream()
+        .map(e -> Map.entry(keyPrefix + e.getKey(), e.getValue()))
+        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
   }
 }
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisRestApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisRestApi.java
@@ -39,8 +39,7 @@ public class PolarisRestApi extends RestApi {
   }
 
   protected Map<String, String> defaultHeaders() {
-    Map<String, String> headers = new HashMap<>();
-    headers.put(endpoints.realmHeaderName(), endpoints.realmId());
+    Map<String, String> headers = new HashMap<>(endpoints.extraHeaders());
     if (authToken != null) {
       headers.put("Authorization", "Bearer " + authToken);
     }
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java
@@ -19,6 +19,7 @@
 package org.apache.polaris.service.it.env;
 
 import java.net.URI;
+import java.util.Map;
 
 /**
  * This is a holder for access information to a particular Polaris Server. Test cases may use only
@@ -37,6 +38,10 @@ default String realmHeaderName() {
     return DEFAULT_REALM_HEADER;
   }
 
+  default Map<String, String> headers() {
+    return Map.of(realmHeaderName(), realmId());
+  }
+
   /**
    * The base URI to all Polaris APIs (e.g. the common base of the Iceberg REST API endpoints and
    * Polaris Management API endpoints).
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java
@@ -92,7 +92,7 @@ private static class Env implements AutoCloseable {
     private Env(Server server) {
       this.server = server;
       this.endpoints =
-          new PolarisApiEndpoints(server.baseUri(), server.realmId(), server.realmHeaderName());
+          new PolarisApiEndpoints(server.baseUri(), server.realmId(), server.headers());
     }
 
     PolarisApiEndpoints endpoints() {
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
@@ -259,15 +259,12 @@ private static RESTSessionCatalog newSessionCatalog(String catalog) {
     RESTSessionCatalog sessionCatalog = new RESTSessionCatalog();
     sessionCatalog.initialize(
         "polaris_catalog_test",
-        Map.of(
-            "uri",
-            endpoints.catalogApiEndpoint().toString(),
-            OAuth2Properties.TOKEN,
-            authToken,
-            "warehouse",
-            catalog,
-            "header." + endpoints.realmHeaderName(),
-            realm));
+        ImmutableMap.<String, String>builder()
+            .put("uri", endpoints.catalogApiEndpoint().toString())
+            .put(OAuth2Properties.TOKEN, authToken)
+            .put("warehouse", catalog)
+            .putAll(endpoints.extraHeaders("header."))
+            .build());
     return sessionCatalog;
   }
 
@@ -590,15 +587,12 @@ public void testWarehouseNotSpecified() throws IOException {
               () ->
                   sessionCatalog.initialize(
                       "polaris_catalog_test",
-                      Map.of(
-                          "uri",
-                          endpoints.catalogApiEndpoint().toString(),
-                          OAuth2Properties.TOKEN,
-                          authToken,
-                          "warehouse",
-                          emptyEnvironmentVariable,
-                          "header." + endpoints.realmHeaderName(),
-                          realm)))
+                      ImmutableMap.<String, String>builder()
+                          .put("uri", endpoints.catalogApiEndpoint().toString())
+                          .put(OAuth2Properties.TOKEN, authToken)
+                          .put("warehouse", emptyEnvironmentVariable)
+                          .putAll(endpoints.extraHeaders("header."))
+                          .build()))
           .isInstanceOf(BadRequestException.class)
           .hasMessage("Malformed request: Please specify a warehouse");
     }
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java
@@ -136,6 +136,10 @@ public void tearDown() {
     client.cleanUp(authToken);
   }
 
+  private static String newRandomString(int length) {
+    return RandomStringUtils.insecure().next(length, true, true);
+  }
+
   @Test
   public void testCatalogSerializing() throws IOException {
     CatalogProperties props = new CatalogProperties("s3://my-old-bucket/path/to/data");
@@ -215,7 +219,7 @@ public void testCreateCatalogWithInvalidName() {
             .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data"))
             .build();
 
-    String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true);
+    String goodName = newRandomString(MAX_IDENTIFIER_LENGTH);
 
     Catalog catalog =
         PolarisCatalog.builder()
@@ -228,7 +232,7 @@ public void testCreateCatalogWithInvalidName() {
       assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus);
     }
 
-    String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true);
+    String longInvalidName = newRandomString(MAX_IDENTIFIER_LENGTH + 1);
     List<String> invalidCatalogNames =
         Arrays.asList(
             longInvalidName,
@@ -751,7 +755,7 @@ public void testGetCatalogNotFound() {
 
   @Test
   public void testGetCatalogInvalidName() {
-    String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true);
+    String longInvalidName = newRandomString(MAX_IDENTIFIER_LENGTH + 1);
     List<String> invalidCatalogNames =
         Arrays.asList(
             longInvalidName,
@@ -784,7 +788,7 @@ public void testCatalogRoleInvalidName() {
             .build();
     managementApi.createCatalog(catalog);
 
-    String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true);
+    String longInvalidName = newRandomString(MAX_IDENTIFIER_LENGTH + 1);
     List<String> invalidCatalogRoleNames =
         Arrays.asList(
             longInvalidName,
@@ -1038,15 +1042,15 @@ public void testCreateListUpdateAndDeletePrincipal() {
 
   @Test
   public void testCreatePrincipalWithInvalidName() {
-    String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true);
+    String goodName = newRandomString(MAX_IDENTIFIER_LENGTH);
     Principal principal =
         Principal.builder()
             .setName(goodName)
             .setProperties(Map.of("custom-tag", "good_principal"))
             .build();
     managementApi.createPrincipal(new CreatePrincipalRequest(principal, null));
 
-    String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true);
+    String longInvalidName = newRandomString(MAX_IDENTIFIER_LENGTH + 1);
     List<String> invalidPrincipalNames =
         Arrays.asList(
             longInvalidName,
@@ -1078,7 +1082,7 @@ public void testCreatePrincipalWithInvalidName() {
 
   @Test
   public void testGetPrincipalWithInvalidName() {
-    String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true);
+    String longInvalidName = newRandomString(MAX_IDENTIFIER_LENGTH + 1);
     List<String> invalidPrincipalNames =
         Arrays.asList(
             longInvalidName,
@@ -1193,12 +1197,12 @@ public void testCreateListUpdateAndDeletePrincipalRole() {
 
   @Test
   public void testCreatePrincipalRoleInvalidName() {
-    String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true);
+    String goodName = newRandomString(MAX_IDENTIFIER_LENGTH);
     PrincipalRole principalRole =
         new PrincipalRole(goodName, false, Map.of("custom-tag", "good_principal_role"), 0L, 0L, 1);
     managementApi.createPrincipalRole(principalRole);
 
-    String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true);
+    String longInvalidName = newRandomString(MAX_IDENTIFIER_LENGTH + 1);
     List<String> invalidPrincipalRoleNames =
         Arrays.asList(
             longInvalidName,
@@ -1233,7 +1237,7 @@ public void testCreatePrincipalRoleInvalidName() {
 
   @Test
   public void testGetPrincipalRoleInvalidName() {
-    String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true);
+    String longInvalidName = newRandomString(MAX_IDENTIFIER_LENGTH + 1);
     List<String> invalidPrincipalRoleNames =
         Arrays.asList(
             longInvalidName,
diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts
@@ -86,7 +86,7 @@ dependencies {
   implementation("com.azure:azure-identity")
   implementation("com.azure:azure-storage-file-datalake")
   constraints {
-    implementation("io.netty:netty-codec-http2:4.2.5.Final") {
+    implementation("io.netty:netty-codec-http2:4.2.6.Final") {
       because("Vulnerability detected in 4.1.72")
     }
     implementation("io.projectreactor.netty:reactor-netty-http:1.2.9") {
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java
diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/ApplicationIntegrationTest.java

Original file line number	Diff line number	Diff line change
`@@ -82,6 +82,7 @@`
`82`	`82`	`Path("README.md"),`
`83`	`83`	`Path("generate_clients.py"),`
`84`	`84`	`Path(".venv"),`
	`85`	`+ Path("dist/"),`
`85`	`86`	`]`
`86`	`87`	`EXCLUDE_EXTENSIONS = [`
`87`	`88`	`"json",`