merge from main, address comments from @adutra

Author: adnanhemani

.github/workflows/stale.yml

@@ -22,7 +22,7 @@ jobs:
   stale:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/stale@8f717f0dfca33b78d3c933452e42558e4456c8e7
+      - uses: actions/stale@3a9db7e6a41a89f618792c92c0e97cc736e1b13f
         with:
           days-before-close: 5
           days-before-stale: 30

CHANGELOG.md

@@ -88,6 +88,8 @@ automatic storage credential refresh per table on the client side. Java client v
 The endpoint path is always returned when using vended credentials, but clients must enable the 
 refresh-credentials flag for the desired storage provider.
 
+- Added a Management API endpoint to reset principal credentials, controlled by the `ENABLE_CREDENTIAL_RESET` (default: true) feature flag.
+
 ### Changes
 
 - Polaris Management API clients must be prepared to deal with new attributes in `AwsStorageConfigInfo` objects.

gradle/libs.versions.toml

@@ -24,6 +24,7 @@ hive = "3.1.3"
 iceberg = "1.9.2" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
 quarkus = "3.25.4"
 immutables = "2.11.3"
+jmh = "1.37"
 picocli = "4.7.7"
 scala212 = "2.12.19"
 spark35 = "3.5.6"
@@ -41,7 +42,7 @@ swagger = "1.6.16"
 antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.strictly = "4.9.3" } # spark integration tests
 assertj-core = { module = "org.assertj:assertj-core", version = "3.27.4" }
 auth0-jwt = { module = "com.auth0:java-jwt", version = "4.5.0" }
-awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.32.29" }
+awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.33.0" }
 awaitility = { module = "org.awaitility:awaitility", version = "4.3.0" }
 azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.37" }
 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.2.2" }
@@ -75,7 +76,11 @@ jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0
 jandex = { module = "io.smallrye.jandex:jandex", version ="3.4.0" }
 javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" }
 junit-bom = { module = "org.junit:junit-bom", version = "5.13.4" }
+localstack = { module = "org.testcontainers:localstack", version = "1.19.7" }
 keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version = "26.0.6" }
+jcstress-core = { module = "org.openjdk.jcstress:jcstress-core", version = "0.16" }
+jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" }
+jmh-generator-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" }
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.15.3" }
 microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" }
@@ -102,6 +107,8 @@ testcontainers-keycloak = { module = "com.github.dasniko:testcontainers-keycloak
 threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" }
 
 [plugins]
+jcstress = { id = "io.github.reyerizo.gradle.jcstress", version = "0.8.15" }
+jmh = { id = "me.champeau.jmh", version = "0.7.3" }
 openapi-generator = { id = "org.openapi.generator", version = "7.12.0" }
 quarkus = { id = "io.quarkus", version.ref = "quarkus" }
 rat = { id = "org.nosphere.apache.rat", version = "0.8.1" }

gradle/projects.main.properties

@@ -48,3 +48,9 @@ polaris-extensions-federation-hive=extensions/federation/hive
 polaris-config-docs-annotations=tools/config-docs/annotations
 polaris-config-docs-generator=tools/config-docs/generator
 polaris-config-docs-site=tools/config-docs/site
+
+# id generation
+polaris-idgen-api=persistence/nosql/idgen/api
+polaris-idgen-impl=persistence/nosql/idgen/impl
+polaris-idgen-mocks=persistence/nosql/idgen/mocks
+polaris-idgen-spi=persistence/nosql/idgen/spi

integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java

@@ -80,6 +80,20 @@ public PrincipalWithCredentials createPrincipal(CreatePrincipalRequest request)
     }
   }
 
+  /**
+   * Retrieves a Principal by name via the management API.
+   *
+   * @param principalName the name of the principal to fetch
+   * @return the Principal object
+   */
+  public Principal getPrincipal(String principalName) {
+    try (Response response =
+        request("v1/principals/{principalName}", Map.of("principalName", principalName)).get()) {
+      assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+      return response.readEntity(Principal.class);
+    }
+  }
+
   public void createPrincipalRole(String name) {
     createPrincipalRole(new PrincipalRole(name));
   }

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java

@@ -880,6 +880,57 @@ public void testCreatePrincipalAndRotateCredentials() {
     // rotation that makes the old secret fall off retention.
   }
 
+  @Test
+  public void testCreatePrincipalAndResetCredentialsWithCustomValues() {
+    //  Create a new principal using root user
+    Principal principal =
+        Principal.builder()
+            .setName(client.newEntityName("myprincipal-reset"))
+            .setProperties(Map.of("custom-tag", "bar"))
+            .build();
+
+    PrincipalWithCredentials creds =
+        managementApi.createPrincipal(new CreatePrincipalRequest(principal, true));
+
+    Map<String, String> customBody =
+        Map.of(
+            "clientId", "f174b76a7e1a99e2",
+            "clientSecret", "27029d236abc08e204922b0a07031bc2");
+
+    PrincipalWithCredentials resetCreds;
+    try (Response response =
+        managementApi
+            .request("v1/principals/{p}/reset", Map.of("p", principal.getName()))
+            .post(Entity.json(customBody))) {
+
+      assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+      resetCreds = response.readEntity(PrincipalWithCredentials.class);
+    }
+
+    assertThat(resetCreds.getCredentials().getClientId()).isEqualTo("f174b76a7e1a99e2");
+    assertThat(resetCreds.getCredentials().getClientSecret())
+        .isEqualTo("27029d236abc08e204922b0a07031bc2");
+
+    // Validate that the principal entity itself is updated in sync with credentials
+    Principal updatedPrincipal = managementApi.getPrincipal(principal.getName());
+    assertThat(updatedPrincipal.getClientId()).isEqualTo("f174b76a7e1a99e2");
+
+    // Principal itself tries to reset with custom creds → should fail (403 Forbidden)
+    String principalToken = client.obtainToken(resetCreds);
+    customBody =
+        Map.of(
+            "clientId", "a174b76a7e1a99e3",
+            "clientSecret", "37029d236abc08e204922b0a07031bc3");
+    try (Response response =
+        client
+            .managementApi(principalToken)
+            .request("v1/principals/{p}/reset", Map.of("p", principal.getName()))
+            .post(Entity.json(customBody))) {
+
+      assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus);
+    }
+  }
+
   @Test
   public void testCreateFederatedPrincipalRoleSucceeds() {
     // Create a federated Principal Role

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java

@@ -20,7 +20,6 @@
 
 import static org.apache.polaris.service.it.env.PolarisClient.polarisClient;
 
-import com.google.common.collect.ImmutableMap;
 import java.lang.reflect.Method;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -88,6 +87,7 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT
   private static ManagementApi managementApi;
 
   private RESTCatalog restCatalog;
+  private StorageConfigInfo storageConfig;
 
   @BeforeAll
   static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) {
@@ -114,7 +114,7 @@ public void before(TestInfo testInfo) {
     Method method = testInfo.getTestMethod().orElseThrow();
     String catalogName = client.newEntityName(method.getName());
 
-    StorageConfigInfo storageConfig = getStorageConfigInfo();
+    storageConfig = getStorageConfigInfo();
     String defaultBaseLocation =
         storageConfig.getAllowedLocations().getFirst()
             + "/"
@@ -201,16 +201,10 @@ public void createViewWithCustomMetadataLocationUsingPolaris(@TempDir Path tempD
     TableIdentifier identifier = TableIdentifier.of("ns", "view");
 
     String location = Paths.get(tempDir.toUri().toString()).toString();
-    String customLocation = Paths.get(tempDir.toUri().toString(), "custom-location").toString();
-    String customLocation2 = Paths.get(tempDir.toUri().toString(), "custom-location2").toString();
-    String customLocationChild =
-        Paths.get(tempDir.toUri().toString(), "custom-location/child").toString();
+    String customLocation =
+        Paths.get(storageConfig.getAllowedLocations().getFirst(), "/custom-location1").toString();
 
-    catalog()
-        .createNamespace(
-            identifier.namespace(),
-            ImmutableMap.of(
-                IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY, location));
+    catalog().createNamespace(identifier.namespace());
 
     Assertions.assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse();
 
@@ -234,35 +228,5 @@ public void createViewWithCustomMetadataLocationUsingPolaris(@TempDir Path tempD
     Assertions.assertThat(((BaseView) view).operations().current().metadataFileLocation())
         .isNotNull()
         .startsWith(customLocation);
-
-    // CANNOT update the view with a new metadata location `baseLocation/customLocation2`,
-    // even though the new location is still under the parent namespace's
-    // `write.metadata.path=baseLocation`.
-    Assertions.assertThatThrownBy(
-            () ->
-                catalog()
-                    .loadView(identifier)
-                    .updateProperties()
-                    .set(
-                        IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY,
-                        customLocation2)
-                    .commit())
-        .isInstanceOf(ForbiddenException.class)
-        .hasMessageContaining("Forbidden: Invalid locations");
-
-    // CANNOT update the view with a child metadata location `baseLocation/customLocation/child`,
-    // even though it is a subpath of the original view's
-    // `write.metadata.path=baseLocation/customLocation`.
-    Assertions.assertThatThrownBy(
-            () ->
-                catalog()
-                    .loadView(identifier)
-                    .updateProperties()
-                    .set(
-                        IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY,
-                        customLocationChild)
-                    .commit())
-        .isInstanceOf(ForbiddenException.class)
-        .hasMessageContaining("Forbidden: Invalid locations");
   }
 }

persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java

@@ -747,4 +747,15 @@ Optional<Optional<String>> hasOverlappingSiblings(
           @Nonnull PolarisCallContext callContext, T entity) {
     return Optional.empty();
   }
+
+  @Nullable
+  @Override
+  public PolarisPrincipalSecrets storePrincipalSecrets(
+      @Nonnull PolarisCallContext callCtx,
+      long principalId,
+      @Nonnull String resolvedClientId,
+      String customClientSecret) {
+    throw new UnsupportedOperationException(
+        "This method is not supported for EclipseLink as metastore");
+  }
 }

persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java

@@ -89,7 +89,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() {
             diagServices, store, Mockito.mock(), realmContext, null, "polaris", RANDOM_SECRETS);
     TransactionalMetaStoreManagerImpl metaStoreManager =
         new TransactionalMetaStoreManagerImpl(clock, diagServices);
-    PolarisCallContext callCtx = new PolarisCallContext(realmContext, session, diagServices);
+    PolarisCallContext callCtx = new PolarisCallContext(realmContext, session);
     return new PolarisTestMetaStoreManager(metaStoreManager, callCtx);
   }
 

persistence/nosql/idgen/README.md

@@ -0,0 +1,55 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Unique ID generation framework and monotonic clock
+
+Provides a framework and implementations for unique ID generation, including a monotonically increasing timestamp/clock
+source.
+
+Provides a
+[Snowflake-IDs](https://medium.com/@jitenderkmr/demystifying-snowflake-ids-a-unique-identifier-in-distributed-computing-72796a827c9d)
+implementation.
+
+Consuming production should primarily leverage the `IdGenerator` and `MonotonicClock` interfaces.
+
+## Snowflake ID source
+
+The Snowflake ID source is configurable for each backend instance, but cannot be modified for an existing backend
+instance to prevent ID conflicts.
+
+The epoch of these timestamps is 2025-03-01-00:00:00.0 GMT. Timestamps occupy 41 bits at
+millisecond precision, which lasts for about 69 years. Node-IDs are 10 bits, which allows 1024 concurrently active
+"JVMs running Polaris". 12 bits are used by the sequence number, which then allows each node to generate 4096 IDs per
+millisecond. One bit is reserved for future use.
+
+Node IDs are leased by every "JVM running Polaris" for a period of time. The ID generator implementation guarantees
+that no IDs will be generated for a timestamp that exceeds the "lease time". Leases can be extended. The implementation
+leverages atomic database operations (CAS) for the lease implementation.
+
+ID generators must not use timestamps before or after the lease period nor must they re-use an older timestamp. This
+requirement is satisfied using a monotonic clock implementation.
+
+## Code structure
+
+The code is structured into multiple modules. Consuming code should almost always pull in only the API module.
+
+* `polaris-idgen-api` provides the necessary Java interfaces and immutable types.
+* `polaris-idgen-impl` provides the storage agnostic implementation.
+* `polaris-idgen-mocks` provides mocks for testing.
+* `polaris-idgen-spi` provides the necessary interfaces to construct ID generators.