diff --git a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 492f61668d..9f7af1a268 100644 --- a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -25,6 +25,7 @@ import jakarta.inject.Inject; import java.nio.file.Path; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -46,12 +47,13 @@ public class EclipseLinkPolarisMetaStoreManagerFactory @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; protected EclipseLinkPolarisMetaStoreManagerFactory() { - this(null); + this(null, null); } @Inject - protected EclipseLinkPolarisMetaStoreManagerFactory(PolarisDiagnostics diagnostics) { - super(diagnostics); + protected EclipseLinkPolarisMetaStoreManagerFactory( + PolarisDiagnostics diagnostics, PolarisConfigurationStore configurationStore) { + super(diagnostics, configurationStore); } @Override diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index f944654142..4b369953dc 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -74,6 +75,7 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject Instance dataSource; @Inject RelationalJdbcConfiguration relationalJdbcConfiguration; + @Inject PolarisConfigurationStore configurationStore; protected JdbcMetaStoreManagerFactory() {} @@ -205,7 +207,8 @@ public synchronized StorageCredentialCache getOrCreateStorageCredentialCache( RealmContext realmContext) { if (!storageCredentialCacheMap.containsKey(realmContext.getRealmIdentifier())) { storageCredentialCacheMap.put( - realmContext.getRealmIdentifier(), new StorageCredentialCache()); + realmContext.getRealmIdentifier(), + new StorageCredentialCache(realmContext, configurationStore)); } return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); @@ -216,7 +219,8 @@ public synchronized EntityCache getOrCreateEntityCache(RealmContext realmContext if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); entityCacheMap.put( - realmContext.getRealmIdentifier(), new InMemoryEntityCache(metaStoreManager)); + realmContext.getRealmIdentifier(), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); } return entityCacheMap.get(realmContext.getRealmIdentifier()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java index adb66daf55..31ae187955 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/PolarisConfiguration.java @@ -24,7 +24,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -210,25 +209,6 @@ public BehaviorChangeConfiguration buildBehaviorChangeConfiguration() { } } - /** - * Returns the value of a `PolarisConfiguration`, or the default if it cannot be loaded. This - * method does not need to be used when a `CallContext` is already available - */ - public static T loadConfig(PolarisConfiguration configuration) { - var callContext = CallContext.getCurrentContext(); - if (callContext == null) { - LOGGER.warn( - String.format( - "Unable to load current call context; using %s = %s", - configuration.key, configuration.defaultValue)); - return configuration.defaultValue; - } - return callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration(callContext.getRealmContext(), configuration); - } - public static Builder builder() { return new Builder<>(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index a995883304..f975ae7392 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -43,6 +43,7 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -80,7 +81,7 @@ public static CatalogEntity of(PolarisBaseEntity sourceEntity) { return null; } - public static CatalogEntity fromCatalog(Catalog catalog) { + public static CatalogEntity fromCatalog(CallContext callContext, Catalog catalog) { Builder builder = new Builder() .setName(catalog.getName()) @@ -90,7 +91,7 @@ public static CatalogEntity fromCatalog(Catalog catalog) { internalProperties.put(CATALOG_TYPE_PROPERTY, catalog.getType().name()); builder.setInternalProperties(internalProperties); builder.setStorageConfigurationInfo( - catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); + callContext, catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); return builder.build(); } @@ -247,7 +248,7 @@ public Builder setReplaceNewLocationPrefixWithCatalogDefault(String value) { } public Builder setStorageConfigurationInfo( - StorageConfigInfo storageConfigModel, String defaultBaseLocation) { + CallContext callContext, StorageConfigInfo storageConfigModel, String defaultBaseLocation) { if (storageConfigModel != null) { PolarisStorageConfigurationInfo config; Set allowedLocations = new HashSet<>(storageConfigModel.getAllowedLocations()); @@ -261,7 +262,7 @@ public Builder setStorageConfigurationInfo( throw new BadRequestException("Must specify default base location"); } allowedLocations.add(defaultBaseLocation); - validateMaxAllowedLocations(allowedLocations); + validateMaxAllowedLocations(callContext, allowedLocations); switch (storageConfigModel.getStorageType()) { case S3: AwsStorageConfigInfo awsConfigModel = (AwsStorageConfigInfo) storageConfigModel; @@ -305,10 +306,15 @@ public Builder setStorageConfigurationInfo( } /** Validate the number of allowed locations not exceeding the max value. */ - private void validateMaxAllowedLocations(Collection allowedLocations) { + private void validateMaxAllowedLocations( + CallContext callContext, Collection allowedLocations) { int maxAllowedLocations = - BehaviorChangeConfiguration.loadConfig( - BehaviorChangeConfiguration.STORAGE_CONFIGURATION_MAX_LOCATIONS); + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration( + callContext.getRealmContext(), + BehaviorChangeConfiguration.STORAGE_CONFIGURATION_MAX_LOCATIONS); if (maxAllowedLocations != -1 && allowedLocations.size() > maxAllowedLocations) { throw new IllegalArgumentException( String.format( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java index 060a908e15..4f5a98ce3c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java @@ -1619,7 +1619,7 @@ private void revokeGrantRecord( try { EnumMap creds = storageIntegration.getSubscopedCreds( - callCtx.getDiagServices(), + callCtx, storageConfigurationInfo, allowListOperation, allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 1cfa89d0f0..747991636a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -26,6 +26,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; @@ -63,10 +64,14 @@ public abstract class LocalPolarisMetaStoreManagerFactory LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class); private final PolarisDiagnostics diagnostics; + private final PolarisConfigurationStore configurationStore; private boolean bootstrap; - protected LocalPolarisMetaStoreManagerFactory(@Nonnull PolarisDiagnostics diagnostics) { + protected LocalPolarisMetaStoreManagerFactory( + @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisConfigurationStore configurationStore) { this.diagnostics = diagnostics; + this.configurationStore = configurationStore; } protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); @@ -177,7 +182,8 @@ public synchronized StorageCredentialCache getOrCreateStorageCredentialCache( RealmContext realmContext) { if (!storageCredentialCacheMap.containsKey(realmContext.getRealmIdentifier())) { storageCredentialCacheMap.put( - realmContext.getRealmIdentifier(), new StorageCredentialCache()); + realmContext.getRealmIdentifier(), + new StorageCredentialCache(realmContext, configurationStore)); } return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); @@ -188,7 +194,8 @@ public synchronized EntityCache getOrCreateEntityCache(RealmContext realmContext if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); entityCacheMap.put( - realmContext.getRealmIdentifier(), new InMemoryEntityCache(metaStoreManager)); + realmContext.getRealmIdentifier(), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); } return entityCacheMap.get(realmContext.getRealmIdentifier()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java index 4599f4aed1..3e1fc3ec66 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCache.java @@ -30,7 +30,8 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.config.PolarisConfiguration; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; @@ -58,7 +59,10 @@ public class InMemoryEntityCache implements EntityCache { * * @param polarisMetaStoreManager the meta store manager implementation */ - public InMemoryEntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { + public InMemoryEntityCache( + @Nonnull RealmContext realmContext, + @Nonnull PolarisConfigurationStore configurationStore, + @Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { // by name cache this.byName = new ConcurrentHashMap<>(); @@ -76,7 +80,8 @@ public InMemoryEntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreMana }; long weigherTarget = - PolarisConfiguration.loadConfig(FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); + configurationStore.getConfiguration( + realmContext, FeatureConfiguration.ENTITY_CACHE_WEIGHER_TARGET); Caffeine byIdBuilder = Caffeine.newBuilder() .maximumWeight(weigherTarget) @@ -84,7 +89,10 @@ public InMemoryEntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreMana .expireAfterAccess(1, TimeUnit.HOURS) // Expire entries after 1 hour of no access .removalListener(removalListener); // Set the removal listener - if (PolarisConfiguration.loadConfig(BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES)) { + boolean useSoftValues = + configurationStore.getConfiguration( + realmContext, BehaviorChangeConfiguration.ENTITY_CACHE_SOFT_VALUES); + if (useSoftValues) { byIdBuilder.softValues(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 0d6111d54b..71686165dd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -2053,7 +2053,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( try { EnumMap creds = storageIntegration.getSubscopedCreds( - callCtx.getDiagServices(), + callCtx, storageConfigurationInfo, allowListOperation, allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java index e549dd1f8a..147ec5c66e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java @@ -22,7 +22,7 @@ import java.util.EnumMap; import java.util.Map; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; /** * Abstract of Polaris Storage Integration. It holds the reference to an object that having the @@ -45,7 +45,7 @@ public String getStorageIdentifierOrId() { /** * Subscope the creds against the allowed read and write locations. * - * @param diagnostics the diagnostics service + * @param callContext the call context * @param storageConfig storage configuration * @param allowListOperation whether to allow LIST on all the provided allowed read/write * locations @@ -54,7 +54,7 @@ public String getStorageIdentifierOrId() { * @return An enum map including the scoped credentials */ public abstract EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull T storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index bc8f5e33c5..04c1970dd9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -19,7 +19,6 @@ package org.apache.polaris.core.storage.aws; import static org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS; -import static org.apache.polaris.core.config.PolarisConfiguration.loadConfig; import jakarta.annotation.Nonnull; import java.net.URI; @@ -29,7 +28,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.StorageUtil; @@ -63,11 +62,16 @@ public AwsCredentialsStorageIntegration( /** {@inheritDoc} */ @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull AwsStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, @Nonnull Set allowedWriteLocations) { + int storageCredentialDurationSeconds = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration(callContext.getRealmContext(), STORAGE_CREDENTIAL_DURATION_SECONDS); AssumeRoleRequest.Builder request = AssumeRoleRequest.builder() .externalId(storageConfig.getExternalId()) @@ -80,7 +84,7 @@ public EnumMap getSubscopedCreds( allowedReadLocations, allowedWriteLocations) .toJson()) - .durationSeconds(loadConfig(STORAGE_CREDENTIAL_DURATION_SECONDS)); + .durationSeconds(storageCredentialDurationSeconds); credentialsProvider.ifPresent( cp -> request.overrideConfiguration(b -> b.credentialsProvider(cp))); AssumeRoleResponse response = stsClient.assumeRole(request.build()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java index bcbc91a5c8..39bd363d49 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.core.storage.azure; +import static org.apache.polaris.core.config.FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS; + import com.azure.core.credential.AccessToken; import com.azure.core.credential.TokenRequestContext; import com.azure.identity.DefaultAzureCredential; @@ -45,8 +47,7 @@ import java.util.HashSet; import java.util.Objects; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.StorageAccessProperty; import org.slf4j.Logger; @@ -71,7 +72,7 @@ public AzureCredentialsStorageIntegration() { @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull AzureStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, @@ -126,7 +127,10 @@ public EnumMap getSubscopedCreds( // clock skew between the client and server, OffsetDateTime startTime = start.truncatedTo(ChronoUnit.SECONDS).atOffset(ZoneOffset.UTC); int intendedDurationSeconds = - FeatureConfiguration.loadConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration(callContext.getRealmContext(), STORAGE_CREDENTIAL_DURATION_SECONDS); OffsetDateTime intendedEndTime = start.plusSeconds(intendedDurationSeconds).atOffset(ZoneOffset.UTC); OffsetDateTime maxAllowedEndTime = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java index ef92973600..36f666db04 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java @@ -32,7 +32,8 @@ import org.apache.iceberg.exceptions.UnprocessableEntityException; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.config.PolarisConfiguration; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; @@ -47,10 +48,16 @@ public class StorageCredentialCache { private static final Logger LOGGER = LoggerFactory.getLogger(StorageCredentialCache.class); private static final long CACHE_MAX_NUMBER_OF_ENTRIES = 10_000L; + private final LoadingCache cache; + private final RealmContext realmContext; + private final PolarisConfigurationStore configurationStore; /** Initialize the creds cache */ - public StorageCredentialCache() { + public StorageCredentialCache( + RealmContext realmContext, PolarisConfigurationStore configurationStore) { + this.realmContext = realmContext; + this.configurationStore = configurationStore; cache = Caffeine.newBuilder() .maximumSize(CACHE_MAX_NUMBER_OF_ENTRIES) @@ -62,7 +69,7 @@ public StorageCredentialCache() { 0, Math.min( (entry.getExpirationTime() - System.currentTimeMillis()) / 2, - maxCacheDurationMs())); + this.maxCacheDurationMs())); return Duration.ofMillis(expireAfterMillis); })) .build( @@ -73,12 +80,13 @@ public StorageCredentialCache() { } /** How long credentials should remain in the cache. */ - private static long maxCacheDurationMs() { + private long maxCacheDurationMs() { var cacheDurationSeconds = - PolarisConfiguration.loadConfig( - FeatureConfiguration.STORAGE_CREDENTIAL_CACHE_DURATION_SECONDS); + configurationStore.getConfiguration( + realmContext, FeatureConfiguration.STORAGE_CREDENTIAL_CACHE_DURATION_SECONDS); var credentialDurationSeconds = - PolarisConfiguration.loadConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); + configurationStore.getConfiguration( + realmContext, FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); if (cacheDurationSeconds >= credentialDurationSeconds) { throw new IllegalArgumentException( String.format( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java index 3a7c85780d..2fc1438fe2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java @@ -38,7 +38,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.StorageAccessProperty; @@ -70,7 +70,7 @@ public GcpCredentialsStorageIntegration( @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull GcpStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java index 1d0564be9e..945f1ccb6b 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java @@ -103,7 +103,8 @@ public InMemoryEntityCacheTest() { * @return new cache for the entity store */ InMemoryEntityCache allocateNewCache() { - return new InMemoryEntityCache(this.metaStoreManager); + return new InMemoryEntityCache( + callCtx.getRealmContext(), callCtx.getConfigurationStore(), this.metaStoreManager); } @Test diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java new file mode 100644 index 0000000000..e008abf74b --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.storage; + +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.BasePersistence; +import org.mockito.Mockito; + +public abstract class BaseStorageIntegrationTest { + protected CallContext newCallContext() { + return new PolarisCallContext( + () -> "realm", Mockito.mock(BasePersistence.class), Mockito.mock(PolarisDiagnostics.class)); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java index a4e58860de..e679d3e32a 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java @@ -27,7 +27,6 @@ import java.util.Set; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; @@ -209,7 +208,7 @@ public MockInMemoryStorageIntegration() { @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull PolarisStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index b1d1789dac..3d889b7657 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -72,12 +72,16 @@ public StorageCredentialCacheTest() { new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); + } + + private StorageCredentialCache newStorageCredentialCache() { + return new StorageCredentialCache(callCtx.getRealmContext(), callCtx.getConfigurationStore()); } @Test public void testBadResult() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); ScopedCredentialsResult badResult = new ScopedCredentialsResult( BaseResult.ReturnStatus.SUBSCOPE_CREDS_ERROR, "extra_error_info"); @@ -110,7 +114,7 @@ public void testBadResult() { @Test public void testCacheHit() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); Mockito.when( @@ -153,7 +157,7 @@ public void testCacheHit() { @RepeatedTest(10) public void testCacheEvict() throws InterruptedException { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ true); Mockito.when( @@ -211,7 +215,7 @@ public void testCacheEvict() throws InterruptedException { @Test public void testCacheGenerateNewEntries() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); Mockito.when( @@ -298,7 +302,7 @@ public void testCacheGenerateNewEntries() { @Test public void testCacheNotAffectedBy() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List mockedScopedCreds = getFakeScopedCreds(3, /* expireSoon= */ false); @@ -443,7 +447,7 @@ private static List getPolarisEntities() { @Test public void testAzureCredentialFormatting() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); List mockedScopedCreds = List.of( new ScopedCredentialsResult( @@ -532,7 +536,7 @@ public void testAzureCredentialFormatting() { @Test public void testExtraProperties() { - storageCredentialCache = new StorageCredentialCache(); + storageCredentialCache = newStorageCredentialCache(); ScopedCredentialsResult properties = new ScopedCredentialsResult( new EnumMap<>( diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java index b837033e17..93f96b3589 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java @@ -25,7 +25,7 @@ import java.util.EnumMap; import java.util.List; import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.storage.BaseStorageIntegrationTest; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; @@ -48,7 +48,7 @@ import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; import software.amazon.awssdk.services.sts.model.Credentials; -class AwsCredentialsStorageIntegrationTest { +class AwsCredentialsStorageIntegrationTest extends BaseStorageIntegrationTest { public static final Instant EXPIRE_TIME = Instant.now().plusMillis(3600_000); @@ -83,7 +83,7 @@ public void testGetSubscopedCreds() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(warehouseDir), @@ -231,7 +231,7 @@ public void testGetSubscopedCredsInlinePolicy(String awsPartition) { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -248,7 +248,7 @@ public void testGetSubscopedCredsInlinePolicy(String awsPartition) { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -349,7 +349,7 @@ public void testGetSubscopedCredsInlinePolicyWithoutList() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -444,7 +444,7 @@ public void testGetSubscopedCredsInlinePolicyWithoutWrites() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( storageType, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -509,7 +509,7 @@ public void testGetSubscopedCredsInlinePolicyWithEmptyReadAndWrite() { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -549,7 +549,7 @@ public void testClientRegion(String awsPartition) { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -566,7 +566,7 @@ public void testClientRegion(String awsPartition) { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -604,7 +604,7 @@ public void testNoClientRegion(String awsPartition) { EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), @@ -622,7 +622,7 @@ public void testNoClientRegion(String awsPartition) { () -> new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( - Mockito.mock(PolarisDiagnostics.class), + newCallContext(), new AwsStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3, List.of(s3Path(bucket, warehouseKeyPrefix)), diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java index f4738d73e1..9f43d42bd1 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java @@ -47,7 +47,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.storage.BaseStorageIntegrationTest; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; @@ -61,7 +61,7 @@ import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; -public class AzureCredentialStorageIntegrationTest { +public class AzureCredentialStorageIntegrationTest extends BaseStorageIntegrationTest { private final String clientId = System.getenv("AZURE_CLIENT_ID"); private final String clientSecret = System.getenv("AZURE_CLIENT_SECRET"); @@ -349,7 +349,7 @@ private Map subscopedCredsForOperations( new AzureCredentialsStorageIntegration(); EnumMap credsMap = azureCredsIntegration.getSubscopedCreds( - new PolarisDefaultDiagServiceImpl(), + newCallContext(), azureConfig, allowListAction, new HashSet<>(allowedReadLoc), diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java index fc88bd8170..861ad43acf 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java @@ -48,7 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.storage.BaseStorageIntegrationTest; import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; @@ -59,7 +59,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -class GcpCredentialsStorageIntegrationTest { +class GcpCredentialsStorageIntegrationTest extends BaseStorageIntegrationTest { private final String gcsServiceKeyJsonFileLocation = System.getenv("GOOGLE_APPLICATION_CREDENTIALS"); @@ -171,7 +171,7 @@ private Map subscopedCredsForOperations( ServiceOptions.getFromServiceLoader(HttpTransportFactory.class, NetHttpTransport::new)); EnumMap credsMap = gcpCredsIntegration.getSubscopedCreds( - new PolarisDefaultDiagServiceImpl(), + newCallContext(), gcpConfig, allowListAction, new HashSet<>(allowedReadLoc), diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java index 728187a38d..1a7a827315 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BaseResolverTest.java @@ -468,7 +468,9 @@ private Resolver allocateResolver( // create a new cache if needs be if (cache == null) { - this.cache = new InMemoryEntityCache(metaStoreManager()); + this.cache = + new InMemoryEntityCache( + callCtx().getRealmContext(), callCtx().getConfigurationStore(), metaStoreManager()); } boolean allRoles = principalRolesScope == null; Optional> roleEntities = diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 593853c409..e1b197005a 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -95,8 +95,9 @@ public Clock clock() { @Produces @ApplicationScoped - public StorageCredentialCache storageCredentialCache() { - return new StorageCredentialCache(); + public StorageCredentialCache storageCredentialCache( + RealmContext realmContext, PolarisConfigurationStore configurationStore) { + return new StorageCredentialCache(realmContext, configurationStore); } @Produces diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 4e8748de55..1c232bf54c 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -284,7 +284,7 @@ public void before(TestInfo testInfo) { .setName(CATALOG_NAME) .setCatalogType("INTERNAL") .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo(callContext, storageConfigModel, storageLocation) .build() .asCatalog())); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 5c488809d1..4d89d55234 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -1726,7 +1726,7 @@ public void testSendNotificationSufficientPrivileges() { new CatalogEntity.Builder() .setName(externalCatalog) .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo(callContext, storageConfigModel, storageLocation) .setCatalogType("EXTERNAL") .build() .asCatalog())); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 2b4c4205c9..614fab7255 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -225,6 +225,10 @@ public Map getConfigOverrides() { private TestPolarisEventListener testPolarisEventListener; private ReservedProperties reservedProperties; + protected String getRealmName() { + return realmName; + } + @BeforeAll public static void setUpMocks() { PolarisStorageIntegrationProviderImpl mock = @@ -257,7 +261,9 @@ public void before(TestInfo testInfo) { entityManager = new PolarisEntityManager( - metaStoreManager, new StorageCredentialCache(), createEntityCache(metaStoreManager)); + metaStoreManager, + new StorageCredentialCache(realmContext, configurationStore), + createEntityCache(metaStoreManager)); PrincipalEntity rootEntity = new PrincipalEntity( @@ -313,7 +319,8 @@ public void before(TestInfo testInfo) { "true") .addProperty( FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, storageConfigModel, storageLocation) .build() .asCatalog())); @@ -423,12 +430,12 @@ public Supplier getOrCreateSessionSupplier( @Override public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + return new StorageCredentialCache(realmContext, configurationStore); } @Override public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager); } @Override @@ -1590,7 +1597,8 @@ public void testDropTableWithPurgeDisabled() { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") - .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, noPurgeStorageConfigModel, storageLocation) .build() .asCatalog())); PolarisPassthroughResolutionView passthroughView = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 0f06a02250..6ea276cc09 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -179,8 +179,8 @@ public void before(TestInfo testInfo) { PolarisEntityManager entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(realmContext, configurationStore), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); CallContext.setCurrentContext(polarisContext); @@ -224,6 +224,7 @@ public void before(TestInfo testInfo) { .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true") .setDefaultBaseLocation("file://tmp") .setStorageConfigurationInfo( + polarisContext, new FileStorageConfigInfo( StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), "file://tmp") diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java index 079b02befc..3c59dff186 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java @@ -31,6 +31,6 @@ public class PolarisCatalogWithEntityCacheTest extends IcebergCatalogTest { @Nullable @Override protected InMemoryEntityCache createEntityCache(PolarisMetaStoreManager metaStoreManager) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(() -> getRealmName(), configurationStore, metaStoreManager); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java index 2aba1773b9..c9196f1750 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java @@ -177,8 +177,8 @@ public void before(TestInfo testInfo) { entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(realmContext, configurationStore), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); PrincipalEntity rootEntity = new PrincipalEntity( @@ -233,7 +233,8 @@ public void before(TestInfo testInfo) { "true") .addProperty( FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, storageConfigModel, storageLocation) .build() .asCatalog())); @@ -302,12 +303,12 @@ public Supplier getOrCreateSessionSupplier( @Override public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + return new StorageCredentialCache(realmContext, configurationStore); } @Override public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index 93ee060723..c0d4b8b463 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -203,8 +203,8 @@ public void before(TestInfo testInfo) { entityManager = new PolarisEntityManager( metaStoreManager, - new StorageCredentialCache(), - new InMemoryEntityCache(metaStoreManager)); + new StorageCredentialCache(realmContext, configurationStore), + new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager)); callContext = polarisContext; @@ -259,7 +259,8 @@ public void before(TestInfo testInfo) { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setStorageConfigurationInfo( + polarisContext, storageConfigModel, storageLocation) .build() .asCatalog())); @@ -326,12 +327,12 @@ public Supplier getOrCreateSessionSupplier( @Override public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { - return new StorageCredentialCache(); + return new StorageCredentialCache(realmContext, configurationStore); } @Override public InMemoryEntityCache getOrCreateEntityCache(RealmContext realmContext) { - return new InMemoryEntityCache(metaStoreManager); + return new InMemoryEntityCache(realmContext, configurationStore, metaStoreManager); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java index 0a6dea8fed..ed44510a9d 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java @@ -34,15 +34,17 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; public class CatalogEntityTest { - @BeforeAll - public static void setup() { + private CallContext callContext; + + @BeforeEach + public void setup() { MetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); RealmContext realmContext = () -> "realm"; PolarisCallContext polarisCallContext = @@ -50,6 +52,7 @@ public static void setup() { realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(() -> "realm").get(), new PolarisDefaultDiagServiceImpl()); + this.callContext = polarisCallContext; CallContext.setCurrentContext(polarisCallContext); } @@ -72,7 +75,7 @@ public void testInvalidAllowedLocationPrefix() { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); @@ -93,7 +96,7 @@ public void testInvalidAllowedLocationPrefix() { new CatalogProperties("abfs://container@storageaccount.blob.windows.net/path")) .setStorageConfigInfo(azureStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, azureCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid azure location uri unsupportPrefix://mybucket/path"); @@ -110,7 +113,7 @@ public void testInvalidAllowedLocationPrefix() { .setProperties(new CatalogProperties("gs://externally-owned-bucket")) .setStorageConfigInfo(gcpStorageConfigModel) .build(); - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, gcpCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining( "Location prefix not allowed: 'unsupportPrefix://mybucket/path', expected prefixes"); @@ -142,7 +145,7 @@ public void testExceedMaxAllowedLocations() { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatCode(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatCode(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)) .doesNotThrowAnyException(); } @@ -166,7 +169,8 @@ public void testValidAllowedLocationPrefix() { .setProperties(prop) .setStorageConfigInfo(awsStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)); basedLocation = "abfs://container@storageaccount.blob.windows.net/path"; prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); @@ -183,7 +187,8 @@ public void testValidAllowedLocationPrefix() { .setProperties(new CatalogProperties(basedLocation)) .setStorageConfigInfo(azureStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(azureCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(callContext, azureCatalog)); basedLocation = "gs://externally-owned-bucket"; prop.put(CatalogEntity.DEFAULT_BASE_LOCATION_KEY, basedLocation); @@ -199,7 +204,8 @@ public void testValidAllowedLocationPrefix() { .setProperties(new CatalogProperties(basedLocation)) .setStorageConfigInfo(gcpStorageConfigModel) .build(); - Assertions.assertThatNoException().isThrownBy(() -> CatalogEntity.fromCatalog(gcpCatalog)); + Assertions.assertThatNoException() + .isThrownBy(() -> CatalogEntity.fromCatalog(callContext, gcpCatalog)); } @ParameterizedTest @@ -234,7 +240,7 @@ public void testInvalidArn(String roleArn) { expectedMessage = "Invalid role ARN format"; } ; - Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(awsCatalog)) + Assertions.assertThatThrownBy(() -> CatalogEntity.fromCatalog(callContext, awsCatalog)) .isInstanceOf(IllegalArgumentException.class) .hasMessage(expectedMessage); } diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 4495d96fd8..1a1914ad10 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -716,7 +716,7 @@ public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG; authorizeBasicRootOperationOrThrow(op); - CatalogEntity entity = CatalogEntity.fromCatalog(catalogRequest.getCatalog()); + CatalogEntity entity = CatalogEntity.fromCatalog(callContext, catalogRequest.getCatalog()); checkArgument(entity.getId() == -1, "Entity to be created must have no ID assigned"); @@ -898,7 +898,7 @@ private void validateUpdateCatalogDiffOrThrow( } if (updateRequest.getStorageConfigInfo() != null) { updateBuilder.setStorageConfigurationInfo( - updateRequest.getStorageConfigInfo(), defaultBaseLocation); + callContext, updateRequest.getStorageConfigInfo(), defaultBaseLocation); } CatalogEntity updatedEntity = updateBuilder.build(); diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java index 3f0b1a3552..83908f0178 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java @@ -22,6 +22,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -36,13 +37,15 @@ public class InMemoryAtomicOperationMetaStoreManagerFactory extends InMemoryPolarisMetaStoreManagerFactory { public InMemoryAtomicOperationMetaStoreManagerFactory() { - super(null, null); + super(null, null, null); } @Inject public InMemoryAtomicOperationMetaStoreManagerFactory( - PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics) { - super(storageIntegration, diagnostics); + PolarisStorageIntegrationProvider storageIntegration, + PolarisDiagnostics diagnostics, + PolarisConfigurationStore configurationStore) { + super(storageIntegration, diagnostics, configurationStore); } @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 895e5b51b1..367a57de63 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.function.Supplier; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -48,13 +49,15 @@ public class InMemoryPolarisMetaStoreManagerFactory private final Set bootstrappedRealms = new HashSet<>(); public InMemoryPolarisMetaStoreManagerFactory() { - this(null, null); + this(null, null, null); } @Inject public InMemoryPolarisMetaStoreManagerFactory( - PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics) { - super(diagnostics); + PolarisStorageIntegrationProvider storageIntegration, + PolarisDiagnostics diagnostics, + PolarisConfigurationStore configurationStore) { + super(diagnostics, configurationStore); this.storageIntegration = storageIntegration; } diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 2d84e3747d..253ab5d03a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -31,7 +31,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -99,7 +99,7 @@ public PolarisStorageIntegrationProviderImpl( new PolarisStorageIntegration<>("file") { @Override public EnumMap getSubscopedCreds( - @Nonnull PolarisDiagnostics diagnostics, + @Nonnull CallContext callContext, @Nonnull T storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index d3622126fb..8f63ecfb83 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -154,7 +154,7 @@ public TestServices build() { () -> GoogleCredentials.create(new AccessToken(GCP_ACCESS_TOKEN, new Date()))); InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory( - storageIntegrationProvider, polarisDiagnostics); + storageIntegrationProvider, polarisDiagnostics, configurationStore); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory) {}; UserSecretsManagerFactory userSecretsManagerFactory =