Merge branch 'main' into log-2-logger

Author: eric-maynard

.github/workflows/gradle.yml

@@ -48,6 +48,10 @@ jobs:
       # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4
+        with:
+          # The setup-gradle action fails, if the wrapper is not using the right version or is not present.
+          # Our `gradlew` validates the integrity of the `gradle-wrapper.jar`, so it's safe to disable this.
+          validate-wrappers: false
 
       - name: Check formatting
         run: ./gradlew check

build.gradle

@@ -29,6 +29,7 @@ plugins {
     id "idea"
     id "eclipse"
     id "org.jetbrains.gradle.plugin.idea-ext" version "1.1.8"
+    id "net.ltgt.errorprone" version "4.0.1"
 }
 
 allprojects {
@@ -50,10 +51,14 @@ subprojects {
     apply plugin: "com.diffplug.spotless"
     apply plugin: "jacoco-report-aggregation"
     apply plugin: "groovy"
+    apply plugin: "net.ltgt.errorprone"
 
     tasks.withType(JavaCompile).configureEach {
         options.compilerArgs << "-Xlint:unchecked"
         options.compilerArgs << "-Xlint:deprecation"
+        options.errorprone.disableAllWarnings = true
+        options.errorprone.disableWarningsInGeneratedCode = true
+        options.errorprone.error("StringCaseLocaleUsage")
 
         // TODO Disabled until the code is only Java 11/17, see #76
         // options.release = 17
@@ -76,6 +81,7 @@ subprojects {
         testImplementation(libs.assertj.core)
         testImplementation(libs.mockito.core)
 
+        errorprone("com.google.errorprone:error_prone_core:${libs.errorprone.get().version}")
         testRuntimeOnly("org.junit.platform:junit-platform-launcher")
     }
 

docs/configuring-polaris-for-production.md

@@ -0,0 +1,82 @@
+<!--
+ Copyright (c) 2024 Snowflake Computing Inc.
+ 
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ 
+      http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+# Configuring Polaris for Production
+
+The default `polaris-server.yml` configuration is intended for develoment and testing. When deploying Polaris in production, there are several best practices to keep in mind.
+
+## Security
+
+### Configurations
+
+Notable configuration used to secure a Polaris deployment are outlined below.
+
+#### oauth2
+
+> [!WARNING]  
+> Ensure that the `tokenBroker` setting reflects the token broker specified in `authenticator` below.
+
+* Configure [OAuth](https://oauth.net/2/) with this setting. Remove the `TestInlineBearerTokenPolarisAuthenticator` option and uncomment the `DefaultPolarisAuthenticator` authenticator option beneath it.
+* Then, configure the token broker. You can configure the token broker to use either [asymmetric](https://github.com/polaris-catalog/polaris/blob/b482617bf8cc508b37dbedf3ebc81a9408160a5e/polaris-service/src/main/java/io/polaris/service/auth/JWTRSAKeyPair.java#L24) or [symmetric](https://github.com/polaris-catalog/polaris/blob/b482617bf8cc508b37dbedf3ebc81a9408160a5e/polaris-service/src/main/java/io/polaris/service/auth/JWTSymmetricKeyBroker.java#L23) keys.
+
+#### authenticator.tokenBroker
+
+> [!WARNING]  
+> Ensure that the `tokenBroker` setting reflects the token broker specified in `oauth2` above.
+
+#### callContextResolver** & **realmContextResolver
+* Use these configurations to specify a service that can resolve a realm from bearer tokens.
+* The service(s) used here must implement the relevant interfaces (i.e. [CallContextResolver](https://github.com/polaris-catalog/polaris/blob/8290019c10290a600e40b35ddb1e2f54bf99e120/polaris-service/src/main/java/io/polaris/service/context/CallContextResolver.java#L27) and [RealmContextResolver](https://github.com/polaris-catalog/polaris/blob/7ce86f10a68a3b56aed766235c88d6027c0de038/polaris-service/src/main/java/io/polaris/service/context/RealmContextResolver.java)).
+
+## Metastore Management
+
+> [!IMPORTANT]  
+> The default `in-memory` implementation for `metastoreManager` is meant for testing and not suitable for production usage. Instead, consider an implementation such as `eclipse-link` which allows you to store metadata in a remote database.
+
+A Metastore Manger should be configured with an implementation that durably persists Polaris entities. Use the configuration `metaStoreManager` to configure a [MetastoreManager](https://github.com/polaris-catalog/polaris/blob/627dc602eb15a3258dcc32babf8def34cf6de0e9/polaris-core/src/main/java/io/polaris/core/persistence/PolarisMetaStoreManager.java#L47) implementation where Polaris entities will be persisted. 
+
+Be sure to secure your metastore backend since it will be storing credentials and catalog metadata.
+
+### Configuring EclipseLink
+
+To use [EclipseLink](https://eclipse.dev/eclipselink/) for metastore management, specify the configuration `metaStoreManager.conf-file` to point to an [EclipseLink `persistence.xml` file](https://eclipse.dev/eclipselink/documentation/2.5/solutions/testingjpa002.htm). This file, local to the Polaris service, will contain information on what database to use for metastore management and how to connect to it.
+
+### Bootstrapping
+
+Before using Polaris when using a metastore manager other than `in-memory`, you must **bootstrap** the metastore manager. This is a manual operation that must be performed **only once** in order to prepare the metastore manager to integrate with Polaris. When the metastore manager is bootstrapped, any existing Polaris entities in the metastore manager may be **purged**.
+
+To bootstrap Polaris, run:
+
+```bash
+java -jar /path/to/jar/polaris-service-1.0.0-all.jar bootstrap polaris-server.yml
+```
+
+Afterwards, Polaris can be launched normally:
+
+```bash
+java -jar /path/to/jar/polaris-service-1.0.0-all.jar server polaris-server.yml
+```
+
+## Other Configurations
+
+When deploying Polaris in production, consider adjusting the following configurations:
+
+#### featureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES
+  - By default Polaris catalogs are allowed to be located in local filesystem with the `FILE` storage type. This should be disabled for production systems.
+  - Use this configuration to additionally disable any other storage types that will not be in use.
+
+

docs/index.html

@@ -33,6 +33,7 @@ commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.0" }
 commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.14.0" }
 dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version = "4.0.7" }
 eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.3" }
+errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.29.2" }
 google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.40.1" }
 guava = { module = "com.google.guava:guava", version = "33.2.1-jre" }
 h2 = { module = "com.h2database:h2", version = "2.2.224" }

gradle/libs.versions.toml

@@ -15,30 +15,136 @@
  */
 package io.polaris.core;
 
-public class PolarisConfiguration {
+import java.util.Optional;
 
-  public static final String ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING =
-      "ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING";
-  public static final String ALLOW_TABLE_LOCATION_OVERLAP = "ALLOW_TABLE_LOCATION_OVERLAP";
-  public static final String ALLOW_NAMESPACE_LOCATION_OVERLAP = "ALLOW_NAMESPACE_LOCATION_OVERLAP";
-  public static final String ALLOW_EXTERNAL_METADATA_FILE_LOCATION =
-      "ALLOW_EXTERNAL_METADATA_FILE_LOCATION";
+public class PolarisConfiguration<T> {
 
-  public static final String ALLOW_OVERLAPPING_CATALOG_URLS = "ALLOW_OVERLAPPING_CATALOG_URLS";
+  public final String key;
+  public final String description;
+  public final T defaultValue;
+  private final Optional<String> catalogConfigImpl;
+  private final Class<T> typ;
 
-  public static final String CATALOG_ALLOW_UNSTRUCTURED_TABLE_LOCATION =
-      "allow.unstructured.table.location";
-  public static final String CATALOG_ALLOW_EXTERNAL_TABLE_LOCATION =
-      "allow.external.table.location";
+  @SuppressWarnings("unchecked")
+  public PolarisConfiguration(
+      String key, String description, T defaultValue, Optional<String> catalogConfig) {
+    this.key = key;
+    this.description = description;
+    this.defaultValue = defaultValue;
+    this.catalogConfigImpl = catalogConfig;
+    this.typ = (Class<T>) defaultValue.getClass();
+  }
 
-  /*
-   * Default values for the configuration properties
-   */
+  public boolean hasCatalogConfig() {
+    return catalogConfigImpl.isPresent();
+  }
 
-  public static final boolean DEFAULT_ALLOW_OVERLAPPING_CATALOG_URLS = false;
-  public static final boolean DEFAULT_ALLOW_TABLE_LOCATION_OVERLAP = false;
-  public static final boolean DEFAULT_ALLOW_EXTERNAL_METADATA_FILE_LOCATION = false;
-  public static final boolean DEFAULT_ALLOW_NAMESPACE_LOCATION_OVERLAP = false;
+  public String catalogConfig() {
+    return catalogConfigImpl.orElseThrow(
+        () ->
+            new IllegalStateException(
+                "Attempted to read a catalog config key from a configuration that doesn't have one."));
+  }
 
-  private PolarisConfiguration() {}
+  T cast(Object value) {
+    return this.typ.cast(value);
+  }
+
+  public static class Builder<T> {
+    private String key;
+    private String description;
+    private T defaultValue;
+    private Optional<String> catalogConfig = Optional.empty();
+
+    public Builder<T> key(String key) {
+      this.key = key;
+      return this;
+    }
+
+    public Builder<T> description(String description) {
+      this.description = description;
+      return this;
+    }
+
+    public Builder<T> defaultValue(T defaultValue) {
+      this.defaultValue = defaultValue;
+      return this;
+    }
+
+    public Builder<T> catalogConfig(String catalogConfig) {
+      this.catalogConfig = Optional.of(catalogConfig);
+      return this;
+    }
+
+    public PolarisConfiguration<T> build() {
+      if (key == null || description == null || defaultValue == null) {
+        throw new IllegalArgumentException("key, description, and defaultValue are required");
+      }
+      return new PolarisConfiguration<>(key, description, defaultValue, catalogConfig);
+    }
+  }
+
+  public static <T> Builder<T> builder() {
+    return new Builder<>();
+  }
+
+  public static final PolarisConfiguration<Boolean>
+      ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING =
+          PolarisConfiguration.<Boolean>builder()
+              .key("ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING")
+              .description(
+                  "If set to true, require that principals must rotate their credentials before being used "
+                      + "for anything else.")
+              .defaultValue(false)
+              .build();
+
+  public static final PolarisConfiguration<Boolean> ALLOW_TABLE_LOCATION_OVERLAP =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ALLOW_TABLE_LOCATION_OVERLAP")
+          .description(
+              "If set to true, allow one table's location to reside within another table's location. "
+                  + "This is only enforced within a given namespace.")
+          .defaultValue(false)
+          .build();
+
+  public static final PolarisConfiguration<Boolean> ALLOW_NAMESPACE_LOCATION_OVERLAP =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ALLOW_NAMESPACE_LOCATION_OVERLAP")
+          .description(
+              "If set to true, allow one table's location to reside within another table's location. "
+                  + "This is only enforced within a parent catalog or namespace.")
+          .defaultValue(false)
+          .build();
+
+  public static final PolarisConfiguration<Boolean> ALLOW_EXTERNAL_METADATA_FILE_LOCATION =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ALLOW_EXTERNAL_METADATA_FILE_LOCATION")
+          .description(
+              "If set to true, allows metadata files to be located outside the default metadata directory.")
+          .defaultValue(false)
+          .build();
+
+  public static final PolarisConfiguration<Boolean> ALLOW_OVERLAPPING_CATALOG_URLS =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ALLOW_OVERLAPPING_CATALOG_URLS")
+          .description("If set to true, allows catalog URLs to overlap.")
+          .defaultValue(false)
+          .build();
+
+  public static final PolarisConfiguration<Boolean> ALLOW_UNSTRUCTURED_TABLE_LOCATION =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ALLOW_UNSTRUCTURED_TABLE_LOCATION")
+          .catalogConfig("allow.unstructured.table.location")
+          .description("If set to true, allows unstructured table locations.")
+          .defaultValue(false)
+          .build();
+
+  public static final PolarisConfiguration<Boolean> ALLOW_EXTERNAL_TABLE_LOCATION =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ALLOW_EXTERNAL_TABLE_LOCATION")
+          .catalogConfig("allow.external.table.location")
+          .description(
+              "If set to true, allows tables to have external locations outside the default structure.")
+          .defaultValue(false)
+          .build();
 }

polaris-core/src/main/java/io/polaris/core/PolarisConfiguration.java

@@ -16,14 +16,18 @@
 package io.polaris.core;
 
 import com.google.common.base.Preconditions;
+import io.polaris.core.entity.CatalogEntity;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Dynamic configuration store used to retrieve runtime parameters, which may vary by realm or by
  * request.
  */
 public interface PolarisConfigurationStore {
+  Logger LOGGER = LoggerFactory.getLogger(PolarisConfigurationStore.class);
 
   /**
    * Retrieve the current value for a configuration key. May be null if not set.
@@ -53,4 +57,56 @@ public interface PolarisConfigurationStore {
     T configValue = getConfiguration(ctx, configName);
     return configValue != null ? configValue : defaultValue;
   }
+
+  /**
+   * In some cases, we may extract a value that doesn't match the expected type for a config. This
+   * method can be used to attempt to force-cast it using `String.valueOf`
+   */
+  private <T> @NotNull T tryCast(PolarisConfiguration<T> config, Object value) {
+    if (value == null) {
+      return config.defaultValue;
+    }
+
+    if (config.defaultValue instanceof Boolean) {
+      return config.cast(Boolean.valueOf(String.valueOf(value)));
+    } else {
+      return config.cast(value);
+    }
+  }
+
+  /**
+   * Retrieve the current value for a configuration.
+   *
+   * @param ctx the current call context
+   * @param config the configuration to load
+   * @return the current value set for the configuration key or null if not set
+   * @param <T> the type of the configuration value
+   */
+  default <T> @NotNull T getConfiguration(PolarisCallContext ctx, PolarisConfiguration<T> config) {
+    T result = getConfiguration(ctx, config.key, config.defaultValue);
+    return tryCast(config, result);
+  }
+
+  /**
+   * Retrieve the current value for a configuration, overriding with a catalog config if it is
+   * present.
+   *
+   * @param ctx the current call context
+   * @param catalogEntity the catalog to check for an override
+   * @param config the configuration to load
+   * @return the current value set for the configuration key or null if not set
+   * @param <T> the type of the configuration value
+   */
+  default <T> @NotNull T getConfiguration(
+      PolarisCallContext ctx,
+      @NotNull CatalogEntity catalogEntity,
+      PolarisConfiguration<T> config) {
+    if (config.hasCatalogConfig()
+        && catalogEntity.getPropertiesAsMap().containsKey(config.catalogConfig())) {
+      LOGGER.debug("Loaded config from catalog: {}", config.catalogConfig());
+      return tryCast(config, catalogEntity.getPropertiesAsMap().get(config.catalogConfig()));
+    } else {
+      return getConfiguration(ctx, config);
+    }
+  }
 }

polaris-core/src/main/java/io/polaris/core/PolarisConfigurationStore.java

@@ -500,8 +500,7 @@ public void authorizeOrThrow(
     boolean enforceCredentialRotationRequiredState =
         featureConfig.getConfiguration(
             CallContext.getCurrentContext().getPolarisCallContext(),
-            PolarisConfiguration.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING,
-            false);
+            PolarisConfiguration.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING);
     if (enforceCredentialRotationRequiredState
         && authenticatedPrincipal
             .getPrincipalEntity()