Add support for S3 request signing

Fixes #32.

Author: adutra

LICENSE

@@ -216,6 +216,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 This product includes code from Apache Iceberg.
 
 * spec/iceberg-rest-catalog-open-api.yaml
+* spec/iceberg-s3-signer-open-api.yaml
 * spec/polaris-catalog-apis/oauth-tokens-api.yaml
 * integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java
 * runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java

api/iceberg-service/build.gradle.kts

@@ -31,6 +31,7 @@ dependencies {
   implementation(platform(libs.iceberg.bom))
   implementation("org.apache.iceberg:iceberg-api")
   implementation("org.apache.iceberg:iceberg-core")
+  implementation("org.apache.iceberg:iceberg-aws")
 
   implementation(libs.jakarta.annotation.api)
   implementation(libs.jakarta.inject.api)
@@ -49,6 +50,9 @@ dependencies {
   implementation("com.fasterxml.jackson.core:jackson-databind")
 
   compileOnly(libs.microprofile.fault.tolerance.api)
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
 }
 
 val rootDir = rootProject.layout.projectDirectory
@@ -70,7 +74,7 @@ openApiGenerate {
   ignoreFileOverride.set(provider { rootDir.file(".openapi-generator-ignore").asFile.absolutePath })
   templateDir.set(provider { templatesDir.asFile.absolutePath })
   removeOperationIdPrefix.set(true)
-  globalProperties.put("apis", "CatalogApi,ConfigurationApi,OAuth2Api")
+  globalProperties.put("apis", "CatalogApi,ConfigurationApi,OAuth2Api,S3SignerApi")
   globalProperties.put("models", "false")
   globalProperties.put("apiDocs", "false")
   globalProperties.put("modelTests", "false")
@@ -112,6 +116,10 @@ openApiGenerate {
       "IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse",
       "OAuthError" to "org.apache.iceberg.rest.responses.ErrorResponse",
 
+      // S3 Signing
+      "S3SignRequest" to "org.apache.polaris.service.types.S3SignRequest",
+      "SignS3Request200Response" to "org.apache.polaris.service.types.S3SignResponse",
+
       // Custom types defined below
       "CommitViewRequest" to "org.apache.polaris.service.types.CommitViewRequest",
       "TokenType" to "org.apache.polaris.service.types.TokenType",

api/iceberg-service/src/main/java/org/apache/polaris/service/types/S3SignRequest.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.types;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import jakarta.annotation.Nullable;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.rest.RESTRequest;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+/**
+ * Request for S3 signing requests.
+ *
+ * <p>Copy of {@link org.apache.iceberg.aws.s3.signer.S3SignRequest}, because the original does not
+ * have Jackson annotations.
+ */
+@PolarisImmutable
+@JsonDeserialize(as = ImmutableS3SignRequest.class)
+@JsonSerialize(as = ImmutableS3SignRequest.class)
+public interface S3SignRequest extends RESTRequest {
+
+  String region();
+
+  String method();
+
+  URI uri();
+
+  Map<String, List<String>> headers();
+
+  // unused by Polaris, but required by Iceberg
+  Map<String, String> properties();
+
+  @Value.Default
+  @Nullable
+  default String body() {
+    return null;
+  }
+
+  @Override
+  default void validate() {}
+}

api/iceberg-service/src/main/java/org/apache/polaris/service/types/S3SignResponse.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.types;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.rest.RESTResponse;
+import org.apache.polaris.immutables.PolarisImmutable;
+
+/**
+ * Response for S3 signing requests.
+ *
+ * <p>Copy of {@link org.apache.iceberg.aws.s3.signer.S3SignResponse}, because the original does not
+ * have Jackson annotations.
+ */
+@PolarisImmutable
+@JsonDeserialize(as = ImmutableS3SignResponse.class)
+@JsonSerialize(as = ImmutableS3SignResponse.class)
+public interface S3SignResponse extends RESTResponse {
+
+  URI uri();
+
+  Map<String, List<String>> headers();
+
+  @Override
+  default void validate() {}
+}

integration-tests/build.gradle.kts

@@ -33,6 +33,7 @@ dependencies {
   implementation(platform(libs.iceberg.bom))
   implementation("org.apache.iceberg:iceberg-api")
   implementation("org.apache.iceberg:iceberg-core")
+  implementation("org.apache.iceberg:iceberg-aws")
 
   implementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests")
   implementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests")

integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogConfig.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.it.env;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.polaris.core.admin.model.Catalog;
+
+/**
+ * Annotation to configure the catalog type and properties for integration tests.
+ *
+ * <p>This is a server-side setting; it is used to specify the Polaris Catalog type (e.g., INTERNAL,
+ * EXTERNAL) and any additional properties required for the catalog configuration.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Inherited
+public @interface CatalogConfig {
+
+  /** The type of the catalog. Defaults to INTERNAL. */
+  Catalog.TypeEnum value() default Catalog.TypeEnum.INTERNAL;
+
+  /** Additional properties for the catalog configuration. */
+  String[] properties() default {};
+}

integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java

@@ -20,6 +20,7 @@
 
 import com.google.common.collect.ImmutableMap;
 import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
 import org.apache.iceberg.rest.RESTCatalog;
 import org.apache.iceberg.rest.auth.OAuth2Properties;
 import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
@@ -38,12 +39,8 @@ public static RESTCatalog restCatalog(
 
     ImmutableMap.Builder<String, String> propertiesBuilder =
         ImmutableMap.<String, String>builder()
-            .put(
-                org.apache.iceberg.CatalogProperties.URI, endpoints.catalogApiEndpoint().toString())
+            .put(CatalogProperties.URI, endpoints.catalogApiEndpoint().toString())
             .put(OAuth2Properties.TOKEN, authToken)
-            .put(
-                org.apache.iceberg.CatalogProperties.FILE_IO_IMPL,
-                "org.apache.iceberg.inmemory.InMemoryFileIO")
             .put("warehouse", catalog)
             .put("header." + endpoints.realmHeaderName(), endpoints.realmId())
             .putAll(extraProperties);

integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java

@@ -19,9 +19,17 @@
 package org.apache.polaris.service.it.env;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
 import java.net.URI;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.TestInfo;
 
 public final class IntegrationTestsHelper {
 
@@ -54,4 +62,60 @@ static URI getTemporaryDirectory(Function<String, String> getenv, Path defaultLo
     envVar = envVar.startsWith("/") ? "file://" + envVar : envVar;
     return URI.create(envVar + "/").normalize();
   }
+
+  /**
+   * Extract a value from the annotated elements of the test method and class.
+   *
+   * <p>This method looks for annotations of the specified type on both the test method and the test
+   * class, extracts the value using the provided extractor function, and returns it. If the value
+   * is not present in either the method or class annotations, the provided default value is
+   * returned.
+   */
+  public static <A extends Annotation, T> T extractFromAnnotatedElements(
+      TestInfo testInfo, Class<A> annotationClass, Function<A, T> extractor, T def) {
+    Method method = testInfo.getTestMethod().orElseThrow();
+    Optional<T> methodCatalogConfig =
+        Optional.ofNullable(method.getAnnotation(annotationClass)).map(extractor);
+    if (methodCatalogConfig.isPresent()) {
+      return methodCatalogConfig.get();
+    }
+    Optional<T> classCatalogConfig =
+        testInfo.getTestClass().map(clz -> clz.getAnnotation(annotationClass)).map(extractor);
+    return classCatalogConfig.orElse(def);
+  }
+
+  /**
+   * Collect properties from annotated elements in the test method and class.
+   *
+   * <p>This method looks for annotations of the specified type on both the test method and the test
+   * class, extracts properties using the provided extractor function, and combines them into a map.
+   * If a property appears in both the method and class annotations, the value from the method
+   * annotation will be used.
+   */
+  public static <A extends Annotation> Map<String, String> mergeFromAnnotatedElements(
+      TestInfo testInfo,
+      Class<A> annotationClass,
+      Function<A, String[]> propertiesExtractor,
+      Map<String, String> defaults) {
+    Method method = testInfo.getTestMethod().orElseThrow();
+    Optional<A> methodCatalogConfig = Optional.ofNullable(method.getAnnotation(annotationClass));
+    Optional<A> classCatalogConfig =
+        testInfo.getTestClass().map(clz -> clz.getAnnotation(annotationClass));
+    String[] methodProperties = methodCatalogConfig.map(propertiesExtractor).orElse(new String[0]);
+    String[] classProperties = classCatalogConfig.map(propertiesExtractor).orElse(new String[0]);
+    String[] properties =
+        Stream.concat(Arrays.stream(methodProperties), Arrays.stream(classProperties))
+            .toArray(String[]::new);
+    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+    builder.putAll(defaults);
+    if (properties.length % 2 != 0) {
+      throw new IllegalArgumentException(
+          "Properties must be in key-value pairs, but found an odd number of elements: "
+              + Arrays.toString(properties));
+    }
+    for (int i = 0; i < properties.length; i += 2) {
+      builder.put(properties[i], properties[i + 1]);
+    }
+    return builder.buildKeepingLast();
+  }
 }

integration-tests/src/main/java/org/apache/polaris/service/it/env/RestCatalogConfig.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.it.env;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to configure the REST catalog for integration tests.
+ *
+ * <p>This is a client-side setting; it is used to configure the client-side REST catalog that is
+ * used in test code to connect to the Polaris REST API and to the storage layer.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Inherited
+public @interface RestCatalogConfig {
+  String[] value() default {};
+}