Remote S3 configs

trinodb · Jul 23, 2024 · ee38d30 · ee38d30
1 parent b764af8
commit ee38d30
Show file tree

Hide file tree

Showing 10 changed files with 268 additions and 31 deletions.
diff --git a/pom.xml b/pom.xml
@@ -55,6 +55,8 @@
         <dep.minio.version>8.5.9</dep.minio.version>
         <dep.docker.version>3.3.6</dep.docker.version>
         <dep.awaitility.version>4.1.1</dep.awaitility.version>
+        <dep.apache.velocity>2.3</dep.apache.velocity>
+
     </properties>
 
     <dependencyManagement>
@@ -105,6 +107,12 @@
                 <version>${dep.minio.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.velocity</groupId>
+                <artifactId>velocity-engine-core</artifactId>
+                <version>${dep.apache.velocity}</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.awaitility</groupId>
                 <artifactId>awaitility</artifactId>

diff --git a/trino-aws-proxy/pom.xml b/trino-aws-proxy/pom.xml
@@ -109,6 +109,11 @@
             <artifactId>jakarta.ws.rs-api</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.velocity</groupId>
+            <artifactId>velocity-engine-core</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.glassfish.jersey.core</groupId>
             <artifactId>jersey-common</artifactId>

diff --git a/...-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/AwsRemoteS3FacadeConstants.java b/...-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/AwsRemoteS3FacadeConstants.java
diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/PathStyleRemoteS3Facade.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/PathStyleRemoteS3Facade.java
@@ -13,12 +13,12 @@
  */
 package io.trino.aws.proxy.server.remote;
 
+import com.google.inject.Inject;
 import jakarta.ws.rs.core.UriBuilder;
 
 import java.net.URI;
 import java.util.Optional;
 
-import static io.trino.aws.proxy.server.remote.AwsRemoteS3FacadeConstants.PATH_BUILDER;
 import static java.util.Objects.requireNonNull;
 
 public class PathStyleRemoteS3Facade
@@ -28,9 +28,10 @@ public class PathStyleRemoteS3Facade
     private final boolean https;
     private final Optional<Integer> port;
 
-    public PathStyleRemoteS3Facade()
+    @Inject
+    public PathStyleRemoteS3Facade(RemoteS3Config remoteS3Config)
     {
-        this(PATH_BUILDER, true, Optional.empty());
+        this((bucket, region) -> RemoteS3HostBuilder.getHostName(bucket, region, remoteS3Config.getDomain(), remoteS3Config.getPathStyleTemplate()), remoteS3Config.getHttps(), remoteS3Config.getPort());
     }
 
     public PathStyleRemoteS3Facade(RemoteS3HostBuilder hostBuilder, boolean https, Optional<Integer> port)

diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3Config.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3Config.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import io.airlift.configuration.Config;
+import jakarta.validation.constraints.NotNull;
+
+import java.util.Optional;
+
+public class RemoteS3Config
+{
+    private Boolean https = true;
+    private String domain = "amazonaws.com";
+    private Optional<Integer> port = Optional.empty();
+    private String virtualHostStyleTemplate = "${bucket}.s3.${region}.${domain}";
+    private String pathStyleTemplate = "s3.${region}.${domain}";
+
+    @Config("remoteS3.https")
+    public RemoteS3Config setHttps(boolean https)
+    {
+        this.https = https;
+        return this;
+    }
+
+    @Config("remoteS3.domain")
+    public RemoteS3Config setDomain(String s3Domain)
+    {
+        this.domain = s3Domain;
+        return this;
+    }
+
+    @Config("remoteS3.port")
+    public RemoteS3Config setPort(Integer port)
+    {
+        this.port = Optional.ofNullable(port);
+        return this;
+    }
+
+    @Config("remoteS3.virtualhoststyle.template")
+    public RemoteS3Config setVirtualHostStyleTemplate(String hostnameTemplate)
+    {
+        this.virtualHostStyleTemplate = hostnameTemplate;
+        return this;
+    }
+
+    @Config("remoteS3.pathstyle.template")
+    public RemoteS3Config setPathStyleTemplate(String hostnameTemplate)
+    {
+        this.pathStyleTemplate = hostnameTemplate;
+        return this;
+    }
+
+    public boolean getHttps()
+    {
+        return https;
+    }
+
+    @NotNull
+    public String getDomain()
+    {
+        return domain;
+    }
+
+    @NotNull
+    public Optional<Integer> getPort()
+    {
+        return port;
+    }
+
+    @NotNull
+    public String getVirtualHostStyleTemplate()
+    {
+        return virtualHostStyleTemplate;
+    }
+
+    @NotNull
+    public String getPathStyleTemplate()
+    {
+        return pathStyleTemplate;
+    }
+}
diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3HostBuilder.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3HostBuilder.java
@@ -13,7 +13,23 @@
  */
 package io.trino.aws.proxy.server.remote;
 
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.Velocity;
+
+import java.io.StringWriter;
+
 public interface RemoteS3HostBuilder
 {
+    static String getHostName(String bucket, String region, String domain, String hostnameTemplate)
+    {
+        VelocityContext context = new VelocityContext();
+        context.put("domain", domain);
+        context.put("bucket", bucket);
+        context.put("region", region);
+        StringWriter stringWriter = new StringWriter();
+        Velocity.evaluate(context, stringWriter, "", hostnameTemplate);
+        return stringWriter.toString();
+    }
+
     String build(String bucket, String region);
 }
diff --git a/...-proxy/src/main/java/io/trino/aws/proxy/server/remote/VirtualHostStyleRemoteS3Facade.java b/...-proxy/src/main/java/io/trino/aws/proxy/server/remote/VirtualHostStyleRemoteS3Facade.java
@@ -13,12 +13,12 @@
  */
 package io.trino.aws.proxy.server.remote;
 
+import com.google.inject.Inject;
 import jakarta.ws.rs.core.UriBuilder;
 
 import java.net.URI;
 import java.util.Optional;
 
-import static io.trino.aws.proxy.server.remote.AwsRemoteS3FacadeConstants.VIRTUAL_HOST_BUILDER;
 import static java.util.Objects.requireNonNull;
 
 public class VirtualHostStyleRemoteS3Facade
@@ -28,9 +28,10 @@ public class VirtualHostStyleRemoteS3Facade
     private final boolean https;
     private final Optional<Integer> port;
 
-    public VirtualHostStyleRemoteS3Facade()
+    @Inject
+    public VirtualHostStyleRemoteS3Facade(RemoteS3Config remoteS3Config)
     {
-        this(VIRTUAL_HOST_BUILDER, true, Optional.empty());
+        this((bucket, region) -> RemoteS3HostBuilder.getHostName(bucket, region, remoteS3Config.getDomain(), remoteS3Config.getVirtualHostStyleTemplate()), remoteS3Config.getHttps(), remoteS3Config.getPort());
     }
 
     public VirtualHostStyleRemoteS3Facade(RemoteS3HostBuilder hostBuilder, boolean https, Optional<Integer> port)

diff --git a/...aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestPathStyleRemoteS3Facade.java b/...aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestPathStyleRemoteS3Facade.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import jakarta.ws.rs.core.UriBuilder;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TestPathStyleRemoteS3Facade
+{
+    @Test
+    public void testBuildEndpoint()
+    {
+        RemoteS3Config remoteS3Config = new RemoteS3Config().setHttps(false).setDomain("testS3Domain.com").setPort(80).setPathStyleTemplate("s3.${region}.${domain}");
+        RemoteS3Facade remoteS3Facade = new PathStyleRemoteS3Facade(remoteS3Config);
+        URI expectedEndpoint = UriBuilder.fromUri("http://s3.us-east-1.testS3Domain.com:80/test_bucket/object_path/foo").build();
+        URI actual = remoteS3Facade.buildEndpoint(UriBuilder.newInstance(), "object_path/foo", "test_bucket", "us-east-1");
+        assertThat(actual).isEqualTo(expectedEndpoint);
+    }
+
+    @Test
+    public void testBuildEndpointWithoutRegion()
+    {
+        RemoteS3Config remoteS3Config = new RemoteS3Config().setHttps(true).setDomain("testS3Domain.com").setPort(80).setPathStyleTemplate("s3.${domain}");
+        RemoteS3Facade remoteS3Facade = new PathStyleRemoteS3Facade(remoteS3Config);
+        URI expectedEndpoint = UriBuilder.fromUri("https://s3.testS3Domain.com:80/test_bucket/object_path/foo").build();
+        URI actual = remoteS3Facade.buildEndpoint(UriBuilder.newInstance(), "object_path/foo", "test_bucket", "us-east-1");
+        assertThat(actual).isEqualTo(expectedEndpoint);
+    }
+}
diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestRemoteS3Config.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestRemoteS3Config.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
+import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
+import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
+
+public class TestRemoteS3Config
+{
+    @Test
+    public void testDefaults()
+    {
+        assertRecordedDefaults(recordDefaults(RemoteS3Config.class)
+                .setDomain("amazonaws.com").setHttps(true).setPort(null)
+                .setVirtualHostStyleTemplate("${bucket}.s3.${region}.${domain}")
+                .setPathStyleTemplate("s3.${region}.${domain}"));
+    }
+
+    @Test
+    public void testExplicitPropertyMappings()
+    {
+        Map<String, String> properties = ImmutableMap.of(
+                "remoteS3.https", "false",
+                "remoteS3.domain", "testS3Domain.com",
+                "remoteS3.port", "80",
+                "remoteS3.virtualhoststyle.template", "${bucket}.s3a.${region}.${domain}",
+                "remoteS3.pathstyle.template", "s3a.${region}.${domain}");
+
+        RemoteS3Config expected = new RemoteS3Config().setHttps(false).setDomain("testS3Domain.com").setPort(80)
+                .setVirtualHostStyleTemplate("${bucket}.s3a.${region}.${domain}")
+                .setPathStyleTemplate("s3a.${region}.${domain}");
+        assertFullMapping(properties, expected);
+    }
+}
diff --git a/...xy/src/test/java/io/trino/aws/proxy/server/remote/TestVirtualHostStyleRemoteS3Facade.java b/...xy/src/test/java/io/trino/aws/proxy/server/remote/TestVirtualHostStyleRemoteS3Facade.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import jakarta.ws.rs.core.UriBuilder;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TestVirtualHostStyleRemoteS3Facade
+{
+    @Test
+    public void testBuildEndpoint()
+    {
+        RemoteS3Config remoteS3Config = new RemoteS3Config().setHttps(false).setDomain("testS3Domain.com").setPort(80).setVirtualHostStyleTemplate("${bucket}.s3.${region}.${domain}");
+        RemoteS3Facade remoteS3Facade = new VirtualHostStyleRemoteS3Facade(remoteS3Config);
+        URI expectedEndpoint = UriBuilder.fromUri("http://test_bucket.s3.us-east-1.testS3Domain.com:80/object_path/foo").build();
+        URI actual = remoteS3Facade.buildEndpoint(UriBuilder.newInstance(), "object_path/foo", "test_bucket", "us-east-1");
+        assertThat(actual).isEqualTo(expectedEndpoint);
+    }
+
+    @Test
+    public void testBuildEndpointWithoutRegion()
+    {
+        RemoteS3Config remoteS3Config = new RemoteS3Config().setHttps(true).setDomain("testS3Domain.com").setPort(80).setVirtualHostStyleTemplate("${bucket}.s3.${domain}");
+        RemoteS3Facade remoteS3Facade = new VirtualHostStyleRemoteS3Facade(remoteS3Config);
+        URI expectedEndpoint = UriBuilder.fromUri("https://test_bucket.s3.testS3Domain.com:80/object_path/foo").build();
+        URI actual = remoteS3Facade.buildEndpoint(UriBuilder.newInstance(), "object_path/foo", "test_bucket", "us-east-1");
+        assertThat(actual).isEqualTo(expectedEndpoint);
+    }
+}