Remote S3 configs

trinodb · Jul 24, 2024 · a2f80dc · a2f80dc
1 parent 0dcb6f8
commit a2f80dc
Show file tree

Hide file tree

Showing 10 changed files with 318 additions and 34 deletions.
diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java
@@ -25,8 +25,7 @@
 import io.airlift.jaxrs.JaxrsBinder;
 import io.airlift.log.Logger;
 import io.trino.aws.proxy.server.credentials.CredentialsController;
-import io.trino.aws.proxy.server.remote.RemoteS3Facade;
-import io.trino.aws.proxy.server.remote.VirtualHostStyleRemoteS3Facade;
+import io.trino.aws.proxy.server.remote.RemoteS3Module;
 import io.trino.aws.proxy.server.rest.RequestFilter;
 import io.trino.aws.proxy.server.rest.RequestLoggerController;
 import io.trino.aws.proxy.server.rest.TrinoS3ProxyClient;
@@ -116,7 +115,7 @@ protected void moduleSpecificBinding(Binder binder)
     {
         binder.bind(S3SecurityController.class).in(Scopes.SINGLETON);
         newOptionalBinder(binder, S3SecurityFacadeProvider.class).setDefault().toInstance(_ -> _ -> SecurityResponse.DEFAULT);
-        binder.bind(RemoteS3Facade.class).to(VirtualHostStyleRemoteS3Facade.class).in(Scopes.SINGLETON);
+        install(new RemoteS3Module());
     }
 
     private void installPlugins()

diff --git a/...-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/AwsRemoteS3FacadeConstants.java b/...-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/AwsRemoteS3FacadeConstants.java
diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/PathStyleRemoteS3Facade.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/PathStyleRemoteS3Facade.java
@@ -13,12 +13,12 @@
  */
 package io.trino.aws.proxy.server.remote;
 
+import com.google.inject.Inject;
 import jakarta.ws.rs.core.UriBuilder;
 
 import java.net.URI;
 import java.util.Optional;
 
-import static io.trino.aws.proxy.server.remote.AwsRemoteS3FacadeConstants.PATH_BUILDER;
 import static java.util.Objects.requireNonNull;
 
 public class PathStyleRemoteS3Facade
@@ -28,9 +28,10 @@ public class PathStyleRemoteS3Facade
     private final boolean https;
     private final Optional<Integer> port;
 
-    public PathStyleRemoteS3Facade()
+    @Inject
+    public PathStyleRemoteS3Facade(RemoteS3Config remoteS3Config)
     {
-        this(PATH_BUILDER, true, Optional.empty());
+        this((bucket, region) -> RemoteS3HostBuilder.getHostName(bucket, region, remoteS3Config.getDomain(), remoteS3Config.getHostnameTemplate()), remoteS3Config.getHttps(), remoteS3Config.getPort());
     }
 
     public PathStyleRemoteS3Facade(RemoteS3HostBuilder hostBuilder, boolean https, Optional<Integer> port)

diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3Config.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3Config.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import io.airlift.configuration.Config;
+import jakarta.validation.constraints.Max;
+import jakarta.validation.constraints.Min;
+import jakarta.validation.constraints.NotNull;
+
+import java.util.Optional;
+
+public class RemoteS3Config
+{
+    private boolean https = true;
+    private String domain = "amazonaws.com";
+    private Optional<@Min(1) @Max(65535) Integer> port = Optional.empty();
+    private boolean virtualHostStyle = true;
+    private String hostnameTemplate = "${bucket}.s3.${region}.${domain}";
+
+    @Config("remoteS3.https")
+    public RemoteS3Config setHttps(boolean https)
+    {
+        this.https = https;
+        return this;
+    }
+
+    @Config("remoteS3.domain")
+    public RemoteS3Config setDomain(String s3Domain)
+    {
+        this.domain = s3Domain;
+        return this;
+    }
+
+    @Config("remoteS3.port")
+    public RemoteS3Config setPort(Integer port)
+    {
+        this.port = Optional.ofNullable(port);
+        return this;
+    }
+
+    @Config("remoteS3.virtual-host-style")
+    public RemoteS3Config setVirtualHostStyle(boolean virtualHostStyle)
+    {
+        this.virtualHostStyle = virtualHostStyle;
+        return this;
+    }
+
+    @Config("remoteS3.hostname.template")
+    public RemoteS3Config setHostnameTemplate(String hostnameTemplate)
+    {
+        this.hostnameTemplate = hostnameTemplate;
+        return this;
+    }
+
+    public boolean getHttps()
+    {
+        return https;
+    }
+
+    @NotNull
+    public String getDomain()
+    {
+        return domain;
+    }
+
+    @NotNull
+    public Optional<Integer> getPort()
+    {
+        return port;
+    }
+
+    public boolean getVirtualHostStyle()
+    {
+        return virtualHostStyle;
+    }
+
+    @NotNull
+    public String getHostnameTemplate()
+    {
+        return hostnameTemplate;
+    }
+}
diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3HostBuilder.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3HostBuilder.java
@@ -13,7 +13,28 @@
  */
 package io.trino.aws.proxy.server.remote;
 
+import com.google.common.base.Strings;
+
 public interface RemoteS3HostBuilder
 {
+    static String getHostName(String bucket, String region, String domain, String hostnameTemplate)
+    {
+        String hostname = hostnameTemplate
+                .replace("${bucket}", bucket)
+                .replace("${region}", region)
+                .replace("${domain}", domain);
+
+        // Handles usecases when the bucket is empty (Eg: List buckets request)
+        if (Strings.isNullOrEmpty(bucket)) {
+            if (hostname.startsWith(".")) {         /* Handles cases when ${bucket} is leading. Eg: "${bucket}.{}.{}" */
+                hostname = hostname.substring(1);
+            }
+            else if (hostname.contains("..")) {     /* Handles cases when ${bucket} is in the midst. Eg: "{}.${bucket}.{}" */
+                hostname = hostname.replace("..", ".");
+            }
+        }
+        return hostname;
+    }
+
     String build(String bucket, String region);
 }
diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3Module.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3Module.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import com.google.inject.Binder;
+import com.google.inject.Scopes;
+import io.airlift.configuration.AbstractConfigurationAwareModule;
+
+import static io.airlift.configuration.ConditionalModule.conditionalModule;
+import static io.airlift.configuration.ConfigBinder.configBinder;
+
+public class RemoteS3Module
+        extends AbstractConfigurationAwareModule
+{
+    @Override
+    protected void setup(Binder binder)
+    {
+        configBinder(binder).bindConfig(RemoteS3Config.class);
+        install(conditionalModule(
+                RemoteS3Config.class,
+                RemoteS3Config::getVirtualHostStyle,
+                innerBinder -> innerBinder.bind(RemoteS3Facade.class).to(VirtualHostStyleRemoteS3Facade.class).in(Scopes.SINGLETON),
+                innerBinder -> innerBinder.bind(RemoteS3Facade.class).to(PathStyleRemoteS3Facade.class).in(Scopes.SINGLETON)));
+    }
+}
diff --git a/...-proxy/src/main/java/io/trino/aws/proxy/server/remote/VirtualHostStyleRemoteS3Facade.java b/...-proxy/src/main/java/io/trino/aws/proxy/server/remote/VirtualHostStyleRemoteS3Facade.java
@@ -13,12 +13,12 @@
  */
 package io.trino.aws.proxy.server.remote;
 
+import com.google.inject.Inject;
 import jakarta.ws.rs.core.UriBuilder;
 
 import java.net.URI;
 import java.util.Optional;
 
-import static io.trino.aws.proxy.server.remote.AwsRemoteS3FacadeConstants.VIRTUAL_HOST_BUILDER;
 import static java.util.Objects.requireNonNull;
 
 public class VirtualHostStyleRemoteS3Facade
@@ -28,9 +28,10 @@ public class VirtualHostStyleRemoteS3Facade
     private final boolean https;
     private final Optional<Integer> port;
 
-    public VirtualHostStyleRemoteS3Facade()
+    @Inject
+    public VirtualHostStyleRemoteS3Facade(RemoteS3Config remoteS3Config)
     {
-        this(VIRTUAL_HOST_BUILDER, true, Optional.empty());
+        this((bucket, region) -> RemoteS3HostBuilder.getHostName(bucket, region, remoteS3Config.getDomain(), remoteS3Config.getHostnameTemplate()), remoteS3Config.getHttps(), remoteS3Config.getPort());
     }
 
     public VirtualHostStyleRemoteS3Facade(RemoteS3HostBuilder hostBuilder, boolean https, Optional<Integer> port)

diff --git a/...aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestPathStyleRemoteS3Facade.java b/...aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestPathStyleRemoteS3Facade.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import jakarta.ws.rs.core.UriBuilder;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TestPathStyleRemoteS3Facade
+{
+    @Test
+    public void testBuildEndpoint()
+    {
+        RemoteS3Config remoteS3Config = new RemoteS3Config().setHttps(false).setDomain("testS3Domain.com").setPort(80).setVirtualHostStyle(false).setHostnameTemplate("s3.${region}.${domain}");
+        RemoteS3Facade remoteS3Facade = new PathStyleRemoteS3Facade(remoteS3Config);
+        URI expectedEndpoint = UriBuilder.fromUri("http://s3.us-east-1.testS3Domain.com:80/test_bucket/object_path/foo").build();
+        URI actual = remoteS3Facade.buildEndpoint(UriBuilder.newInstance(), "object_path/foo", "test_bucket", "us-east-1");
+        assertThat(actual).isEqualTo(expectedEndpoint);
+    }
+
+    @Test
+    public void testBuildEndpointWithoutRegion()
+    {
+        RemoteS3Config remoteS3Config = new RemoteS3Config().setHttps(true).setDomain("testS3Domain.com").setPort(80).setVirtualHostStyle(false).setHostnameTemplate("s3.${domain}");
+        RemoteS3Facade remoteS3Facade = new PathStyleRemoteS3Facade(remoteS3Config);
+        URI expectedEndpoint = UriBuilder.fromUri("https://s3.testS3Domain.com:80/test_bucket/object_path/foo").build();
+        URI actual = remoteS3Facade.buildEndpoint(UriBuilder.newInstance(), "object_path/foo", "test_bucket", "us-east-1");
+        assertThat(actual).isEqualTo(expectedEndpoint);
+    }
+}
diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestRemoteS3Config.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/TestRemoteS3Config.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
+import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
+import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
+
+public class TestRemoteS3Config
+{
+    @Test
+    public void testDefaults()
+    {
+        assertRecordedDefaults(recordDefaults(RemoteS3Config.class)
+                .setDomain("amazonaws.com").setHttps(true).setPort(null)
+                .setVirtualHostStyle(true)
+                .setHostnameTemplate("${bucket}.s3.${region}.${domain}"));
+    }
+
+    @Test
+    public void testExplicitPropertyMappings()
+    {
+        Map<String, String> properties = ImmutableMap.of(
+                "remoteS3.https", "false",
+                "remoteS3.domain", "testS3Domain.com",
+                "remoteS3.port", "80",
+                "remoteS3.virtual-host-style", "false",
+                "remoteS3.hostname.template", "s3.${region}.${domain}");
+
+        RemoteS3Config expected = new RemoteS3Config().setHttps(false).setDomain("testS3Domain.com").setPort(80).setVirtualHostStyle(false)
+                .setHostnameTemplate("s3.${region}.${domain}");
+        assertFullMapping(properties, expected);
+    }
+}