Skip to content

Commit

Permalink
Support runtime reload for TLS resources (apache#12277)
Browse files Browse the repository at this point in the history
* build swappable tls resource bundle when possible

* fix format

* remove the limitation that secrets are from files

* simplify logic
  • Loading branch information
zhtaoxiang authored Jan 22, 2024
1 parent ced6bc2 commit f1fec06
Show file tree
Hide file tree
Showing 9 changed files with 242 additions and 50 deletions.
4 changes: 4 additions & 0 deletions pinot-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,10 @@
<groupId>com.github.seancfoley</groupId>
<artifactId>ipaddress</artifactId>
</dependency>
<dependency>
<groupId>io.github.hakky54</groupId>
<artifactId>sslcontext-kickstart-for-netty</artifactId>
</dependency>
</dependencies>
<profiles>
<profile>
Expand Down
114 changes: 79 additions & 35 deletions pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.pinot.common.utils;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import io.netty.handler.ssl.ClientAuth;
import io.netty.handler.ssl.SslContext;
Expand All @@ -28,15 +29,15 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.exception.GenericSSLContextException;
import org.apache.commons.lang.StringUtils;
import org.apache.http.ssl.SSLContexts;
import org.apache.pinot.common.config.TlsConfig;
Expand All @@ -60,6 +61,10 @@ public final class TlsUtils {
private static final String TRUSTSTORE_PASSWORD = "truststore.password";
private static final String SSL_PROVIDER = "ssl.provider";

private static final String FILE_SCHEME = "file";
private static final String FILE_SCHEME_PREFIX = FILE_SCHEME + "://";
private static final String FILE_SCHEME_PREFIX_WITHOUT_SLASH = FILE_SCHEME + ":";

private static final AtomicReference<SSLContext> SSL_CONTEXT_REF = new AtomicReference<>();

private TlsUtils() {
Expand Down Expand Up @@ -136,7 +141,7 @@ public static KeyManagerFactory createKeyManagerFactory(String keyStorePath, Str

try {
KeyStore keyStore = KeyStore.getInstance(keyStoreType);
try (InputStream is = makeKeyStoreUrl(keyStorePath).openStream()) {
try (InputStream is = makeKeyOrTrustStoreUrl(keyStorePath).openStream()) {
keyStore.load(is, keyStorePassword.toCharArray());
}

Expand Down Expand Up @@ -176,7 +181,7 @@ public static TrustManagerFactory createTrustManagerFactory(String trustStorePat

try {
KeyStore keyStore = KeyStore.getInstance(trustStoreType);
try (InputStream is = makeKeyStoreUrl(trustStorePath).openStream()) {
try (InputStream is = makeKeyOrTrustStoreUrl(trustStorePath).openStream()) {
keyStore.load(is, trustStorePassword.toCharArray());
}

Expand Down Expand Up @@ -213,25 +218,14 @@ public static void installDefaultSSLSocketFactory(TlsConfig tlsConfig) {
*/
public static void installDefaultSSLSocketFactory(String keyStoreType, String keyStorePath, String keyStorePassword,
String trustStoreType, String trustStorePath, String trustStorePassword) {
KeyManager[] keyManagers = null;
if (keyStorePath != null) {
keyManagers = createKeyManagerFactory(keyStorePath, keyStorePassword, keyStoreType).getKeyManagers();
}

TrustManager[] trustManagers = null;
if (trustStorePath != null) {
trustManagers = createTrustManagerFactory(trustStorePath, trustStorePassword, trustStoreType).getTrustManagers();
}

try {
SSLContext sc = SSLContext.getInstance("SSL");
sc.init(keyManagers, trustManagers, new java.security.SecureRandom());

SSLFactory sslFactory = createSSLFactory(keyStoreType, keyStorePath, keyStorePassword,
trustStoreType, trustStorePath, trustStorePassword,
"SSL", new java.security.SecureRandom());
// HttpsURLConnection
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());

setSslContext(sc);
} catch (GeneralSecurityException e) {
HttpsURLConnection.setDefaultSSLSocketFactory(sslFactory.getSslSocketFactory());
setSslContext(sslFactory.getSslContext());
} catch (GenericSSLContextException e) {
throw new IllegalStateException("Could not initialize SSL support", e);
}
}
Expand All @@ -240,14 +234,14 @@ private static String key(String namespace, String suffix) {
return namespace + "." + suffix;
}

public static URL makeKeyStoreUrl(String storePath)
public static URL makeKeyOrTrustStoreUrl(String storePath)
throws URISyntaxException, MalformedURLException {
URI inputUri = new URI(storePath);
if (StringUtils.isBlank(inputUri.getScheme())) {
if (storePath.startsWith("/")) {
return new URL("file://" + storePath);
return new URL(FILE_SCHEME_PREFIX + storePath);
}
return new URL("file://./" + storePath);
return new URL(FILE_SCHEME_PREFIX + "./" + storePath);
}
return inputUri.toURL();
}
Expand Down Expand Up @@ -293,14 +287,11 @@ private static final class SSLContextHolder {
* @param tlsConfig TLS config
*/
public static SslContext buildClientContext(TlsConfig tlsConfig) {
SSLFactory sslFactory = createSSLFactory(tlsConfig);
SslContextBuilder sslContextBuilder =
SslContextBuilder.forClient().sslProvider(SslProvider.valueOf(tlsConfig.getSslProvider()));
if (tlsConfig.getKeyStorePath() != null) {
sslContextBuilder.keyManager(createKeyManagerFactory(tlsConfig));
}
if (tlsConfig.getTrustStorePath() != null) {
sslContextBuilder.trustManager(createTrustManagerFactory(tlsConfig));
}
sslFactory.getKeyManagerFactory().ifPresent(sslContextBuilder::keyManager);
sslFactory.getTrustManagerFactory().ifPresent(sslContextBuilder::trustManager);
try {
return sslContextBuilder.build();
} catch (Exception e) {
Expand All @@ -317,11 +308,10 @@ public static SslContext buildServerContext(TlsConfig tlsConfig) {
if (tlsConfig.getKeyStorePath() == null) {
throw new IllegalArgumentException("Must provide key store path for secured server");
}
SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(createKeyManagerFactory(tlsConfig))
SSLFactory sslFactory = createSSLFactory(tlsConfig);
SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(sslFactory.getKeyManagerFactory().get())
.sslProvider(SslProvider.valueOf(tlsConfig.getSslProvider()));
if (tlsConfig.getTrustStorePath() != null) {
sslContextBuilder.trustManager(createTrustManagerFactory(tlsConfig));
}
sslFactory.getTrustManagerFactory().ifPresent(sslContextBuilder::trustManager);
if (tlsConfig.isClientAuthEnabled()) {
sslContextBuilder.clientAuth(ClientAuth.REQUIRE);
}
Expand All @@ -331,4 +321,58 @@ public static SslContext buildServerContext(TlsConfig tlsConfig) {
throw new RuntimeException(e);
}
}

/**
* Create a {@link SSLFactory} instance with identity material and trust material swappable for a given TlsConfig
* @param tlsConfig {@link TlsConfig}
* @return a {@link SSLFactory} instance with identity material and trust material swappable
*/
public static SSLFactory createSSLFactory(TlsConfig tlsConfig) {
return createSSLFactory(
tlsConfig.getKeyStoreType(), tlsConfig.getKeyStorePath(), tlsConfig.getKeyStorePassword(),
tlsConfig.getTrustStoreType(), tlsConfig.getTrustStorePath(), tlsConfig.getTrustStorePassword(),
null, null);
}

@VisibleForTesting
static SSLFactory createSSLFactory(
String keyStoreType, String keyStorePath, String keyStorePassword,
String trustStoreType, String trustStorePath, String trustStorePassword,
String sslContextProtocol, SecureRandom secureRandom) {
try {
SSLFactory.Builder sslFactoryBuilder = SSLFactory.builder();
InputStream keyStoreStream = null;
InputStream trustStoreStream = null;
if (keyStorePath != null) {
Preconditions.checkNotNull(keyStorePassword, "key store password must not be null");
keyStoreStream = makeKeyOrTrustStoreUrl(keyStorePath).openStream();
sslFactoryBuilder
.withSwappableIdentityMaterial()
.withIdentityMaterial(keyStoreStream, keyStorePassword.toCharArray(), keyStoreType);
}
if (trustStorePath != null) {
Preconditions.checkNotNull(trustStorePassword, "trust store password must not be null");
trustStoreStream = makeKeyOrTrustStoreUrl(trustStorePath).openStream();
sslFactoryBuilder
.withSwappableTrustMaterial()
.withTrustMaterial(trustStoreStream, trustStorePassword.toCharArray(), trustStoreType);
}
if (sslContextProtocol != null) {
sslFactoryBuilder.withSslContextAlgorithm(sslContextProtocol);
}
if (secureRandom != null) {
sslFactoryBuilder.withSecureRandom(secureRandom);
}
SSLFactory sslFactory = sslFactoryBuilder.build();
if (keyStoreStream != null) {
keyStoreStream.close();
}
if (trustStoreStream != null) {
trustStoreStream.close();
}
return sslFactory;
} catch (Exception e) {
throw new IllegalStateException(e);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,8 @@
import java.util.Collections;
import java.util.Iterator;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLException;
import javax.net.ssl.TrustManagerFactory;
import nl.altindag.ssl.SSLFactory;
import org.apache.pinot.common.config.GrpcConfig;
import org.apache.pinot.common.proto.PinotQueryServerGrpc;
import org.apache.pinot.common.proto.Server;
Expand All @@ -56,15 +55,10 @@ public GrpcQueryClient(String host, int port, GrpcConfig config) {
.usePlaintext().build();
} else {
try {
SSLFactory sslFactory = TlsUtils.createSSLFactory(config.getTlsConfig());
SslContextBuilder sslContextBuilder = SslContextBuilder.forClient();
if (config.getTlsConfig().getKeyStorePath() != null) {
KeyManagerFactory keyManagerFactory = TlsUtils.createKeyManagerFactory(config.getTlsConfig());
sslContextBuilder.keyManager(keyManagerFactory);
}
if (config.getTlsConfig().getTrustStorePath() != null) {
TrustManagerFactory trustManagerFactory = TlsUtils.createTrustManagerFactory(config.getTlsConfig());
sslContextBuilder.trustManager(trustManagerFactory);
}
sslFactory.getKeyManagerFactory().ifPresent(sslContextBuilder::keyManager);
sslFactory.getTrustManagerFactory().ifPresent(sslContextBuilder::trustManager);
if (config.getTlsConfig().getSslProvider() != null) {
sslContextBuilder =
GrpcSslContexts.configure(sslContextBuilder, SslProvider.valueOf(config.getTlsConfig().getSslProvider()));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.pinot.common.utils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509KeyManager;
import javax.net.ssl.X509TrustManager;
import nl.altindag.ssl.SSLFactory;
import org.apache.commons.io.FileUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import static org.testng.Assert.assertEquals;


public class TlsUtilsTest {
private static final String TLS_RESOURCE_FOLDER = "tls/";
private static final String TLS_KEYSTORE_FILE = "keystore.p12";
private static final String TLS_TRUSTSTORE_FILE = "truststore.p12";
private static final String[] TLS_RESOURCE_FILES = {TLS_KEYSTORE_FILE, TLS_TRUSTSTORE_FILE};
private static final String PASSWORD = "changeit";
private static final String KEYSTORE_TYPE = "PKCS12";
private static final String TRUSTSTORE_TYPE = "PKCS12";
private static final String DEFAULT_TEST_TLS_DIR
= new File(FileUtils.getTempDirectoryPath(), "test-tls-dir" + System.currentTimeMillis()).getAbsolutePath();

@BeforeClass
public void setUp()
throws IOException, URISyntaxException {
copyResourceFilesToTempFolder();
}

private static void copyResourceFilesToTempFolder()
throws URISyntaxException, IOException {
// Create the destination folder if it doesn't exist
Files.createDirectories(Paths.get(DEFAULT_TEST_TLS_DIR));
for (String fileName : TLS_RESOURCE_FILES) {
// Use the class loader to get the InputStream of the resource file
try (InputStream resourceStream
= TlsUtilsTest.class.getClassLoader().getResourceAsStream(TLS_RESOURCE_FOLDER + fileName)) {
if (resourceStream == null) {
throw new IOException("Resource file not found: " + fileName);
}
// Specify the destination path
Path destinationPath = Paths.get(DEFAULT_TEST_TLS_DIR, fileName);
// Use Files.copy to copy the file to the destination folder
Files.copy(resourceStream, destinationPath, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
e.printStackTrace(); // Handle the exception as needed
}
}
}

@AfterClass
public void tearDown() {
FileUtils.deleteQuietly(new File(DEFAULT_TEST_TLS_DIR));
}

@Test
public void swappableSSLFactoryHasSameAsStaticOnes()
throws NoSuchAlgorithmException, KeyManagementException, IOException, URISyntaxException {
SecureRandom secureRandom = new SecureRandom();
KeyManagerFactory keyManagerFactory =
TlsUtils.createKeyManagerFactory(DEFAULT_TEST_TLS_DIR + "/" + TLS_KEYSTORE_FILE, PASSWORD,
KEYSTORE_TYPE);
TrustManagerFactory trustManagerFactory =
TlsUtils.createTrustManagerFactory(DEFAULT_TEST_TLS_DIR + "/" + TLS_TRUSTSTORE_FILE, PASSWORD,
TRUSTSTORE_TYPE);
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), secureRandom);
SSLFactory sslFactory =
TlsUtils.createSSLFactory(KEYSTORE_TYPE, DEFAULT_TEST_TLS_DIR + "/" + TLS_KEYSTORE_FILE, PASSWORD,
TRUSTSTORE_TYPE, DEFAULT_TEST_TLS_DIR + "/" + TLS_TRUSTSTORE_FILE, PASSWORD,
"TLS", secureRandom);
KeyManagerFactory swappableKeyManagerFactory = sslFactory.getKeyManagerFactory().get();
assertEquals(swappableKeyManagerFactory.getKeyManagers().length, keyManagerFactory.getKeyManagers().length);
assertEquals(swappableKeyManagerFactory.getKeyManagers().length, 1);
assertSSLKeyManagersEqual(swappableKeyManagerFactory.getKeyManagers()[0], keyManagerFactory.getKeyManagers()[0]);
TrustManagerFactory swappableTrustManagerFactory = sslFactory.getTrustManagerFactory().get();
assertEquals(swappableTrustManagerFactory.getTrustManagers().length,
trustManagerFactory.getTrustManagers().length);
assertEquals(swappableTrustManagerFactory.getTrustManagers().length, 1);
assertSSLTrustManagersEqual(swappableTrustManagerFactory.getTrustManagers()[0],
trustManagerFactory.getTrustManagers()[0]);
SSLContext swappableSSLContext = sslFactory.getSslContext();
assertEquals(swappableSSLContext.getProtocol(), sslContext.getProtocol());
assertEquals(swappableSSLContext.getProvider(), sslContext.getProvider());
}

private static void assertSSLKeyManagersEqual(KeyManager km1, KeyManager km2) {
X509KeyManager x509KeyManager1 = (X509KeyManager) km1;
X509KeyManager x509KeyManager2 = (X509KeyManager) km2;
assertEquals(x509KeyManager1.getPrivateKey("mykey"), x509KeyManager2.getPrivateKey("mykey"));

Certificate[] certs1 = x509KeyManager1.getCertificateChain("mykey");
Certificate[] certs2 = x509KeyManager2.getCertificateChain("mykey");
assertEquals(certs1.length, certs2.length);
assertEquals(certs1.length, 1);
assertEquals(certs1[0], certs2[0]);
}

private static void assertSSLTrustManagersEqual(TrustManager tm1, TrustManager tm2) {
X509TrustManager x509TrustManager1 = (X509TrustManager) tm1;
X509TrustManager x509TrustManager2 = (X509TrustManager) tm2;

assertEquals(x509TrustManager1.getAcceptedIssuers().length, x509TrustManager2.getAcceptedIssuers().length);
assertEquals(x509TrustManager1.getAcceptedIssuers().length, 1);
assertEquals(x509TrustManager1.getAcceptedIssuers()[0], x509TrustManager2.getAcceptedIssuers()[0]);
}
}
Binary file added pinot-common/src/test/resources/tls/keystore.p12
Binary file not shown.
Binary file not shown.
Loading

0 comments on commit f1fec06

Please sign in to comment.