SOLR-16963: Fix usage of clientHostnameVerification

solr/CHANGES.txt

@@ -171,6 +171,9 @@ Bug Fixes
 * SOLR-16415: asyncId must not have '/'; enforce this.  Enhance ZK cleanup to process directories
   instead of fail.  (David Smiley, Paul McArthur)
 
+* SOLR-16963: The "solr.jetty.ssl.verifyClientHostName" sysProp and "SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION" envVar have been fixed,
+  and the setting once again tells the server to check the originating client hostname against the client certificate when doing mTLS. (Houston Putman, Tomás Fernández Löbbe)
+
 Dependency Upgrades
 ---------------------
 

solr/bin/solr

@@ -229,8 +229,8 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
     SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore.type=$SOLR_SSL_TRUST_STORE_TYPE"
   fi
 
-  if [ "${SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION:-true}" == "false" ]; then
-    SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.verifyClientHostName=false"
+  if [ "${SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION:true}" == "true" ] ; then
+    SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.verifyClientHostName=HTTPS"
   fi
 
   if [ -n "$SOLR_SSL_NEED_CLIENT_AUTH" ]; then

solr/bin/solr.cmd

@@ -105,8 +105,8 @@ IF "%SOLR_SSL_ENABLED%"=="true" (
   IF NOT DEFINED SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION (
     set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=true
   )
-  IF "%SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION%"=="false" (
-    set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.ssl.verifyClientHostName=false"
+  IF "%SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION%"=="true" (
+    set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.ssl.verifyClientHostName=HTTPS"
   )
 
   IF DEFINED SOLR_SSL_NEED_CLIENT_AUTH (

solr/packaging/test/test_ssl.bats

@@ -48,7 +48,6 @@ teardown() {
   export SOLR_SSL_TRUST_STORE_PASSWORD=secret
   export SOLR_SSL_NEED_CLIENT_AUTH=false
   export SOLR_SSL_WANT_CLIENT_AUTH=false
-  export SOLR_SSL_CHECK_PEER_NAME=true
   export SOLR_HOST=localhost
 
   solr start -c
@@ -57,7 +56,7 @@ teardown() {
   run solr create -c test -s 2
   assert_output --partial "Created collection 'test'"
 
-  run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" 'https://127.0.0.1:8983/solr/test/select?q=*:*'
+  run solr api -get 'https://localhost:8983/solr/test/select?q=*:*'
   assert_output --partial '"numFound":0'
 }
 
@@ -82,8 +81,6 @@ teardown() {
   export SOLR_SSL_NEED_CLIENT_AUTH=false
   export SOLR_SSL_WANT_CLIENT_AUTH=false
   export SOLR_SSL_CHECK_PEER_NAME=false
-  # Remove later when SOLR-16963 is resolved
-  export SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
   export SOLR_HOST=localhost
 
   solr start -c
@@ -92,12 +89,15 @@ teardown() {
   run solr create -c test -s 2
   assert_output --partial "Created collection 'test'"
 
+  # Just test that curl can connect via insecure or via a custom host header
   run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" -k 'https://localhost:8983/solr/test/select?q=*:*'
   assert_output --partial '"numFound":0'
 
+  run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" -H "Host: test.solr.apache.org" 'https://127.0.0.1:8983/solr/test/select?q=*:*'
+  assert_output --partial '"numFound":0'
+
+  # This is a client setting, so we don't need to restart Solr to make sure that it fails
   export SOLR_SSL_CHECK_PEER_NAME=true
-  # Remove later when SOLR-16963 is resolved
-  export SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=true
 
   # This should fail the peername check
   run ! solr api -get 'https://localhost:8983/solr/test/select?q=*:*'
@@ -207,6 +207,7 @@ teardown() {
 
   export SOLR_SECURITY_MANAGER_ENABLED=true
   export SOLR_OPTS="-Djava.io.tmpdir=${test_tmp_dir}"
+  export SOLR_LOG_LEVEL="DEBUG"
   export SOLR_TOOL_OPTS="-Djava.io.tmpdir=${test_tmp_dir} -Djavax.net.debug=SSL,keymanager,trustmanager,ssl:handshake"
 
   export ssl_dir="${BATS_TEST_TMPDIR}/ssl"
@@ -243,7 +244,7 @@ teardown() {
     # Trust the keystore cert with the CA
     keytool -storepass server-key -keystore solr-server.keystore.p12 -storetype PKCS12 -certreq -alias server | \
         keytool -storepass secret -keystore "$ssl_dir/ca.p12" -storetype PKCS12 -gencert -alias ca \
-        -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=serverAuth -rfc > server.pem
+        -ext "SAN=DNS:localhost,IP:127.0.0.1" -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=serverAuth -rfc > server.pem
     keytool -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -importcert -alias root -file "$ssl_dir/root.pem"  -noprompt
     keytool -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -importcert -alias ca -file "$ssl_dir/ca.pem"  -noprompt
     keytool -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -importcert -alias server -file server.pem
@@ -253,18 +254,19 @@ teardown() {
     keytool -keystore solr-server.truststore.p12 -storetype PKCS12 -keypass server-trust -storepass server-trust -importcert -alias ca -file "$ssl_dir/ca.pem"  -noprompt
   )
   # Create a client keystore & truststore
+  # The client cert will have a bogus DNS name and IP address, as we want the clientHostnameVerification to fail
   mkdir -p "$client_ssl_dir"
   (
     cd "$client_ssl_dir"
     rm -f solr-client.keystore.p12 client.pem solr-client.truststore.p12
 
     # Create a keystore and certificate
-    keytool -genkeypair -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -alias client -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
+    keytool -genkeypair -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -alias client -ext SAN=DNS:test.solr.apache.org,IP:127.0.0.2 -dname "CN=test.solr.apache.org, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
 
     # Trust the keystore cert with the CA
     keytool -storepass client-key -keystore solr-client.keystore.p12 -storetype PKCS12 -certreq -alias client | \
         keytool -storepass secret -keystore "$ssl_dir/ca.p12" -storetype PKCS12 -gencert -alias ca \
-        -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=clientAuth -rfc > client.pem
+        -ext "SAN=DNS:test.solr.apache.org,IP:127.0.0.2" -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=clientAuth -rfc > client.pem
     keytool -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -importcert -alias root -file "$ssl_dir/root.pem"  -noprompt
     keytool -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -importcert -alias ca -file "$ssl_dir/ca.pem"  -noprompt
     keytool -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -importcert -alias client -file client.pem
@@ -291,33 +293,51 @@ teardown() {
   export SOLR_SSL_NEED_CLIENT_AUTH=true
   export SOLR_SSL_WANT_CLIENT_AUTH=false
   export SOLR_SSL_CHECK_PEER_NAME=true
-  export SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=true
+  # Cannot set this to true, because the client certificate does not have the right hostname ("localhost") or IP
+  export SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
   export SOLR_HOST=localhost
 
   solr start -c
   solr start -c -z localhost:9983 -p 8984
 
-  export SOLR_SSL_KEY_STORE=
-  export SOLR_SSL_KEY_STORE_PASSWORD=
-  export SOLR_SSL_TRUST_STORE=
-  export SOLR_SSL_TRUST_STORE_PASSWORD=
+  # Test Client connections, which do not need the server keystore/truststore
+  (
+    export SOLR_SSL_KEY_STORE=
+    export SOLR_SSL_KEY_STORE_PASSWORD=
+    export SOLR_SSL_TRUST_STORE=
+    export SOLR_SSL_TRUST_STORE_PASSWORD=
 
-  solr assert --started https://localhost:8983/solr --timeout 5000
-  solr assert --started https://localhost:8984/solr --timeout 5000
+    solr assert --started https://localhost:8983/solr --timeout 5000
+    solr assert --started https://localhost:8984/solr --timeout 5000
 
-  run solr create -c test -s 2
-  assert_output --partial "Created collection 'test'"
+    run solr create -c test -s 2
+    assert_output --partial "Created collection 'test'"
 
-  run solr api -get 'https://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS'
-  assert_output --partial '"urlScheme":"https"'
+    run solr api -get 'https://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS'
+    assert_output --partial '"urlScheme":"https"'
 
-  run solr api -get 'https://localhost:8984/solr/test/select?q=*:*&rows=0'
-  assert_output --partial '"numFound":0'
+    run solr api -get 'https://localhost:8984/solr/test/select?q=*:*&rows=0'
+    assert_output --partial '"numFound":0'
 
-  export SOLR_SSL_CLIENT_KEY_STORE=
-  export SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
+    (
+      # Try connecting with just the client truststore, which should fail because mTLS requires a keystore and mTLS is needed
+      export SOLR_SSL_CLIENT_KEY_STORE=
+      export SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
+
+      run ! solr api -get 'https://localhost:8983/solr/test/select?q=*:*&rows=0'
+      assert_output --partial 'Server refused connection'
+    )
+  )
+
+  # Turn on client hostname verification, and start a new Solr node since the property is a server setting.
+  # Test that it fails because the client cert does not use "localhost"
+  export SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=true
+  solr start -c -z localhost:9983 -p 8985
 
-  run ! solr api -get 'https://localhost:8983/solr/test/select?q=*:*&rows=0'
+  # We can't check if the server has come up, because we can't connect to it, so just wait
+  sleep 5
+
+  run ! solr api -get 'https://localhost:8985/solr/test/select?q=*:*&rows=0'
   assert_output --partial 'Server refused connection'
 }
 
@@ -342,8 +362,8 @@ teardown() {
     cd "$ssl_dir"
     rm -f root.p12 root.pem ca.p12 ca.pem
 
-    keytool -genkeypair -keystore root.p12 -storetype PKCS12 -keypass secret -storepass secret -alias root -ext bc:c -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
-    keytool -genkeypair -keystore ca.p12 -storetype PKCS12 -keypass secret -storepass secret -alias ca -ext bc:c -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
+    keytool -genkeypair -keystore root.p12 -storetype PKCS12 -keypass secret -storepass secret -alias root -ext bc:c -ext "SAN=DNS:localhost,IP:127.0.0.1" -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
+    keytool -genkeypair -keystore ca.p12 -storetype PKCS12 -keypass secret -storepass secret -alias ca -ext bc:c -ext "SAN=DNS:localhost,IP:127.0.0.1" -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
 
     keytool -keystore root.p12 -storetype PKCS12 -storepass secret -alias root -exportcert -rfc > root.pem
 
@@ -360,12 +380,12 @@ teardown() {
     rm -f solr-server.keystore.p12 server.pem solr-server.truststore.p12
 
     # Create a keystore and certificate
-    keytool -genkeypair -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -alias server -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
+    keytool -genkeypair -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -alias server -ext "SAN=DNS:localhost,IP:127.0.0.1" -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
 
     # Trust the keystore cert with the CA
     keytool -storepass server-key -keystore solr-server.keystore.p12 -storetype PKCS12 -certreq -alias server | \
      keytool -storepass secret -keystore "$ssl_dir/ca.p12" -storetype PKCS12 -gencert -alias ca \
-     -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=serverAuth -rfc > server.pem
+     -ext "SAN=DNS:localhost,IP:127.0.0.1" -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=serverAuth -rfc > server.pem
     keytool -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -importcert -alias root -file "$ssl_dir/root.pem"  -noprompt
     keytool -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -importcert -alias ca -file "$ssl_dir/ca.pem"  -noprompt
     keytool -keystore solr-server.keystore.p12 -storetype PKCS12 -keypass server-key -storepass server-key -importcert -alias server -file server.pem
@@ -381,12 +401,12 @@ teardown() {
     rm -f solr-client.keystore.p12 client.pem solr-client.truststore.p12
 
     # Create a keystore and certificate
-    keytool -genkeypair -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -alias client -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
+    keytool -genkeypair -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -alias client -ext "SAN=DNS:localhost,IP:127.0.0.1" -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" -keyalg rsa
 
     # Trust the keystore cert with the CA
     keytool -storepass client-key -keystore solr-client.keystore.p12 -storetype PKCS12 -certreq -alias client | \
      keytool -storepass secret -keystore "$ssl_dir/ca.p12" -storetype PKCS12 -gencert -alias ca \
-     -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=clientAuth -rfc > client.pem
+     -ext "SAN=DNS:localhost,IP:127.0.0.1" -ext "ku:c=nonRepudiation,digitalSignature,keyEncipherment" -ext eku:c=clientAuth -rfc > client.pem
     keytool -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -importcert -alias root -file "$ssl_dir/root.pem"  -noprompt
     keytool -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -importcert -alias ca -file "$ssl_dir/ca.pem"  -noprompt
     keytool -keystore solr-client.keystore.p12 -storetype PKCS12 -keypass client-key -storepass client-key -importcert -alias client -file client.pem
@@ -439,6 +459,14 @@ teardown() {
   export SOLR_SSL_CLIENT_KEY_STORE=
   export SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
 
+  # mTLS requires a keyStore, so just using the truststore would fail if mTLS was "NEED"ed, however it is only "WANT"ed, so its ok
   run solr api -get 'https://localhost:8983/solr/test/select?q=*:*&rows=0'
   assert_output --partial '"numFound":0'
+
+  export SOLR_SSL_CLIENT_TRUST_STORE=
+  export SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
+
+  # TLS cannot work if a truststore and keystore are not provided (either Server or Client)
+  run solr api -get 'https://localhost:8983/solr/test/select?q=*:*&rows=0'
+  assert_output --partial 'Server refused connection'
 }

solr/server/etc/jetty-ssl.xml

@@ -22,6 +22,7 @@
   <Set name="WantClientAuth"><Property name="solr.jetty.ssl.wantClientAuth" default="false"/></Set>
   <Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="PKCS12"/></Set>
   <Set name="TrustStoreType"><Property name="solr.jetty.truststore.type" default="PKCS12"/></Set>
+  <Set name="EndpointIdentificationAlgorithm"><Property name="solr.jetty.ssl.verifyClientHostName"/></Set>
 
   <!-- =========================================================== -->
   <!-- Create a TLS specific HttpConfiguration based on the        -->

solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc

@@ -125,7 +125,7 @@ set SOLR_SSL_CHECK_PEER_NAME=true
 .Client Authentication Settings
 WARNING: Enable either `SOLR_SSL_NEED_CLIENT_AUTH` or `SOLR_SSL_WANT_CLIENT_AUTH` but not both at the same time.
 They are mutually exclusive and Jetty will select one of them which may not be what you expect.
-`SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION` should be set to false if you want to disable hostname verification.
+`SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION` should be set to false if you want to disable hostname verification for client certificates.
 
 When you start Solr, the `bin/solr` script includes these settings and will pass them as system properties to the JVM.
 

solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc

@@ -81,6 +81,11 @@ Therefore, when using the default settings, nodes that were previously excluded
 see the  xref:deployment-guide:securing-solr.adoc#network-configuration[Network Configuration documentation] for more information.
 
 === Security
+* Since Solr 8.4.1/8.5.0, the `solr.jetty.ssl.verifyClientHostName` sysProp and `SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION` envVar have been used incorrectly.
+It has instead been used to override the `solr.ssl.checkPeerName` sysProp in the `HTTP2SolrClient`.
+This has been fixed, and the setting once again tells the server to check the originating client hostname against the client certificate when doing mTLS.
+This option is still enabled by default.
+
 * The `solr.jetty.ssl.sniHostCheck` option now defaults to the value of `SOLR_SSL_CHECK_PEER_NAME`, if it is provided.
 This will enable client and server hostName check settings to be governed by the same environment variable.
 If users want separate client/server settings, they can manually override the `solr.jetty.ssl.sniHostCheck` option in `SOLR_OPTS`.

solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java

@@ -1386,12 +1386,6 @@ static SslContextFactory.Client getDefaultSslContextFactory() {
       sslContextFactory.setTrustStoreType(System.getProperty("javax.net.ssl.trustStoreType"));
     }
 
-    if (Boolean.parseBoolean(System.getProperty("solr.jetty.ssl.verifyClientHostName", "true"))) {
-      sslContextFactory.setEndpointIdentificationAlgorithm("HTTPS");
-    } else {
-      sslContextFactory.setEndpointIdentificationAlgorithm(null);
-    }
-
     return sslContextFactory;
   }
 

solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientTest.java

@@ -769,25 +769,25 @@ public void testGetDefaultSslContextFactory() {
     System.clearProperty("javax.net.ssl.keyStoreType");
     System.clearProperty("javax.net.ssl.trustStoreType");
 
-    System.setProperty("solr.jetty.ssl.verifyClientHostName", "true");
+    System.setProperty("solr.ssl.checkPeerName", "true");
     System.setProperty("javax.net.ssl.keyStoreType", "foo");
     System.setProperty("javax.net.ssl.trustStoreType", "bar");
     SslContextFactory.Client sslContextFactory2 = Http2SolrClient.getDefaultSslContextFactory();
     assertEquals("HTTPS", sslContextFactory2.getEndpointIdentificationAlgorithm());
     assertEquals("foo", sslContextFactory2.getKeyStoreType());
     assertEquals("bar", sslContextFactory2.getTrustStoreType());
-    System.clearProperty("solr.jetty.ssl.verifyClientHostName");
+    System.clearProperty("solr.ssl.checkPeerName");
     System.clearProperty("javax.net.ssl.keyStoreType");
     System.clearProperty("javax.net.ssl.trustStoreType");
 
-    System.setProperty("solr.jetty.ssl.verifyClientHostName", "false");
+    System.setProperty("solr.ssl.checkPeerName", "false");
     System.setProperty("javax.net.ssl.keyStoreType", "foo");
     System.setProperty("javax.net.ssl.trustStoreType", "bar");
     SslContextFactory.Client sslContextFactory3 = Http2SolrClient.getDefaultSslContextFactory();
     assertNull(sslContextFactory3.getEndpointIdentificationAlgorithm());
     assertEquals("foo", sslContextFactory3.getKeyStoreType());
     assertEquals("bar", sslContextFactory3.getTrustStoreType());
-    System.clearProperty("solr.jetty.ssl.verifyClientHostName");
+    System.clearProperty("solr.ssl.checkPeerName");
     System.clearProperty("javax.net.ssl.keyStoreType");
     System.clearProperty("javax.net.ssl.trustStoreType");
   }