From 5026a30ba79a2a980efb9c25ece9a2fd3f09fa3c Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Tue, 16 Apr 2024 12:49:58 -0700 Subject: [PATCH 1/9] HADOOP-19152. Do not hard code security providers. --- hadoop-common-project/hadoop-common/pom.xml | 1 + .../org/apache/hadoop/crypto/JceCtrCryptoCodec.java | 5 ----- .../org/apache/hadoop/crypto/key/KeyProvider.java | 5 ----- .../org/apache/hadoop/crypto/TestCryptoCodec.java | 12 ++++++++---- .../TestCryptoStreamsWithJceSm4CtrCryptoCodec.java | 5 +---- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index a7dcbb24a9b40..63c6ffd68dd87 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -375,6 +375,7 @@ org.bouncycastle bcprov-jdk18on + test org.apache.kerby diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java index 7aae65d47ccd2..a522023532332 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.crypto; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -27,7 +26,6 @@ import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.SecureRandom; -import java.security.Security; import javax.crypto.Cipher; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; @@ -83,9 +81,6 @@ public Configuration getConf() { public void setConf(Configuration conf) { this.conf = conf; setProvider(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY)); - if (BouncyCastleProvider.PROVIDER_NAME.equals(provider)) { - Security.addProvider(new BouncyCastleProvider()); - } final String secureRandomAlg = conf.get( HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index 5e207251805fc..6d54416dc3535 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -26,7 +26,6 @@ import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; -import java.security.Security; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -35,7 +34,6 @@ import java.util.Map; import java.util.Objects; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; import org.apache.hadoop.classification.InterfaceAudience; @@ -411,9 +409,6 @@ public KeyProvider(Configuration conf) { System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter); } String jceProvider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY); - if (BouncyCastleProvider.PROVIDER_NAME.equals(jceProvider)) { - Security.addProvider(new BouncyCastleProvider()); - } } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index c0fdc51b1389b..e97a8bcd0b4f7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -32,6 +32,7 @@ import java.math.BigInteger; import java.security.GeneralSecurityException; import java.security.SecureRandom; +import java.security.Security; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -104,6 +105,11 @@ public void testJceAesCtrCryptoCodec() throws Exception { jceAesCodecClass, opensslAesCodecClass, iv); } + static void initBouncyCastleProvider(Configuration conf) { + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, BouncyCastleProvider.PROVIDER_NAME); + Security.addProvider(new BouncyCastleProvider()); + } + @Test(timeout=120000) public void testJceSm4CtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); @@ -114,8 +120,7 @@ public void testJceSm4CtrCryptoCodec() throws Exception { conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, JceSm4CtrCryptoCodec.class.getName()); - conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, - BouncyCastleProvider.PROVIDER_NAME); + initBouncyCastleProvider(conf); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, jceSm4CodecClass, jceSm4CodecClass, iv); @@ -164,8 +169,7 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { LOG.warn("Skipping test since openSSL library not loaded"); Assume.assumeTrue(false); } - conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, - BouncyCastleProvider.PROVIDER_NAME); + initBouncyCastleProvider(conf); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, opensslSm4CodecClass, opensslSm4CodecClass, iv); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java index 62573ede7d1ea..c0e71bae85d0e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.crypto; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.junit.BeforeClass; @@ -26,7 +25,6 @@ HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; import static org.assertj.core.api.Assertions.assertThat; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; public class TestCryptoStreamsWithJceSm4CtrCryptoCodec extends TestCryptoStreams { @@ -35,8 +33,7 @@ public class TestCryptoStreamsWithJceSm4CtrCryptoCodec extends public static void init() throws Exception { Configuration conf = new Configuration(); conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); - conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, - BouncyCastleProvider.PROVIDER_NAME); + TestCryptoCodec.initBouncyCastleProvider(conf); conf.set( CommonConfigurationKeysPublic. HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, From c6db80c1e02152f4577be24a1ae7283543b62103 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Tue, 16 Apr 2024 13:07:40 -0700 Subject: [PATCH 2/9] Update core-default.xml --- .../hadoop-common/src/main/resources/core-default.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 075c7e02e8111..92df71fb19676 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -3623,6 +3623,9 @@ The switch to turn S3A auditing on or off. The JCE provider name used in CryptoCodec. + If this value is set, the corresponding provider must be added to the provider list. + The provider may be added statically in the java.security file, or + added dynamically by calling the java.security.Security.addProvider(..) method. From 698e48e5638ea210c7e04a96f31d5d11bfde3a2f Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Tue, 16 Apr 2024 18:07:23 -0700 Subject: [PATCH 3/9] Remove unused code. --- .../src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index 6d54416dc3535..826b567a5b9c5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -43,7 +43,6 @@ import javax.crypto.KeyGenerator; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER; /** @@ -408,7 +407,6 @@ public KeyProvider(Configuration conf) { JCEKS_KEY_SERIALFILTER_DEFAULT); System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter); } - String jceProvider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY); } /** From 84e10b88587f0a0a8c2502d7610dee903e617735 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Mon, 22 Apr 2024 10:55:36 -0700 Subject: [PATCH 4/9] Use reflection. --- hadoop-common-project/hadoop-common/pom.xml | 1 - .../org/apache/hadoop/crypto/CryptoUtils.java | 71 +++++++++++++++++++ .../hadoop/crypto/JceCtrCryptoCodec.java | 8 +-- .../apache/hadoop/crypto/key/KeyProvider.java | 2 + .../fs/CommonConfigurationKeysPublic.java | 3 + .../src/main/resources/core-default.xml | 14 +++- .../apache/hadoop/crypto/TestCryptoCodec.java | 12 ++-- ...CryptoStreamsWithJceSm4CtrCryptoCodec.java | 5 +- 8 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 63c6ffd68dd87..a7dcbb24a9b40 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -375,7 +375,6 @@ org.bouncycastle bcprov-jdk18on - test org.apache.kerby diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java new file mode 100644 index 0000000000000..64964d82daf50 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.security.Provider; +import java.security.Security; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; + +@InterfaceAudience.Private +public class CryptoUtils { + static final Logger LOG = LoggerFactory.getLogger(CryptoUtils.class); + + private static final String BOUNCY_CASTLE_PROVIDER_CLASS + = "org.bouncycastle.jce.provider.BouncyCastleProvider"; + private static final String PROVIDER_NAME_FIELD = "PROVIDER_NAME"; + + private static void addProvider(String provider) { + if (provider == null || provider.isEmpty()) { + return; + } + try { + // For backward compatible, try to auto-add BOUNCY_CASTLE_PROVIDER_CLASS. + final Class clazz = Class.forName(BOUNCY_CASTLE_PROVIDER_CLASS); + final Field provider_name = clazz.getField("PROVIDER_NAME"); + if (provider.equals(provider_name.get(null))) { + Security.addProvider((Provider) clazz.getConstructor().newInstance()); + LOG.debug("Successfully added security provider {}", provider); + } + } catch (ClassNotFoundException e) { + LOG.warn("Failed to load " + BOUNCY_CASTLE_PROVIDER_CLASS, e); + } catch (NoSuchFieldException e) { + LOG.warn("Failed to get field " + PROVIDER_NAME_FIELD + + " from class " + BOUNCY_CASTLE_PROVIDER_CLASS, e); + } catch (Exception e) { + LOG.warn("Failed to add security provider for {}", provider, e); + } + } + + public static String getJceProvider(Configuration conf) { + final String jceProvider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY); + if (conf.getBoolean(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_KEY, + HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_DEFAULT)) { + CryptoUtils.addProvider(jceProvider); + } + return jceProvider; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java index a522023532332..f4c8a382cac88 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java @@ -31,7 +31,6 @@ import javax.crypto.spec.SecretKeySpec; import org.slf4j.Logger; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT; @@ -46,10 +45,6 @@ public String getProvider() { return provider; } - public void setProvider(String provider) { - this.provider = provider; - } - public void calculateIV(byte[] initIV, long counter, byte[] iv, int blockSize) { Preconditions.checkArgument(initIV.length == blockSize); @@ -80,7 +75,8 @@ public Configuration getConf() { public void setConf(Configuration conf) { this.conf = conf; - setProvider(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY)); + this.provider = CryptoUtils.getJceProvider(conf); + final String secureRandomAlg = conf.get( HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index 826b567a5b9c5..5cdd2f8d53f16 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -39,6 +39,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CryptoUtils; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import javax.crypto.KeyGenerator; @@ -407,6 +408,7 @@ public KeyProvider(Configuration conf) { JCEKS_KEY_SERIALFILTER_DEFAULT); System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter); } + CryptoUtils.getJceProvider(conf); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 24a3167b3db2d..4a39a36599b79 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -773,6 +773,9 @@ public class CommonConfigurationKeysPublic { */ public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY = "hadoop.security.crypto.jce.provider"; + public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_KEY = + "hadoop.security.crypto.jce.provider.add"; + public static final boolean HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_DEFAULT = true; /** * @see * diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 92df71fb19676..649080640749e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -3625,7 +3625,19 @@ The switch to turn S3A auditing on or off. The JCE provider name used in CryptoCodec. If this value is set, the corresponding provider must be added to the provider list. The provider may be added statically in the java.security file, or - added dynamically by calling the java.security.Security.addProvider(..) method. + dynamically by calling the java.security.Security.addProvider(..) method, or + automatically (only for org.bouncycastle.jce.provider.BouncyCastleProvider) + by setting "hadoop.security.crypto.jce.provider.add" to true + + + + + hadoop.security.crypto.jce.provider.add + true + + Automatically add the org.bouncycastle.jce.provider.BouncyCastleProvider + when the value in "hadoop.security.crypto.jce.provider" is set to + BouncyCastleProvider.PROVIDER_NAME. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index e97a8bcd0b4f7..c0fdc51b1389b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -32,7 +32,6 @@ import java.math.BigInteger; import java.security.GeneralSecurityException; import java.security.SecureRandom; -import java.security.Security; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -105,11 +104,6 @@ public void testJceAesCtrCryptoCodec() throws Exception { jceAesCodecClass, opensslAesCodecClass, iv); } - static void initBouncyCastleProvider(Configuration conf) { - conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, BouncyCastleProvider.PROVIDER_NAME); - Security.addProvider(new BouncyCastleProvider()); - } - @Test(timeout=120000) public void testJceSm4CtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); @@ -120,7 +114,8 @@ public void testJceSm4CtrCryptoCodec() throws Exception { conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, JceSm4CtrCryptoCodec.class.getName()); - initBouncyCastleProvider(conf); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, + BouncyCastleProvider.PROVIDER_NAME); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, jceSm4CodecClass, jceSm4CodecClass, iv); @@ -169,7 +164,8 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { LOG.warn("Skipping test since openSSL library not loaded"); Assume.assumeTrue(false); } - initBouncyCastleProvider(conf); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, + BouncyCastleProvider.PROVIDER_NAME); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, opensslSm4CodecClass, opensslSm4CodecClass, iv); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java index c0e71bae85d0e..62573ede7d1ea 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.crypto; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.junit.BeforeClass; @@ -25,6 +26,7 @@ HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; import static org.assertj.core.api.Assertions.assertThat; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; public class TestCryptoStreamsWithJceSm4CtrCryptoCodec extends TestCryptoStreams { @@ -33,7 +35,8 @@ public class TestCryptoStreamsWithJceSm4CtrCryptoCodec extends public static void init() throws Exception { Configuration conf = new Configuration(); conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); - TestCryptoCodec.initBouncyCastleProvider(conf); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, + BouncyCastleProvider.PROVIDER_NAME); conf.set( CommonConfigurationKeysPublic. HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, From 07cebad3573d47d09d790c3eed3f1faaed75900d Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Tue, 23 Apr 2024 08:55:19 -0700 Subject: [PATCH 5/9] Address review comments. --- .../org/apache/hadoop/crypto/CryptoUtils.java | 70 +++++++++++-------- .../fs/CommonConfigurationKeysPublic.java | 6 +- .../src/main/resources/core-default.xml | 8 +-- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java index 64964d82daf50..cc35d6522e7f5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java @@ -19,6 +19,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.store.LogExactlyOnce; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,46 +28,54 @@ import java.security.Provider; import java.security.Security; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_DEFAULT; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_KEY; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; - +/** Utility methods for the crypto related features. */ @InterfaceAudience.Private public class CryptoUtils { static final Logger LOG = LoggerFactory.getLogger(CryptoUtils.class); + private static final LogExactlyOnce LOG_FAILED_TO_LOAD_CLASS = new LogExactlyOnce(LOG); + private static final LogExactlyOnce LOG_FAILED_TO_GET_FIELD = new LogExactlyOnce(LOG); + private static final LogExactlyOnce LOG_FAILED_TO_ADD_PROVIDER = new LogExactlyOnce(LOG); private static final String BOUNCY_CASTLE_PROVIDER_CLASS = "org.bouncycastle.jce.provider.BouncyCastleProvider"; private static final String PROVIDER_NAME_FIELD = "PROVIDER_NAME"; - private static void addProvider(String provider) { - if (provider == null || provider.isEmpty()) { - return; - } - try { - // For backward compatible, try to auto-add BOUNCY_CASTLE_PROVIDER_CLASS. - final Class clazz = Class.forName(BOUNCY_CASTLE_PROVIDER_CLASS); - final Field provider_name = clazz.getField("PROVIDER_NAME"); - if (provider.equals(provider_name.get(null))) { - Security.addProvider((Provider) clazz.getConstructor().newInstance()); - LOG.debug("Successfully added security provider {}", provider); + /** + * Get the security provider value specified in + * {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY} + * from the given conf. + * + * @param conf the configuration + * @return the configured provider, if there is any; otherwise, return an empty string. + */ + public static String getJceProvider(Configuration conf) { + final String provider = conf.getTrimmed( + CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, ""); + final boolean autoAdd = conf.getBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY, + CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT); + + // For backward compatible, auto-add BOUNCY_CASTLE_PROVIDER_CLASS. + if (autoAdd && !provider.isEmpty()) { + try { + // Use reflection in order to avoid statically loading the class. + final Class clazz = Class.forName(BOUNCY_CASTLE_PROVIDER_CLASS); + final Field provider_name = clazz.getField("PROVIDER_NAME"); + if (provider.equals(provider_name.get(null))) { + Security.addProvider((Provider) clazz.getConstructor().newInstance()); + LOG.debug("Successfully added security provider {}", provider); + } + } catch (ClassNotFoundException e) { + LOG_FAILED_TO_LOAD_CLASS.warn("Failed to load " + BOUNCY_CASTLE_PROVIDER_CLASS, e); + } catch (NoSuchFieldException e) { + LOG_FAILED_TO_GET_FIELD.warn("Failed to get field " + PROVIDER_NAME_FIELD + + " from class " + BOUNCY_CASTLE_PROVIDER_CLASS, e); + } catch (Exception e) { + LOG_FAILED_TO_ADD_PROVIDER.warn("Failed to add security provider for {}", provider, e); } - } catch (ClassNotFoundException e) { - LOG.warn("Failed to load " + BOUNCY_CASTLE_PROVIDER_CLASS, e); - } catch (NoSuchFieldException e) { - LOG.warn("Failed to get field " + PROVIDER_NAME_FIELD - + " from class " + BOUNCY_CASTLE_PROVIDER_CLASS, e); - } catch (Exception e) { - LOG.warn("Failed to add security provider for {}", provider, e); } + return provider; } - public static String getJceProvider(Configuration conf) { - final String jceProvider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY); - if (conf.getBoolean(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_KEY, - HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_DEFAULT)) { - CryptoUtils.addProvider(jceProvider); - } - return jceProvider; - } + private CryptoUtils() {} } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 4a39a36599b79..955a085120ed5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -773,9 +773,9 @@ public class CommonConfigurationKeysPublic { */ public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY = "hadoop.security.crypto.jce.provider"; - public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_KEY = - "hadoop.security.crypto.jce.provider.add"; - public static final boolean HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_ADD_DEFAULT = true; + public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY = + "hadoop.security.crypto.jce.provider.auto-add"; + public static final boolean HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT = true; /** * @see * diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 649080640749e..bd91de0f080fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -3627,17 +3627,17 @@ The switch to turn S3A auditing on or off. The provider may be added statically in the java.security file, or dynamically by calling the java.security.Security.addProvider(..) method, or automatically (only for org.bouncycastle.jce.provider.BouncyCastleProvider) - by setting "hadoop.security.crypto.jce.provider.add" to true + by setting "hadoop.security.crypto.jce.provider.auto-add" to true - hadoop.security.crypto.jce.provider.add + hadoop.security.crypto.jce.provider.auto-add true Automatically add the org.bouncycastle.jce.provider.BouncyCastleProvider - when the value in "hadoop.security.crypto.jce.provider" is set to - BouncyCastleProvider.PROVIDER_NAME. + when the value in "hadoop.security.crypto.jce.provider" is set + to BouncyCastleProvider.PROVIDER_NAME. From d1daa7ade1a1f83d6851c42743a146f4b9aa52dc Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Wed, 24 Apr 2024 14:03:18 -0700 Subject: [PATCH 6/9] Use "BC". --- .../org/apache/hadoop/crypto/CryptoUtils.java | 30 +++-- .../hadoop/crypto/JceCtrCryptoCodec.java | 2 +- .../apache/hadoop/crypto/package-info.java | 18 +++ .../fs/CommonConfigurationKeysPublic.java | 2 +- .../apache/hadoop/crypto/TestCryptoCodec.java | 121 ++++++++++-------- .../apache/hadoop/crypto/TestCryptoUtils.java | 85 ++++++++++++ 6 files changed, 189 insertions(+), 69 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java index cc35d6522e7f5..421e66789d332 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java @@ -17,20 +17,21 @@ */ package org.apache.hadoop.crypto; +import java.lang.reflect.Field; +import java.security.Provider; +import java.security.Security; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.store.LogExactlyOnce; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Field; -import java.security.Provider; -import java.security.Security; /** Utility methods for the crypto related features. */ @InterfaceAudience.Private -public class CryptoUtils { +public final class CryptoUtils { static final Logger LOG = LoggerFactory.getLogger(CryptoUtils.class); private static final LogExactlyOnce LOG_FAILED_TO_LOAD_CLASS = new LogExactlyOnce(LOG); private static final LogExactlyOnce LOG_FAILED_TO_GET_FIELD = new LogExactlyOnce(LOG); @@ -40,6 +41,8 @@ public class CryptoUtils { = "org.bouncycastle.jce.provider.BouncyCastleProvider"; private static final String PROVIDER_NAME_FIELD = "PROVIDER_NAME"; + static final String PROVIDER_NAME = "BC"; + /** * Get the security provider value specified in * {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY} @@ -55,15 +58,18 @@ public static String getJceProvider(Configuration conf) { CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY, CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT); - // For backward compatible, auto-add BOUNCY_CASTLE_PROVIDER_CLASS. - if (autoAdd && !provider.isEmpty()) { + // For backward compatible, auto-add BOUNCY_CASTLE_PROVIDER_CLASS when the provider is "BC". + if (autoAdd && PROVIDER_NAME.equals(provider)) { try { // Use reflection in order to avoid statically loading the class. final Class clazz = Class.forName(BOUNCY_CASTLE_PROVIDER_CLASS); - final Field provider_name = clazz.getField("PROVIDER_NAME"); - if (provider.equals(provider_name.get(null))) { + final Field providerName = clazz.getField("PROVIDER_NAME"); + if (provider.equals(providerName.get(null))) { Security.addProvider((Provider) clazz.getConstructor().newInstance()); LOG.debug("Successfully added security provider {}", provider); + if (LOG.isTraceEnabled()) { + LOG.trace("Trace", new Throwable()); + } } } catch (ClassNotFoundException e) { LOG_FAILED_TO_LOAD_CLASS.warn("Failed to load " + BOUNCY_CASTLE_PROVIDER_CLASS, e); @@ -77,5 +83,5 @@ public static String getJceProvider(Configuration conf) { return provider; } - private CryptoUtils() {} + private CryptoUtils() { } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java index f4c8a382cac88..8d15a8262e9b7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java @@ -83,7 +83,7 @@ public void setConf(Configuration conf) { HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT); try { - random = (provider != null) + random = (provider != null && !provider.isEmpty()) ? SecureRandom.getInstance(secureRandomAlg, provider) : SecureRandom.getInstance(secureRandomAlg); } catch(GeneralSecurityException e) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java new file mode 100644 index 0000000000000..38471f27e244a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 955a085120ed5..d01ddd30f4705 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -774,7 +774,7 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY = "hadoop.security.crypto.jce.provider"; public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY = - "hadoop.security.crypto.jce.provider.auto-add"; + "hadoop.security.crypto.jce.provider.auto-add"; public static final boolean HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT = true; /** * @see diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index c0fdc51b1389b..f8a6030a4808e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -55,12 +55,12 @@ import org.slf4j.LoggerFactory; public class TestCryptoCodec { - private static final Logger LOG= LoggerFactory.getLogger(TestCryptoCodec + private static final Logger LOG = LoggerFactory.getLogger(TestCryptoCodec .class); private static byte[] key = new byte[16]; private static byte[] iv = new byte[16]; private static final int bufferSize = 4096; - + private Configuration conf = new Configuration(); private int count = 10000; private int seed = new Random().nextInt(); @@ -72,7 +72,7 @@ public class TestCryptoCodec { "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"; private final String opensslSm4CodecClass = "org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec"; - + @Before public void setUp() throws IOException { Random random = new SecureRandom(); @@ -80,7 +80,7 @@ public void setUp() throws IOException { random.nextBytes(iv); } - @Test(timeout=120000) + @Test(timeout = 120000) public void testJceAesCtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -95,7 +95,7 @@ public void testJceAesCtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, jceAesCodecClass, opensslAesCodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for(int i = 0; i < 8; i++) { + for (int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -104,7 +104,7 @@ public void testJceAesCtrCryptoCodec() throws Exception { jceAesCodecClass, opensslAesCodecClass, iv); } - @Test(timeout=120000) + @Test(timeout = 120000) public void testJceSm4CtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -115,7 +115,7 @@ public void testJceSm4CtrCryptoCodec() throws Exception { conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, JceSm4CtrCryptoCodec.class.getName()); conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, - BouncyCastleProvider.PROVIDER_NAME); + BouncyCastleProvider.PROVIDER_NAME); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, jceSm4CodecClass, jceSm4CodecClass, iv); @@ -124,7 +124,7 @@ public void testJceSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, jceSm4CodecClass, opensslSm4CodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for(int i = 0; i < 8; i++) { + for (int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -132,8 +132,8 @@ public void testJceSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, jceSm4CodecClass, opensslSm4CodecClass, iv); } - - @Test(timeout=120000) + + @Test(timeout = 120000) public void testOpensslAesCtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -148,7 +148,7 @@ public void testOpensslAesCtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, opensslAesCodecClass, jceAesCodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for(int i = 0; i < 8; i++) { + for (int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -157,7 +157,7 @@ public void testOpensslAesCtrCryptoCodec() throws Exception { opensslAesCodecClass, jceAesCodecClass, iv); } - @Test(timeout=120000) + @Test(timeout = 120000) public void testOpensslSm4CtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -165,7 +165,7 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { Assume.assumeTrue(false); } conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, - BouncyCastleProvider.PROVIDER_NAME); + BouncyCastleProvider.PROVIDER_NAME); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, opensslSm4CodecClass, opensslSm4CodecClass, iv); @@ -174,7 +174,7 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, opensslSm4CodecClass, jceSm4CodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for(int i = 0; i < 8; i++) { + for (int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -182,77 +182,77 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, opensslSm4CodecClass, jceSm4CodecClass, iv); } - - private void cryptoCodecTest(Configuration conf, int seed, int count, - String encCodecClass, String decCodecClass, byte[] iv) throws IOException, + + private void cryptoCodecTest(Configuration conf, int seed, int count, + String encCodecClass, String decCodecClass, byte[] iv) throws IOException, GeneralSecurityException { CryptoCodec encCodec = null; try { - encCodec = (CryptoCodec)ReflectionUtils.newInstance( + encCodec = (CryptoCodec) ReflectionUtils.newInstance( conf.getClassByName(encCodecClass), conf); } catch (ClassNotFoundException cnfe) { throw new IOException("Illegal crypto codec!"); } LOG.info("Created a Codec object of type: " + encCodecClass); - + // Generate data DataOutputBuffer data = new DataOutputBuffer(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); - for(int i = 0; i < count; ++i) { + for (int i = 0; i < count; ++i) { generator.next(); RandomDatum key = generator.getKey(); RandomDatum value = generator.getValue(); - + key.write(data); value.write(data); } LOG.info("Generated " + count + " records"); - + // Encrypt data DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer(); - CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer, + CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer, encCodec, bufferSize, key, iv); out.write(data.getData(), 0, data.getLength()); out.flush(); out.close(); LOG.info("Finished encrypting data"); - + CryptoCodec decCodec = null; try { - decCodec = (CryptoCodec)ReflectionUtils.newInstance( + decCodec = (CryptoCodec) ReflectionUtils.newInstance( conf.getClassByName(decCodecClass), conf); } catch (ClassNotFoundException cnfe) { throw new IOException("Illegal crypto codec!"); } LOG.info("Created a Codec object of type: " + decCodecClass); - + // Decrypt data DataInputBuffer decryptedDataBuffer = new DataInputBuffer(); - decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, + decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, encryptedDataBuffer.getLength()); - CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer, + CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize, key, iv); DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in)); - + // Check DataInputBuffer originalData = new DataInputBuffer(); originalData.reset(data.getData(), 0, data.getLength()); DataInputStream originalIn = new DataInputStream( new BufferedInputStream(originalData)); - - for(int i=0; i < count; ++i) { + + for (int i = 0; i < count; ++i) { RandomDatum k1 = new RandomDatum(); RandomDatum v1 = new RandomDatum(); k1.readFields(originalIn); v1.readFields(originalIn); - + RandomDatum k2 = new RandomDatum(); RandomDatum v2 = new RandomDatum(); k2.readFields(dataIn); v2.readFields(dataIn); assertTrue("original and encrypted-then-decrypted-output not equal", - k1.equals(k2) && v1.equals(v2)); - + k1.equals(k2) && v1.equals(v2)); + // original and encrypted-then-decrypted-output have the same hashCode Map m = new HashMap(); m.put(k1, k1.toString()); @@ -265,9 +265,9 @@ private void cryptoCodecTest(Configuration conf, int seed, int count, // Decrypt data byte-at-a-time originalData.reset(data.getData(), 0, data.getLength()); - decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, + decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, encryptedDataBuffer.getLength()); - in = new CryptoInputStream(decryptedDataBuffer, + in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize, key, iv); // Check @@ -276,9 +276,9 @@ private void cryptoCodecTest(Configuration conf, int seed, int count, do { expected = originalIn.read(); assertEquals("Decrypted stream read by byte does not match", - expected, in.read()); + expected, in.read()); } while (expected != -1); - + // Seek to a certain position and decrypt originalData.reset(data.getData(), 0, data.getLength()); decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, @@ -287,24 +287,26 @@ private void cryptoCodecTest(Configuration conf, int seed, int count, decryptedDataBuffer), decCodec, bufferSize, key, iv); int seekPos = data.getLength() / 3; in.seek(seekPos); - + // Check - TestCryptoStreams.FakeInputStream originalInput = + TestCryptoStreams.FakeInputStream originalInput = new TestCryptoStreams.FakeInputStream(originalData); originalInput.seek(seekPos); do { expected = originalInput.read(); assertEquals("Decrypted stream read by byte does not match", - expected, in.read()); + expected, in.read()); } while (expected != -1); LOG.info("SUCCESS! Completed checking " + count + " records"); - + // Check secure random generator testSecureRandom(encCodec); } - - /** Test secure random generator */ + + /** + * Test secure random generator + */ private void testSecureRandom(CryptoCodec codec) { // len = 16 checkSecureRandom(codec, 16); @@ -313,22 +315,22 @@ private void testSecureRandom(CryptoCodec codec) { // len = 128 checkSecureRandom(codec, 128); } - + private void checkSecureRandom(CryptoCodec codec, int len) { byte[] rand = new byte[len]; byte[] rand1 = new byte[len]; codec.generateSecureRandom(rand); codec.generateSecureRandom(rand1); - + Assert.assertEquals(len, rand.length); Assert.assertEquals(len, rand1.length); Assert.assertFalse(Arrays.equals(rand, rand1)); } - + /** * Regression test for IV calculation, see HADOOP-11343 */ - @Test(timeout=120000) + @Test(timeout = 120000) public void testCalculateIV() throws Exception { JceAesCtrCryptoCodec codec = new JceAesCtrCryptoCodec(); codec.setConf(conf); @@ -341,28 +343,28 @@ public void testCalculateIV() throws Exception { long counter = 10000; // Overflow test, IV: 00 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff - for(int i = 0; i < 8; i++) { - initIV[8 + i] = (byte)0xff; + for (int i = 0; i < 8; i++) { + initIV[8 + i] = (byte) 0xff; } - for(long j = 0; j < counter; j++) { + for (long j = 0; j < counter; j++) { assertIVCalculation(codec, initIV, j, IV); } // Random IV and counter sequence test - for(long i = 0; i < iterations; i++) { + for (long i = 0; i < iterations; i++) { sr.nextBytes(initIV); - for(long j = 0; j < counter; j++) { + for (long j = 0; j < counter; j++) { assertIVCalculation(codec, initIV, j, IV); } } // Random IV and random counter test - for(long i = 0; i < iterations; i++) { + for (long i = 0; i < iterations; i++) { sr.nextBytes(initIV); - for(long j = 0; j < counter; j++) { + for (long j = 0; j < counter; j++) { long c = sr.nextLong(); assertIVCalculation(codec, initIV, c, IV); } @@ -384,4 +386,13 @@ private static BigInteger calculateRef(byte[] initIV, long counter) { BigInteger bi = new BigInteger(1, initIV); return bi.add(new BigInteger(1, cb)); } + + @Test(timeout = 10_000) + public void testBC() throws Exception { + JceAesCtrCryptoCodec codec = new JceAesCtrCryptoCodec(); + Configuration conf = new Configuration(); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, "BC"); + + + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java new file mode 100644 index 0000000000000..c358e3a3311e4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.event.Level; + +import java.security.Provider; +import java.security.Security; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; + +/** Test {@link CryptoUtils}. */ +public class TestCryptoUtils { + static { + GenericTestUtils.setLogLevel(CryptoUtils.LOG, Level.TRACE); + } + + @Test(timeout = 1_000) + public void testProviderName() { + Assert.assertEquals(CryptoUtils.PROVIDER_NAME, BouncyCastleProvider.PROVIDER_NAME); + } + + static void assertRemoveProvider() { + Security.removeProvider(BouncyCastleProvider.PROVIDER_NAME); + Assert.assertNull(Security.getProvider(BouncyCastleProvider.PROVIDER_NAME)); + } + + static void assertSetProvider(Configuration conf) { + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.PROVIDER_NAME); + final String providerFromConf = CryptoUtils.getJceProvider(conf); + Assert.assertEquals(CryptoUtils.PROVIDER_NAME, providerFromConf); + } + + @Test(timeout = 5_000) + public void testAutoAddDisabled() { + assertRemoveProvider(); + + final Configuration conf = new Configuration(); + conf.setBoolean(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY, false); + + assertSetProvider(conf); + + Assert.assertNull(Security.getProvider(BouncyCastleProvider.PROVIDER_NAME)); + } + + @Test(timeout = 5_000) + public void testAutoAddEnabled() { + assertRemoveProvider(); + + final Configuration conf = new Configuration(); + Assert.assertTrue("true".equalsIgnoreCase(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY))); + Assert.assertTrue(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT); + + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.PROVIDER_NAME); + final String providerFromConf = CryptoUtils.getJceProvider(conf); + Assert.assertEquals(CryptoUtils.PROVIDER_NAME, providerFromConf); + + final Provider provider = Security.getProvider(BouncyCastleProvider.PROVIDER_NAME); + Assert.assertTrue(provider instanceof BouncyCastleProvider); + + assertRemoveProvider(); + } +} From a1411f11ea8c85b47d37e8d5fa53920b3824bb62 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 25 Apr 2024 09:11:15 -0700 Subject: [PATCH 7/9] Revert writespace changes. --- .../apache/hadoop/crypto/package-info.java | 2 + .../apache/hadoop/crypto/TestCryptoCodec.java | 121 ++++++++---------- .../apache/hadoop/crypto/TestCryptoUtils.java | 3 +- 3 files changed, 59 insertions(+), 67 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java index 38471f27e244a..fe947dc263020 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java @@ -15,4 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/** Crypto related classes. */ package org.apache.hadoop.crypto; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index f8a6030a4808e..c0fdc51b1389b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -55,12 +55,12 @@ import org.slf4j.LoggerFactory; public class TestCryptoCodec { - private static final Logger LOG = LoggerFactory.getLogger(TestCryptoCodec + private static final Logger LOG= LoggerFactory.getLogger(TestCryptoCodec .class); private static byte[] key = new byte[16]; private static byte[] iv = new byte[16]; private static final int bufferSize = 4096; - + private Configuration conf = new Configuration(); private int count = 10000; private int seed = new Random().nextInt(); @@ -72,7 +72,7 @@ public class TestCryptoCodec { "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"; private final String opensslSm4CodecClass = "org.apache.hadoop.crypto.OpensslSm4CtrCryptoCodec"; - + @Before public void setUp() throws IOException { Random random = new SecureRandom(); @@ -80,7 +80,7 @@ public void setUp() throws IOException { random.nextBytes(iv); } - @Test(timeout = 120000) + @Test(timeout=120000) public void testJceAesCtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -95,7 +95,7 @@ public void testJceAesCtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, jceAesCodecClass, opensslAesCodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for (int i = 0; i < 8; i++) { + for(int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -104,7 +104,7 @@ public void testJceAesCtrCryptoCodec() throws Exception { jceAesCodecClass, opensslAesCodecClass, iv); } - @Test(timeout = 120000) + @Test(timeout=120000) public void testJceSm4CtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -115,7 +115,7 @@ public void testJceSm4CtrCryptoCodec() throws Exception { conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, JceSm4CtrCryptoCodec.class.getName()); conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, - BouncyCastleProvider.PROVIDER_NAME); + BouncyCastleProvider.PROVIDER_NAME); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, jceSm4CodecClass, jceSm4CodecClass, iv); @@ -124,7 +124,7 @@ public void testJceSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, jceSm4CodecClass, opensslSm4CodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for (int i = 0; i < 8; i++) { + for(int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -132,8 +132,8 @@ public void testJceSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, jceSm4CodecClass, opensslSm4CodecClass, iv); } - - @Test(timeout = 120000) + + @Test(timeout=120000) public void testOpensslAesCtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -148,7 +148,7 @@ public void testOpensslAesCtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, opensslAesCodecClass, jceAesCodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for (int i = 0; i < 8; i++) { + for(int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -157,7 +157,7 @@ public void testOpensslAesCtrCryptoCodec() throws Exception { opensslAesCodecClass, jceAesCodecClass, iv); } - @Test(timeout = 120000) + @Test(timeout=120000) public void testOpensslSm4CtrCryptoCodec() throws Exception { GenericTestUtils.assumeInNativeProfile(); if (!NativeCodeLoader.buildSupportsOpenssl()) { @@ -165,7 +165,7 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { Assume.assumeTrue(false); } conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, - BouncyCastleProvider.PROVIDER_NAME); + BouncyCastleProvider.PROVIDER_NAME); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, opensslSm4CodecClass, opensslSm4CodecClass, iv); @@ -174,7 +174,7 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, opensslSm4CodecClass, jceSm4CodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff - for (int i = 0; i < 8; i++) { + for(int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, @@ -182,77 +182,77 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { cryptoCodecTest(conf, seed, count, opensslSm4CodecClass, jceSm4CodecClass, iv); } - - private void cryptoCodecTest(Configuration conf, int seed, int count, - String encCodecClass, String decCodecClass, byte[] iv) throws IOException, + + private void cryptoCodecTest(Configuration conf, int seed, int count, + String encCodecClass, String decCodecClass, byte[] iv) throws IOException, GeneralSecurityException { CryptoCodec encCodec = null; try { - encCodec = (CryptoCodec) ReflectionUtils.newInstance( + encCodec = (CryptoCodec)ReflectionUtils.newInstance( conf.getClassByName(encCodecClass), conf); } catch (ClassNotFoundException cnfe) { throw new IOException("Illegal crypto codec!"); } LOG.info("Created a Codec object of type: " + encCodecClass); - + // Generate data DataOutputBuffer data = new DataOutputBuffer(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); - for (int i = 0; i < count; ++i) { + for(int i = 0; i < count; ++i) { generator.next(); RandomDatum key = generator.getKey(); RandomDatum value = generator.getValue(); - + key.write(data); value.write(data); } LOG.info("Generated " + count + " records"); - + // Encrypt data DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer(); - CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer, + CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer, encCodec, bufferSize, key, iv); out.write(data.getData(), 0, data.getLength()); out.flush(); out.close(); LOG.info("Finished encrypting data"); - + CryptoCodec decCodec = null; try { - decCodec = (CryptoCodec) ReflectionUtils.newInstance( + decCodec = (CryptoCodec)ReflectionUtils.newInstance( conf.getClassByName(decCodecClass), conf); } catch (ClassNotFoundException cnfe) { throw new IOException("Illegal crypto codec!"); } LOG.info("Created a Codec object of type: " + decCodecClass); - + // Decrypt data DataInputBuffer decryptedDataBuffer = new DataInputBuffer(); - decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, + decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, encryptedDataBuffer.getLength()); - CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer, + CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize, key, iv); DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in)); - + // Check DataInputBuffer originalData = new DataInputBuffer(); originalData.reset(data.getData(), 0, data.getLength()); DataInputStream originalIn = new DataInputStream( new BufferedInputStream(originalData)); - - for (int i = 0; i < count; ++i) { + + for(int i=0; i < count; ++i) { RandomDatum k1 = new RandomDatum(); RandomDatum v1 = new RandomDatum(); k1.readFields(originalIn); v1.readFields(originalIn); - + RandomDatum k2 = new RandomDatum(); RandomDatum v2 = new RandomDatum(); k2.readFields(dataIn); v2.readFields(dataIn); assertTrue("original and encrypted-then-decrypted-output not equal", - k1.equals(k2) && v1.equals(v2)); - + k1.equals(k2) && v1.equals(v2)); + // original and encrypted-then-decrypted-output have the same hashCode Map m = new HashMap(); m.put(k1, k1.toString()); @@ -265,9 +265,9 @@ private void cryptoCodecTest(Configuration conf, int seed, int count, // Decrypt data byte-at-a-time originalData.reset(data.getData(), 0, data.getLength()); - decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, + decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, encryptedDataBuffer.getLength()); - in = new CryptoInputStream(decryptedDataBuffer, + in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize, key, iv); // Check @@ -276,9 +276,9 @@ private void cryptoCodecTest(Configuration conf, int seed, int count, do { expected = originalIn.read(); assertEquals("Decrypted stream read by byte does not match", - expected, in.read()); + expected, in.read()); } while (expected != -1); - + // Seek to a certain position and decrypt originalData.reset(data.getData(), 0, data.getLength()); decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, @@ -287,26 +287,24 @@ private void cryptoCodecTest(Configuration conf, int seed, int count, decryptedDataBuffer), decCodec, bufferSize, key, iv); int seekPos = data.getLength() / 3; in.seek(seekPos); - + // Check - TestCryptoStreams.FakeInputStream originalInput = + TestCryptoStreams.FakeInputStream originalInput = new TestCryptoStreams.FakeInputStream(originalData); originalInput.seek(seekPos); do { expected = originalInput.read(); assertEquals("Decrypted stream read by byte does not match", - expected, in.read()); + expected, in.read()); } while (expected != -1); LOG.info("SUCCESS! Completed checking " + count + " records"); - + // Check secure random generator testSecureRandom(encCodec); } - - /** - * Test secure random generator - */ + + /** Test secure random generator */ private void testSecureRandom(CryptoCodec codec) { // len = 16 checkSecureRandom(codec, 16); @@ -315,22 +313,22 @@ private void testSecureRandom(CryptoCodec codec) { // len = 128 checkSecureRandom(codec, 128); } - + private void checkSecureRandom(CryptoCodec codec, int len) { byte[] rand = new byte[len]; byte[] rand1 = new byte[len]; codec.generateSecureRandom(rand); codec.generateSecureRandom(rand1); - + Assert.assertEquals(len, rand.length); Assert.assertEquals(len, rand1.length); Assert.assertFalse(Arrays.equals(rand, rand1)); } - + /** * Regression test for IV calculation, see HADOOP-11343 */ - @Test(timeout = 120000) + @Test(timeout=120000) public void testCalculateIV() throws Exception { JceAesCtrCryptoCodec codec = new JceAesCtrCryptoCodec(); codec.setConf(conf); @@ -343,28 +341,28 @@ public void testCalculateIV() throws Exception { long counter = 10000; // Overflow test, IV: 00 00 00 00 00 00 00 00 ff ff ff ff ff ff ff ff - for (int i = 0; i < 8; i++) { - initIV[8 + i] = (byte) 0xff; + for(int i = 0; i < 8; i++) { + initIV[8 + i] = (byte)0xff; } - for (long j = 0; j < counter; j++) { + for(long j = 0; j < counter; j++) { assertIVCalculation(codec, initIV, j, IV); } // Random IV and counter sequence test - for (long i = 0; i < iterations; i++) { + for(long i = 0; i < iterations; i++) { sr.nextBytes(initIV); - for (long j = 0; j < counter; j++) { + for(long j = 0; j < counter; j++) { assertIVCalculation(codec, initIV, j, IV); } } // Random IV and random counter test - for (long i = 0; i < iterations; i++) { + for(long i = 0; i < iterations; i++) { sr.nextBytes(initIV); - for (long j = 0; j < counter; j++) { + for(long j = 0; j < counter; j++) { long c = sr.nextLong(); assertIVCalculation(codec, initIV, c, IV); } @@ -386,13 +384,4 @@ private static BigInteger calculateRef(byte[] initIV, long counter) { BigInteger bi = new BigInteger(1, initIV); return bi.add(new BigInteger(1, cb)); } - - @Test(timeout = 10_000) - public void testBC() throws Exception { - JceAesCtrCryptoCodec codec = new JceAesCtrCryptoCodec(); - Configuration conf = new Configuration(); - conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, "BC"); - - - } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java index c358e3a3311e4..4f4fd923bd710 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java @@ -70,7 +70,8 @@ public void testAutoAddEnabled() { assertRemoveProvider(); final Configuration conf = new Configuration(); - Assert.assertTrue("true".equalsIgnoreCase(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY))); + Assert.assertTrue("true".equalsIgnoreCase( + conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY))); Assert.assertTrue(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT); conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.PROVIDER_NAME); From 6494370d68379abf2a4adbd0753f9a798abaad81 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Mon, 29 Apr 2024 08:10:25 -0700 Subject: [PATCH 8/9] Rename field check. --- .../org/apache/hadoop/crypto/CryptoUtils.java | 22 +++++-------------- .../apache/hadoop/crypto/TestCryptoUtils.java | 10 ++++----- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java index 421e66789d332..3d118457df7cd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.crypto; -import java.lang.reflect.Field; import java.security.Provider; import java.security.Security; @@ -34,14 +33,11 @@ public final class CryptoUtils { static final Logger LOG = LoggerFactory.getLogger(CryptoUtils.class); private static final LogExactlyOnce LOG_FAILED_TO_LOAD_CLASS = new LogExactlyOnce(LOG); - private static final LogExactlyOnce LOG_FAILED_TO_GET_FIELD = new LogExactlyOnce(LOG); private static final LogExactlyOnce LOG_FAILED_TO_ADD_PROVIDER = new LogExactlyOnce(LOG); private static final String BOUNCY_CASTLE_PROVIDER_CLASS = "org.bouncycastle.jce.provider.BouncyCastleProvider"; - private static final String PROVIDER_NAME_FIELD = "PROVIDER_NAME"; - - static final String PROVIDER_NAME = "BC"; + static final String BOUNCY_CASTLE_PROVIDER_NAME = "BC"; /** * Get the security provider value specified in @@ -59,23 +55,17 @@ public static String getJceProvider(Configuration conf) { CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT); // For backward compatible, auto-add BOUNCY_CASTLE_PROVIDER_CLASS when the provider is "BC". - if (autoAdd && PROVIDER_NAME.equals(provider)) { + if (autoAdd && BOUNCY_CASTLE_PROVIDER_NAME.equals(provider)) { try { // Use reflection in order to avoid statically loading the class. final Class clazz = Class.forName(BOUNCY_CASTLE_PROVIDER_CLASS); - final Field providerName = clazz.getField("PROVIDER_NAME"); - if (provider.equals(providerName.get(null))) { - Security.addProvider((Provider) clazz.getConstructor().newInstance()); - LOG.debug("Successfully added security provider {}", provider); - if (LOG.isTraceEnabled()) { - LOG.trace("Trace", new Throwable()); - } + Security.addProvider((Provider) clazz.getConstructor().newInstance()); + LOG.debug("Successfully added security provider {}", provider); + if (LOG.isTraceEnabled()) { + LOG.trace("Trace", new Throwable()); } } catch (ClassNotFoundException e) { LOG_FAILED_TO_LOAD_CLASS.warn("Failed to load " + BOUNCY_CASTLE_PROVIDER_CLASS, e); - } catch (NoSuchFieldException e) { - LOG_FAILED_TO_GET_FIELD.warn("Failed to get field " + PROVIDER_NAME_FIELD - + " from class " + BOUNCY_CASTLE_PROVIDER_CLASS, e); } catch (Exception e) { LOG_FAILED_TO_ADD_PROVIDER.warn("Failed to add security provider for {}", provider, e); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java index 4f4fd923bd710..e0b6832d26f1a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java @@ -39,7 +39,7 @@ public class TestCryptoUtils { @Test(timeout = 1_000) public void testProviderName() { - Assert.assertEquals(CryptoUtils.PROVIDER_NAME, BouncyCastleProvider.PROVIDER_NAME); + Assert.assertEquals(CryptoUtils.BOUNCY_CASTLE_PROVIDER_NAME, BouncyCastleProvider.PROVIDER_NAME); } static void assertRemoveProvider() { @@ -48,9 +48,9 @@ static void assertRemoveProvider() { } static void assertSetProvider(Configuration conf) { - conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.PROVIDER_NAME); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.BOUNCY_CASTLE_PROVIDER_NAME); final String providerFromConf = CryptoUtils.getJceProvider(conf); - Assert.assertEquals(CryptoUtils.PROVIDER_NAME, providerFromConf); + Assert.assertEquals(CryptoUtils.BOUNCY_CASTLE_PROVIDER_NAME, providerFromConf); } @Test(timeout = 5_000) @@ -74,9 +74,9 @@ public void testAutoAddEnabled() { conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY))); Assert.assertTrue(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT); - conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.PROVIDER_NAME); + conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.BOUNCY_CASTLE_PROVIDER_NAME); final String providerFromConf = CryptoUtils.getJceProvider(conf); - Assert.assertEquals(CryptoUtils.PROVIDER_NAME, providerFromConf); + Assert.assertEquals(CryptoUtils.BOUNCY_CASTLE_PROVIDER_NAME, providerFromConf); final Provider provider = Security.getProvider(BouncyCastleProvider.PROVIDER_NAME); Assert.assertTrue(provider instanceof BouncyCastleProvider); From 20b511efc294d7d76b8ec6f3ae533a65e200d0ab Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Mon, 13 May 2024 11:47:08 -0700 Subject: [PATCH 9/9] Use Assertions.assertThat. --- .../java/org/apache/hadoop/crypto/TestCryptoUtils.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java index e0b6832d26f1a..be3695472409c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.test.GenericTestUtils; +import org.assertj.core.api.Assertions; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.Assert; import org.junit.Test; @@ -70,8 +71,9 @@ public void testAutoAddEnabled() { assertRemoveProvider(); final Configuration conf = new Configuration(); - Assert.assertTrue("true".equalsIgnoreCase( - conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY))); + Assertions.assertThat(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY)) + .describedAs("conf: " + HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_KEY) + .isEqualToIgnoringCase("true"); Assert.assertTrue(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_AUTO_ADD_DEFAULT); conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, CryptoUtils.BOUNCY_CASTLE_PROVIDER_NAME); @@ -79,7 +81,8 @@ public void testAutoAddEnabled() { Assert.assertEquals(CryptoUtils.BOUNCY_CASTLE_PROVIDER_NAME, providerFromConf); final Provider provider = Security.getProvider(BouncyCastleProvider.PROVIDER_NAME); - Assert.assertTrue(provider instanceof BouncyCastleProvider); + Assertions.assertThat(provider) + .isInstanceOf(BouncyCastleProvider.class); assertRemoveProvider(); }